fix: check the provider only on node tables and capture iSCSI state when the EBS gate fails

An ocibsd API-key run seeds spinifex.toml with the credential before init,
so a non-empty file no longer shows that formation ran, and a checkpoint
left by an earlier run skipped init on fresh nodes.

.github/workflows/e2e-cloudvendor-nightly.yml

@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
     inputs:
       topologies:
-        description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
+        description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal vm-single-ocibsd vm-multi-ocibsd). One name runs one topology."
         required: false
       workbooks:
         description: "Space-separated workbooks (blank = the driver's default list)"
@@ -62,9 +62,10 @@ jobs:
     # a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
     # lifts the six-hour cap a hosted job carries.
     runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
-    # Three topologies including a bare-metal build, each with its own install,
-    # formation and workbook pass. The sweep still runs if this fires.
-    timeout-minutes: 300
+    # Six topologies including a bare-metal build, each with its own install,
+    # formation and workbook pass, so this is the hosted runner's cap. The sweep
+    # still runs if this fires, ocibsd volumes and backups included.
+    timeout-minutes: 360
     steps:
       # Set here rather than in a job-level env block: the runner context is not
       # available at job level, only from a step onwards, and a workflow that
@@ -236,8 +237,8 @@ jobs:
 
       # always(), and before the artifact upload: an OCI bare-metal host left
       # running by a cancelled or timed-out job is the most expensive way this
-      # workflow can fail. The sweep works from the Terraform state files, so it
-      # can only destroy what this run created.
+      # workflow can fail. It destroys by state file, then deletes OCI volumes and
+      # backups by the cluster tag each ocibsd topology recorded.
       - name: Sweep anything left behind
         if: always() && !inputs.keep_on_fail
         working-directory: spinifex/scripts/terraform/oci-spx

.github/workflows/publish-system-ami.yml

@@ -1,21 +1,20 @@
 name: Publish System AMIs
 
-# Builds the system AMIs (libguestfs, on the runner) and publishes each
-# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Six images ship:
-#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
-#   spinifex-ecs-node       Alpine ECS container instance
-#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
-#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
-#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
-#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
+# Builds the system AMIs and publishes each compressed qcow2 + .sha256 sidecar
+# to the Cloudflare R2 bucket that backs iso.mulgadc.com/system-ami/. The Alpine
+# images build with libguestfs (build-system-image.sh); the Ubuntu GPU images
+# build with mkosi, which installs from resolute-updates and resolute-security,
+# so each release ships the newest kernel. Eight images ship:
+#   spinifex-eks-node               Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node               Alpine ECS container instance
+#   spinifex-rds-postgres           Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb            Alpine RDS MariaDB instance
+#   spinifex-eks-node-gpu           Ubuntu EKS K3s GPU worker (mkosi)
+#   spinifex-ecs-node-gpu           Ubuntu ECS GPU container instance (mkosi)
+#   ubuntu-26.04-nvidia-gpu-x86_64  Ubuntu NVIDIA GPU base image (mkosi)
+#   ubuntu-26.04-amd-gpu-x86_64     Ubuntu AMD ROCm GPU base image (mkosi)
 # Operators then install them on any cluster with:
-#   spx admin images import --name spinifex-eks-node
-#   spx admin images import --name spinifex-ecs-node
-#   spx admin images import --name spinifex-eks-node-gpu
-#   spx admin images import --name spinifex-ecs-node-gpu
-#   spx admin images import --name spinifex-rds-postgres
-#   spx admin images import --name spinifex-rds-mariadb
+#   spx admin images import --name <catalog key>
 #
 # Publishing overwrites the mirror artifact, so it runs only from release.yml
 # once a tag has passed its gates, with siblings at the tags go.mod requires.
@@ -54,9 +53,12 @@ jobs:
           VARIANT: ${{ inputs.variant }}
         run: |
           set -euo pipefail
-          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
-          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
-          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
+          # builder=legacy entries carry a manifest for build-system-image.sh;
+          # builder=mkosi entries carry the mkosi-build.sh --image name and the
+          # R2 object base, which must equal the catalog URL basename.
+          cpu='{"name":"spinifex-eks-node","builder":"legacy","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","builder":"legacy","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","builder":"mkosi","image":"spinifex-eks-node-gpu","object":"spinifex-eks-node-gpu-x86_64"},{"name":"spinifex-ecs-node-gpu","builder":"mkosi","image":"spinifex-ecs-node-gpu","object":"spinifex-ecs-node-gpu-x86_64"},{"name":"ubuntu-26.04-nvidia-gpu","builder":"mkosi","image":"ubuntu-gpu-nvidia","object":"ubuntu-26.04-nvidia-gpu-x86_64"},{"name":"ubuntu-26.04-amd-gpu","builder":"mkosi","image":"ubuntu-gpu-amd","object":"ubuntu-26.04-amd-gpu-x86_64"}'
+          rds='{"name":"spinifex-rds-postgres","builder":"legacy","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","builder":"legacy","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
@@ -132,6 +134,8 @@ jobs:
           go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Install build + upload tooling
+        # mkosi legs build and upload inside the pinned builder container.
+        if: matrix.builder == 'legacy'
         run: |
           set -euo pipefail
           sudo apt-get update -qq
@@ -144,6 +148,7 @@ jobs:
           fi
 
       - name: Build + publish ${{ matrix.name }} to R2
+        if: matrix.builder == 'legacy'
         working-directory: spinifex
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
@@ -168,3 +173,34 @@ jobs:
             FLAGS+=(--dry-run)
           fi
           ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"
+
+      - name: Build + publish ${{ matrix.name }} to R2 (mkosi)
+        if: matrix.builder == 'mkosi'
+        working-directory: spinifex
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
+          R2_BUCKET: spinifex-iso
+          DRY_RUN: ${{ inputs.dry_run }}
+          IMAGE: ${{ matrix.image }}
+          OBJECT: ${{ matrix.object }}
+          # On-disk scratch for the same reason as the legacy legs: the GPU raw
+          # and its qcow2 do not fit a RAM-backed /tmp.
+          MKOSI_OUTPUT_DIR: ${{ runner.temp }}/mkosi-output
+        run: |
+          set -euo pipefail
+          mkdir -p "${MKOSI_OUTPUT_DIR}"
+          ./scripts/mkosi-build.sh --image "${IMAGE}" -- --manifest-format=json
+          # The kernel this image boots, from mkosi's package manifest.
+          KERNEL=$(jq -r '.packages[] | select(.name | test("^linux-image-[0-9]")) | "\(.name) \(.version)"' "${MKOSI_OUTPUT_DIR}/${IMAGE}.manifest")
+          if [[ -z "${KERNEL}" ]]; then
+            echo "::error::no linux-image-<version> package in ${IMAGE}.manifest" >&2
+            exit 1
+          fi
+          echo "kernel: ${KERNEL}"
+          FLAGS=(--image "${IMAGE}" --name "${OBJECT}")
+          if [[ "${DRY_RUN}" == "true" ]]; then
+            FLAGS+=(--dry-run)
+          fi
+          ./scripts/mkosi-publish.sh "${FLAGS[@]}"

Makefile

@@ -515,9 +515,9 @@ install-system:
 	fi
 
 install-go:
-	@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.4 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS builder
+FROM golang:1.27.4-bookworm@sha256:4fa7c968ba93f1635670522dbb01a2c78150443e1385746924566079f36570ea AS builder
 
 ARG VERSION
 
@@ -51,6 +51,8 @@ RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
 RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
 RUN cd spinifex/scripts && cp install-node.sh smoke-test.sh node-reset.sh uninstall-spx.sh /out/
 RUN mkdir -p /out/udev && cp spinifex/build/udev/* /out/udev/
+# Staged by setup.sh on an OCI VM; spx admin init or join installs it on ocibsd nodes.
+RUN cp -r spinifex/build/ocibsd /out/ocibsd
 RUN mkdir -p /out/lib && cp spinifex/scripts/lib/distro.sh /out/lib/distro.sh
 RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
 RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/

build/ocibsd/99-spinifex-ocibsd.rules

@@ -0,0 +1,4 @@
+# Give QEMU (group spinifex) read-write on the disks of OCI Block Volume
+# attachments, named by ID_PATH, which is also the /dev/disk/by-path link.
+# Partitions and every other block device keep their default ownership.
+SUBSYSTEM=="block", ACTION!="remove", ENV{DEVTYPE}=="disk", ENV{ID_PATH}=="ip-*-iscsi-iqn.2015-12.com.oracleiaas:*", GROUP="spinifex", MODE="0660"

build/ocibsd/spinifex-daemon.service.d/ocibsd.conf

@@ -0,0 +1,7 @@
+# ocibsd nodes only: QEMU, in the daemon's device cgroup, opens the SCSI disk
+# an iSCSI login creates, including one attached after the daemon started.
+[Unit]
+After=spinifex-ocibsd.service
+
+[Service]
+DeviceAllow=block-sd rw

build/ocibsd/spinifex-ocibsd.service

@@ -0,0 +1,65 @@
+# spinifex-unit-version: 1
+[Unit]
+Description=Spinifex ocibsd (OCI Block Volumes)
+PartOf=spinifex.target
+After=spinifex-nats.service iscsid.socket iscsid.service
+Wants=spinifex-nats.service iscsid.socket
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+# Root without capabilities: iscsid answers iscsiadm only for uid 0, and the
+# node records, lock and devices it touches are root-owned. iscsid does the
+# privileged session and module work.
+User=root
+Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/usr/local/bin/spx service ocibsd start
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+OOMScoreAdjust=-500
+MemoryHigh=10%
+MemoryMax=20%
+EnvironmentFile=/etc/spinifex/systemd.env
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/ocibsd/
+StateDirectory=spinifex/ocibsd
+
+# Filesystem access: iscsiadm writes node records and its lock
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=no
+DevicePolicy=closed
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem
+ReadWritePaths=/var/log/spinifex -/etc/iscsi -/var/lib/iscsi -/run/lock
+# read-only, not yes: a config_file credential may default to ~/.oci/config
+ProtectHome=read-only
+
+# Hardening
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+AmbientCapabilities=
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
+[Install]
+WantedBy=spinifex.target

docs/oci-integration/README.md

@@ -41,6 +41,8 @@ resources:
 - [Harden it before production](#harden-it-before-production)
 - [Next steps](#next-steps)
 - [Removing the deployment](#removing-the-deployment)
+- [EBS on OCI Block Volumes](#ebs-on-oci-block-volumes)
+- [Testing the OCI Block Volumes provider live](#testing-the-oci-block-volumes-provider-live)
 - [Troubleshooting](#troubleshooting)
 
 ---
@@ -773,6 +775,116 @@ oci limits resource-availability get --service-name vcn \
     --limit-name reserved-public-ip-count --compartment-id "$TENANCY_OCID"
 ```
 
+## EBS on OCI Block Volumes
+
+On an OCI virtual machine, EBS volumes can be backed by OCI Block Volumes through the `ocibsd` provider instead of Viperblock. Each EBS volume is one OCI block volume, attached over iSCSI to the node that runs the instance using it.
+
+### How the provider is chosen
+
+There is no flag. `spx admin init` takes the provider from the first of these that gives one:
+
+1. The existing `/etc/spinifex/spinifex.toml`, whose provider the node keeps; a node table that omits it is `viperblockd`.
+2. The `SPINIFEX_EBS_PROVIDER` environment variable: `viperblockd`, `qemunbd` or `ocibsd`.
+3. The OCI instance metadata service (IMDS v2), read with a one-second timeout. An OCI VM shape selects `ocibsd`; a `BM.*` shape, or no answer, selects `viperblockd`.
+
+Init prints its choice and where it came from, for example `EBS provider: ocibsd (detected: OCI virtual machine <shape>)`. Joining nodes take the provider from formation rather than choosing their own. The choice is written once for every node and cannot be changed without wiping and re-forming the cluster.
+
+For `ocibsd`, init fills the compartment and availability domain from instance metadata unless the existing config names them, generates a cluster ID that tags every OCI volume and backup the cluster makes, and defaults `auth` to `instance_principal` when the existing config names no credential. It then reads the compartment's volumes with that credential, and stops with an error naming the needed policy, `manage volume-family` and `use instances`, if the read is refused. It never falls back to Viperblock.
+
+The settings land in the node's table in `spinifex.toml`:
+
+```toml
+[nodes.<node>.ebs]
+provider = "ocibsd"
+
+[nodes.<node>.ebs.oci]
+compartment_id = "<compartment-ocid>"
+availability_domain = "<availability-domain>"
+cluster_id = "<generated-cluster-id>"
+auth = "instance_principal"
+```
+
+On an OCI VM, `setup.sh` installs the iSCSI initiator (`open-iscsi` with apt, `iscsi-initiator-utils` with dnf) and stages the `ocibsd` files under `/usr/local/share/spinifex/ocibsd`. When the provider resolves to `ocibsd`, init and join install the `spinifex-ocibsd.service` unit, its udev rule and a `spinifex-daemon` drop-in, mask `spinifex-viperblock.service`, and enable `spinifex-ocibsd.service`, so exactly one service answers for EBS on the node.
+
+To keep Viperblock on an OCI VM, name it before init:
+
+```bash
+sudo env SPINIFEX_EBS_PROVIDER=viperblockd spx admin init <your usual flags>
+```
+
+`validate-topology.sh` does this for every topology except `vm-single-ocibsd` and `vm-multi-ocibsd`, which rely on detection and need `--distro`.
+
+### One availability domain per cluster
+
+Every volume is created in the availability domain the cluster recorded at init, whichever node handles the request. Config validation refuses nodes that name different compartments or availability domains, and `spinifex-ocibsd` refuses to start on a node whose instance metadata places it in another availability domain. Deploy every node of an `ocibsd` cluster into one availability domain.
+
+### IAM policy
+
+The node's credential needs two grants in the compartment, on top of the four the address allocator needs:
+
+```text
+Allow dynamic-group <dynamic-group-name> to manage volume-family in compartment id <compartment-ocid>
+Allow dynamic-group <dynamic-group-name> to use instances in compartment id <compartment-ocid>
+```
+
+`manage volume-family` covers volumes, backups and attachments, and `use instances` covers attaching them to the node. The instance-principal policy that `--setup-instance-principal` on an `ocibsd` topology (or `setup-identity.sh --ebs-provider ocibsd`) creates grants both, for a Dynamic Group matching `ALL {instance.compartment.id = '<compartment-ocid>'}`. Built for `viperblockd`, the default, it grants neither, and because the policy is shared by every deployment in the compartment, re-running `setup-identity.sh` without the flag removes them.
+
+For an API key, grant the same two verbs to the key user's group, alongside the four from [section 2.2](#grant-the-group-its-four-permissions):
+
+```text
+Allow group <group-name> to manage volume-family in compartment id <compartment-ocid>
+Allow group <group-name> to use instances in compartment id <compartment-ocid>
+```
+
+`terraform output -json api_key_policy_statements` prints all six for the group named by `api_key_group_name` (default `spinifex-nodes`) when `ebs_provider = "ocibsd"`, and only the four address grants otherwise. Because init defaults to the instance principal, an API-key node needs its credential named in `spinifex.toml` before init, which init then keeps:
+
+```toml
+[nodes.<node>.ebs]
+provider = "ocibsd"
+
+[nodes.<node>.ebs.oci]
+auth = "config_file"
+config_file = "/etc/spinifex/oci/config"
+config_profile = "spinifex"
+```
+
+`validate-topology.sh` writes this for you on an API-key run of an `ocibsd` topology, after the credential hook has installed the key.
+
+### The 50 GB floor
+
+OCI creates no block volume smaller than 50 GB, where OCI's GB is 2^30 bytes. The OCI volume is the requested size rounded up to a whole GB, and never less than 50 GB. Spinifex records the requested size in a `spinifex-capacity-bytes` tag, reports it as the EBS volume's size, and presents exactly that size to the guest, so a 1 GiB EBS volume is 1 GiB to the API and the guest. OCI bills for the 50 GB volume underneath it.
+
+## Testing the OCI Block Volumes provider live
+
+`providers/ebs/ocibsd/live_test.go` runs the EBS provider conformance suite, in process and over NATS, against a real compartment and one OCI instance.
+It also checks that OCI's GB is 2^30 bytes on the attached device, and that recreating a deleted volume under the same ID never answers with the deleted volume.
+The test runs on your workstation with the API-key profile from `~/.oci/config`, and drives `sudo iscsiadm` and the device check on the instance over SSH, so no credential is placed on the instance.
+Every subtest skips unless `SPINIFEX_OCIBSD_LIVE` is set.
+
+| Variable | Meaning |
+| --- | --- |
+| `SPINIFEX_OCIBSD_LIVE` | Any value runs the live tests |
+| `SPINIFEX_OCIBSD_COMPARTMENT_ID` | Compartment the volumes and backups are created in |
+| `SPINIFEX_OCIBSD_AVAILABILITY_DOMAIN` | The instance's availability domain, where every volume is created |
+| `SPINIFEX_OCIBSD_INSTANCE_ID` | OCID of the instance volumes are attached to |
+| `SPINIFEX_OCIBSD_SSH_HOST` | Address of that instance |
+| `SPINIFEX_OCIBSD_SSH_USER` | SSH user with passwordless `sudo`, `opc` when unset |
+| `SPINIFEX_OCIBSD_SSH_KEY` | Private key for that user |
+| `SPINIFEX_OCIBSD_SSH_KNOWN_HOSTS` | Known-hosts file, a fresh temporary one when unset |
+| `SPINIFEX_OCIBSD_CLUSTER_ID` | `spinifex-cluster` tag value for the run, random when unset and logged either way |
+| `SPINIFEX_OCIBSD_OCI_PROFILE` | OCI config profile, `DEFAULT` when unset |
+
+The instance needs `iscsi-initiator-utils` and the Oracle Cloud Agent "Block Volume Management" plugin disabled, so no agent logs sessions in or out behind the test.
+The profile needs `manage volume-family` and `use instances` in the compartment.
+
+```bash
+cd spinifex
+SPINIFEX_OCIBSD_LIVE=1 go test ./providers/ebs/ocibsd/ -run TestLive -count=1 -v -timeout 60m
+```
+
+The suite deletes what it creates, but a failed run can leave volumes and backups behind, and OCI bills for them.
+Sweep everything tagged `spinifex-cluster=<cluster ID>` in the compartment after every run.
+
 ## Troubleshooting
 
 The deploy stops at the first failure and names the log it wrote, all under `.validate-<topology>/`. Add `--keep-on-fail` to leave a failed deployment up so you can log in and look. **It keeps billing** until you run `--destroy-only`.

docs/terraform-workbooks/demo-app/Dockerfile

@@ -4,7 +4,7 @@
 # distroless base (no shell, no package manager) for a small, production-shaped
 # image suitable for pushing to ECR and pulling onto EKS workers.
 
-FROM golang:1.27.3-alpine@sha256:d200d2dd76b03a99b52651fd1325aff7dcd35422c35c01bac56571b0507f3deb AS build
+FROM golang:1.27.4-alpine@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS build
 WORKDIR /src
 COPY go.mod ./
 COPY main.go ./

docs/terraform-workbooks/demo-app/go.mod

@@ -1,3 +1,3 @@
 module github.com/mulgadc/spinifex-demo
 
-go 1.27.3
+go 1.27.4

docs/testing/e2e-coverage.md

@@ -140,6 +140,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 ### Storage and durability
 
 - **`storagefault`** — what a guest does when its storage backend stops answering, and what stop/start does when the seal failed. **The claim is about corruption, not availability.** SIGSTOPs a cluster-wide service, so it always runs last and in its own cell. Three-node, because the RS(2,1) shard floor and cross-node takeover after a failed seal do not exist on one node.
+- **No suite reaches a `pending` snapshot.** Every e2e cell runs the default `viperblockd` provider, which finishes a snapshot before `CreateSnapshot` answers, so `single`'s `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch` and `TestCreateImage`, and `rds`'s `TestSnapshotRestore` and `TestDelete`, run the create that writes the snapshot document `pending` before the provider call and settle it `completed` within the request, and the pending-snapshot reaper finds nothing to settle. The `pending` paths are unit-tested against the memory provider answering `pending`: the reaper settling snapshots and AMIs and carrying out deferred deletes, the request's rollback and its ambiguous calls, and `DeleteSnapshot` of a pending snapshot in `domains/ec2/snapshot`; `CreateImage` and the readiness gates in `domains/ec2/image`, `domains/ec2/instance` and `domains/ec2/volume`; DB snapshots and the final snapshot waiting in `creating` in `domains/rds/instance`; and the import's wait in `operator/admin`. The contract's `pending` answer, its filtered `ListSnapshots` and the `state` it reports are in the conformance suite (`providers/ebs/conformance`), which every provider runs.
 - **`partialblock`** — concurrent sub-block writes to the same block surviving a cold reread through NBD from a real guest.
 - **`storagegrowth`** — volume growth and image pull. Requires `external_mode=pool`; skips on a nat env, which has no EIP and therefore no guest SSH.
 - **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest forced off and relaunched once its budget expires. Proves a real reboot by requiring a *new* boot ID.

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex
 
-go 1.27.3
+go 1.27.4
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8

scripts/dev-install.sh

@@ -40,6 +40,7 @@ mkdir -p "$STAGING/scripts"
 cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
 mkdir -p "$STAGING/udev"
 cp "$PROJECT_ROOT/build/udev/"* "$STAGING/udev/"
+cp -r "$PROJECT_ROOT/build/ocibsd" "$STAGING/ocibsd"
 cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
 tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .
 

scripts/install-node.sh

@@ -50,6 +50,11 @@
 #   --yes                    Skip the confirmation prompt
 #   --dry-run                Print what would happen and touch nothing
 #
+# Environment:
+#   SPINIFEX_EBS_PROVIDER    Passed to spx admin init on the first host, naming
+#                            what backs EBS in place of its detection; joining
+#                            nodes take the provider from formation.
+#
 # THE HOST FIREWALL
 #
 # Nodes installed from the ISO boot armed, scoped to the only cluster member
@@ -159,7 +164,7 @@ while [[ $# -gt 0 ]]; do
         --no-firewall)  MANAGE_FIREWALL=false; shift ;;
         --yes|-y)   ASSUME_YES=true; shift ;;
         --dry-run)  DRY_RUN=true; shift ;;
-        -h|--help)  sed -n '2,67p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+        -h|--help)  sed -n '2,72p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
         -*)         echo "ERROR: unknown option: $1" >&2; exit 2 ;;
         *)          HOSTS+=("$1"); shift ;;
     esac
@@ -210,6 +215,18 @@ else
     done
 fi
 
+# init_env prints the env prefix that carries SPINIFEX_EBS_PROVIDER to init, or
+# nothing when it is unset. It is joined into a remote shell, so the value is
+# limited to a provider name's characters rather than quoted.
+init_env() {
+    case "$1" in
+        "") ;;
+        *[!a-z]*) echo "SPINIFEX_EBS_PROVIDER must be a provider name such as viperblockd, got: $1"; return 1 ;;
+        *) printf 'env SPINIFEX_EBS_PROVIDER=%s ' "$1" ;;
+    esac
+}
+INIT_ENV="$(init_env "${SPINIFEX_EBS_PROVIDER:-}")" || fail "$INIT_ENV"
+
 case "$IPSEC" in
     on)  IPSEC_ENABLED=true ;;
     off) IPSEC_ENABLED=false ;;
@@ -671,7 +688,7 @@ log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s
 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid ${INIT_ENV}sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
     exit 0" || fail "${HOSTS[0]}: could not start spx admin init"
 
 TOKEN=""

scripts/install-node_test.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
-# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
-# sudo's log (which holds the command it was given) never contains it, and the other values must
-# arrive as plain arguments even when they hold shell syntax.
+# Tests remote_join_cmd and init_env from install-node.sh: the join token reaches spx only on
+# stdin, so sudo's log never holds it; other values arrive as plain arguments even when they
+# hold shell syntax; and a SPINIFEX_EBS_PROVIDER that is not a provider name is refused.
 # preflight runs this with sh
 [ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
 set -euo pipefail
@@ -38,4 +38,15 @@ check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--t
 check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
 check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
 
+# init_env: its value is joined into a remote sh -c unquoted, so anything beyond
+# a provider name's characters is refused, not passed on.
+eval "$(sed -n '/^init_env()/,/^}/p' "$here/install-node.sh")"
+check "no provider sets nothing" "$(init_env "")" ""
+check "a provider is carried to init" "$(init_env viperblockd)" "env SPINIFEX_EBS_PROVIDER=viperblockd "
+check "a provider with shell syntax is refused" \
+    "$(init_env 'ocibsd;touch PWNED3' >/dev/null 2>&1 && echo accepted || echo refused)" refused
+check "a provider with a space is refused" \
+    "$(init_env 'ocibsd $(touch PWNED4)' >/dev/null 2>&1 && echo accepted || echo refused)" refused
+check "no refused value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
+
 [ "$fails" -eq 0 ]

scripts/setup.sh

@@ -17,7 +17,7 @@
 #   SETUP_STAGES               Comma-separated subset of stages to run:
 #                                deps, aws, dns, users, sudoers, firewall, timesync, files,
 #                                directories, env, systemd, sysctl, needrestart, modules,
-#                                logrotate, udev, fixown, migrations
+#                                logrotate, udev, ocibsd, fixown, migrations
 #                              Unset = run every stage appropriate for the current mode.
 
 set -e
@@ -43,6 +43,13 @@ FIREWALL_TRACK="${FIREWALL_DIR}/conntrack.nft"
 FIREWALL_MODE_FILE="${SPINIFEX_FIREWALL_MODE_FILE:-${FIREWALL_DIR}/mode}"
 FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"
 
+# The OCI instance metadata service spx admin init detects ocibsd by. v2 needs
+# the bearer header; it is a confused-deputy guard, not a credential.
+OCI_IMDS_SHAPE_URL="http://169.254.169.254/opc/v2/instance/shape"
+# Where the tarball's ocibsd/ is staged; init and join install it from here.
+OCIBSD_STAGE_DIR="/usr/local/share/spinifex/ocibsd"
+SPINIFEX_TOML="/etc/spinifex/spinifex.toml"
+
 # The daemon runs with ProtectKernelModules=yes, so every module it needs must
 # already be loaded when it starts.
 SPINIFEX_KERNEL_MODULES="openvswitch geneve vhost_net tun"
@@ -1351,6 +1358,7 @@ install_deps() {
             apt) install_apt_deps ;;
             dnf) install_dnf_deps ;;
         esac
+        install_iscsi_initiator "$pm"
     fi
 
     # Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
@@ -1385,6 +1393,53 @@ install_apt_deps() {
     fi
 }
 
+# is_oci_vm is spx admin init's detection: the OCI metadata service answers
+# within a second with a shape that is not bare metal. Asked once per run,
+# because off OCI each ask costs the full second.
+is_oci_vm() {
+    if [ -z "${OCI_VM:-}" ]; then
+        local shape=""
+        OCI_VM=no
+        if command -v curl >/dev/null 2>&1 \
+            && shape=$(curl -sf -m 1 -H 'Authorization: Bearer Oracle' "$OCI_IMDS_SHAPE_URL" 2>/dev/null); then
+            case "$shape" in
+                "" | BM.*) ;;
+                *) OCI_VM=yes ;;
+            esac
+        fi
+    fi
+    [ "$OCI_VM" = yes ]
+}
+
+# node_runs_ocibsd reports whether this node's existing config names ocibsd in
+# a [nodes.<name>.ebs] table, the provider being the same on every node.
+node_runs_ocibsd() {
+    local text
+    [ -e "$SPINIFEX_TOML" ] || return 1
+    if [ -r "$SPINIFEX_TOML" ]; then
+        text=$(cat "$SPINIFEX_TOML") || return 1
+    else
+        text=$($SUDO cat "$SPINIFEX_TOML") || return 1
+    fi
+    printf '%s\n' "$text" | awk '
+        /^[[:space:]]*\[/ { ebs = ($0 ~ /^[[:space:]]*\[nodes\.[^].]+\.ebs\][[:space:]]*(#.*)?$/); next }
+        ebs && /^[[:space:]]*provider[[:space:]]*=[[:space:]]*"ocibsd"/ { found = 1 }
+        END { exit !found }'
+}
+
+# The initiator ocibsd drives, on the only host it can back EBS on.
+install_iscsi_initiator() {
+    is_oci_vm || return 0
+    info "OCI virtual machine detected, installing the iSCSI initiator..."
+    case "$1" in
+        apt) DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq open-iscsi > /dev/null ;;
+        dnf) $SUDO dnf install -y -q iscsi-initiator-utils > /dev/null ;;
+    esac
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO systemctl enable --now iscsid.socket
+    fi
+}
+
 # The package table is the other place keyed on the OS ID: dnf alone does not
 # say which repositories or package names a host has.
 install_dnf_deps() {
@@ -2135,6 +2190,45 @@ install_udev() {
     info "udev rules installed"
 }
 
+# install_ocibsd stages the tarball's ocibsd/ on an OCI virtual machine, where
+# spx admin init or join installs it once the provider resolves to ocibsd. On
+# a node whose config already runs ocibsd it also reinstalls the staged files.
+install_ocibsd() {
+    stage "staging the ocibsd files"
+    local ocibsd_node=false f
+    node_runs_ocibsd && ocibsd_node=true
+    if ! $ocibsd_node && ! is_oci_vm; then
+        return 0
+    fi
+    if [ ! -d "$EXTRACT_DIR/ocibsd" ]; then
+        $ocibsd_node && fatal "This node runs ocibsd and the tarball has no ocibsd/ directory to reinstall it from"
+        warn "The tarball has no ocibsd/ directory, so OCI Block Volumes cannot back EBS on this node"
+        return 0
+    fi
+    $SUDO rm -rf "$OCIBSD_STAGE_DIR"
+    $SUDO install -d -m 0755 "$OCIBSD_STAGE_DIR/spinifex-daemon.service.d"
+    for f in spinifex-ocibsd.service 99-spinifex-ocibsd.rules spinifex-daemon.service.d/ocibsd.conf; do
+        $SUDO install -m 0644 "$EXTRACT_DIR/ocibsd/$f" "$OCIBSD_STAGE_DIR/$f"
+    done
+    info "  $OCIBSD_STAGE_DIR (staged)"
+    $ocibsd_node || return 0
+
+    $SUDO install -m 0644 "$OCIBSD_STAGE_DIR/spinifex-ocibsd.service" /etc/systemd/system/spinifex-ocibsd.service
+    $SUDO install -d -m 0755 /etc/systemd/system/spinifex-daemon.service.d
+    $SUDO install -m 0644 "$OCIBSD_STAGE_DIR/spinifex-daemon.service.d/ocibsd.conf" \
+        /etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf
+    $SUDO install -d /etc/udev/rules.d
+    $SUDO install -m 0644 "$OCIBSD_STAGE_DIR/99-spinifex-ocibsd.rules" /etc/udev/rules.d/99-spinifex-ocibsd.rules
+    # install_systemd has just written the viperblock unit over its mask.
+    $SUDO ln -sf /dev/null /etc/systemd/system/spinifex-viperblock.service
+    info "  ocibsd unit, device drop-in and udev rule reinstalled; spinifex-viperblock.service masked"
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO systemctl daemon-reload
+        $SUDO systemctl enable spinifex-ocibsd.service
+        $SUDO udevadm control --reload-rules
+    fi
+}
+
 # --- Upgrade handling ---
 handle_upgrade() {
     if $SUDO systemctl is-active --quiet spinifex.target 2>/dev/null; then
@@ -2273,12 +2367,12 @@ main() {
     detect_os
     detect_arch
 
-    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev.
+    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev, ocibsd.
     # Only download/extract when at least one such stage is enabled, and before
     # anything on the host changes, so the hand-off finds it untouched.
     if stage_enabled files || stage_enabled directories \
         || stage_enabled systemd || stage_enabled logrotate \
-        || stage_enabled udev; then
+        || stage_enabled udev || stage_enabled ocibsd; then
         download_spinifex
         hand_off_to_packaged_setup "${args[@]}"
     fi
@@ -2306,6 +2400,7 @@ main() {
     stage_enabled modules    && install_kernel_modules
     stage_enabled logrotate  && install_logrotate
     stage_enabled udev       && install_udev
+    stage_enabled ocibsd     && install_ocibsd
     stage_enabled swap       && setup_swap
 
     # Migrations are only safe on a live system (need a running NATS and a

scripts/setup_ocibsd_test.sh

@@ -0,0 +1,157 @@
+#!/bin/sh
+# setup_ocibsd_test.sh — setup.sh's systemd and udev stages never install the
+# tarball's ocibsd/ files; install_ocibsd stages them on an OCI VM, and
+# reinstalls them on upgrade only where the node's config already runs ocibsd.
+# A stub SUDO records every privileged command and runs the file ones under a
+# scratch root, so what each case leaves on disk is checked as well.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+SETUP="$SCRIPT_DIR/setup.sh"
+BUILD="$(dirname -- "$SCRIPT_DIR")/build"
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+# The tarball layout the release image and dev-install.sh assemble.
+EXTRACT="$TMP/extract"
+mkdir -p "$EXTRACT/systemd" "$EXTRACT/udev"
+cp "$BUILD"/systemd/* "$EXTRACT/systemd/"
+cp "$BUILD"/udev/* "$EXTRACT/udev/"
+cp -r "$BUILD/ocibsd" "$EXTRACT/ocibsd"
+
+# install, ln, rm and mkdir run with /etc and /usr/local/share/spinifex moved
+# under $ROOT; anything else is only recorded.
+cat > "$TMP/sudo" <<'EOF'
+#!/usr/bin/env bash
+echo "$*" >> "$CALLS"
+case "$1" in
+    install | ln | rm | mkdir) ;;
+    tee) cat > /dev/null; exit 0 ;;
+    *) exit 0 ;;
+esac
+args=()
+for a in "$@"; do
+    case "$a" in
+        /etc/* | /usr/local/share/spinifex/*) a="$ROOT$a" ;;
+    esac
+    args+=("$a")
+done
+exec "${args[@]}"
+EOF
+chmod +x "$TMP/sudo"
+
+# run_setup <case> <oci_vm yes|no> <config body or -> <functions...>
+# runs the named setup.sh functions in a fresh root for one case.
+run_setup() {
+    name="$1" vm="$2" body="$3"
+    shift 3
+    ROOT="$TMP/$name/root"
+    CALLS="$TMP/$name/calls"
+    mkdir -p "$ROOT/etc/systemd/system" "$ROOT/etc/spinifex" "$ROOT/usr/local/share/spinifex"
+    : > "$CALLS"
+    [ "$body" = - ] || printf '%s\n' "$body" > "$ROOT/etc/spinifex/spinifex.toml"
+    export ROOT CALLS
+    OUT=$(INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '
+        . "$1"; SUDO="$2"; EXTRACT_DIR="$3"; OCI_VM="$4"; SPINIFEX_TOML="$ROOT/etc/spinifex/spinifex.toml"
+        shift 4
+        for f in "$@"; do $f; done' _ "$SETUP" "$TMP/sudo" "$EXTRACT" "$vm" "$@" 2>&1) \
+        || fail "$name: $* failed: $OUT"
+}
+
+echo "-- the systemd and udev stages leave the tarball's ocibsd/ alone"
+run_setup units yes - install_systemd install_udev
+if grep -q " /etc/systemd/system/spinifex-viperblock.service$" "$CALLS" && grep -q " /etc/udev/rules.d/99-spinifex-vfio.rules$" "$CALLS"; then
+    pass "installs the systemd/ units and udev/ rules"
+else
+    fail "did not install the systemd/ and udev/ files: $(cat "$CALLS")"
+fi
+if grep -q "ocibsd" "$CALLS"; then
+    fail "installed an ocibsd/ file: $(grep ocibsd "$CALLS")"
+else
+    pass "installs nothing from ocibsd/"
+fi
+
+STAGED="usr/local/share/spinifex/ocibsd"
+FILES="spinifex-ocibsd.service 99-spinifex-ocibsd.rules spinifex-daemon.service.d/ocibsd.conf"
+installed_count() {
+    n=0
+    for p in etc/systemd/system/spinifex-ocibsd.service etc/udev/rules.d/99-spinifex-ocibsd.rules \
+        etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf; do
+        [ ! -e "$ROOT/$p" ] || n=$((n + 1))
+    done
+    echo "$n"
+}
+staged_ok() {
+    for f in $FILES; do
+        cmp -s "$EXTRACT/ocibsd/$f" "$ROOT/$STAGED/$f" || return 1
+    done
+}
+
+echo "-- a fresh install on an OCI VM stages ocibsd/ and installs none of it"
+run_setup fresh yes - install_systemd install_udev install_ocibsd
+if staged_ok; then pass "the three files are staged"; else fail "the stage does not match the tarball: $(ls -R "$ROOT/$STAGED" 2>&1)"; fi
+[ "$(installed_count)" = 0 ] && pass "nothing is installed" || fail "installed $(installed_count) ocibsd file(s) on a node init has not resolved"
+[ -L "$ROOT/etc/systemd/system/spinifex-viperblock.service" ] && fail "masked viperblock" || pass "viperblock is not masked"
+
+echo "-- an upgrade of an ocibsd node replaces the installed files with the new stage"
+OCIBSD_TOML='[nodes.node1.ebs]
+provider = "ocibsd"
+
+[nodes.node1.ebs.oci]
+auth = "instance_principal"'
+ROOT="$TMP/upgrade/root"
+mkdir -p "$ROOT/$STAGED/spinifex-daemon.service.d" "$ROOT/etc/systemd/system/spinifex-daemon.service.d" "$ROOT/etc/udev/rules.d"
+for f in $FILES; do echo "# the previous release" > "$ROOT/$STAGED/$f"; done
+echo "# the previous release" > "$ROOT/etc/systemd/system/spinifex-ocibsd.service"
+echo "# the previous release" > "$ROOT/etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf"
+echo "# the previous release" > "$ROOT/etc/udev/rules.d/99-spinifex-ocibsd.rules"
+ln -s /dev/null "$ROOT/etc/systemd/system/spinifex-viperblock.service"
+run_setup upgrade yes "$OCIBSD_TOML" install_systemd install_udev install_ocibsd
+if staged_ok; then pass "the stage holds the new files"; else fail "the stage kept the previous release"; fi
+if cmp -s "$EXTRACT/ocibsd/spinifex-ocibsd.service" "$ROOT/etc/systemd/system/spinifex-ocibsd.service" \
+    && cmp -s "$EXTRACT/ocibsd/spinifex-daemon.service.d/ocibsd.conf" "$ROOT/etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf" \
+    && cmp -s "$EXTRACT/ocibsd/99-spinifex-ocibsd.rules" "$ROOT/etc/udev/rules.d/99-spinifex-ocibsd.rules"; then
+    pass "the unit, drop-in and udev rule are the new ones"
+else
+    fail "an installed ocibsd file is not the new one"
+fi
+if [ "$(readlink "$ROOT/etc/systemd/system/spinifex-viperblock.service")" = /dev/null ]; then
+    pass "viperblock is masked again after the unit reinstall"
+else
+    fail "the reinstalled viperblock unit is not masked"
+fi
+grep -q '^systemctl daemon-reload$' "$CALLS" && grep -q '^udevadm control --reload-rules$' "$CALLS" \
+    && pass "systemd and udev are reloaded" || fail "no reload: $(cat "$CALLS")"
+
+echo "-- an ocibsd node is refreshed even when the metadata service does not answer"
+run_setup upgrade-no-imds no "$OCIBSD_TOML" install_ocibsd
+[ "$(installed_count)" = 3 ] && pass "the three files are installed" || fail "installed $(installed_count) of 3"
+
+echo "-- an upgrade of a viperblock node on an OCI VM installs none of them"
+run_setup viperblock yes '[nodes.node1.ebs]
+provider = "viperblockd"' install_systemd install_udev install_ocibsd
+[ "$(installed_count)" = 0 ] && pass "nothing is installed" || fail "installed $(installed_count) ocibsd file(s) on a viperblock node"
+[ -f "$ROOT/etc/systemd/system/spinifex-viperblock.service" ] && [ ! -L "$ROOT/etc/systemd/system/spinifex-viperblock.service" ] \
+    && pass "the viperblock unit is installed and not masked" || fail "the viperblock unit was masked"
+
+echo "-- off OCI nothing is staged"
+run_setup off-oci no '[nodes.node1.daemon]' install_ocibsd
+[ ! -e "$ROOT/$STAGED" ] && pass "no stage" || fail "staged ocibsd/ off OCI"
+
+echo "-- the initiator is installed on an OCI VM only"
+run_setup initiator-apt yes - "install_iscsi_initiator apt"
+grep -q 'apt-get install -y -qq open-iscsi' "$CALLS" && grep -q '^systemctl enable --now iscsid.socket$' "$CALLS" \
+    && pass "apt installs open-iscsi and enables iscsid.socket" || fail "apt: $(cat "$CALLS")"
+run_setup initiator-dnf yes - "install_iscsi_initiator dnf"
+grep -q 'dnf install -y -q iscsi-initiator-utils' "$CALLS" && pass "dnf installs iscsi-initiator-utils" || fail "dnf: $(cat "$CALLS")"
+run_setup initiator-off no - "install_iscsi_initiator apt"
+[ -s "$CALLS" ] && fail "installed an initiator off OCI: $(cat "$CALLS")" || pass "no initiator off OCI"
+
+if [ "$fails" -gt 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

scripts/terraform/oci-spx/README.md

@@ -230,11 +230,11 @@ Create them once per tenancy, then every deployment and rebuild afterwards uses
 
 **`adopt` references the dynamic group by nothing at all.** Its matching rule is `instance.compartment.id`, so it covers every instance in the compartment and names no OCID — which is why adopting needs no read on an identity resource and no tenancy rights. The cost is that a missing policy is invisible at apply time: the node forms, passes every health check, and then refuses every launch wanting a public address. The allocator gate in `validate-topology.sh` is what catches that, by requiring `ocinet credential authorised to allocate` in each node's `spinifex-daemon` journal.
 
-The policy grants four verbs in one compartment: `use vnics`, `manage private-ips`, `manage public-ips` and `use subnets`.
-That is exactly what allocating an external address does and nothing more.
+The policy grants `use vnics`, `manage private-ips`, `manage public-ips` and `use subnets` in one compartment for the allocator, and with `ebs_provider = "ocibsd"` also `manage volume-family` and `use instances` for the `ocibsd` EBS provider. `setup-identity.sh --ebs-provider ocibsd` creates it with them; the policy is shared by every deployment in the compartment, so running it without the flag removes them.
+The first four are exactly what allocating an external address does and nothing more; the last two are what creating, attaching, backing up and deleting a block volume does.
 `use subnets` looks unrelated and is not: `CreatePrivateIp` is checked against `SUBNET_ATTACH` and `CreatePrivateIp` is how an address is registered, so a policy without it authorises nothing and every allocation returns a 404.
 
-**An API key is the fallback**, and the default because it needs nothing from a tenancy admin. Grant that user only the operations Spinifex performs on addresses and nothing else; it is not a tenancy admin, and broader rights widen the blast radius of a node compromise for no benefit.
+**An API key is the fallback**, and the default because it needs nothing from a tenancy admin. Grant that user only the operations Spinifex performs on addresses and volumes and nothing else: `terraform output -json api_key_policy_statements` prints the same statements for the group named by `api_key_group_name` (default `spinifex-nodes`), for a tenancy admin to apply; it is not a tenancy admin, and broader rights widen the blast radius of a node compromise for no benefit.
 
 `spx-oci-config.sh` installs it on **every** node, as `/etc/spinifex/oci/{oci_api_key.pem,config}` — not a home directory, because the daemon's unit sets `ProtectHome=yes`. Both files are needed: `oci_config_file` names an ordinary OCI SDK config, so a node holding only the PEM forms and then fails every allocation. The profile it writes is `spinifex`, matching `oci_config_profile`:
 
@@ -275,6 +275,20 @@ The count that later gates read is the number of addresses the `hosts_file` outp
 
 **A clean teardown can still leave public IPs behind, and they count against the tenancy.** The addresses Spinifex allocates for guests are created by the running node, so they are in no Terraform state and `destroy` neither sees nor removes them. The node's own reconcile collects detached ones every ten minutes, which is exactly the sweep a destroy takes away. Twenty accumulated in `spxbm` over four runs and exhausted the tenancy's 50 reserved-public-IP limit, at which point every later launch in every compartment failed. [Check for leftover public IPs afterwards](../../../docs/oci-integration/README.md#check-for-leftover-public-ips-afterwards) has the query and the rule for reading it.
 
+### The ocibsd topologies
+
+`vm-single-ocibsd` and `vm-multi-ocibsd` are `vm-single` and `vm-multi` with EBS backed by OCI Block Volumes. They build with `ebs_provider = "ocibsd"`, which disables the Block Volume Management plugin and attaches each node's data volume paravirtualized, because every iSCSI session on the node is `ocibsd`'s to manage. They need `--distro`, because a published release predates the `ocibsd` files that `setup.sh` stages, and an external pool, because an API-key run gives `ocibsd` the pool's credential.
+
+The ocibsd topologies name no EBS provider. `spx admin init` detects that it is on an OCI virtual machine, reads the compartment, availability domain and region from the instance metadata, and generates the cluster's `cluster_id`. Every other topology sets `SPINIFEX_EBS_PROVIDER=viperblockd`, so its viperblock baseline is not replaced by detection on OCI. The run checks the `EBS provider:` line init prints: `detected` on the ocibsd topologies under the instance principal, and `from SPINIFEX_EBS_PROVIDER` on the rest.
+
+Under the instance principal, `ocibsd` uses the instance's own certificate. An API-key run runs the credential hook before formation instead, and seeds each node's `spinifex.toml` with `provider = "ocibsd"` and the hook's `/etc/spinifex/oci/config`, which init keeps as an existing config, so its `EBS provider:` line reads `from the existing config`.
+
+Every volume and backup `ocibsd` creates is tagged `spinifex-cluster=<cluster_id>`, and none of them is in any Terraform state, so the run records the ID in `.validate-<topology>/ocibsd-clusters` before any volume exists.
+
+The `ebs provider` gate then checks the units, the viperblock mask, the daemon's device drop-in, and that the compartment and availability domain init detected are the ones the state reports on every node, imports an AMI onto OCI and boots from it, and drives stop and start, attach and detach, snapshot and restore with a guest-written marker read back over the serial console. On `vm-multi-ocibsd` it also starts the stopped instance on another node. It ends by terminating everything and polling OCI until nothing tagged with the instance's volumes or snapshot is left.
+
+`e2e-oci.sh` deletes everything tagged with each recorded `cluster_id` once that topology's nodes are destroyed, including from `--sweep-only`, and fails the run if anything tagged remains. A standalone `validate-topology.sh` run does not sweep, so follow it with `./e2e-oci.sh --sweep-only`.
+
 The workbook runs **on the node** against `127.0.0.1`, because the node certificate carries no SAN for its public address — a workbook driven from outside the VCN is still blocked.
 
 ### Validate infrastructure Terraform already created

scripts/terraform/oci-spx/compute.tf

@@ -45,6 +45,11 @@ locals {
   # after the boot disk (mulga-poc has oraclevda1 pointing at sdb1).
   data_device = "/dev/oracleoci/oraclevdb"
 
+  # The Block Volume Management plugin logs iSCSI sessions in and out by itself,
+  # and on an ocibsd node those sessions are ocibsd's to manage. With the plugin
+  # off, the node's data volume attaches paravirtualized, which needs no login.
+  ocibsd_nodes = var.ebs_provider == "ocibsd"
+
   # adopt and create differ only in who creates the dynamic group and policy; a node
   # authenticates identically either way, because the matching rule covers the whole
   # compartment and so names no instance.
@@ -106,11 +111,12 @@ resource "oci_core_instance" "mulgadc" {
     user_data           = base64encode(local.user_data)
   }
 
-  # Required for automatic UHP iSCSI multipath setup and login.
+  # Required for automatic UHP iSCSI multipath setup and login, and disabled on
+  # an ocibsd node, where ocibsd owns every iSCSI session (see ocibsd_nodes).
   agent_config {
     plugins_config {
       name          = "Block Volume Management"
-      desired_state = "ENABLED"
+      desired_state = local.ocibsd_nodes ? "DISABLED" : "ENABLED"
     }
   }
 
@@ -165,10 +171,10 @@ resource "oci_core_volume" "mulgadc_data" {
 
 resource "oci_core_volume_attachment" "mulgadc_data" {
   count           = var.node_count
-  attachment_type = "iscsi"
+  attachment_type = local.ocibsd_nodes ? "paravirtualized" : "iscsi"
   # Required by OCI for Ultra High Performance iSCSI volume attachments.
   device                            = local.data_device
-  is_agent_auto_iscsi_login_enabled = true
+  is_agent_auto_iscsi_login_enabled = local.ocibsd_nodes ? null : true
   instance_id                       = oci_core_instance.mulgadc[count.index].id
   volume_id                         = oci_core_volume.mulgadc_data[count.index].id
 }

scripts/terraform/oci-spx/e2e-oci.sh

@@ -8,8 +8,10 @@
 #
 #   SPX_REF           Ref to prove. Default: the spinifex checkout's current branch.
 #   OCI_TOPOLOGIES    Space-separated. Default: "vm-single vm-multi bm
-#                     vm-single-principal". Set one name to test one thing: a
-#                     single VM topology is roughly a quarter of the full run.
+#                     vm-single-principal vm-single-ocibsd vm-multi-ocibsd". Set
+#                     one name to test one thing. The two ocibsd topologies back
+#                     EBS with OCI Block Volumes, need OCI_SOURCE=tree and an
+#                     external pool, and are recorded SKIPPED without them.
 #   OCI_SOURCE        tree (build SPX_REF) or release (the published installer).
 #   OCI_VERSION       With OCI_SOURCE=release, install this exact release tag and
 #                     ignore OCI_CHANNEL. Preferred for anything repeatable: a tag
@@ -82,7 +84,7 @@ SPINIFEX_ROOT="$(cd "$HERE/../../.." && pwd)"
 # under the workspace. Deriving one from the other is only right in the first case.
 MULGA_ROOT="${MULGA_ROOT:-$SPINIFEX_ROOT/..}"
 
-TOPOLOGIES="${OCI_TOPOLOGIES:-vm-single vm-multi bm vm-single-principal}"
+TOPOLOGIES="${OCI_TOPOLOGIES:-vm-single vm-multi bm vm-single-principal vm-single-ocibsd vm-multi-ocibsd}"
 SOURCE="${OCI_SOURCE:-tree}"
 CHANNEL="${OCI_CHANNEL:-latest}"
 INSTALL_VERSION="${OCI_VERSION:-}"
@@ -289,6 +291,35 @@ publish_summary() {
     fi
 }
 
+# ocibsd's volumes and backups live in no Terraform state, so a destroy leaves any
+# a failed run made. Each cluster_id a topology recorded is swept by its tag once
+# the nodes are gone, and anything still tagged afterwards fails the run.
+sweep_ocibsd() {
+    local dir="$1" record="$1/ocibsd-clusters" cluster compartment region left
+    [ -s "$record" ] || return 0
+    left="$(mktemp)"
+    while IFS=$'\t' read -r cluster compartment region; do
+        [ -n "$cluster" ] || continue
+        log "sweep: OCI volumes and backups of ocibsd cluster $cluster"
+        if OCIBSD_COMPARTMENT="$compartment" python3 "$HERE/scripts/oci_env.py" \
+            ${OCI_PROFILE:+--profile "$OCI_PROFILE"} --region "$region" -- \
+            python3 "$HERE/scripts/oci_volumes.py" sweep --cluster "$cluster" \
+            < /dev/null >> "$ARTIFACT_DIR/sweep.log" 2>&1; then
+            log "sweep: nothing tagged with ocibsd cluster $cluster remains"
+        else
+            log "sweep: OCI still holds volumes or backups of ocibsd cluster $cluster. See $ARTIFACT_DIR/sweep.log"
+            echo "sweep ${dir##*/.validate-} ocibsd cluster $cluster: FAIL" >> "$VERDICT"
+            printf '%s\t%s\t%s\n' "$cluster" "$compartment" "$region" >> "$left"
+        fi
+    done < "$record"
+    # Kept only for what is left, so the next sweep retries exactly that.
+    if [ -s "$left" ]; then
+        mv "$left" "$record"
+    else
+        rm -f "$left" "$record"
+    fi
+}
+
 # A topology destroys itself from its own EXIT trap, so this only catches a run
 # the trap never reached: a killed process, a runner reclaimed mid-job, a reboot.
 # It works from the state files rather than by name filter, so it can only ever
@@ -297,12 +328,16 @@ sweep() {
     local found=0 dir topology
     local sweep_env=(env)
     for dir in "$STATE_ROOT"/.validate-*; do
-        [ -s "$dir/terraform.tfstate" ] || continue
+        [ -d "$dir" ] || continue
+        if [ ! -s "$dir/terraform.tfstate" ]; then
+            sweep_ocibsd "$dir"
+            continue
+        fi
         python3 -c '
 import json, sys
 state = json.load(open(sys.argv[1]))
 sys.exit(0 if any(r["instances"] for r in state.get("resources", [])) else 1)
-' "$dir/terraform.tfstate" 2>/dev/null || continue
+' "$dir/terraform.tfstate" 2>/dev/null || { sweep_ocibsd "$dir"; continue; }
         topology="${dir##*/.validate-}"
         # The child reads OCI_STATE_ROOT from the environment, so it looks in the
         # same place this loop found the state rather than beside its own script.
@@ -319,6 +354,7 @@ sys.exit(0 if any(r["instances"] for r in state.get("resources", [])) else 1)
         if "${sweep_env[@]}" "$HERE/validate-topology.sh" --topology "$topology" --destroy-only \
             >> "$ARTIFACT_DIR/sweep.log" 2>&1; then
             log "sweep: $topology destroyed"
+            sweep_ocibsd "$dir"
         else
             log "sweep: $topology WOULD NOT DESTROY -- it is still billing. See $ARTIFACT_DIR/sweep.log"
             echo "sweep $topology: FAIL" >> "$VERDICT"
@@ -342,6 +378,10 @@ sweep
 
 if [ "$SWEEP_ONLY" = 1 ]; then
     log "--sweep-only: done"
+    if grep -q FAIL "$VERDICT"; then
+        cat "$VERDICT"
+        exit 1
+    fi
     exit 0
 fi
 
@@ -394,6 +434,22 @@ for topology in $TOPOLOGIES; do
         build_principal_env
         run_env=(env "${PRINCIPAL_ENV[@]}")
     fi
+    # ocibsd's files ship in the distro this checkout builds, and its volumes
+    # need the credential the pool uses, so neither can be stood in for.
+    case "$topology" in
+        *-ocibsd)
+            if [ "$SOURCE" != tree ]; then
+                log "$topology: SKIPPED -- the ocibsd files ship only with a tree build"
+                echo "$topology: SKIPPED (OCI_SOURCE=$SOURCE)" >> "$VERDICT"
+                continue
+            fi
+            if [ "${OCI_NO_EXTERNAL_POOL:-0}" = 1 ]; then
+                log "$topology: SKIPPED -- ocibsd needs the OCI credential the external pool installs"
+                echo "$topology: SKIPPED (OCI_NO_EXTERNAL_POOL=1)" >> "$VERDICT"
+                continue
+            fi
+            ;;
+    esac
     args=(--topology "$topology" --ssh-public-key "$SSH_PUBLIC_KEY" --ssh-private-key "$SSH_PRIVATE_KEY")
     [ "$DRY_RUN" = 1 ] && args+=(--dry-run)
     # Two separate choices. --no-external-pool is the one that forms without any
@@ -460,5 +516,8 @@ if [ "$RUN_RC" = 0 ] && ! grep -q FAIL "$VERDICT"; then
 else
     log "FAIL"
     printf 'FAIL\te2e-oci\t%d.000s\n' "$SECONDS"
+    # A failed sweep is a FAIL in the verdict with every topology passing, and is
+    # still resources left billing, so it fails the run as well.
+    RUN_RC=1
 fi
 exit "$RUN_RC"

scripts/terraform/oci-spx/instance-principal.tf

@@ -29,26 +29,44 @@ resource "oci_identity_dynamic_group" "nodes" {
 }
 
 # Narrower than it looks. The allocator creates, moves and deletes secondary
-# private IPs on a node's own VNIC and attaches public IPs to them, which is what
-# these four verbs cover; nothing here grants compute, storage or identity.
+# private IPs on a node's own VNIC and attaches public IPs to them, and ocibsd
+# creates, attaches and backs up EBS volumes. Nothing here grants identity.
+locals {
+  network_grants = [
+    "use vnics",
+    "manage private-ips",
+    "manage public-ips",
+
+    # CreatePrivateIp needs SUBNET_ATTACH and DeletePrivateIp needs SUBNET_DETACH,
+    # both of which only the subnets `use` verb carries. Without it every allocation
+    # is refused as a 404, which reads as a missing resource rather than a missing grant.
+    "use subnets",
+  ]
+
+  # ocibsd: volumes, backups and attachments, and attaching them to an instance.
+  # Without these every ocibsd call is refused and its service fails at start.
+  # A viperblockd deployment touches no OCI volume, so it is not granted them.
+  ocibsd_grants = [
+    "manage volume-family",
+    "use instances",
+  ]
+
+  node_grants = concat(local.network_grants, var.ebs_provider == "ocibsd" ? local.ocibsd_grants : [])
+}
+
 resource "oci_identity_policy" "nodes" {
   provider       = oci.home
   count          = var.instance_principal == "create" ? 1 : 0
   compartment_id = var.compartment_ocid
   name           = "${var.deployment_name}-nodes-network"
-  description    = "Allows Spinifex nodes to manage the private and public IPs backing external addresses."
+  description    = "Allows Spinifex nodes to manage the IPs backing external addresses and the OCI volumes backing EBS."
 
   statements = [
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use vnics in compartment id ${var.compartment_ocid}",
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage private-ips in compartment id ${var.compartment_ocid}",
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage public-ips in compartment id ${var.compartment_ocid}",
-
-    # CreatePrivateIp needs SUBNET_ATTACH and DeletePrivateIp needs SUBNET_DETACH,
-    # both of which only the subnets `use` verb carries. Without it every allocation
-    # is refused as a 404, which reads as a missing resource rather than a missing grant.
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use subnets in compartment id ${var.compartment_ocid}",
+    for grant in local.node_grants :
+    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to ${grant} in compartment id ${var.compartment_ocid}"
   ]
-    lifecycle {
+
+  lifecycle {
     precondition {
       condition     = local.tenancy_home_region != ""
       error_message = "OCI did not report a tenancy home region; cannot create instance-principal IAM resources."

scripts/terraform/oci-spx/outputs.tf

@@ -31,3 +31,20 @@ output "nodes" {
 output "hosts_file" {
   value = join("\n", [for instance in oci_core_instance.mulgadc : instance.public_ip])
 }
+
+# The api_key route's equivalent of the instance-principal policy, for a tenancy
+# admin to apply to the API user's group. Sensitive only to keep the compartment
+# OCID out of apply.log; read it with terraform output api_key_policy_statements.
+output "api_key_policy_statements" {
+  sensitive = true
+  value = [
+    for grant in local.node_grants :
+    "Allow group ${var.api_key_group_name} to ${grant} in compartment id ${var.compartment_ocid}"
+  ]
+}
+
+# Where spx admin init should detect the nodes on ocibsd, so the run can check
+# what init read from the instance metadata and knows where to sweep afterwards.
+output "availability_domain" { value = local.compute_availability_domain }
+output "region" { value = var.region }
+output "ebs_provider" { value = var.ebs_provider }

scripts/terraform/oci-spx/scripts/oci_volumes.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""Find and sweep the OCI volumes and backups ocibsd tags, which no Terraform state holds.
+
+Run under oci_env.py for the credential, with the compartment in OCIBSD_COMPARTMENT.
+Standard library and the openssl binary only; the key reaches openssl through a pipe.
+
+  oci_volumes.py volumes --tag spinifex-volume-id=vol-0abc
+  oci_volumes.py backups --tag spinifex-snapshot-id=snap-0abc
+  oci_volumes.py sweep --cluster <cluster_id> [--timeout 900]
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import email.utils
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+API = "/20160918"
+# Gone for good; OCI keeps the record for a while and it cannot be deleted again.
+GONE = {"TERMINATED"}
+# OCI's own answers to a delete it will accept later: an attachment or a backup
+# still in progress (409), or a precondition such as a busy lifecycle (412).
+RETRY_LATER = {409, 412, 429, 500, 502, 503, 504}
+
+
+class OCIError(Exception):
+    def __init__(self, status: int, body: str):
+        super().__init__(f"OCI answered {status}: {body[:300]}")
+        self.status = status
+
+
+class Client:
+    def __init__(self, env: dict[str, str]):
+        missing = [k for k in ("TF_VAR_tenancy_ocid", "TF_VAR_user_ocid", "TF_VAR_fingerprint", "TF_VAR_region") if not env.get(k)]
+        if missing:
+            raise ValueError(f"run this under oci_env.py: {', '.join(missing)} unset")
+        self.key_id = f"{env['TF_VAR_tenancy_ocid']}/{env['TF_VAR_user_ocid']}/{env['TF_VAR_fingerprint']}"
+        self.host = f"iaas.{env['TF_VAR_region']}.oraclecloud.com"
+        if env.get("TF_VAR_private_key"):
+            self.key = env["TF_VAR_private_key"].encode()
+        elif env.get("TF_VAR_private_key_path"):
+            self.key = Path(env["TF_VAR_private_key_path"]).read_bytes()
+        else:
+            raise ValueError("run this under oci_env.py: no private key in TF_VAR_private_key or TF_VAR_private_key_path")
+
+    def _sign(self, data: bytes) -> str:
+        read_end, write_end = os.pipe()
+        try:
+            os.write(write_end, self.key)
+        finally:
+            os.close(write_end)
+        try:
+            result = subprocess.run(
+                ["openssl", "dgst", "-sha256", "-sign", f"/dev/fd/{read_end}"],
+                input=data,
+                capture_output=True,
+                pass_fds=(read_end,),
+                check=False,
+            )
+        finally:
+            os.close(read_end)
+        if result.returncode != 0:
+            raise ValueError(f"openssl could not sign with the OCI key: {result.stderr.decode(errors='replace').strip()}")
+        return base64.b64encode(result.stdout).decode()
+
+    def request(self, method: str, path: str, query: dict[str, str] | None = None) -> tuple[dict[str, str], object]:
+        target = API + path
+        if query:
+            target += "?" + urllib.parse.urlencode(query)
+        date = email.utils.formatdate(usegmt=True)
+        headers = {"date": date, "host": self.host}
+        signed = ["(request-target)", "date", "host"]
+        lines = [f"(request-target): {method.lower()} {target}", f"date: {date}", f"host: {self.host}"]
+        # A PUT or POST also signs its body, which this signer does not do.
+        if method not in ("GET", "DELETE"):
+            raise ValueError(f"unsupported method {method}")
+        signature = self._sign("\n".join(lines).encode())
+        headers["authorization"] = (
+            f'Signature version="1",keyId="{self.key_id}",algorithm="rsa-sha256",'
+            f'headers="{" ".join(signed)}",signature="{signature}"'
+        )
+        req = urllib.request.Request(f"https://{self.host}{target}", method=method, headers=headers)
+        try:
+            with urllib.request.urlopen(req, timeout=60) as resp:
+                body = resp.read()
+                return dict(resp.headers), json.loads(body) if body else None
+        except urllib.error.HTTPError as error:
+            raise OCIError(error.code, error.read().decode(errors="replace")) from None
+
+    def list_all(self, kind: str, compartment: str) -> list[dict]:
+        items: list[dict] = []
+        query = {"compartmentId": compartment, "limit": "100"}
+        while True:
+            headers, page = self.request("GET", f"/{kind}", query)
+            items.extend(page or [])
+            next_page = {k.lower(): v for k, v in headers.items()}.get("opc-next-page")
+            if not next_page:
+                return items
+            query["page"] = next_page
+
+
+def matching(items: list[dict], tags: dict[str, str]) -> list[dict]:
+    """The live items carrying every tag, with no OCID beyond the item's own."""
+    found = []
+    for item in items:
+        freeform = item.get("freeformTags") or {}
+        if item.get("lifecycleState") in GONE:
+            continue
+        if all(freeform.get(k) == v for k, v in tags.items()):
+            found.append(
+                {
+                    "id": item["id"],
+                    "displayName": item.get("displayName"),
+                    "lifecycleState": item.get("lifecycleState"),
+                    "freeformTags": freeform,
+                }
+            )
+    return found
+
+
+def parse_tags(pairs: list[str]) -> dict[str, str]:
+    tags = {}
+    for pair in pairs:
+        key, sep, value = pair.partition("=")
+        if not sep or not key:
+            raise ValueError(f"--tag wants key=value, got {pair!r}")
+        tags[key] = value
+    return tags
+
+
+def sweep(client: Client, compartment: str, cluster: str, timeout: int, interval: int) -> int:
+    """Delete every live volume and backup tagged with this cluster until none is left.
+
+    OCI refuses both deletes while a backup is CREATING, so a refusal is retried
+    until the deadline, which is what makes a run killed mid-snapshot sweepable.
+    """
+    tags = {"spinifex-cluster": cluster}
+    deadline = time.monotonic() + timeout
+    while True:
+        left = []
+        for kind in ("volumeBackups", "volumes"):
+            for item in matching(client.list_all(kind, compartment), tags):
+                left.append((kind, item))
+                if item["lifecycleState"] == "TERMINATING":
+                    continue
+                try:
+                    client.request("DELETE", f"/{kind}/{item['id']}")
+                    print(f"sweep: deleting {kind[:-1]} {item['displayName']} ({item['lifecycleState']})")
+                except OCIError as error:
+                    if error.status == 404:
+                        continue
+                    if error.status not in RETRY_LATER:
+                        raise
+                    print(f"sweep: {kind[:-1]} {item['displayName']} not deletable yet: {error.status}")
+        if not left:
+            print(f"sweep: nothing tagged spinifex-cluster={cluster} is left")
+            return 0
+        if time.monotonic() >= deadline:
+            for kind, item in left:
+                print(f"sweep: STILL PRESENT {kind[:-1]} {item['displayName']} {item['lifecycleState']}")
+            print(f"sweep: {len(left)} tagged resource(s) left after {timeout}s")
+            return 1
+        time.sleep(interval)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = parser.add_subparsers(dest="command", required=True)
+    for name in ("volumes", "backups"):
+        listing = sub.add_parser(name, help=f"print the live {name} carrying every --tag, as JSON")
+        listing.add_argument("--tag", action="append", default=[], required=True)
+    sweeping = sub.add_parser("sweep", help="delete every live volume and backup tagged with --cluster")
+    sweeping.add_argument("--cluster", required=True)
+    sweeping.add_argument("--timeout", type=int, default=900)
+    sweeping.add_argument("--interval", type=int, default=15)
+    args = parser.parse_args()
+
+    compartment = os.environ.get("OCIBSD_COMPARTMENT", "")
+    if not compartment.startswith("ocid1."):
+        raise ValueError("OCIBSD_COMPARTMENT is not set to a compartment OCID")
+    client = Client(dict(os.environ))
+
+    if args.command == "sweep":
+        if not args.cluster.strip():
+            raise ValueError("--cluster is empty, which would match nothing and report clean")
+        return sweep(client, compartment, args.cluster, args.timeout, args.interval)
+    kind = "volumes" if args.command == "volumes" else "volumeBackups"
+    print(json.dumps(matching(client.list_all(kind, compartment), parse_tags(args.tag))))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (OCIError, ValueError, OSError) as error:
+        print(f"oci_volumes.py: {error}", file=sys.stderr)
+        raise SystemExit(1)

scripts/terraform/oci-spx/setup-identity.sh

@@ -21,6 +21,7 @@ STATE_DIR="${OCI_STATE_ROOT:-$HERE}/.identity"
 SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}"
 DESTROY=0
 DRY_RUN=0
+EBS_PROVIDER=viperblockd
 
 log() { printf '[setup-identity] %s\n' "$*"; }
 die() {
@@ -30,14 +31,17 @@ die() {
 
 usage() {
     cat >&2 <<EOF
-usage: ${0##*/} [--dry-run] [--destroy]
+usage: ${0##*/} [--dry-run] [--destroy] [--ebs-provider viperblockd|ocibsd]
 
 Creates the OCI dynamic group and policy for instance-principal authentication.
 Needs a tenancy-admin OCI credential; the compartment-scoped user the rest of this
 configuration runs as cannot create either resource.
 
-  --dry-run   Show the plan, change nothing.
-  --destroy   Remove them again. Every node using "adopt" loses its credential.
+  --dry-run       Show the plan, change nothing.
+  --destroy       Remove them again. Every node using "adopt" loses its credential.
+  --ebs-provider  ocibsd also grants manage volume-family and use instances. The
+                  policy is shared by every deployment in the compartment, so a
+                  later run without it removes those grants. Default viperblockd.
 EOF
     exit 2
 }
@@ -46,11 +50,16 @@ while [ $# -gt 0 ]; do
     case "$1" in
         --dry-run) DRY_RUN=1; shift ;;
         --destroy) DESTROY=1; shift ;;
+        --ebs-provider) EBS_PROVIDER="${2:?}"; shift 2 ;;
         -h | --help) usage ;;
         *) die "unknown option $1" ;;
     esac
 done
 
+case "$EBS_PROVIDER" in
+    viperblockd | ocibsd) ;;
+    *) die "--ebs-provider must be viperblockd or ocibsd, got '$EBS_PROVIDER'" ;;
+esac
 [ -r "$SSH_PUBLIC_KEY" ] || die "no readable SSH public key at $SSH_PUBLIC_KEY (set OCI_SSH_PUBLIC_KEY)"
 mkdir -p "$STATE_DIR"
 
@@ -75,7 +84,7 @@ if [ "$DESTROY" = 1 ]; then
     exit 0
 fi
 
-args=(-var 'instance_principal=create' -state "$STATE_DIR/terraform.tfstate" "${targets[@]}")
+args=(-var 'instance_principal=create' -var "ebs_provider=$EBS_PROVIDER" -state "$STATE_DIR/terraform.tfstate" "${targets[@]}")
 
 if [ "$DRY_RUN" = 1 ]; then
     log "dry run: planning the dynamic group and policy only"

scripts/terraform/oci-spx/validate-topology.sh

@@ -49,7 +49,7 @@ WORKBOOKS=""
 # set -u an unset array is an error rather than an empty one.
 HOSTS=()
 
-TOPOLOGIES="bm vm-single vm-multi vm-single-principal"
+TOPOLOGIES="bm vm-single vm-multi vm-single-principal vm-single-ocibsd vm-multi-ocibsd"
 
 # What a topology is worth when nothing says otherwise, so a CI run of a named
 # topology is the same every time. Supplied as TF_VAR_, which terraform.auto.tfvars
@@ -61,8 +61,14 @@ TOPOLOGIES="bm vm-single vm-multi vm-single-principal"
 #
 # A case rather than an associative array: macOS ships bash 3.2, where declare -A
 # is an indexed assignment and the key is read as an arithmetic variable.
+#
+# TOPO_EBS_PROVIDER is what backs EBS. The -ocibsd topologies are their viperblock
+# twins on OCI Block Volumes, so a difference between a pair is the provider's.
+# They set no provider and rely on init's detection on an OCI VM; every other
+# topology names viperblockd in SPINIFEX_EBS_PROVIDER to keep its baseline.
 topo_defaults() {
     TOPO_FORCE_PRINCIPAL=0
+    TOPO_EBS_PROVIDER=viperblockd
     case "$1" in
         bm)        TOPO_DEFAULT_SHAPE="BM.Standard.E2.64";  TOPO_DEFAULT_NODES=1 ;;
         vm-single) TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=1 ;;
@@ -74,6 +80,10 @@ topo_defaults() {
             TOPO_DEFAULT_SHAPE="${OCI_PRINCIPAL_SHAPE:-VM.Standard.E6.Flex}"
             TOPO_DEFAULT_NODES=1
             TOPO_FORCE_PRINCIPAL=1 ;;
+        vm-single-ocibsd)
+            TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=1; TOPO_EBS_PROVIDER=ocibsd ;;
+        vm-multi-ocibsd)
+            TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=3; TOPO_EBS_PROVIDER=ocibsd ;;
         *) return 1 ;;
     esac
 }
@@ -86,7 +96,7 @@ die() {
 
 usage() {
     cat >&2 <<EOF
-usage: ${0##*/} --topology <bm|vm-single|vm-multi|vm-single-principal> [options]
+usage: ${0##*/} --topology <bm|vm-single|vm-multi|vm-single-principal|vm-single-ocibsd|vm-multi-ocibsd> [options]
 
 Builds a topology, or installs on existing OCI hosts, then installs Spinifex,
 forms the cluster, and verifies the OCI address allocator.
@@ -95,6 +105,13 @@ forms the cluster, and verifies the OCI address allocator.
                           topology is the easiest expensive mistake here.
                           vm-single-principal is vm-single authenticating as the
                           instance, and implies --instance-principal.
+                          vm-single-ocibsd and vm-multi-ocibsd set no EBS
+                          provider, so init detects ocibsd on the OCI VM, and
+                          add the ebs provider gate; the others set
+                          SPINIFEX_EBS_PROVIDER=viperblockd. The ocibsd pair
+                          need --distro. Their OCI volumes and backups are
+                          in no Terraform state: the destroy here leaves any a
+                          failed run made, and e2e-oci.sh sweeps them by tag.
   --ssh-public-key PATH   Public half installed on each node. Default $SSH_PUBLIC_KEY
   --ssh-private-key PATH  Private half used to reach them. Default $SSH_PRIVATE_KEY
   --instance-principal    Configure the pool with oci_auth="instance_principal"
@@ -106,7 +123,9 @@ forms the cluster, and verifies the OCI address allocator.
                           Create the dynamic group and policy before deployment.
                           Needs a tenancy-admin profile. Opt-in because these are
                           tenancy-level resources. Use --identity-oci-profile to
-                          use a different profile for this one-time setup.
+                          use a different profile for this one-time setup. The
+                          policy carries the volume grants only on an ocibsd
+                          topology, and it is shared, so run it from one of those.
   --identity-oci-profile NAME
                           Tenancy-admin profile for --setup-instance-principal.
                           Defaults to --oci-profile / OCI_PROFILE.
@@ -209,6 +228,19 @@ fi
 
 [ -z "$EXISTING_HOSTS_FILE" ] || [ -z "$EXISTING_STATE_FILE" ] \
     || die "--existing-hosts-file and --existing-state-file are mutually exclusive"
+
+# ocibsd needs a node credential, which a run with no pool never installs, and the
+# compartment and region from the state, which a hosts file does not carry, to
+# sweep and check its volumes. setup.sh stages its files from the tarball, and a
+# published release predates them.
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ] && [ "$DESTROY_ONLY" = 0 ]; then
+    [ "$SKIP_POOL" != 1 ] \
+        || die "--no-external-pool installs no OCI credential, so ocibsd on $TOPOLOGY could not reach the OCI API"
+    [ -z "$EXISTING_HOSTS_FILE" ] \
+        || die "$TOPOLOGY reads its compartment and availability domain from Terraform state; use --existing-state-file"
+    [ -n "$DISTRO" ] || [ "$DRY_RUN" = 1 ] \
+        || die "$TOPOLOGY needs --distro: a published release predates the ocibsd files setup.sh stages and init installs"
+fi
 if [ -n "$EXISTING_HOSTS_FILE" ] || [ -n "$EXISTING_STATE_FILE" ]; then
     EXISTING_INFRA=1
     [ "$DESTROY_ONLY" = 0 ] \
@@ -281,7 +313,7 @@ tf() {
     # deployment sizes itself in that file and the topology only supplies the
     # default a CI run wants.
     TF_VAR_compute_shape="$TOPO_DEFAULT_SHAPE" TF_VAR_node_count="$TOPO_DEFAULT_NODES" \
-        python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" \
+        TF_VAR_ebs_provider="$TOPO_EBS_PROVIDER" python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" \
         ${OCI_PROFILE:+--profile "$OCI_PROFILE"} -- \
         terraform -chdir="$HERE" "$@"
 }
@@ -383,18 +415,18 @@ identity_resources() {
                 "$HERE/setup-identity.sh" --destroy
         elif [ "$DRY_RUN" = 1 ]; then
             OCI_CLI_PROFILE="$identity_profile" OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \
-                "$HERE/setup-identity.sh" --dry-run
+                "$HERE/setup-identity.sh" --dry-run --ebs-provider "$TOPO_EBS_PROVIDER"
         else
             OCI_CLI_PROFILE="$identity_profile" OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \
-                "$HERE/setup-identity.sh"
+                "$HERE/setup-identity.sh" --ebs-provider "$TOPO_EBS_PROVIDER"
         fi
     else
         if [ "$action" = destroy ]; then
             OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --destroy
         elif [ "$DRY_RUN" = 1 ]; then
-            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --dry-run
+            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --dry-run --ebs-provider "$TOPO_EBS_PROVIDER"
         else
-            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh"
+            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --ebs-provider "$TOPO_EBS_PROVIDER"
         fi
     fi
 }
@@ -443,8 +475,10 @@ mark_stage() { : > "$STATE_DIR/stage-$1.done"; }
 formation_complete() {
     local host
     stage_done formation || return 1
+    # A line only init writes, not a non-empty file: an ocibsd API-key run seeds the
+    # file with the credential before init, so a fresh node already has one.
     for host in "${HOSTS[@]}"; do
-        ssh_node "$host" 'test -s /etc/spinifex/spinifex.toml' || return 1
+        ssh_node "$host" "grep -q '^data_dir = ' /etc/spinifex/spinifex.toml" || return 1
     done
 }
 
@@ -634,6 +668,7 @@ if [ "$DRY_RUN" = 1 ]; then
         | sed -E 's/ +=/ =/' | sort -u | sed 's/^/  /'
     log "full plan in $STATE_DIR/plan.txt"
     plan="install Spinifex on $NODES node(s), form the cluster"
+    [ "$TOPO_EBS_PROVIDER" = ocibsd ] && plan="$plan on ocibsd, run the ebs provider gate"
     [ "$SKIP_POOL" = 1 ] && plan="$plan, configure no external pool"
     if [ "$SKIP_WORKLOAD" = 1 ]; then
         plan="$plan, launch no guests"
@@ -669,6 +704,29 @@ else
         || die "read instance_principal='$PRINCIPAL_MODE' from the tfvars and the apply built '$built_principal'; set it in one place"
 fi
 
+# Where ocibsd should keep the cluster's volumes, read back from what was built
+# rather than from this run's inputs, to check what init detected and to sweep.
+# Never logged: the compartment is an account's OCID.
+topo_output() {
+    if [ "$EXISTING_INFRA" = 1 ]; then
+        state_output "$1"
+    else
+        tf output -raw "${tf_state[@]}" "$1"
+    fi
+}
+OCIBSD_COMPARTMENT=""
+OCIBSD_AD=""
+OCIBSD_REGION=""
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+    built_provider="$(topo_output ebs_provider 2>/dev/null)" \
+        || die "the state has no ebs_provider output; it predates ocibsd, so rebuild it"
+    [ "$built_provider" = ocibsd ] \
+        || die "$TOPOLOGY needs nodes built for ocibsd and the state says ebs_provider='$built_provider'; a tfvars override wins over the topology"
+    OCIBSD_COMPARTMENT="$(topo_output compartment_ocid)" || die "could not read compartment_ocid from the state"
+    OCIBSD_AD="$(topo_output availability_domain)" || die "could not read availability_domain from the state"
+    OCIBSD_REGION="$(topo_output region)" || die "could not read region from the state"
+fi
+
 # Written for every topology, not just the multi-node one that passes it to
 # install-node.sh: it is also how an operator gets the addresses afterwards
 # without a terraform invocation and the right -state path.
@@ -768,6 +826,42 @@ REMOTE
 done
 record install PASS "$INSTALLED_VERSION"
 
+
+# spx admin init's EBS choice, written to every node once (ADR-0018:S3). The ocibsd
+# topologies set nothing, so init detects ocibsd on the OCI VM; the rest name
+# viperblockd, which keeps the viperblock baseline on OCI.
+INIT_EBS_PROVIDER=""
+[ "$TOPO_EBS_PROVIDER" = ocibsd ] || INIT_EBS_PROVIDER=viperblockd
+
+# ocibsd defaults to the instance principal. An API-key run gives init the key
+# instead: the hook installs it first, and each node's config is seeded with
+# provider and credential, which init keeps as an existing config's values.
+API_KEY_EBS=0
+[ "$TOPO_EBS_PROVIDER" = ocibsd ] && [ "$PRINCIPAL_MODE" = off ] && API_KEY_EBS=1
+PRESEEDED_EBS=0
+if [ "$API_KEY_EBS" = 1 ] && ! formation_complete; then
+    log "running the credential hook before formation, so init can check the ocibsd credential"
+    "$CREDENTIAL_HOOK" "$SSH_PRIVATE_KEY" "${HOSTS[@]}" > "$STATE_DIR/credential-hook.log" 2>&1 \
+        || die "the credential hook failed; see $STATE_DIR/credential-hook.log"
+    for i in "${!HOSTS[@]}"; do
+        # Named as init and install-node.sh name them: the host on one node,
+        # node1..nodeN on several.
+        if [ "$NODES" = 1 ]; then name='$(hostname -s)'; else name="node$((i + 1))"; fi
+        ssh_node "${HOSTS[$i]}" "sed \"s/@NODE@/$name/\" | sudo tee /etc/spinifex/spinifex.toml >/dev/null" <<'TOML' \
+            || die "could not seed the ocibsd credential on ${HOSTS[$i]}"
+[nodes.@NODE@.ebs]
+provider = "ocibsd"
+
+[nodes.@NODE@.ebs.oci]
+auth = "config_file"
+config_file = "/etc/spinifex/oci/config"
+config_profile = "spinifex"
+TOML
+    done
+    PRESEEDED_EBS=1
+    record "ocibsd credential" PASS "API key installed and seeded before init"
+fi
+
 # A single node is its own documented path: install-node.sh refuses fewer than two
 # hosts, because there is nothing to join. Both branches use the flags the guide
 # names, so what is validated is what the guide tells a customer to type.
@@ -776,19 +870,29 @@ if formation_complete; then
     record formation SKIP "checkpoint verified"
 elif [ "$NODES" = 1 ]; then
     log "initializing the single node"
-    ssh_node "${HOSTS[0]}" '
-        set -e
-        sudo spx admin init --node "$(hostname -s)" --nodes 1 \
-            --region ap-southeast-2 --az ap-southeast-2a \
-            --external-mode=nat --ipsec=false
-        sudo systemctl start spinifex.target
-    ' > "$STATE_DIR/form.log" 2>&1 \
+    ssh_node "${HOSTS[0]}" "INIT_EBS_PROVIDER=$INIT_EBS_PROVIDER PRESEEDED_EBS=$PRESEEDED_EBS bash -s" \
+        > "$STATE_DIR/form.log" 2>&1 <<'REMOTE' \
         || die "single-node init failed; see $STATE_DIR/form.log"
+set -e
+ebs_env=()
+[ -z "$INIT_EBS_PROVIDER" ] || ebs_env=("SPINIFEX_EBS_PROVIDER=$INIT_EBS_PROVIDER")
+# The seeded config is the only one there, so replacing it needs --force.
+force=()
+[ "$PRESEEDED_EBS" = 0 ] || force=(--force)
+sudo env -u SPINIFEX_EBS_PROVIDER ${ebs_env[@]+"${ebs_env[@]}"} spx admin init ${force[@]+"${force[@]}"} \
+    --node "$(hostname -s)" --nodes 1 \
+    --region ap-southeast-2 --az ap-southeast-2a \
+    --external-mode=nat --ipsec=false
+sudo systemctl start spinifex.target
+REMOTE
     mark_stage formation
     record formation PASS
 else
     log "forming the cluster"
-    "$REPO_ROOT/scripts/install-node.sh" \
+    # env -u first, so a SPINIFEX_EBS_PROVIDER in this shell cannot stand in for
+    # detection on the ocibsd topologies.
+    env -u SPINIFEX_EBS_PROVIDER ${INIT_EBS_PROVIDER:+SPINIFEX_EBS_PROVIDER=$INIT_EBS_PROVIDER} \
+        "$REPO_ROOT/scripts/install-node.sh" \
         --hosts-file "$STATE_DIR/hosts" \
         --user ubuntu \
         --identity "$SSH_PRIVATE_KEY" \
@@ -800,6 +904,53 @@ else
     record formation PASS
 fi
 
+# What init printed about its EBS choice, from the log formation left: init's own
+# output on one node, the init log install-node.sh keeps on the first host on several.
+init_ebs_line() {
+    if [ "$NODES" = 1 ]; then
+        grep -m1 '^EBS provider: ' "$STATE_DIR/form.log"
+    else
+        ssh_node "${HOSTS[0]}" "sudo grep -m1 '^EBS provider: ' /var/log/spx-init.log"
+    fi
+}
+# The provider has to be the topology's for the reason the topology names: the
+# ocibsd pair detected on the VM, the rest told viperblockd by the environment,
+# so a baseline flipped by detection or an ocibsd run that was configured fails.
+case "$TOPO_EBS_PROVIDER:$API_KEY_EBS" in
+    ocibsd:0) want_ebs='EBS provider: ocibsd (detected: OCI virtual machine ' ;;
+    ocibsd:1) want_ebs='EBS provider: ocibsd (from the existing config)' ;;
+    *)        want_ebs='EBS provider: viperblockd (from SPINIFEX_EBS_PROVIDER)' ;;
+esac
+ebs_line="$(init_ebs_line)" || die "init printed no EBS provider line; see $STATE_DIR/form.log"
+case "$ebs_line" in
+    "$want_ebs"*) record "ebs provider choice" PASS "${ebs_line#EBS provider: }" ;;
+    *) die "init chose '$ebs_line', and $TOPOLOGY needs '$want_ebs'" ;;
+esac
+
+# ocibsd tags every volume and backup with the cluster_id spx admin init generated,
+# and no Terraform state holds them, so e2e-oci.sh sweeps by that tag after the
+# destroy. Recorded before any volume exists, so a killed run is still sweepable.
+OCIBSD_CLUSTERS="$STATE_DIR/ocibsd-clusters"
+OCIBSD_CLUSTER_ID=""
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+    OCIBSD_CLUSTER_ID="$(ssh_node "${HOSTS[0]}" 'sudo python3 -' <<'PY'
+import sys, tomllib
+nodes = tomllib.load(open("/etc/spinifex/spinifex.toml", "rb")).get("nodes", {})
+# A leader's config also lists the other nodes, without their EBS tables.
+ids = {n["ebs"]["oci"].get("cluster_id", "") for n in nodes.values() if "oci" in n.get("ebs", {})}
+if len(ids) != 1 or "" in ids:
+    sys.exit("expected one cluster_id across [nodes.*.ebs.oci], found %r" % sorted(ids))
+print(ids.pop())
+PY
+)" || die "could not read the ocibsd cluster_id from ${HOSTS[0]}"
+    [[ "$OCIBSD_CLUSTER_ID" =~ ^[0-9a-f]{32}$ ]] \
+        || die "${HOSTS[0]} reports cluster_id '$OCIBSD_CLUSTER_ID', which is not the 128-bit hex value init writes"
+    # Appended: a record a failed sweep left behind must survive this run's.
+    line="$(printf '%s\t%s\t%s' "$OCIBSD_CLUSTER_ID" "$OCIBSD_COMPARTMENT" "$OCIBSD_REGION")"
+    grep -qxF "$line" "$OCIBSD_CLUSTERS" 2>/dev/null || printf '%s\n' "$line" >> "$OCIBSD_CLUSTERS"
+    log "ocibsd cluster_id $OCIBSD_CLUSTER_ID, recorded for the sweep"
+fi
+
 # Runs after formation and before the pool is configured, which is the only window
 # where a node has /etc/spinifex but has not yet started the allocator. Nothing in
 # this repository knows what it does: an API-key deployment needs a credential on
@@ -809,7 +960,7 @@ if pool_complete; then
     log "OCI pool configuration already completed (resuming; pool setup skipped)"
     record pool SKIP "checkpoint verified"
 else
-    if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then
+    if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ] && [ "$PRESEEDED_EBS" = 0 ]; then
         log "running the credential hook"
         # Arguments, not a file: the hook is told where the nodes are and how to reach
         # them, and decides for itself what to put there.
@@ -936,9 +1087,9 @@ fi
 
 # Parse the STATUS column, never grep for Ready: NotReady contains it, and that
 # read passed a single-node run whose only node was NotReady on 0.0.0.0. The table
-# is colourised, so the escapes come off first.
+# is colourised and drawn with box characters, so both come off first.
 count_ready() {
-    sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/nodes.txt" \
+    sed -e 's/\x1b\[[0-9;]*m//g' -e 's/│/|/g' "$STATE_DIR/nodes.txt" \
         | awk -F'|' 'NR > 1 { gsub(/^[ \t]+|[ \t]+$/, "", $2); if ($2 == "Ready") n++ } END { print n + 0 }'
 }
 
@@ -962,10 +1113,382 @@ fi
 
 if [ "$SKIP_WORKLOAD" = 1 ]; then
     log "--skip-workload: stopping before the workbook"
+    if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+        record "ebs provider" SKIPPED "--skip-workload"
+    fi
     record workbooks SKIPPED "--skip-workload"
     exit 0
 fi
 
+# The ebs provider gate, ocibsd only. The workbooks reach EBS only through RDS,
+# ECS and EKS, so this boots from an imported AMI, stops, attaches, snapshots,
+# restores and terminates, checking each against OCI's own view of the volumes.
+
+GATE_IMAGE=""
+GATE_LOG="$STATE_DIR/ebs-gate.log"
+
+# The AWS CLI on node 1, as the workbook driver runs it. Each argument is quoted
+# for the remote shell, so a JMESPath expression arrives as one word.
+node_aws() {
+    ssh_node "${HOSTS[0]}" "AWS_PROFILE=spinifex aws --output text $(printf '%q ' "$@")"
+}
+
+# OCI's live volumes or backups carrying every key=value given, as JSON, read from
+# this runner with the credential the apply used; nothing on a node is trusted.
+oci_tagged() {
+    local kind="$1" tag tags=()
+    shift
+    for tag in "$@"; do tags+=(--tag "$tag"); done
+    OCIBSD_COMPARTMENT="$OCIBSD_COMPARTMENT" python3 "$HERE/scripts/oci_env.py" \
+        ${OCI_PROFILE:+--profile "$OCI_PROFILE"} --region "$OCIBSD_REGION" -- \
+        python3 "$HERE/scripts/oci_volumes.py" "$kind" "${tags[@]}" 2>>"$GATE_LOG"
+}
+
+oci_count() {
+    oci_tagged "$@" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)))'
+}
+
+# get-console-output is base64 on the wire and the CLI may or may not decode it.
+console_text() {
+    node_aws ec2 get-console-output --instance-id "$1" --query Output 2>/dev/null \
+        | python3 -c '
+import base64, binascii, sys
+raw = sys.stdin.read().strip()
+try:
+    sys.stdout.write(base64.b64decode(raw, validate=True).decode("utf-8", "replace"))
+except (binascii.Error, ValueError):
+    sys.stdout.write(raw)
+' | tr -d '\r'
+}
+
+# wait_console <instance> <ERE> <seconds> prints the last console line matching.
+wait_console() {
+    local id="$1" pattern="$2" deadline=$((SECONDS + $3)) line
+    while [ "$SECONDS" -lt "$deadline" ]; do
+        line="$(console_text "$id" | grep -E "$pattern" | tail -1 || true)"
+        if [ -n "$line" ]; then
+            printf '%s\n' "$line"
+            return 0
+        fi
+        sleep 10
+    done
+    return 1
+}
+
+# wait_new_boot <instance> <token> <boots seen> <seconds> prints the boot ID of
+# the first boot not yet seen that found the root marker, proving a restart kept it.
+wait_new_boot() {
+    local id="$1" tok="$2" seen="$3" deadline=$((SECONDS + $4)) boot
+    while [ "$SECONDS" -lt "$deadline" ]; do
+        for boot in $(console_text "$id" \
+            | sed -n "s/.*SPXGATE boot=\([0-9a-f-]*\) root-present serial=[^ ]* $tok\$/\1/p"); do
+            case " $seen " in
+                *" $boot "*) ;;
+                *) printf '%s\n' "$boot"; return 0 ;;
+            esac
+        done
+        sleep 10
+    done
+    return 1
+}
+
+# The node whose QEMU runs the instance, by process, as the multinode suite finds it.
+hosting_node() {
+    local host found=""
+    for host in "${HOSTS[@]}"; do
+        if ssh_node "$host" "ps -eo args | grep -F -- '$1' | grep -q '[q]emu-system'"; then
+            [ -z "$found" ] || die "$1 runs on both $found and $host"
+            found="$host"
+        fi
+    done
+    [ -n "$found" ] || die "no node runs a QEMU for $1"
+    printf '%s\n' "$found"
+}
+
+wait_gone_from_oci() {
+    local deadline=$((SECONDS + 600)) left vol
+    while :; do
+        left=0
+        for vol in "$@"; do
+            case "$vol" in
+                snap-*) left=$((left + $(oci_count backups "spinifex-snapshot-id=$vol"))) ;;
+                *) left=$((left + $(oci_count volumes "spinifex-volume-id=$vol"))) ;;
+            esac
+        done
+        [ "$left" = 0 ] && return 0
+        [ "$SECONDS" -lt "$deadline" ] || return 1
+        sleep 15
+    done
+}
+
+# Reports on the serial console, which get-console-output reads: the root marker
+# at each boot, then each Spinifex disk (serial vol...) as it appears, writing a
+# marker to a blank one and reading one back from a restored one.
+write_gate_user_data() {
+    cat > "$1" <<'USERDATA'
+#cloud-config
+write_files:
+  - path: /usr/local/sbin/spx-ebs-gate
+    permissions: "0755"
+    content: |
+      #!/bin/sh
+      BOOT=$(cat /proc/sys/kernel/random/boot_id)
+      say() { echo "SPXGATE boot=$BOOT $*" > /dev/ttyS0; }
+      serial() { cat "/sys/block/$1/serial" 2>/dev/null || cat "/sys/block/$1/device/serial" 2>/dev/null || true; }
+      src=$(findmnt -no SOURCE /)
+      rootdisk=$(lsblk -no PKNAME "$src" | head -1)
+      [ -n "$rootdisk" ] || rootdisk=${src##*/}
+      mark=/var/lib/spx-ebs-gate/root
+      if [ -s "$mark" ]; then
+          say "root-present serial=$(serial "$rootdisk") $(cat "$mark")"
+      else
+          mkdir -p /var/lib/spx-ebs-gate
+          cat /proc/sys/kernel/random/uuid > "$mark.tmp" && sync && mv "$mark.tmp" "$mark" && sync
+          say "root-written serial=$(serial "$rootdisk") $(cat "$mark")"
+      fi
+      seen=" "
+      while :; do
+          next=" "
+          for sys in /sys/block/*; do
+              d=${sys##*/}
+              [ "$d" != "$rootdisk" ] || continue
+              s=$(serial "$d")
+              case "$s" in vol*) ;; *) continue ;; esac
+              case "$seen" in *" $d "*) next="$next$d "; continue ;; esac
+              head=$(dd if="/dev/$d" bs=4096 count=1 iflag=direct 2>/dev/null | head -c 64 | tr -d '\000')
+              case "$head" in
+                  "SPXGATE-DATA "*)
+                      say "data-present serial=$s ${head#SPXGATE-DATA }"; next="$next$d " ;;
+                  "")
+                      if blkid -p "/dev/$d" >/dev/null 2>&1; then
+                          say "data-foreign serial=$s"; next="$next$d "
+                      else
+                          tok=$(cat /proc/sys/kernel/random/uuid)
+                          if printf 'SPXGATE-DATA %s\n' "$tok" \
+                              | dd of="/dev/$d" bs=4096 count=1 conv=sync oflag=direct,sync 2>/dev/null; then
+                              say "data-written serial=$s $tok"; next="$next$d "
+                          fi
+                      fi ;;
+                  *) say "data-foreign serial=$s"; next="$next$d " ;;
+              esac
+          done
+          seen=$next
+          sleep 3
+      done
+  - path: /etc/systemd/system/spx-ebs-gate.service
+    content: |
+      [Unit]
+      Description=Spinifex EBS provider gate reporter
+      After=local-fs.target
+      [Service]
+      ExecStart=/usr/local/sbin/spx-ebs-gate
+      Restart=always
+      [Install]
+      WantedBy=multi-user.target
+runcmd:
+  - [systemctl, daemon-reload]
+  - [systemctl, enable, --now, spx-ebs-gate.service]
+USERDATA
+}
+
+# What the gate leaves for a failure it cannot explain from its own output: each
+# node's iSCSI sessions and records, block devices and their holders, and the
+# provider's and kernel's recent log. Best effort, so a failed read never masks
+# the gate's own error.
+gate_diagnose() {
+    local host
+    for host in "${HOSTS[@]}"; do
+        {
+            printf '\n=== diagnostics from %s ===\n' "$host"
+            ssh_node "$host" 'sudo bash -s' <<'DIAG' 2>&1
+for c in "iscsiadm -m session -P 3" "iscsiadm -m node" "lsblk -o NAME,SIZE,TYPE,MOUNTPOINTS,HCTL" "ls -l /dev/disk/by-path"; do
+    printf -- '--- %s\n' "$c"; $c 2>&1
+done
+printf -- '--- holders of iSCSI devices\n'
+for d in /dev/disk/by-path/*iqn.2015-12.com.oracleiaas:*; do
+    [ -e "$d" ] || continue
+    dev=$(readlink -f "$d"); printf '%s -> %s\n' "$d" "$dev"
+    ls /sys/class/block/"${dev##*/}"/holders 2>&1; fuser -v "$dev" 2>&1
+done
+printf -- '--- journalctl -u spinifex-ocibsd\n'; journalctl -u spinifex-ocibsd -n 300 --no-pager 2>&1
+printf -- '--- journalctl -k\n'; journalctl -k -n 150 --no-pager 2>&1
+DIAG
+        } >> "$GATE_LOG" || true
+    done
+}
+gate_die() {
+    gate_diagnose
+    die "$@"
+}
+
+ebs_provider_gate() {
+    local host out ami snap_ami itype subnet iid az root_vol dot tok boot boots line
+    local data_vol data_serial data_tok snap restored from to
+    : > "$GATE_LOG"
+
+    log "ebs provider gate: units, masks, drop-ins and the detected placement on every node"
+    for host in "${HOSTS[@]}"; do
+        # The placement init read from the instance metadata must be where this run
+        # built the nodes; compared on the node, so the compartment is never printed.
+        ssh_node "$host" "WANT_COMPARTMENT='$OCIBSD_COMPARTMENT' WANT_AD='$OCIBSD_AD' bash -s" >> "$GATE_LOG" 2>&1 <<'REMOTE' \
+            || gate_die "$host is not an ocibsd node as installed; see $GATE_LOG"
+fail() { echo "FAIL $(hostname -s): $*"; exit 1; }
+[ "$(systemctl is-enabled spinifex-viperblock.service 2>&1)" = masked ] \
+    || fail "spinifex-viperblock.service is not masked"
+if systemctl is-active --quiet spinifex-viperblock.service; then fail "spinifex-viperblock.service is running"; fi
+systemctl is-active --quiet spinifex-ocibsd.service || fail "spinifex-ocibsd.service is not active"
+[ "$(systemctl is-enabled spinifex-ocibsd.service)" = enabled ] || fail "spinifex-ocibsd.service is not enabled"
+systemctl show -p DropInPaths --value spinifex-daemon.service \
+    | grep -q /etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf \
+    || fail "the daemon's ocibsd device drop-in is not loaded"
+systemctl show -p DeviceAllow --value spinifex-daemon.service | grep -q 'block-sd rw' \
+    || fail "the daemon's device cgroup does not allow block-sd"
+test -f /etc/udev/rules.d/99-spinifex-ocibsd.rules || fail "no ocibsd udev rule"
+sudo WANT_COMPARTMENT="$WANT_COMPARTMENT" WANT_AD="$WANT_AD" python3 - <<"PY" \
+    || fail "a node in spinifex.toml is not on ocibsd where the nodes were built"
+import os, sys, tomllib
+nodes = tomllib.load(open("/etc/spinifex/spinifex.toml", "rb")).get("nodes", {})
+# Only tables with an ebs section: a leader also lists the other nodes, bare.
+# Each node's own table always has one, and this runs on every node.
+nodes = {n: v for n, v in nodes.items() if "ebs" in v}
+other = {n: v["ebs"].get("provider") for n, v in nodes.items() if v["ebs"].get("provider") != "ocibsd"}
+if not nodes or other:
+    sys.exit("providers other than ocibsd: %r" % other)
+for n, v in sorted(nodes.items()):
+    oci = v["ebs"].get("oci", {})
+    if oci.get("compartment_id") != os.environ["WANT_COMPARTMENT"]:
+        sys.exit("%s: compartment_id is not the compartment the nodes were built in" % n)
+    if oci.get("availability_domain") != os.environ["WANT_AD"]:
+        sys.exit("%s: availability_domain %r, and the nodes were built in %r" % (n, oci.get("availability_domain"), os.environ["WANT_AD"]))
+print("ebs providers:", ", ".join("%s=ocibsd" % n for n in sorted(nodes)))
+PY
+echo "ok $(hostname -s)"
+REMOTE
+    done
+    record "ebs provider units" PASS "ocibsd active, viperblock masked, device drop-in, placement from metadata on $NODES node(s)"
+
+    GATE_IMAGE=ubuntu-26.04-x86_64
+    log "ebs provider gate: importing $GATE_IMAGE onto an OCI volume"
+    out="$(ssh_node "${HOSTS[0]}" "sudo spx admin images import --name $GATE_IMAGE --config /etc/spinifex/spinifex.toml" 2>&1)" \
+        || { printf '%s\n' "$out" >> "$GATE_LOG"; gate_die "importing $GATE_IMAGE failed; see $GATE_LOG"; }
+    printf '%s\n' "$out" >> "$GATE_LOG"
+    ami="$(printf '%s\n' "$out" | sed -n 's/.*Image-ID (AMI): \(ami-[0-9a-f]*\).*/\1/p' | tail -1)"
+    [ -n "$ami" ] || gate_die "the import printed no AMI ID; see $GATE_LOG"
+    snap_ami="$(node_aws ec2 describe-images --image-ids "$ami" --query 'Images[0].BlockDeviceMappings[0].Ebs.SnapshotId')"
+    [ "$(oci_count backups "spinifex-snapshot-id=$snap_ami" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$ami's snapshot $snap_ami is not one OCI backup tagged with this cluster"
+    record "ami import" PASS "$ami, snapshot $snap_ami is an OCI backup"
+
+    itype="$(node_aws ec2 describe-instance-types \
+        --query 'sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus>=`2` && MemoryInfo.SizeInMiB>=`2048`], &MemoryInfo.SizeInMiB)[0].InstanceType')"
+    { [ -n "$itype" ] && [ "$itype" != None ]; } || gate_die "no instance type with 2 vCPU and 2 GiB on this cluster"
+    subnet="$(node_aws ec2 describe-subnets --filters Name=default-for-az,Values=true \
+        --query 'Subnets[0].SubnetId' 2>>"$GATE_LOG")" || subnet=""
+    [ "$subnet" != None ] || subnet=""
+    write_gate_user_data "$STATE_DIR/ebs-gate-user-data.yaml"
+    scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o LogLevel=ERROR -q "$STATE_DIR/ebs-gate-user-data.yaml" "ubuntu@${HOSTS[0]}:/tmp/ebs-gate-user-data.yaml" \
+        || gate_die "could not copy the gate's user-data"
+
+    log "ebs provider gate: booting $itype from $ami"
+    iid="$(node_aws ec2 run-instances --image-id "$ami" --instance-type "$itype" --count 1 \
+        ${subnet:+--subnet-id "$subnet"} --user-data file:///tmp/ebs-gate-user-data.yaml \
+        --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=ebs-provider-gate}]' \
+        --query 'Instances[0].InstanceId')" || gate_die "run-instances from $ami failed"
+    node_aws ec2 wait instance-running --instance-ids "$iid" || gate_die "$iid never reached running"
+    read -r az root_vol dot <<EOF
+$(node_aws ec2 describe-instances --instance-ids "$iid" \
+    --query 'Reservations[0].Instances[0].[Placement.AvailabilityZone,BlockDeviceMappings[0].Ebs.VolumeId,BlockDeviceMappings[0].Ebs.DeleteOnTermination]')
+EOF
+    [ "$dot" = True ] || gate_die "$iid's root volume $root_vol is not delete-on-termination"
+    line="$(wait_console "$iid" 'SPXGATE boot=[0-9a-f-]+ root-written ' 900)" \
+        || gate_die "$iid never reported its root marker on the console"
+    boot="$(printf '%s\n' "$line" | sed -n 's/.*boot=\([0-9a-f-]*\) .*/\1/p')"
+    tok="${line##* }"
+    boots="$boot"
+    [ "$(oci_count volumes "spinifex-volume-id=$root_vol" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$iid's root volume $root_vol is not one OCI volume tagged with this cluster"
+    record "boot from ocibsd" PASS "$iid on $root_vol, an OCI volume tagged with this cluster"
+
+    log "ebs provider gate: stop and start $iid"
+    node_aws ec2 stop-instances --instance-ids "$iid" >/dev/null || gate_die "stop-instances $iid failed"
+    node_aws ec2 wait instance-stopped --instance-ids "$iid" || gate_die "$iid never stopped"
+    node_aws ec2 start-instances --instance-ids "$iid" >/dev/null || gate_die "start-instances $iid failed"
+    node_aws ec2 wait instance-running --instance-ids "$iid" || gate_die "$iid never ran again"
+    boot="$(wait_new_boot "$iid" "$tok" "$boots" 900)" \
+        || gate_die "$iid did not find its root marker after a stop and start"
+    boots="$boots $boot"
+    record "stop and start" PASS "the root marker survived"
+
+    log "ebs provider gate: attach and detach a data volume"
+    data_vol="$(node_aws ec2 create-volume --availability-zone "$az" --size 1 --volume-type gp3 \
+        --query VolumeId)" || gate_die "create-volume failed"
+    node_aws ec2 wait volume-available --volume-ids "$data_vol" || gate_die "$data_vol never became available"
+    node_aws ec2 attach-volume --volume-id "$data_vol" --instance-id "$iid" --device /dev/sdf >/dev/null \
+        || gate_die "attach-volume $data_vol failed"
+    node_aws ec2 wait volume-in-use --volume-ids "$data_vol" || gate_die "$data_vol never attached"
+    data_serial="${data_vol//-/}"
+    line="$(wait_console "$iid" "SPXGATE boot=[0-9a-f-]+ data-written serial=$data_serial " 600)" \
+        || gate_die "the guest never wrote to $data_vol"
+    data_tok="${line##* }"
+    [ "$(oci_count volumes "spinifex-volume-id=$data_vol" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$data_vol is not one OCI volume tagged with this cluster"
+    node_aws ec2 detach-volume --volume-id "$data_vol" >/dev/null || gate_die "detach-volume $data_vol failed"
+    node_aws ec2 wait volume-available --volume-ids "$data_vol" || gate_die "$data_vol never detached"
+    record "attach and detach" PASS "$data_vol written in the guest"
+
+    log "ebs provider gate: snapshot, restore and read back"
+    snap="$(node_aws ec2 create-snapshot --volume-id "$data_vol" --query SnapshotId)" \
+        || gate_die "create-snapshot $data_vol failed"
+    node_aws ec2 wait snapshot-completed --snapshot-ids "$snap" || gate_die "$snap never completed"
+    [ "$(oci_count backups "spinifex-snapshot-id=$snap" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$snap is not one OCI backup tagged with this cluster"
+    restored="$(node_aws ec2 create-volume --availability-zone "$az" --snapshot-id "$snap" \
+        --volume-type gp3 --query VolumeId)" || gate_die "create-volume from $snap failed"
+    node_aws ec2 wait volume-available --volume-ids "$restored" || gate_die "$restored never became available"
+    node_aws ec2 attach-volume --volume-id "$restored" --instance-id "$iid" --device /dev/sdg >/dev/null \
+        || gate_die "attach-volume $restored failed"
+    node_aws ec2 wait volume-in-use --volume-ids "$restored" || gate_die "$restored never attached"
+    wait_console "$iid" "SPXGATE boot=[0-9a-f-]+ data-present serial=${restored//-/} $data_tok\$" 600 >/dev/null \
+        || gate_die "the guest did not read $data_vol's marker back from $restored"
+    node_aws ec2 detach-volume --volume-id "$restored" >/dev/null || gate_die "detach-volume $restored failed"
+    node_aws ec2 wait volume-available --volume-ids "$restored" || gate_die "$restored never detached"
+    record "snapshot and restore" PASS "$snap restored to $restored with the marker intact"
+
+    # A stopped instance restarts on its last node unless that node's daemon does
+    # not answer, so the daemon is stopped there for the start. The guest is down
+    # first, so no write is in flight; this is not a fence test.
+    if [ "$NODES" -gt 1 ]; then
+        from="$(hosting_node "$iid")"
+        log "ebs provider gate: moving $iid off $from"
+        node_aws ec2 stop-instances --instance-ids "$iid" >/dev/null || gate_die "stop-instances $iid failed"
+        node_aws ec2 wait instance-stopped --instance-ids "$iid" || gate_die "$iid never stopped"
+        ssh_node "$from" 'sudo systemctl stop spinifex-daemon.service' || gate_die "could not stop the daemon on $from"
+        node_aws ec2 start-instances --instance-ids "$iid" >/dev/null || gate_die "start-instances $iid failed"
+        node_aws ec2 wait instance-running --instance-ids "$iid" || gate_die "$iid never ran on another node"
+        ssh_node "$from" 'sudo systemctl start spinifex-daemon.service' || gate_die "could not restart the daemon on $from"
+        to="$(hosting_node "$iid")"
+        [ "$to" != "$from" ] || gate_die "$iid started on $from again, so nothing moved"
+        boot="$(wait_new_boot "$iid" "$tok" "$boots" 900)" \
+            || gate_die "$iid did not find its root marker on $to"
+        record "cross-node start" PASS "$iid moved from $from to $to with its root marker"
+    fi
+
+    log "ebs provider gate: terminate and delete"
+    node_aws ec2 terminate-instances --instance-ids "$iid" >/dev/null || gate_die "terminate-instances $iid failed"
+    node_aws ec2 wait instance-terminated --instance-ids "$iid" || gate_die "$iid never terminated"
+    node_aws ec2 delete-volume --volume-id "$data_vol" || gate_die "delete-volume $data_vol failed"
+    node_aws ec2 delete-volume --volume-id "$restored" || gate_die "delete-volume $restored failed"
+    node_aws ec2 delete-snapshot --snapshot-id "$snap" || gate_die "delete-snapshot $snap failed"
+    wait_gone_from_oci "$root_vol" "$data_vol" "$restored" "$snap" \
+        || gate_die "OCI still holds a live volume or backup for $root_vol, $data_vol, $restored or $snap after 600s"
+    record "terminate" PASS "no OCI volume or backup left for the instance, its volumes or the snapshot"
+}
+
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+    ebs_provider_gate
+fi
+
 # The same driver the GitHub nightly runs (cell 17), on the same published
 # workbooks, so a workbook that passes on a hypervisor and fails on OCI is a
 # difference in OCI and not in the test. It owns its own per-workbook assertions
@@ -1002,12 +1525,18 @@ scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev
 # to absorb parallel uploads. Three are appliances rather than distros: RDS, ECS
 # and EKS each boot their own, found by a spinifex:managed-by tag and never by name.
 log "importing the images the workbooks need"
-ssh_node "${HOSTS[0]}" '
-    set -e
-    for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do
-        sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml >/dev/null
-    done
-' > "$STATE_DIR/images.log" 2>&1 || die "image import failed; see $STATE_DIR/images.log"
+# The ebs provider gate has already imported its own, and is not asked twice.
+WORKBOOK_IMAGES=""
+for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do
+    [ "$img" = "$GATE_IMAGE" ] || WORKBOOK_IMAGES="$WORKBOOK_IMAGES $img"
+done
+ssh_node "${HOSTS[0]}" "WORKBOOK_IMAGES='$WORKBOOK_IMAGES' bash -s" > "$STATE_DIR/images.log" 2>&1 <<'REMOTE' \
+    || die "image import failed; see $STATE_DIR/images.log"
+set -e
+for img in $WORKBOOK_IMAGES; do
+    sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml >/dev/null
+done
+REMOTE
 
 # Every workbook that is a workbook. The shared driver's own default is five, which
 # leaves ECS and all three EKS variants untested on every platform -- their

scripts/terraform/oci-spx/variables.tf

@@ -68,6 +68,28 @@ variable "instance_principal" {
   }
 }
 
+# What backs EBS on these nodes. ocibsd attaches an OCI block volume per EBS
+# volume at runtime, so the Oracle Cloud Agent's Block Volume Management plugin
+# is disabled and the node's own data volume attaches paravirtualized instead.
+variable "ebs_provider" {
+  description = "EBS provider the nodes are installed with: viperblockd, or ocibsd for OCI Block Volumes."
+  type        = string
+  default     = "viperblockd"
+
+  validation {
+    condition     = contains(["viperblockd", "ocibsd"], var.ebs_provider)
+    error_message = "ebs_provider must be viperblockd or ocibsd."
+  }
+}
+
+# Named only in the api_key_policy_statements output, which a tenancy admin
+# applies to the group holding the API user that spx-oci-config.sh installs.
+variable "api_key_group_name" {
+  description = "IAM group of the API-key user the nodes authenticate as when instance_principal is off."
+  type        = string
+  default     = "spinifex-nodes"
+}
+
 variable "vcn_cidr" {
   description = "VCN address space. A /22 leaves room for the per-node secondary private IPs Spinifex allocates for external addresses."
   type        = string

scripts/uninstall-spx.sh

@@ -349,7 +349,8 @@ rm_path /etc/systemd/system/spinifex-*.service \
         /etc/systemd/system/spinifex.slice \
         /etc/systemd/system/spinifex-system.slice \
         /etc/systemd/system/spinifex-guests.slice \
-        /etc/systemd/system/system.slice.d/spinifex-reserve.conf
+        /etc/systemd/system/system.slice.d/spinifex-reserve.conf \
+        /etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf
 
 # Drop-ins we wrote into *other people's* units. These are the dangerous ones:
 # they are not in a spinifex-named path, so a namespace glob misses them, and

spec/EBSProvider/EBSProvider.tla

@@ -42,6 +42,7 @@ ASSUME Capabilities \in [online_expansion          : BOOLEAN,
                          read_only_publish         : BOOLEAN,
                          volume_enumeration        : BOOLEAN,
                          snapshot_enumeration      : BOOLEAN,
+                         snapshot_copy             : BOOLEAN,
                          owner_routing             : BOOLEAN,
                          exclusion                 : [scope             : {"none", "node", "cluster"},
                                                       claim_ttl_seconds : Nat,

spec/EBSProvider/README.md

@@ -17,6 +17,8 @@ The contract that every implementation of the `EBSProvider` interface must meet.
   A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
 - Is the schema version part of the contract?
   Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
+  A block export arrives at contract version 2: a PublishVolume call below it, for a volume that only a host block device can export, is refused with `unsupported_version`.
+  A snapshot's `state` in a ListSnapshots ref, and the `snapshot_id` filter, arrive at contract version 4: a ref carries no state below it, and a filtered call below it is refused with `unsupported_version` rather than answered with every snapshot.
 - Are `internal` and `unavailable` the only errors that a failed provider may cause?
 - Must a present but malformed ID be refused with `invalid_argument`?
   IDs are opaque here, so only an omitted one is refused.
@@ -100,13 +102,10 @@ The contract that every implementation of the `EBSProvider` interface must meet.
 
 ### Snapshots
 
-- Should a pending snapshot's move to `completed` or `error` be modelled, as a step that happens without a call?
-  Without it a pending snapshot never completes, and no snapshot is ever in `error`.
-
 #### CreateSnapshot
 
-- May success return a `pending` snapshot?
-  AWS starts a snapshot as `pending`, while the memory implementation returns `completed` and the NATS implementation waits for completion and refuses anything else.
+- May a provider forget a snapshot's source?
+  The control plane mints every snapshot ID and records it before the first call, so the ID alone names the request. A provider that no longer knows a snapshot's source, such as qemu-nbd after a restart, answers a retry with the snapshot, and refuses with `already_exists` only when it knows the source differs. The specification models a provider that always knows the source.
 - Must a retry compare the volume handle?
   It is ignored, as in the memory implementation, so a volume deleted and recreated under the same ID returns the old snapshot.
 - Should a copy's ID count as a retry?
@@ -133,7 +132,7 @@ The contract that every implementation of the `EBSProvider` interface must meet.
 - Is the handle check part of the contract?
   The Viperblock implementation ignores the handle and deletes anyway.
 - May a pending snapshot be deleted?
-  AWS allows it.
+  AWS allows it, while OCI refuses to delete a backup that is still being created, so the control plane never asks: it defers a customer's delete of a pending snapshot until the snapshot settles.
 - What does success promise?
   The Go docs say the snapshot is gone, but a snapshot whose handle did not match is left in place.
 - Should a snapshot that does not exist be refused with `not_found`?

spec/EBSProvider/Snapshots.tla

@@ -13,8 +13,9 @@ CreateSnapshotRequests == [snapshot_id   : Optional(SnapshotIDs),
 
 CreateSnapshot(req, val, err) ==
   (*******************************************************************************************************************)
-  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. A retry of a call that         *)
-  (* already succeeded returns the same snapshot.                                                                    *)
+  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. The snapshot holds the         *)
+  (* volume's data as it was when the call returned, even while it is pending, and a pending snapshot settles later  *)
+  (* without a call (SettleSnapshot). A retry of a call that already succeeded returns the same snapshot.            *)
   (*******************************************************************************************************************)
   LET snapID == req.snapshot_id
       volID  == req.volume_id
@@ -69,13 +70,14 @@ CopySnapshot(req, val, err) ==
       volID     == req.volume_id
 
       Errors ==
-        LET checks == [invalid_argument |-> \/ None \in {srcSnapID, dstSnapID, volID}
-                                            \/ srcSnapID = dstSnapID
-                                            \* Only compared, not looked up, since a snapshot outlives its volume.
-                                            \/ /\ KeyExists(snapshots, srcSnapID)
-                                               /\ snapshots[srcSnapID].source_volume_id /= volID,
-                       already_exists   |-> KeyExists(snapshots, dstSnapID),
-                       not_found        |-> srcSnapID \notin Optional(DOMAIN snapshots)]
+        LET checks == [invalid_argument       |-> \/ None \in {srcSnapID, dstSnapID, volID}
+                                                  \/ srcSnapID = dstSnapID
+                                                  \* Only compared, not looked up, since a snapshot outlives its volume.
+                                                  \/ /\ KeyExists(snapshots, srcSnapID)
+                                                     /\ snapshots[srcSnapID].source_volume_id /= volID,
+                       already_exists         |-> KeyExists(snapshots, dstSnapID),
+                       not_found              |-> srcSnapID \notin Optional(DOMAIN snapshots),
+                       unsupported_capability |-> ~Capabilities.snapshot_copy]
         IN {e \in DOMAIN checks : checks[e]}
 
      \* The request was rejected, or the provider failed to answer.
@@ -97,27 +99,45 @@ CopySnapshot(req, val, err) ==
 
 
 ListSnapshotsRequests == [max_results    : -1..(MaxListResults + 1),
-                          starting_token : Optional(ListTokens)]
+                          starting_token : Optional(ListTokens),
+                          snapshot_id    : Optional(SnapshotIDs)]
+
+\* The ref that ListSnapshots returns for a snapshot, carrying its current state.
+SnapshotRef(id) == [id               |-> id,
+                    source_volume_id |-> snapshots[id].source_volume_id,
+                    handle           |-> snapshots[id].handle,
+                    state            |-> snapshots[id].state]
 
 ListSnapshots(req, val, err) ==
   (*******************************************************************************************************************)
   (* Some of the snapshots are listed, a page at a time, and nothing changes. A call with no starting token that     *)
-  (* returns no next token lists every snapshot.                                                                     *)
+  (* returns no next token lists every snapshot. A call that names a snapshot returns that snapshot in its current   *)
+  (* state, or nothing when it does not exist, so an accepted snapshot is never reported absent while it is pending. *)
   (*******************************************************************************************************************)
-  LET Errors ==
-    LET checks == [unsupported_capability |-> ~Capabilities.snapshot_enumeration]
-    IN {e \in DOMAIN checks : checks[e]}
+  LET snapID == req.snapshot_id
+
+      Errors ==
+        \* A call that names a snapshot reads only that one, so it needs no enumeration.
+        LET checks == [unsupported_capability |-> snapID = None /\ ~Capabilities.snapshot_enumeration]
+        IN {e \in DOMAIN checks : checks[e]}
 
      \* The request was rejected, or the provider failed to answer.
   IN \/ /\ val = None
         /\ err \in Errors \union ProviderErrors
         /\ UNCHANGED vars
 
+     \* The request named one snapshot, so paging is ignored.
+     \/ /\ Errors = {}
+        /\ snapID /= None
+        /\ val = [snapshots  |-> IF KeyExists(snapshots, snapID) THEN {SnapshotRef(snapID)} ELSE {},
+                  next_token |-> None]
+        /\ err = None
+        /\ UNCHANGED vars
+
      \* The request was accepted.
      \/ /\ Errors = {}
-        /\ LET refs == {[id               |-> id,
-                         source_volume_id |-> snapshots[id].source_volume_id,
-                         handle           |-> snapshots[id].handle] : id \in DOMAIN snapshots}
+        /\ snapID = None
+        /\ LET refs == {SnapshotRef(id) : id \in DOMAIN snapshots}
            \* The page and the next token are the implementation's choice.
            IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
                 /\ Cardinality(page) <= PageSize(req.max_results)
@@ -156,4 +176,18 @@ DeleteSnapshot(req, err) ==
         /\ err = None
         /\ UNCHANGED <<volumes, publications>>
 
+-----------------------------------------------------------------------------------------------------------------------
+\* Steps without a call
+
+SettleSnapshot(snapID) ==
+  (*******************************************************************************************************************)
+  (* A pending snapshot settles as completed or error, as the provider finishes it. Its data is unchanged, since it  *)
+  (* was fixed when CreateSnapshot returned, and a snapshot that has settled never changes state again.              *)
+  (*******************************************************************************************************************)
+  /\ KeyExists(snapshots, snapID)
+  /\ snapshots[snapID].state = "pending"
+  /\ \E s \in {"completed", "error"} :
+       snapshots' = [snapshots EXCEPT ![snapID].state = s]
+  /\ UNCHANGED <<volumes, publications>>
+
 =======================================================================================================================

spinifex/bootstrap/config/config.go

@@ -322,14 +322,27 @@ type ViperblockConfig struct {
 }
 
 // EBS provider selectors. Viperblockd routes EBS calls to the viperblockd
-// daemon and qemunbd to the qemunbdd daemon, both over the versioned
-// ebs.provider.v1.* NATS contract. Embedded named the removed in-process
+// daemon, qemunbd to the qemunbdd daemon and ocibsd to the ocibsd daemon, all
+// over the versioned ebs.provider.v1.* NATS contract. Embedded named the removed in-process
 // engine and survives solely so a config that still selects it can be
 // rejected by name.
 const (
 	EBSProviderEmbedded    = "embedded"
 	EBSProviderViperblockd = "viperblockd"
 	EBSProviderQEMUNBD     = "qemunbd"
+	EBSProviderOCIBSD      = "ocibsd"
+)
+
+// OCI credential selections, as in oci_auth on an external pool and auth in
+// [nodes.<name>.ebs.oci]. An unset auth reads a config file.
+const (
+	// OCIAuthConfigFile authenticates from an API-key config file. The default,
+	// because it is what an operator already has from `oci setup config`.
+	OCIAuthConfigFile = "config_file"
+	// OCIAuthInstancePrincipal authenticates as the instance, from the
+	// certificate its metadata service serves. No key material on any node and
+	// nothing to rotate, at the cost of a dynamic group and an IAM policy.
+	OCIAuthInstancePrincipal = "instance_principal"
 )
 
 // Per-volume export tunables. Both are applied per nbdkit process, and
@@ -353,10 +366,13 @@ const (
 // boundary, not one provider's settings, so a second provider never needs a
 // rename.
 type EBSConfig struct {
-	// Provider is "viperblockd" or "qemunbd" and may be left unset. Volumes
-	// are persisted in ebsmetadata.
+	// Provider is "viperblockd", "qemunbd" or "ocibsd" and may be left unset.
+	// Volumes are persisted in ebsmetadata.
 	Provider string `json:"Provider" mapstructure:"provider"`
 
+	// OCI is the [nodes.<name>.ebs.oci] table, read only by ocibsd.
+	OCI EBSOCIConfig `json:"OCI" mapstructure:"oci"`
+
 	// DefaultThreads is nbdkit's -t: worker threads per NBD connection, which
 	// bounds how many requests are in flight inside viperblock for one volume.
 	// QEMU opens a single connection per volume, so this is the whole
@@ -370,9 +386,27 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// EBSOCIConfig is where ocibsd keeps the cluster's volumes and how it
+// authenticates. Auth, ConfigFile and ConfigProfile mean what oci_auth,
+// oci_config_file and oci_config_profile mean on an oci external pool.
+type EBSOCIConfig struct {
+	// CompartmentID and AvailabilityDomain are the same on every node: a
+	// volume is created there whichever node answers, and attached anywhere.
+	CompartmentID      string `json:"CompartmentID" mapstructure:"compartment_id"`
+	AvailabilityDomain string `json:"AvailabilityDomain" mapstructure:"availability_domain"`
+	// ClusterID is generated once by spx admin init and tags every OCI
+	// resource ocibsd creates, so one compartment can hold several clusters.
+	ClusterID     string `json:"ClusterID" mapstructure:"cluster_id"`
+	Auth          string `json:"Auth" mapstructure:"auth"`
+	ConfigFile    string `json:"ConfigFile" mapstructure:"config_file"`
+	ConfigProfile string `json:"ConfigProfile" mapstructure:"config_profile"`
+}
+
+func (c EBSOCIConfig) isZero() bool { return c == EBSOCIConfig{} }
+
 // SupportedEBSProvider reports whether name is a provider a node may run.
 func SupportedEBSProvider(name string) bool {
-	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD || name == EBSProviderOCIBSD
 }
 
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
@@ -889,6 +923,63 @@ func validateSingleEBSProvider(cc *ClusterConfig) error {
 	return nil
 }
 
+// validateOCIAuth checks an OCI credential selection, where prefix is what
+// the keys are named with ("oci_" on an external pool, "" in [ebs.oci]).
+func validateOCIAuth(prefix, auth, configFile, configProfile string) error {
+	// Named explicitly rather than inferred, so a node that was meant to
+	// use its own identity and has no dynamic group fails at config load
+	// instead of falling back to a key file nobody installed.
+	switch auth {
+	case "", OCIAuthConfigFile, OCIAuthInstancePrincipal:
+	default:
+		return fmt.Errorf("%sauth must be %q or %q, got %q", prefix, OCIAuthConfigFile, OCIAuthInstancePrincipal, auth)
+	}
+	// Credentials come from the metadata service under instance principal,
+	// so a config file named beside it is one of two sources of truth and
+	// the one that would be silently ignored.
+	if auth == OCIAuthInstancePrincipal && (configFile != "" || configProfile != "") {
+		return fmt.Errorf("%[1]sconfig_file/%[1]sconfig_profile are not valid with %[1]sauth=\"instance_principal\" (the instance's own certificate is the credential)", prefix)
+	}
+	return nil
+}
+
+// validateEBSOCI checks every node's [ebs.oci] table: present and complete
+// exactly when the node runs ocibsd, and naming one compartment and one AD
+// across the cluster, since any node may create a volume another attaches.
+func validateEBSOCI(cc *ClusterConfig) error {
+	var first string
+	var want EBSOCIConfig
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		ebs := cc.Nodes[name].EBS
+		if ebs.ResolvedProvider() != EBSProviderOCIBSD {
+			if !ebs.OCI.isZero() {
+				return fmt.Errorf("config: [nodes.%s.ebs.oci] is only valid with provider = %q", name, EBSProviderOCIBSD)
+			}
+			continue
+		}
+		oci := ebs.OCI
+		if oci.CompartmentID == "" || oci.AvailabilityDomain == "" {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] compartment_id and availability_domain are required with provider = %q", name, EBSProviderOCIBSD)
+		}
+		if err := validateOCIAuth("", oci.Auth, oci.ConfigFile, oci.ConfigProfile); err != nil {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] %w", name, err)
+		}
+		if first == "" {
+			first, want = name, oci
+			continue
+		}
+		if oci.ClusterID != want.ClusterID {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] cluster_id %q differs from [nodes.%s.ebs.oci] cluster_id %q; every node must carry the cluster's one ID",
+				name, oci.ClusterID, first, want.ClusterID)
+		}
+		if oci.CompartmentID != want.CompartmentID || oci.AvailabilityDomain != want.AvailabilityDomain {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] names compartment %q in %q but [nodes.%s.ebs.oci] names %q in %q; every node must name the same compartment and availability domain",
+				first, want.CompartmentID, want.AvailabilityDomain, name, oci.CompartmentID, oci.AvailabilityDomain)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -905,7 +996,7 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			// ebsmetadata one-way without the operator ever asking for it.
 			return fmt.Errorf("config: [nodes.%s.ebs] provider=%q has been removed; set provider = %q or remove the key (this is a one-way switch: volumes move to ebsmetadata)", nodeName, EBSProviderEmbedded, EBSProviderViperblockd)
 		default:
-			return fmt.Errorf("config: [nodes.%s.ebs] provider=%q unsupported; use %q or %q, or remove the key", nodeName, nodeCfg.EBS.Provider, EBSProviderViperblockd, EBSProviderQEMUNBD)
+			return fmt.Errorf("config: [nodes.%s.ebs] provider=%q unsupported; use %q, %q or %q, or remove the key", nodeName, nodeCfg.EBS.Provider, EBSProviderViperblockd, EBSProviderQEMUNBD, EBSProviderOCIBSD)
 		}
 		// Range checks only. Both settings are deliberately unbounded above by
 		// anything host-aware: they are operator tunables, and the host cost is
@@ -923,6 +1014,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 	if err := validateSingleEBSProvider(cc); err != nil {
 		return err
 	}
+	if err := validateEBSOCI(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err
@@ -993,19 +1087,8 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			if (p.OCIVNICID == "") == (p.OCIVNICIface == "") {
 				return fmt.Errorf("config: [[network.external_pools]] %q: source=\"oci\" requires exactly one of oci_vnic_id or oci_vnic_iface", p.Name)
 			}
-			// Named explicitly rather than inferred, so a node that was meant to
-			// use its own identity and has no dynamic group fails at config load
-			// instead of falling back to a key file nobody installed.
-			switch p.OCIAuth {
-			case "", "config_file", "instance_principal":
-			default:
-				return fmt.Errorf("config: [[network.external_pools]] %q: oci_auth must be \"config_file\" or \"instance_principal\", got %q", p.Name, p.OCIAuth)
-			}
-			// Credentials come from the metadata service under instance principal,
-			// so a config file named beside it is one of two sources of truth and
-			// the one that would be silently ignored.
-			if p.OCIAuth == "instance_principal" && (p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
-				return fmt.Errorf("config: [[network.external_pools]] %q: oci_config_file/oci_config_profile are not valid with oci_auth=\"instance_principal\" (the instance's own certificate is the credential)", p.Name)
+			if err := validateOCIAuth("oci_", p.OCIAuth, p.OCIConfigFile, p.OCIConfigProfile); err != nil {
+				return fmt.Errorf("config: [[network.external_pools]] %q: %w", p.Name, err)
 			}
 			if p.BindBridge != "" || p.DHCPMAC != "" {
 				return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)

spinifex/bootstrap/config/config_test.go

@@ -1053,6 +1053,7 @@ region = "us-east-1"
 		{"unset defaults to viperblockd", "", EBSProviderViperblockd},
 		{"viperblockd explicit", "[nodes.n1.ebs]\nprovider = \"viperblockd\"\n", EBSProviderViperblockd},
 		{"qemunbd explicit", "[nodes.n1.ebs]\nprovider = \"qemunbd\"\n", EBSProviderQEMUNBD},
+		{"ocibsd explicit", "[nodes.n1.ebs]\nprovider = \"ocibsd\"\n\n[nodes.n1.ebs.oci]\ncompartment_id = \"c\"\navailability_domain = \"ad\"\n", EBSProviderOCIBSD},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -1148,6 +1149,59 @@ func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
 	}
 }
 
+func TestLoadConfig_EBSOCI(t *testing.T) {
+	oci := func(node, compartment, ad, extra string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = \"ocibsd\"\n\n[nodes.%s.ebs.oci]\ncompartment_id = %q\navailability_domain = %q\n%s", node, node, compartment, ad, extra)
+	}
+	const comp, ad = "ocid1.compartment.oc1..a", "Uocm:US-ASHBURN-AD-1"
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"one node", []string{oci("n1", comp, ad, "")}, ""},
+		{"every node agrees, with unset peers", []string{oci("n1", comp, ad, ""), "", oci("n3", comp, ad, "auth = \"instance_principal\"\n")}, ""},
+		{"config file credentials", []string{oci("n1", comp, ad, "auth = \"config_file\"\nconfig_file = \"/etc/spinifex/oci/config\"\nconfig_profile = \"SPX\"\n")}, ""},
+		{"no compartment", []string{oci("n1", "", ad, "")}, "[nodes.n1.ebs.oci] compartment_id and availability_domain are required"},
+		{"no availability domain", []string{oci("n1", comp, "", "")}, "[nodes.n1.ebs.oci] compartment_id and availability_domain are required"},
+		{"no table at all", []string{"[nodes.n1.ebs]\nprovider = \"ocibsd\"\n"}, "compartment_id and availability_domain are required"},
+		{"unknown auth", []string{oci("n1", comp, ad, "auth = \"api_key\"\n")}, `[nodes.n1.ebs.oci] auth must be "config_file" or "instance_principal", got "api_key"`},
+		{"instance principal beside a config file", []string{oci("n1", comp, ad, "auth = \"instance_principal\"\nconfig_file = \"/x\"\n")},
+			`config_file/config_profile are not valid with auth="instance_principal"`},
+		{"nodes disagree on the AD", []string{oci("n1", comp, ad, ""), oci("n2", comp, "Uocm:US-ASHBURN-AD-2", "")},
+			"every node must name the same compartment and availability domain"},
+		{"nodes agree on the cluster ID", []string{oci("n1", comp, ad, "cluster_id = \"c1\"\n"), oci("n2", comp, ad, "cluster_id = \"c1\"\n")}, ""},
+		{"nodes disagree on the cluster ID", []string{oci("n1", comp, ad, "cluster_id = \"c1\"\n"), oci("n2", comp, ad, "cluster_id = \"c2\"\n")},
+			`[nodes.n2.ebs.oci] cluster_id "c2" differs from [nodes.n1.ebs.oci] cluster_id "c1"`},
+		{"nodes disagree on the compartment", []string{oci("n1", comp, ad, ""), oci("n2", "ocid1.compartment.oc1..b", ad, "")},
+			"every node must name the same compartment and availability domain"},
+		{"the table on another provider", []string{"[nodes.n1.ebs]\nprovider = \"viperblockd\"\n\n[nodes.n1.ebs.oci]\ncompartment_id = \"x\"\n"},
+			`[nodes.n1.ebs.oci] is only valid with provider = "ocibsd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			cfg, err := LoadConfig(path)
+			if tc.wantErr != "" {
+				require.ErrorContains(t, err, tc.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, EBSProviderOCIBSD, cfg.Nodes["n1"].EBS.ResolvedProvider())
+			got := cfg.Nodes["n1"].EBS.OCI
+			assert.Equal(t, comp, got.CompartmentID)
+			assert.Equal(t, ad, got.AvailabilityDomain)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/domains/ec2/ebs/metadata/metadata.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"strings"
 	"time"
@@ -82,6 +83,48 @@ type Snapshot struct {
 	AvailabilityZone string            `json:"availability_zone"`
 	Tags             map[string]string `json:"tags,omitempty"`
 	ProviderHandle   string            `json:"provider_handle,omitempty"`
+	// DeleteRequested is a DeleteSnapshot accepted before the snapshot could be
+	// deleted; the snapshot is hidden from then on and the reaper deletes it.
+	DeleteRequested bool `json:"delete_requested,omitempty"`
+	// AmbiguousCreate is set before every provider CreateSnapshot call and kept
+	// while any call on the document went unanswered, as one may still be accepted.
+	AmbiguousCreate bool `json:"ambiguous_create,omitempty"`
+	// TombstonedAt is when a deleted AmbiguousCreate snapshot's document became
+	// a tombstone, under which a late-accepted snapshot of its ID is deleted.
+	TombstonedAt time.Time `json:"tombstoned_at,omitzero"`
+}
+
+// Hidden reports a snapshot a customer has deleted: it is absent to every
+// request, and only the reaper still acts on its document.
+func (s Snapshot) Hidden() bool {
+	return s.DeleteRequested || !s.TombstonedAt.IsZero()
+}
+
+// Tombstoned reports a document kept only to catch a late-accepted create.
+func (s Snapshot) Tombstoned() bool {
+	return !s.TombstonedAt.IsZero()
+}
+
+// SnapshotStateCompleted is the one snapshot state a consumer may use.
+const SnapshotStateCompleted = "completed"
+
+// The states of an AMI document. A CreateImage's AMI is pending until its
+// snapshot settles; available is the one state an instance may launch from.
+const (
+	AMIStatePending   = "pending"
+	AMIStateAvailable = "available"
+	AMIStateFailed    = "failed"
+)
+
+// Ready reports a snapshot whose contents a consumer may read.
+func (s Snapshot) Ready() bool {
+	return s.State == SnapshotStateCompleted
+}
+
+// NotReadyError is the IncorrectState a consumer answers for a snapshot that
+// is not Ready.
+func (s Snapshot) NotReadyError() error {
+	return awserrors.Errorf(awserrors.ErrorIncorrectState, "Snapshot is in invalid state - %s", s.State)
 }
 
 // RootDeviceName is the device every image this platform serves reports as its
@@ -115,6 +158,18 @@ type AMI struct {
 	UEFIData *UEFIDataRef `json:"uefi_data,omitempty"`
 }
 
+// Available reports an AMI an instance may launch from or a copy may read. An
+// empty State predates the field, and every such image is complete.
+func (a AMI) Available() bool {
+	return a.State == "" || a.State == AMIStateAvailable
+}
+
+// UnavailableError is the InvalidAMIID.Unavailable answer for imageID, whose
+// document is a, when a is not Available.
+func (a AMI) UnavailableError(imageID string) error {
+	return awserrors.Errorf(awserrors.ErrorInvalidAMIIDUnavailable, "The image '%s' is %s, not available.", imageID, a.State)
+}
+
 // UEFIDataRef names an image's UEFI variable store object and the firmware it
 // was written under, so a launch can check the firmware without reading it.
 type UEFIDataRef struct {

spinifex/domains/ec2/image/account_termination_current_behaviour_test.go

@@ -57,6 +57,11 @@ func newStorageServices(t *testing.T) storageServices {
 	snapshots.SetEBSProvider(provider)
 	images := ec2image.NewImageServiceImplWithConfig(cfg, store, nil)
 	images.SetEBSProvider(provider)
+	// CreateImage snapshots through a snapshot service with no index handle.
+	imageSnapshots := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, store, nc)
+	imageSnapshots.SetEBSProvider(provider)
+	images.SetSnapshotCreator(imageSnapshots)
+	images.SetImageClaims(imageSnapshots)
 
 	return storageServices{
 		volumes: volumes, snapshots: snapshots, images: images,
@@ -230,9 +235,9 @@ func TestCurrentBehaviour_AccountTermination_DeleteSnapshotDuringCloneCreationSt
 	assert.EqualError(t, err, awserrors.ErrorInvalidSnapshotNotFound, "a clone started after the delete is refused")
 }
 
-// CreateImage on an instance re-reads the image the instance launched from,
-// after the provider has taken the snapshot, so once that image is
-// deregistered the call fails and leaves a provider snapshot with no document.
+// CreateImage on an instance reads the image the instance launched from before
+// it snapshots the root volume, so once that image is deregistered the call
+// fails and the provider holds no snapshot for it.
 func TestCurrentBehaviour_AccountTermination_CreateImageOfRetainedInstanceNeedsItsLaunchImage(t *testing.T) {
 	s := newStorageServices(t)
 	ctx := context.Background()
@@ -252,17 +257,7 @@ func TestCurrentBehaviour_AccountTermination_CreateImageOfRetainedInstanceNeedsI
 
 	listed, err := s.provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewVersioned()})
 	require.NoError(t, err)
-	var orphans []string
 	for _, ref := range listed.Snapshots {
-		if ref.SourceVolumeID == "vol-retained-root" {
-			orphans = append(orphans, ref.ID)
-		}
-	}
-	require.Len(t, orphans, 1, "the provider kept the snapshot it took before the image read failed")
-
-	docs, err := s.metadata.ListSnapshotsStrict(ctx, ctAccountID)
-	require.NoError(t, err)
-	for _, doc := range docs {
-		assert.NotEqual(t, orphans[0], doc.SnapshotID, "no document names that snapshot")
+		assert.NotEqual(t, "vol-retained-root", ref.SourceVolumeID, "the provider took a snapshot for an image that failed")
 	}
 }

spinifex/domains/ec2/image/create_image_pending_test.go

@@ -0,0 +1,308 @@
+package image
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// pendingImageFixture is an image service wired to a snapshot service over the
+// same store and memory provider, whose reaper settles what CreateImage leaves.
+type pendingImageFixture struct {
+	svc       *ImageServiceImpl
+	snapshots *ec2snapshot.SnapshotServiceImpl
+	mem       *ebsprovider.MemoryProvider
+	volume    ebsmetadata.Volume
+}
+
+func newPendingImageFixture(t *testing.T) *pendingImageFixture {
+	t.Helper()
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	mem, ok := provider.(*ebsprovider.MemoryProvider)
+	require.True(t, ok)
+	snapshots := useProvider(svc, mem)
+	return &pendingImageFixture{
+		svc:       svc,
+		snapshots: snapshots,
+		mem:       mem,
+		volume:    seedProviderVolume(t, svc, provider, "vol-createimage01", 8),
+	}
+}
+
+func (f *pendingImageFixture) createImage(t *testing.T) (*ec2.CreateImageOutput, error) {
+	t.Helper()
+	return f.svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-createimage01"), Name: aws.String("pending-image")},
+		RootVolumeID: f.volume.VolumeID,
+	}, testAccountID)
+}
+
+// onlyAMI is the one AMI document in the store, and its snapshot's document.
+func (f *pendingImageFixture) onlyAMI(t *testing.T) (ebsmetadata.AMI, ebsmetadata.Snapshot) {
+	t.Helper()
+	amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, err)
+	require.Len(t, amis, 1)
+	snap, err := f.svc.MetadataStore().GetSnapshot(context.Background(), testAccountID, amis[0].SnapshotID)
+	require.NoError(t, err)
+	return amis[0], snap
+}
+
+func (f *pendingImageFixture) sweep(t *testing.T) {
+	t.Helper()
+	_, err := f.snapshots.NewPendingSnapshotReaper().Sweep(context.Background())
+	require.NoError(t, err)
+}
+
+// recordingCreateProvider reads the image's documents when the provider is called.
+type recordingCreateProvider struct {
+	ebsprovider.EBSProvider
+
+	store     *ebsmetadata.Store
+	amis      []ebsmetadata.AMI
+	snapshot  ebsmetadata.Snapshot
+	readError error
+}
+
+func (p *recordingCreateProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.amis, p.readError = p.store.ListAMIs(ctx)
+	if p.readError == nil {
+		p.snapshot, p.readError = p.store.GetSnapshot(ctx, testAccountID, req.SnapshotID)
+	}
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+func TestCreateImage_WritesBothDocumentsPendingBeforeTheProviderCall(t *testing.T) {
+	f := newPendingImageFixture(t)
+	recording := &recordingCreateProvider{EBSProvider: f.mem, store: f.svc.MetadataStore()}
+	f.snapshots.SetEBSProvider(recording)
+
+	_, err := f.createImage(t)
+	require.NoError(t, err)
+
+	require.NoError(t, recording.readError)
+	require.Len(t, recording.amis, 1, "the AMI document exists when the provider is called")
+	assert.Equal(t, ebsmetadata.AMIStatePending, recording.amis[0].State)
+	assert.Equal(t, recording.snapshot.SnapshotID, recording.amis[0].SnapshotID)
+	assert.Equal(t, "pending", recording.snapshot.State, "the snapshot document is pending when the provider is called")
+}
+
+func TestCreateImage_DefiniteRefusalRemovesBothDocuments(t *testing.T) {
+	f := newPendingImageFixture(t)
+	f.snapshots.SetEBSProvider(&createInUseProvider{EBSProvider: f.mem})
+
+	_, err := f.createImage(t)
+	code, _, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "unregistered error: %v", err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+
+	amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, amis, "a refused image leaves no AMI document")
+	snaps, err := f.svc.MetadataStore().ListAllSnapshots(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, snaps, "a refused image leaves no snapshot document")
+}
+
+func TestCreateImage_SettlesTheSnapshotAndTheAMI(t *testing.T) {
+	tests := []struct {
+		name    string
+		arrange func(f *pendingImageFixture)
+		wantErr string
+		// settle is how the provider finishes a snapshot it answered pending; empty: it did not.
+		settle      ebsprovider.SnapshotState
+		wantPending bool
+		wantSnap    string
+		wantAMI     string
+	}{
+		{name: "completed in the request", arrange: func(*pendingImageFixture) {}, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable},
+		{
+			name: "answered pending, then completed", arrange: func(f *pendingImageFixture) { f.mem.AnswerSnapshotsPending(true) },
+			wantPending: true, settle: ebsprovider.SnapshotStateCompleted, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable,
+		},
+		{
+			name: "answered pending, then error", arrange: func(f *pendingImageFixture) { f.mem.AnswerSnapshotsPending(true) },
+			wantPending: true, settle: ebsprovider.SnapshotStateError, wantSnap: "error", wantAMI: ebsmetadata.AMIStateFailed,
+		},
+		{
+			name: "both answers lost", arrange: func(f *pendingImageFixture) { f.mem.FailCreateSnapshotsAfterAccept(2, nil) },
+			wantErr: awserrors.ErrorServerInternal, wantPending: true, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingImageFixture(t)
+			tc.arrange(f)
+
+			_, err := f.createImage(t)
+			if tc.wantErr != "" {
+				require.Error(t, err)
+				assert.Equal(t, tc.wantErr, err.Error())
+			} else {
+				require.NoError(t, err)
+			}
+
+			ami, snap := f.onlyAMI(t)
+			if tc.wantPending {
+				assert.Equal(t, ebsmetadata.AMIStatePending, ami.State)
+				assert.Equal(t, "pending", snap.State)
+				if tc.settle != "" {
+					require.NoError(t, f.mem.SettleSnapshot(snap.SnapshotID, tc.settle))
+				}
+				f.sweep(t)
+				ami, snap = f.onlyAMI(t)
+			}
+			assert.Equal(t, tc.wantSnap, snap.State)
+			assert.Equal(t, tc.wantAMI, ami.State)
+		})
+	}
+}
+
+// DeregisterImage is allowed on an image whose snapshot has not completed.
+func TestDeregisterImage_AllowsAPendingAMI(t *testing.T) {
+	f := newPendingImageFixture(t)
+	f.mem.AnswerSnapshotsPending(true)
+	out, err := f.createImage(t)
+	require.NoError(t, err)
+
+	_, err = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: out.ImageId}, testAccountID)
+	require.NoError(t, err)
+	_, err = f.svc.MetadataStore().GetAMI(context.Background(), aws.StringValue(out.ImageId))
+	assert.Error(t, err, "the pending AMI is deregistered")
+}
+
+// putHookStore runs onPut once, before the write to a matching key that
+// follows skip earlier matching writes reaches the store.
+type putHookStore struct {
+	objectstore.ObjectStore
+
+	match func(key string) bool
+	skip  int
+	onPut func()
+
+	mu   sync.Mutex
+	seen int
+}
+
+func (s *putHookStore) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if s.match(aws.StringValue(input.Key)) {
+		s.mu.Lock()
+		fire := s.seen == s.skip
+		s.seen++
+		s.mu.Unlock()
+		if fire {
+			s.onPut()
+		}
+	}
+	return s.ObjectStore.PutObject(ctx, input)
+}
+
+// A DeregisterImage that lands between the reaper's reread of a pending AMI
+// and its write either waits the write out or is refused retryably; one that
+// succeeds is never undone by the reaper writing the settled AMI back.
+func TestDeregisterImage_DuringTheReapersAMIWriteStaysDeleted(t *testing.T) {
+	f := newPendingImageFixture(t)
+	f.mem.AnswerSnapshotsPending(true)
+	out, err := f.createImage(t)
+	require.NoError(t, err)
+	imageID := aws.StringValue(out.ImageId)
+	ami, snap := f.onlyAMI(t)
+	require.Equal(t, ebsmetadata.AMIStatePending, ami.State)
+	require.NoError(t, f.mem.SettleSnapshot(snap.SnapshotID, ebsprovider.SnapshotStateCompleted))
+
+	key, err := ebsmetadata.AMIKey(imageID)
+	require.NoError(t, err)
+	var deregErr error
+	deregistered := false
+	hooked := &putHookStore{ObjectStore: f.svc.store, match: func(k string) bool { return k == key }}
+	hooked.onPut = func() {
+		deregistered = true
+		_, deregErr = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: out.ImageId}, testAccountID)
+	}
+	reaping := ec2snapshot.NewSnapshotServiceImplWithStore(f.svc.config, hooked, nil)
+	reaping.SetEBSProvider(f.mem)
+	f.svc.SetImageClaims(reaping)
+
+	_, err = reaping.NewPendingSnapshotReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	require.True(t, deregistered, "the reaper wrote the settled AMI")
+
+	assertDeregistrationStaysDeleted(t, f, imageID, deregErr, func() {
+		_, err := reaping.NewPendingSnapshotReaper().Sweep(context.Background())
+		require.NoError(t, err)
+	})
+}
+
+// assertDeregistrationStaysDeleted checks a DeregisterImage of imageID that
+// raced a settling write, and answered deregErr: one that succeeded left the
+// AMI gone, and one refused retryably succeeds on retry and stays gone after sweep.
+func assertDeregistrationStaysDeleted(t *testing.T, f *pendingImageFixture, imageID string, deregErr error, sweep func()) {
+	t.Helper()
+	_, getErr := f.svc.MetadataStore().GetAMI(context.Background(), imageID)
+	if deregErr == nil {
+		require.True(t, objectstore.IsNoSuchKeyError(getErr), "a deregistered AMI was written back: %v", getErr)
+		return
+	}
+	code, _, ok := awserrors.ResolveErrorDetail(deregErr)
+	require.True(t, ok, "unregistered error: %v", deregErr)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code, "a refused deregistration is retryable")
+	require.NoError(t, getErr)
+
+	_, err := f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
+	require.NoError(t, err, "the retry succeeds once the settling write is done")
+	sweep()
+	_, getErr = f.svc.MetadataStore().GetAMI(context.Background(), imageID)
+	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the retried deregistration stays deleted: %v", getErr)
+}
+
+// A DeregisterImage that lands between CreateImage's reread of its pending AMI
+// and the write making it available is refused retryably, never undone.
+func TestDeregisterImage_DuringCreateImagesAMIWriteStaysDeleted(t *testing.T) {
+	f := newPendingImageFixture(t)
+	sample, err := ebsmetadata.AMIKey("ami-0")
+	require.NoError(t, err)
+	amiKeyPrefix := sample[:strings.LastIndex(sample, "ami-0")]
+	var imageID string
+	var deregErr error
+	deregistered := false
+	// The Recorded hook's pending write is skipped; the Completed hook's is the one.
+	hooked := &putHookStore{
+		ObjectStore: f.svc.store,
+		skip:        1,
+		match:       func(k string) bool { return strings.HasPrefix(k, amiKeyPrefix) },
+	}
+	hooked.onPut = func() {
+		amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
+		if err != nil || len(amis) != 1 {
+			deregErr = err
+			return
+		}
+		imageID = amis[0].ImageID
+		deregistered = true
+		_, deregErr = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
+	}
+	f.svc.metadata = ebsmetadata.NewStore(hooked, testBucket)
+	settling := ec2snapshot.NewSnapshotServiceImplWithStore(f.svc.config, hooked, nil)
+	settling.SetEBSProvider(f.mem)
+	f.svc.SetSnapshotCreator(settling)
+	f.svc.SetImageClaims(settling)
+
+	out, err := f.createImage(t)
+	require.NoError(t, err)
+	require.True(t, deregistered, "CreateImage wrote the available AMI")
+	require.Equal(t, imageID, aws.StringValue(out.ImageId))
+
+	assertDeregistrationStaysDeleted(t, f, imageID, deregErr, func() { f.sweep(t) })
+}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -40,10 +41,42 @@ func setupProviderSnapshotImageService(t *testing.T) (*ImageServiceImpl, *object
 		},
 	}
 	svc := NewImageServiceImplWithConfig(cfg, store, nil)
-	svc.SetEBSProvider(provider)
+	useProvider(svc, provider)
 	return svc, store, provider
 }
 
+// useProvider makes p svc's provider and wires a snapshot service over svc's
+// store and p, as the daemon wires its own, for CreateImage to snapshot through.
+func useProvider(svc *ImageServiceImpl, p ebsprovider.EBSProvider) *ec2snapshot.SnapshotServiceImpl {
+	svc.SetEBSProvider(p)
+	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(svc.config, svc.store, nil)
+	snapshots.SetEBSProvider(p)
+	svc.SetSnapshotCreator(snapshots)
+	svc.SetImageClaims(snapshots)
+	return snapshots
+}
+
+// createImageOf runs CreateImage of a stopped instance whose root volume is
+// volumeID, and returns its AMI's snapshot ID when one was recorded.
+func createImageOf(t *testing.T, svc *ImageServiceImpl, volumeID, name string) (string, error) {
+	t.Helper()
+	out, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-" + name), Name: aws.String(name)},
+		RootVolumeID: volumeID,
+	}, testAccountID)
+	amis, listErr := svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, listErr)
+	for _, ami := range amis {
+		if ami.Name == name {
+			return ami.SnapshotID, err
+		}
+	}
+	if err == nil {
+		t.Fatalf("CreateImage %s returned %s but recorded no AMI", name, aws.StringValue(out.ImageId))
+	}
+	return "", err
+}
+
 // seedProviderVolume creates a provider-backed volume plus its ebsmetadata
 // document, wiring the document's ProviderHandle to the volume's real handle
 // so the fixture cannot drift from what the provider actually issued.
@@ -100,15 +133,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
+// TestCreateImageFromInstance_Provider_CreatesThroughProvider locks that a
 // root-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
+func TestCreateImageFromInstance_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
+	snapshotID, err := createImageOf(t, svc, doc.VolumeID, "stopped001")
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -116,28 +149,28 @@ func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	// confirms the snapshot genuinely exists in the provider (sourced from
 	// the right volume) instead of the call having silently no-opped.
 	snap, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-stopped001", VolumeID: doc.VolumeID,
+		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: doc.VolumeID,
 	})
 	require.NoError(t, err)
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestCreateImageFromInstance_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestCreateImageFromInstance_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
+	_, err := createImageOf(t, svc, doc.VolumeID, "running001")
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
+	_, err = createImageOf(t, svc, doc.VolumeID, "running002")
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -236,16 +269,21 @@ func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.
 	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
 }
 
-// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
-// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
-func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
+// TestCreateImageFromInstance_Provider_UnavailableLeavesThePendingSnapshot
+// locks that a CreateImage snapshot whose call and repeat both go unanswered
+// returns InternalError and leaves its document pending for the reaper.
+func TestCreateImageFromInstance_Provider_UnavailableLeavesThePendingSnapshot(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
-	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
+	useProvider(svc, &createUnavailableProvider{EBSProvider: provider})
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
+	snapshotID, err := createImageOf(t, svc, doc.VolumeID, "stopped-busy")
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	snap, err := svc.metadata.GetSnapshot(context.Background(), testAccountID, snapshotID)
+	require.NoError(t, err)
+	assert.Equal(t, "pending", snap.State)
+	assert.True(t, snap.AmbiguousCreate)
 }
 
 // createInUseProvider refuses every CreateSnapshot as volume in use.
@@ -257,14 +295,14 @@ func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.Create
 	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
 }
 
-// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
-// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
-func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
+// TestCreateImageFromInstance_Provider_VolumeInUseIsIncorrectState locks that
+// the CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
+func TestCreateImageFromInstance_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
-	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
+	useProvider(svc, &createInUseProvider{EBSProvider: provider})
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
+	_, err := createImageOf(t, svc, doc.VolumeID, "attached-inuse")
 
 	code, message, ok := awserrors.ResolveErrorDetail(err)
 	require.True(t, ok, "unregistered error: %v", err)
@@ -295,7 +333,7 @@ func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *t
 	svc.natsConn = nc
 	svc.config.DataDir = testutil.SocketTempDir(t)
 	counting := &countingSnapshotProvider{EBSProvider: provider}
-	svc.SetEBSProvider(counting)
+	useProvider(svc, counting)
 
 	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
 	doc.State, doc.AttachedInstance = "in-use", instanceID

spinifex/domains/ec2/image/readiness_gate_test.go

@@ -0,0 +1,78 @@
+package image
+
+import (
+	"context"
+	"testing"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegisterImage_RefusesASnapshotThatIsNotCompleted(t *testing.T) {
+	tests := []struct {
+		name     string
+		snapshot ebsmetadata.Snapshot
+		wantCode string
+	}{
+		{name: "pending", snapshot: ebsmetadata.Snapshot{State: "pending"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "error", snapshot: ebsmetadata.Snapshot{State: "error"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "deleted while pending", snapshot: ebsmetadata.Snapshot{State: "pending", DeleteRequested: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+		{name: "deleted after completing", snapshot: ebsmetadata.Snapshot{State: "completed", DeleteRequested: true, AmbiguousCreate: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestImageService(t)
+			snap := tc.snapshot
+			snap.SnapshotID, snap.VolumeID, snap.VolumeSize, snap.OwnerID = "snap-gate000001", "vol-gate000001", 8, testAccountID
+			putTestSnapshotDoc(t, store, snap)
+
+			_, err := svc.RegisterImage(context.Background(), validRegisterImageServiceInput(snap.SnapshotID), testAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			amis, err := svc.MetadataStore().ListAMIs(context.Background())
+			require.NoError(t, err)
+			assert.Empty(t, amis, "a refused registration writes no AMI")
+		})
+	}
+}
+
+// An AMI whose snapshot is gone reads as gone, as an AMI with no snapshot document does.
+func TestCopyImage_RefusesASourceThatIsNotAvailable(t *testing.T) {
+	tests := []struct {
+		name     string
+		amiState string
+		hideSnap bool
+		wantCode string
+	}{
+		{name: "pending", amiState: ebsmetadata.AMIStatePending, wantCode: awserrors.ErrorInvalidAMIIDUnavailable},
+		{name: "failed", amiState: ebsmetadata.AMIStateFailed, wantCode: awserrors.ErrorInvalidAMIIDUnavailable},
+		{name: "snapshot deleted", amiState: "available", hideSnap: true, wantCode: awserrors.ErrorInvalidAMIIDNotFound},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestImageService(t)
+			seedCopyableAMI(t, store, "ami-gate000001", "gated", testAccountID, "snap-gate000001", "vol-gate000001", 8)
+			ctx := context.Background()
+			ami, err := svc.MetadataStore().GetAMI(ctx, "ami-gate000001")
+			require.NoError(t, err)
+			ami.State = tc.amiState
+			require.NoError(t, svc.MetadataStore().PutAMI(ctx, ami))
+			if tc.hideSnap {
+				snap := readTestSnapshotDoc(t, store, testAccountID, "snap-gate000001")
+				snap.DeleteRequested = true
+				putTestSnapshotDoc(t, store, snap)
+			}
+
+			_, err = svc.CopyImage(ctx, validCopyImageServiceInput("ami-gate000001", "copy"), testAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			amis, err := svc.MetadataStore().ListAMIs(ctx)
+			require.NoError(t, err)
+			assert.Len(t, amis, 1, "a refused copy writes no AMI")
+		})
+	}
+}

spinifex/domains/ec2/image/service_impl.go

@@ -55,6 +55,24 @@ type ImageServiceImpl struct {
 
 	uefiRecords UEFIStoreRecords
 	nodeID      string
+
+	// snapshots takes an image's snapshot through the snapshot service, so the
+	// claim and reference are the ones CreateSnapshot uses.
+	snapshots ec2snapshot.ImageSnapshotCreator
+	// imageClaims is the claim DeregisterImage shares with whatever settles a
+	// pending AMI; nil excludes nothing.
+	imageClaims ec2snapshot.ImageClaims
+}
+
+// SetSnapshotCreator wires the snapshot service CreateImage snapshots through.
+func (s *ImageServiceImpl) SetSnapshotCreator(snapshots ec2snapshot.ImageSnapshotCreator) {
+	s.snapshots = snapshots
+}
+
+// SetImageClaims wires the image claims of the snapshot service whose reaper
+// settles this service's pending AMIs.
+func (s *ImageServiceImpl) SetImageClaims(claims ec2snapshot.ImageClaims) {
+	s.imageClaims = claims
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane.
@@ -461,12 +479,14 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		}
 	}
 
-	// Step 2: Snapshot root volume
-	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
-		return nil, err
+	// Step 2: Read source volume config for size
+	if s.config == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if s.snapshots == nil || s.imageClaims == nil {
+		slog.Error("CreateImageFromInstance: no snapshot service wired", "amiId", amiID)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-
-	// Step 3: Read source volume config for size
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, params.RootVolumeID)
 	if err != nil {
 		slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err)
@@ -474,7 +494,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}
 	volumeSizeGiB := volMeta.CapacityGiB
 
-	// Step 4: Read source AMI config for architecture, platform, etc.
+	// Step 3: Read source AMI config for architecture, platform, etc.
 	sourceAMI := ebsmetadata.AMI{
 		Architecture:    "x86_64",
 		PlatformDetails: "Linux/UNIX",
@@ -489,13 +509,6 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		sourceAMI = srcCfg
 	}
 
-	// Step 5: Store snapshot metadata
-	if err := s.putSnapshotMetadata(context.Background(), snapshotID, params.RootVolumeID, volumeSizeGiB, accountID); err != nil {
-		slog.Error("CreateImageFromInstance: failed to write snapshot metadata", "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// Step 6: Build and store AMI config
 	meta := ebsmetadata.AMI{
 		ImageID:         amiID,
 		Name:            name,
@@ -513,57 +526,41 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		EnaSupport:   aws.Bool(true),
 		Distro:       sourceAMI.Distro,
 		DistroFamily: sourceAMI.DistroFamily,
-		// Snapshot succeeded before this point, so the image is complete.
-		State:    "available",
+		// Pending until its snapshot completes, here or in the snapshot reaper.
+		State:    ebsmetadata.AMIStatePending,
 		UEFIData: uefiData,
 	}
 
-	if err := s.putAMIConfig(context.Background(), amiID, meta); err != nil {
+	// Step 4: Snapshot the root volume. Its document and reference, then the
+	// AMI's document, are written pending before the provider is called, so an
+	// accepted snapshot whose answer is lost is still settled by the reaper.
+	ctx := context.Background()
+	snap, err := s.snapshots.CreateImageSnapshot(ctx, volMeta, snapshotID, accountID, "Created by CreateImage("+aws.StringValue(input.InstanceId)+") for "+amiID,
+		ec2snapshot.ImageSnapshotHooks{
+			Recorded:  func(ctx context.Context) error { return s.putAMIConfig(ctx, amiID, meta) },
+			Refused:   func(ctx context.Context) { s.removePendingAMI(ctx, amiID) },
+			Completed: func(ctx context.Context) { s.imageClaims.SettlePendingAMI(ctx, amiID, ebsmetadata.AMIStateAvailable) },
+		})
+	if err != nil {
 		// A UEFI object left under the image's prefix is in the orphan report.
-		slog.Error("CreateImageFromInstance: failed to store AMI config", "amiId", amiID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		slog.Error("CreateImageFromInstance: snapshot failed", "amiId", amiID, "snapshotId", snapshotID, "err", err)
+		return nil, err
 	}
 
-	slog.Info("CreateImageFromInstance completed", "amiId", amiID, "snapshotId", snapshotID)
+	slog.Info("CreateImageFromInstance completed", "amiId", amiID, "snapshotId", snapshotID, "snapshotState", aws.StringValue(snap.State))
 
 	return &ec2.CreateImageOutput{
 		ImageId: aws.String(amiID),
 	}, nil
 }
 
-// snapshotVolume snapshots the root volume through the EBS provider, whether
-// or not the instance is running. A provider that buffers writes drains the
-// volume inside its own CreateSnapshot, so no drain is issued here.
-func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
-	if s.config == nil {
-		return errors.New(awserrors.ErrorServerInternal)
+// removePendingAMI deletes the AMI document of a CreateImage whose snapshot
+// the provider refused, before the snapshot's own document goes.
+func (s *ImageServiceImpl) removePendingAMI(ctx context.Context, amiID string) {
+	if err := s.metadata.DeleteAMI(ctx, amiID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.ErrorContext(ctx, "CreateImageFromInstance: failed to remove a refused image's AMI document; the reaper marks it failed",
+			"amiId", amiID, "err", err)
 	}
-
-	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
-	if err != nil {
-		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
-}
-
-// createSnapshot delegates snapshot creation for volMeta to the EBS provider.
-// op names the calling path so a failure is attributable in the log.
-func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetadata.Volume, snapshotID, op string) error {
-	if s.provider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", op)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volMeta.VolumeID, VolumeHandle: volMeta.ProviderHandle,
-	})
-	if err != nil || created == nil {
-		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
-		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
-	}
-	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
-	return nil
 }
 
 // getVolumeMetadata reads a volume's control-plane document, remapping a
@@ -648,6 +645,9 @@ func (s *ImageServiceImpl) GetAMISourceVolumeID(ctx context.Context, imageID str
 			"imageId", imageID, "snapshotId", meta.SnapshotID, "err", err)
 		return "", errors.New(awserrors.ErrorServerInternal)
 	}
+	if snapCfg.Hidden() {
+		return "", errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
 	if snapCfg.VolumeID == "" {
 		slog.ErrorContext(ctx, "GetAMISourceVolumeID: snapshot metadata has no source volume",
 			"imageId", imageID, "snapshotId", meta.SnapshotID)
@@ -815,6 +815,10 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 	if !callerCanReadAMI(srcMeta, accountID) {
 		return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
 	}
+	// The copy shares the source's snapshot, so it is no readier than it.
+	if !srcMeta.Available() {
+		return nil, srcMeta.UnavailableError(sourceImageID)
+	}
 
 	// Orphaned source (missing SnapshotID, or snapshot gone/corrupt) is
 	// reported as NotFound — don't leak the half-broken state.
@@ -842,6 +846,9 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 	}
+	if srcSnap.Hidden() {
+		return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
+	}
 
 	if exists, err := s.amiNameExists(ctx, name); err != nil {
 		slog.ErrorContext(ctx, "CopyImage: failed to check AMI name uniqueness", "name", name, "err", err)
@@ -1063,6 +1070,14 @@ func (s *ImageServiceImpl) RegisterImage(ctx context.Context, input *ec2.Registe
 		slog.ErrorContext(ctx, "RegisterImage: failed to read snapshot metadata", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
+	if snapCfg.Hidden() {
+		return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
+	// An image registered on a snapshot that is not completed could never be
+	// launched from, and none is written pending outside CreateImage.
+	if !snapCfg.Ready() {
+		return nil, snapCfg.NotReadyError()
+	}
 
 	snapSizeGiB := uint64(0)
 	if snapCfg.VolumeSize > 0 {
@@ -1175,6 +1190,16 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
+	// Held across both deletes, so the reaper cannot write back a settled AMI
+	// it read before them.
+	if s.imageClaims != nil {
+		unlock, err := s.imageClaims.LockImage(ctx, imageID)
+		if err != nil {
+			return nil, err
+		}
+		defer unlock()
+	}
+
 	// Only the image's own prefix: a copy or a source instance has its own.
 	if n, err := uefistore.DeleteObjects(ctx, s.store, s.bucketName, imageID); err != nil {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete the UEFI variable store", "imageId", imageID, "deleted", n, "err", err)

spinifex/domains/ec2/image/service_impl_test.go

@@ -6,8 +6,6 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"io"
 	"log/slog"
-	"net/http"
-	"net/http/httptest"
 	"strconv"
 	"strings"
 	"testing"
@@ -154,6 +152,24 @@ func TestCreateImageFromInstance_RunningInstance_NoNATS(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
+// An image service the daemon gave no snapshot service fails CreateImage with
+// InternalError before it records anything.
+func TestCreateImageFromInstance_NoSnapshotServiceIsInternalError(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	svc := NewImageServiceImplWithConfig(&config.Config{Predastore: config.PredastoreConfig{Bucket: testBucket}}, store, nil)
+	createTestVolumeConfig(t, store, "vol-root123", 10)
+
+	_, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-test123"), Name: aws.String("my-image")},
+		RootVolumeID: "vol-root123",
+	}, testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	amis, err := svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, amis)
+}
+
 func TestCreateImageFromInstance_StoppedInstance_NoConfig(t *testing.T) {
 	svc, _ := setupTestImageService(t)
 
@@ -2550,114 +2566,3 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
-
-func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config:     &config.Config{},
-		store:      store,
-		bucketName: testBucket,
-	}
-	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotVolume_BackendInitFails(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-stopped1", 10)
-
-	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// imageS3MockServer passes ListObjectsV2 (Backend.Init) but returns 404 for everything else.
-func imageS3MockServer(t *testing.T) *httptest.Server {
-	t.Helper()
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if strings.Contains(r.URL.RawQuery, "list-type=2") {
-			w.Header().Set("Content-Type", "application/xml")
-			w.WriteHeader(http.StatusOK)
-			io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?>`+
-				`<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
-				`<Name>test-bucket</Name><KeyCount>0</KeyCount>`+
-				`<MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>`)
-			return
-		}
-		w.Header().Set("Content-Type", "application/xml")
-		w.WriteHeader(http.StatusNotFound)
-		io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?>`+
-			`<Error><Code>NoSuchKey</Code>`+
-			`<Message>The specified key does not exist.</Message></Error>`)
-	}))
-}
-
-func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-			Viperblock: config.ViperblockConfig{
-				EncryptionKeyFile: "/nonexistent-image-stop-key.bin",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
-
-	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotVolume_LoadStateFails(t *testing.T) {
-	srv := imageS3MockServer(t)
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:      srv.URL,
-				Region:    "us-east-1",
-				Bucket:    testBucket,
-				AccessKey: "test-key",
-				SecretKey: "test-secret",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
-
-	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}

spinifex/domains/ec2/image/uefi_test.go

@@ -55,7 +55,7 @@ func newUEFIImageFixture(t *testing.T) *uefiImageFixture {
 	svc.SetUEFIStores(records, "node-a")
 	volume := seedProviderVolume(t, svc, provider, "vol-uefi001", 8)
 	snapshots := &countingSnapshots{EBSProvider: provider}
-	svc.SetEBSProvider(snapshots)
+	useProvider(svc, snapshots)
 	return &uefiImageFixture{svc: svc, snapshots: snapshots, objects: objects, records: records, volume: volume}
 }
 

spinifex/domains/ec2/instance/provider_root_volume_test.go

@@ -76,13 +76,25 @@ func rootVolumeAMILoader() *fakeAMILoader {
 	}
 }
 
+// createCapturingProvider records the last CreateVolume request it served.
+type createCapturingProvider struct {
+	*ebsprovider.MemoryProvider
+
+	last ebsprovider.CreateVolumeRequest
+}
+
+func (p *createCapturingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
+	p.last = req
+	return p.MemoryProvider.CreateVolume(ctx, req)
+}
+
 // TestPrepareRootVolume_Provider_CreatesFromAMISnapshot locks that an injected
 // provider owns root-volume creation: the volume is restored from the AMI's
 // snapshot rather than copied block-by-block by the control plane.
 func TestPrepareRootVolume_Provider_CreatesFromAMISnapshot(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true})
+	provider := &createCapturingProvider{MemoryProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true})}
 	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
 	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
 
@@ -97,6 +109,9 @@ func TestPrepareRootVolume_Provider_CreatesFromAMISnapshot(t *testing.T) {
 	})
 	require.NoError(t, err, "the root volume must exist in the provider")
 	assert.Equal(t, int64(testRootVolumeBytes), created.CapacityBytes)
+	params, err := ebsprovider.ParseVolumeParameters(provider.last.Parameters)
+	require.NoError(t, err)
+	assert.Equal(t, ebspolicy.VolumeTypeGP3, params.VolumeType, "the provider must be told the root volume's type")
 
 	require.Len(t, instance.EBSRequests.Requests, 1)
 	assert.Equal(t, "vol-root", instance.EBSRequests.Requests[0].Name)

spinifex/domains/ec2/instance/service_impl.go

@@ -697,6 +697,12 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec
 		slog.WarnContext(ctx, "PrepareRunInstances: AMI not owned by caller", "imageId", *input.ImageId, "amiOwner", amiOwner, "accountID", accountID)
 		return nil, nil, nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
 	}
+	// A CreateImage AMI is pending until its snapshot completes, and failed if
+	// it errors; neither has a root volume to clone.
+	if !amiMeta.Available() {
+		slog.WarnContext(ctx, "PrepareRunInstances: AMI is not available", "imageId", *input.ImageId, "state", amiMeta.State)
+		return nil, nil, nil, amiMeta.UnavailableError(*input.ImageId)
+	}
 	if err := checkImageArchitecture(instanceType, amiMeta.Architecture); err != nil {
 		slog.WarnContext(ctx, "PrepareRunInstances: AMI architecture does not match instance type", "imageId", *input.ImageId, "InstanceType", *input.InstanceType, "amiArchitecture", amiMeta.Architecture)
 		return nil, nil, nil, err
@@ -1855,6 +1861,12 @@ func (s *InstanceServiceImpl) createRootVolumeViaProvider(ctx context.Context, s
 	slog.InfoContext(ctx, "Creating root volume from AMI snapshot via EBS provider",
 		"imageId", spec.amiID, "volumeId", spec.volumeID, "snapshotId", amiConfig.SnapshotID, "sourceVolumeId", sourceVolumeID)
 
+	params, err := ebsprovider.NewVolumeParameters(ebspolicy.VolumeTypeGP3)
+	if err != nil {
+		slog.ErrorContext(ctx, "Encode root volume provider parameters failed", "volumeId", spec.volumeID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
 	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
 		Versioned:              ebsprovider.NewVersioned(),
 		VolumeID:               spec.volumeID,
@@ -1862,6 +1874,7 @@ func (s *InstanceServiceImpl) createRootVolumeViaProvider(ctx context.Context, s
 		AvailabilityZone:       s.config.AZ,
 		SourceSnapshotID:       amiConfig.SnapshotID,
 		SourceSnapshotVolumeID: sourceVolumeID,
+		Parameters:             params,
 	})
 	if err != nil {
 		slog.ErrorContext(ctx, "Provider root volume creation failed",

spinifex/domains/ec2/instance/service_impl_test.go

@@ -3169,6 +3169,30 @@ func TestPrepareRunInstances_AMINotOwnedByCaller(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error())
 }
 
+func TestPrepareRunInstances_AMINotAvailable(t *testing.T) {
+	for _, state := range []string{ebsmetadata.AMIStatePending, ebsmetadata.AMIStateFailed} {
+		t.Run(state, func(t *testing.T) {
+			types, _ := defaultPrepareInstanceTypes()
+			svc := &InstanceServiceImpl{
+				instanceTypes: types,
+				amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{
+					"ami-1": {ImageOwnerAlias: "acc", State: state},
+				}},
+			}
+			_, _, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
+				InstanceType: aws.String("t3.micro"),
+				ImageId:      aws.String("ami-1"),
+				MinCount:     aws.Int64(1),
+				MaxCount:     aws.Int64(1),
+			}, "acc", "")
+			code, message, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, awserrors.ErrorInvalidAMIIDUnavailable, code)
+			assert.Contains(t, message, state)
+		})
+	}
+}
+
 func TestPrepareRunInstances_KeyPairNotFound(t *testing.T) {
 	types, _ := defaultPrepareInstanceTypes()
 	svc := &InstanceServiceImpl{

spinifex/domains/ec2/snapshot/claims.go

@@ -0,0 +1,197 @@
+package snapshot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+// ClaimPrefix is the kvsync prefix of every snapshot's claim, ClaimPrefix+ID,
+// naming the node that alone may call the provider for it or delete its
+// document. An AMI's claim, keyed by image ID, shares the prefix.
+const ClaimPrefix = "snapshot."
+
+// claimReleaseTimeout bounds the unlock after the last write, which runs even
+// when the request that held the claim has been cancelled.
+const claimReleaseTimeout = 5 * time.Second
+
+// NewClaimGroup returns node's group of snapshot claims over store. The daemon
+// builds it once per process, as kvsync requires of every prefix.
+func NewClaimGroup(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
+	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
+}
+
+// NodeLiveness reports when a node last heartbeated. found is false only for a
+// node with no heartbeat; a read that fails is an error, never a dead node.
+type NodeLiveness interface {
+	LastSeen(ctx context.Context, node string) (seen time.Time, found bool, err error)
+}
+
+// errClaimHeld is a snapshot claim another holder has: a request of this
+// process, or a node whose heartbeat is still fresh.
+var errClaimHeld = errors.New("snapshot claim is held")
+
+// Claims decides who may act on a pending snapshot. A kvsync claim names a
+// node, not a process, so the claims this process holds are also kept here:
+// the reaper on the node of a running request must not adopt that request's.
+type Claims struct {
+	group    *kvsync.ClaimGroup
+	node     string
+	liveness NodeLiveness
+	now      func() time.Time
+
+	mu    sync.Mutex
+	local map[string]struct{}
+}
+
+// NewClaims returns the claims of node over group. A nil liveness never judges
+// a holder dead, so a claim another node holds is never taken over.
+func NewClaims(group *kvsync.ClaimGroup, node string, liveness NodeLiveness) *Claims {
+	return &Claims{group: group, node: node, liveness: liveness, now: time.Now, local: map[string]struct{}{}}
+}
+
+// localClaims excludes only within this process: a service built without the
+// daemon's claim group has no other process to exclude.
+func localClaims() *Claims {
+	return NewClaims(nil, "", nil)
+}
+
+// heldClaim is one acquisition of a snapshot's claim.
+type heldClaim struct {
+	claims     *Claims
+	snapshotID string
+	claim      *kvsync.Claim
+	token      kvsync.Token
+}
+
+// reserve marks snapshotID held by this process, and is false when it already is.
+func (c *Claims) reserve(snapshotID string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if _, held := c.local[snapshotID]; held {
+		return false
+	}
+	c.local[snapshotID] = struct{}{}
+	return true
+}
+
+func (c *Claims) unreserve(snapshotID string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.local, snapshotID)
+}
+
+// lock takes snapshotID's claim for a creating request. It never takes over:
+// the ID is new, so any holder is a fault worth refusing the request for.
+func (c *Claims) lock(ctx context.Context, snapshotID string) (*heldClaim, error) {
+	return c.acquire(ctx, snapshotID, false)
+}
+
+// take acquires snapshotID's claim for the reaper. A claim another node holds
+// is taken over only once that node's heartbeat is LivenessWindow old or
+// absent; while it is fresh, or cannot be read, the claim is refused.
+func (c *Claims) take(ctx context.Context, snapshotID string) (*heldClaim, error) {
+	return c.acquire(ctx, snapshotID, true)
+}
+
+// acquire reserves snapshotID in this process and locks its claim, taking it
+// over from a dead holder only when takeOver allows.
+func (c *Claims) acquire(ctx context.Context, snapshotID string, takeOver bool) (*heldClaim, error) {
+	if !c.reserve(snapshotID) {
+		return nil, fmt.Errorf("%w: %s by this process", errClaimHeld, snapshotID)
+	}
+	held, err := c.acquireReserved(ctx, snapshotID, takeOver)
+	if err != nil {
+		c.unreserve(snapshotID)
+		return nil, err
+	}
+	return held, nil
+}
+
+func (c *Claims) acquireReserved(ctx context.Context, snapshotID string, takeOver bool) (*heldClaim, error) {
+	held := &heldClaim{claims: c, snapshotID: snapshotID}
+	if c.group == nil {
+		return held, nil
+	}
+	held.claim = c.group.Get(snapshotID)
+	tok, err := held.claim.TryLock(ctx)
+	if err == nil {
+		held.token = tok
+		return held, nil
+	}
+	he, ok := errors.AsType[*kvsync.HeldError](err)
+	if !takeOver || !ok || he.Current.Holder == "" {
+		return nil, fmt.Errorf("take snapshot claim %s: %w", snapshotID, err)
+	}
+	holder := he.Current
+	if c.liveness == nil {
+		return nil, fmt.Errorf("%w: %s by %s, and no liveness is wired to judge it", errClaimHeld, snapshotID, holder.Holder)
+	}
+	seen, found, err := c.liveness.LastSeen(ctx, holder.Holder)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s by %s, whose heartbeat cannot be read: %w", errClaimHeld, snapshotID, holder.Holder, err)
+	}
+	now := c.now()
+	if found && clusterv1.LiveAt(seen, now) {
+		return nil, fmt.Errorf("%w: %s by %s, which is still heartbeating", errClaimHeld, snapshotID, holder.Holder)
+	}
+	tok, err = held.claim.TakeOver(ctx, holder)
+	if err != nil {
+		return nil, fmt.Errorf("take over snapshot claim %s from %s: %w", snapshotID, holder.Holder, err)
+	}
+	attrs := []any{"snapshotId", snapshotID, "holder", holder.Holder, "fromGeneration", holder.Generation, "generation", tok.Generation}
+	if found {
+		attrs = append(attrs, "heartbeat_age_ms", otelsetup.Millis(now.Sub(seen)))
+	}
+	slog.WarnContext(ctx, "Took over a pending snapshot's claim from a node whose heartbeat has stopped", attrs...)
+	held.token = tok
+	return held, nil
+}
+
+// release gives the claim up after the holder's last write. A failed unlock
+// leaves the claim with this node until a restart or a takeover frees it, so
+// it is logged and not returned.
+func (h *heldClaim) release(ctx context.Context) {
+	defer h.claims.unreserve(h.snapshotID)
+	if h.claim == nil {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), claimReleaseTimeout)
+	defer cancel()
+	if err := h.claim.Unlock(ctx, h.token); err != nil {
+		slog.WarnContext(ctx, "Failed to release a snapshot claim",
+			"snapshotId", h.snapshotID, "generation", h.token.Generation, "err", err)
+	}
+}
+
+// releaseOwn unlocks snapshotID's claim when it names this node and no request
+// of this process holds it: a claim left by an earlier process of this node,
+// whose requests did not survive its restart.
+func (c *Claims) releaseOwn(ctx context.Context, snapshotID string) (bool, error) {
+	if c.group == nil || !c.reserve(snapshotID) {
+		return false, nil
+	}
+	defer c.unreserve(snapshotID)
+	claim := c.group.Get(snapshotID)
+	tok, ok, err := claim.Holder(ctx)
+	if err != nil {
+		return false, fmt.Errorf("read snapshot claim %s: %w", snapshotID, err)
+	}
+	if !ok || tok.Holder != c.node {
+		return false, nil
+	}
+	if err := claim.Unlock(ctx, tok); err != nil {
+		if errors.Is(err, kvsync.ErrLost) {
+			return false, nil
+		}
+		return false, fmt.Errorf("release snapshot claim %s: %w", snapshotID, err)
+	}
+	return true, nil
+}

spinifex/domains/ec2/snapshot/delete_pending_test.go

@@ -0,0 +1,581 @@
+package snapshot
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fastClaims makes svc's writers retry a held claim without the real wait.
+func fastClaims(svc *SnapshotServiceImpl) *SnapshotServiceImpl {
+	svc.claimRetryDelay = time.Millisecond
+	return svc
+}
+
+func (f *pendingFixture) deleteSnapshot(svc *SnapshotServiceImpl, id string) error {
+	_, err := svc.DeleteSnapshot(f.t.Context(), &ec2.DeleteSnapshotInput{SnapshotId: aws.String(id)}, testAccountID)
+	return err
+}
+
+// assertHidden requires that every request answers id as a deleted snapshot.
+func (f *pendingFixture) assertHidden(svc *SnapshotServiceImpl, id string) {
+	f.t.Helper()
+	listed, err := svc.DescribeSnapshots(f.t.Context(), &ec2.DescribeSnapshotsInput{}, testAccountID)
+	require.NoError(f.t, err)
+	for _, snap := range listed.Snapshots {
+		assert.NotEqual(f.t, id, aws.StringValue(snap.SnapshotId), "DescribeSnapshots lists a deleted snapshot")
+	}
+	_, err = svc.DescribeSnapshots(f.t.Context(), &ec2.DescribeSnapshotsInput{SnapshotIds: aws.StringSlice([]string{id})}, testAccountID)
+	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
+	_, err = svc.DescribeSnapshotsStrict(f.t.Context(), &ec2.DescribeSnapshotsInput{SnapshotIds: aws.StringSlice([]string{id})}, testAccountID)
+	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
+	_, err = svc.CopySnapshot(f.t.Context(), &ec2.CopySnapshotInput{SourceSnapshotId: aws.String(id)}, testAccountID)
+	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
+	assert.EqualError(f.t, f.deleteSnapshot(svc, id), awserrors.ErrorInvalidSnapshotNotFound, "a repeated delete")
+}
+
+// DeleteSnapshot marks a pending or AmbiguousCreate snapshot and returns,
+// leaving the provider alone; any other is deleted as it always was.
+func TestDeleteSnapshot_MarksWhatTheProviderMayStillHold(t *testing.T) {
+	const id = "snap-markdelete0001"
+	cases := []struct {
+		name       string
+		doc        ebsmetadata.Snapshot
+		wantMarked bool
+	}{
+		{name: "pending", doc: ebsmetadata.Snapshot{State: snapshotStatePending}, wantMarked: true},
+		{name: "pending with an unanswered create", doc: ebsmetadata.Snapshot{State: snapshotStatePending, AmbiguousCreate: true}, wantMarked: true},
+		{name: "completed with an unanswered create", doc: ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted, AmbiguousCreate: true}, wantMarked: true},
+		{name: "errored with an unanswered create", doc: ebsmetadata.Snapshot{State: snapshotStateError, AmbiguousCreate: true}, wantMarked: true},
+		{name: "completed", doc: ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted}},
+		{name: "errored", doc: ebsmetadata.Snapshot{State: snapshotStateError}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := f.node(nodeA)
+			tc.doc.SnapshotID = id
+			f.seedDoc(tc.doc, true)
+			providerSnapshot(f, id)
+
+			require.NoError(t, f.deleteSnapshot(svc, id))
+
+			doc, ok := f.doc(id)
+			if !tc.wantMarked {
+				assert.False(t, ok, "a settled snapshot is deleted as it always was")
+				assert.Equal(t, []string{"snapshot " + id}, f.provider.deleteCalls())
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+				return
+			}
+			require.True(t, ok)
+			assert.True(t, doc.DeleteRequested)
+			assert.Equal(t, tc.doc.State, doc.State, "the mark changes nothing else")
+			assert.Empty(t, f.provider.deleteCalls(), "the provider is not asked while the snapshot may be in progress")
+			_, held := f.providerHolds(id)
+			assert.True(t, held)
+			assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID), "the reference stays for the reaper")
+			assert.Empty(t, f.claimHolder(id), "the claim is released once the mark is written")
+			f.assertHidden(svc, id)
+		})
+	}
+}
+
+// The InUse checks run first, unchanged, on a pending snapshot too.
+func TestDeleteSnapshot_PendingInUseIsStillRefused(t *testing.T) {
+	const id = "snap-markinuse00001"
+	cases := []struct {
+		name string
+		seed func(f *pendingFixture)
+	}{
+		{name: "a volume was created from it", seed: func(f *pendingFixture) {
+			require.NoError(f.t, f.metadata.PutVolume(f.t.Context(), ebsmetadata.Volume{
+				VolumeID: "vol-fromsnap000001", TenantID: testAccountID, SnapshotID: id, State: "available",
+			}))
+		}},
+		{name: "a registered AMI names it", seed: func(f *pendingFixture) {
+			require.NoError(f.t, f.metadata.PutAMI(f.t.Context(), ebsmetadata.AMI{ImageID: "ami-0aaaaaaaaaaaaaaa1", SnapshotID: id, State: ebsmetadata.AMIStatePending}))
+		}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := f.node(nodeA)
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+			tc.seed(f)
+
+			err := f.deleteSnapshot(svc, id)
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidSnapshotInUse), "got %v", err)
+			doc, ok := f.doc(id)
+			require.True(t, ok)
+			assert.False(t, doc.DeleteRequested)
+		})
+	}
+}
+
+// A claim another live node holds is retried three times and then answered
+// IncorrectState; one a stopped node holds is taken over.
+func TestDeleteSnapshot_ClaimContention(t *testing.T) {
+	const id = "snap-markheld000001"
+	cases := []struct {
+		name        string
+		holder      string
+		holderSeen  time.Time
+		wantErr     string
+		wantRefused int
+	}{
+		{name: "held by a live node", holder: nodeC, holderSeen: time.Now(), wantErr: awserrors.ErrorIncorrectState, wantRefused: claimRetries + 1},
+		{name: "held by a node whose heartbeat stopped", holder: nodeC, holderSeen: time.Now().Add(-2 * clusterv1.LivenessWindow)},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := fastClaims(f.node(nodeA))
+			refused := 0
+			svc.claimRefused = func(string) { refused++ }
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+			f.holdClaim(tc.holder, id)
+			f.liveness.beat(tc.holder, tc.holderSeen)
+
+			err := f.deleteSnapshot(svc, id)
+
+			assert.Equal(t, tc.wantRefused, refused)
+			doc, ok := f.doc(id)
+			require.True(t, ok)
+			if tc.wantErr != "" {
+				require.Error(t, err)
+				assert.True(t, awserrors.IsErrorCode(err, tc.wantErr), "got %v", err)
+				assert.False(t, doc.DeleteRequested)
+				assert.Equal(t, tc.holder, f.claimHolder(id))
+				return
+			}
+			require.NoError(t, err)
+			assert.True(t, doc.DeleteRequested)
+			assert.Empty(t, f.claimHolder(id))
+		})
+	}
+}
+
+// Each row of the marked column: the reaper deletes a marked snapshot once
+// the provider settles it, never repeats one the provider does not hold, and
+// keeps a tombstone in place of a document that had an unanswered create.
+func TestPendingSnapshotReaper_MarkedColumn(t *testing.T) {
+	const id = "snap-markedrow00001"
+	cases := []struct {
+		name      string
+		ambiguous bool
+		setup     func(f *pendingFixture)
+		// wantKept: the document stays marked, its reference and provider snapshot untouched.
+		wantKept      bool
+		wantTombstone bool
+		wantDeletes   int // provider DeleteSnapshot calls
+	}{
+		{name: "provider pending", setup: func(f *pendingFixture) {
+			f.mem.AnswerSnapshotsPending(true)
+			providerSnapshot(f, id)
+		}, wantKept: true},
+		{name: "provider completed", setup: func(f *pendingFixture) { providerSnapshot(f, id) }, wantDeletes: 1},
+		{name: "provider error", setup: func(f *pendingFixture) {
+			f.mem.AnswerSnapshotsPending(true)
+			providerSnapshot(f, id)
+			require.NoError(f.t, f.mem.SettleSnapshot(id, ebsprovider.SnapshotStateError))
+		}, wantDeletes: 1},
+		{name: "provider completed, create unanswered", ambiguous: true, setup: func(f *pendingFixture) { providerSnapshot(f, id) },
+			wantTombstone: true, wantDeletes: 1},
+		{name: "provider delete answers not_found", setup: func(f *pendingFixture) {
+			providerSnapshot(f, id)
+			f.provider.onDelete = func(ebsprovider.DeleteSnapshotRequest) error { return ebsprovider.ErrNotFound }
+		}, wantDeletes: 1},
+		{name: "absent", wantDeletes: 0},
+		{name: "absent, create unanswered", ambiguous: true, wantTombstone: true},
+		{name: "absent, claim held by a live node", setup: func(f *pendingFixture) { f.holdClaim(nodeC, id) }, wantKept: true},
+		{name: "provider read fails", setup: func(f *pendingFixture) {
+			f.provider.onList = func(ebsprovider.ListSnapshotsRequest) (bool, *ebsprovider.ListSnapshotsResponse, error) {
+				return true, nil, ebsprovider.ErrUnavailable
+			}
+		}, wantKept: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			reaper := f.node(nodeB)
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending, DeleteRequested: true, AmbiguousCreate: tc.ambiguous}, true)
+			if tc.setup != nil {
+				tc.setup(f)
+			}
+			_, heldBefore := f.providerHolds(id)
+
+			f.sweep(reaper)
+
+			assert.Zero(t, f.provider.createCalls(), "a marked snapshot is never repeated")
+			assert.Len(t, f.provider.deleteCalls(), tc.wantDeletes)
+			doc, ok := f.doc(id)
+			switch {
+			case tc.wantKept:
+				require.True(t, ok)
+				assert.True(t, doc.DeleteRequested)
+				assert.False(t, doc.Tombstoned())
+				assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID))
+				_, held := f.providerHolds(id)
+				assert.Equal(t, heldBefore, held)
+			case tc.wantTombstone:
+				require.True(t, ok, "a document that had an unanswered create is kept as a tombstone")
+				assert.True(t, doc.Tombstoned())
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+				f.assertHidden(f.node(nodeA), id)
+			default:
+				assert.False(t, ok)
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+			}
+			if !tc.wantKept && tc.wantDeletes > 0 && f.provider.onDelete == nil {
+				_, held := f.providerHolds(id)
+				assert.False(t, held)
+			}
+		})
+	}
+}
+
+// A failure or interruption between the provider delete, the reference's
+// removal and the document's removal leaves the marked document for the next
+// sweep, which finishes; no reference is ever left without its document.
+func TestPendingSnapshotReaper_CleanupInterrupted(t *testing.T) {
+	const id = "snap-cleanupstep001"
+	cases := []struct {
+		name      string
+		ambiguous bool
+		break_    func(f *pendingFixture, reaper *SnapshotServiceImpl)
+		heal      func(f *pendingFixture, reaper *SnapshotServiceImpl)
+		wantRef   bool // the reference is still there after the broken sweep
+	}{
+		{name: "the provider delete fails", break_: func(f *pendingFixture, _ *SnapshotServiceImpl) {
+			f.provider.onDelete = func(ebsprovider.DeleteSnapshotRequest) error { return ebsprovider.ErrUnavailable }
+		}, heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.provider.onDelete = nil }, wantRef: true},
+		{name: "interrupted after the provider delete", break_: func(_ *pendingFixture, r *SnapshotServiceImpl) {
+			r.reapAt = func(step reapStep, _ string) bool { return step == reapProviderDeleted }
+		}, heal: func(_ *pendingFixture, r *SnapshotServiceImpl) { r.reapAt = nil }, wantRef: true},
+		{name: "the reference removal fails", break_: func(f *pendingFixture, r *SnapshotServiceImpl) {
+			r.snapRefs = f.unreachableRefs()
+		}, heal: func(f *pendingFixture, r *SnapshotServiceImpl) { r.snapRefs = f.node(nodeB).snapRefs }, wantRef: true},
+		{name: "interrupted after the reference removal", break_: func(_ *pendingFixture, r *SnapshotServiceImpl) {
+			r.reapAt = func(step reapStep, _ string) bool { return step == reapRefRemoved }
+		}, heal: func(_ *pendingFixture, r *SnapshotServiceImpl) { r.reapAt = nil }},
+		{name: "the document removal fails", break_: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.failDelete = true },
+			heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.failDelete = false }},
+		{name: "the tombstone write fails", ambiguous: true, break_: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.putsBefore = 0 },
+			heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.putsBefore = -1 }},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			reaper := f.node(nodeB)
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending, DeleteRequested: true, AmbiguousCreate: tc.ambiguous}, true)
+			providerSnapshot(f, id)
+
+			tc.break_(f, reaper)
+			f.sweep(reaper)
+
+			doc, ok := f.doc(id)
+			require.True(t, ok, "the document is the record the next sweep finds the work from")
+			assert.True(t, doc.DeleteRequested)
+			assert.False(t, doc.Tombstoned())
+			if tc.wantRef {
+				assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID))
+			} else {
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+			}
+
+			tc.heal(f, reaper)
+			f.sweep(reaper)
+
+			_, held := f.providerHolds(id)
+			assert.False(t, held)
+			assert.Empty(t, f.refsOf(pendingVolumeID))
+			doc, ok = f.doc(id)
+			if tc.ambiguous {
+				require.True(t, ok)
+				assert.True(t, doc.Tombstoned())
+			} else {
+				assert.False(t, ok)
+			}
+		})
+	}
+}
+
+// A DeleteSnapshot that lands while the reaper holds the claim, between its
+// reread and its write, waits on the claim and ends with the snapshot deleted,
+// whether it runs on another node or on the reaper's own.
+func TestDeleteSnapshot_LandsBetweenTheReapersRereadAndWrite(t *testing.T) {
+	const id = "snap-deleteraced001"
+	for _, sameNode := range []bool{false, true} {
+		name := "another node"
+		if sameNode {
+			name = "the reaper's node"
+		}
+		t.Run(name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			reaper := fastClaims(f.node(nodeB))
+			deleter := fastClaims(f.node(nodeA))
+			if sameNode {
+				deleter = reaper
+			}
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+			providerSnapshot(f, id)
+
+			refused := make(chan struct{}, claimRetries+1)
+			deleter.claimRefused = func(string) { refused <- struct{}{} }
+			var wg sync.WaitGroup
+			var deleteErr error
+			reaper.reapAt = func(step reapStep, _ string) bool {
+				if step == reapReread {
+					wg.Go(func() { deleteErr = f.deleteSnapshot(deleter, id) })
+					<-refused
+				}
+				return false
+			}
+			deleter.claimRetryDelay = 50 * time.Millisecond
+
+			f.sweep(reaper)
+			wg.Wait()
+
+			require.NoError(t, deleteErr, "the delete waits out the reaper's claim")
+			_, ok := f.doc(id)
+			assert.False(t, ok, "the delete is not undone by the reaper's settle")
+			_, held := f.providerHolds(id)
+			assert.False(t, held)
+			assert.Empty(t, f.refsOf(pendingVolumeID))
+		})
+	}
+}
+
+// A request whose provider call returns after its claim was taken over and the
+// snapshot marked leaves the document marked and pending, for the reaper.
+func TestCreateSnapshot_AnswerAfterTheMarkIsNotWritten(t *testing.T) {
+	f := newPendingFixture(t)
+	svc := f.node(nodeA)
+	f.liveness.beat(nodeA, time.Now().Add(-2*clusterv1.LivenessWindow))
+	deleter := fastClaims(f.node(nodeB))
+	f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
+		if n == 1 {
+			require.NoError(t, f.deleteSnapshot(deleter, req.SnapshotID))
+		}
+		return false, nil, nil
+	}
+
+	_, err := f.create(svc)
+	require.NoError(t, err, "the provider answered, so the request succeeds")
+
+	doc := f.onlyDoc()
+	assert.True(t, doc.DeleteRequested)
+	assert.Equal(t, snapshotStatePending, doc.State, "a marked document is never settled by the request")
+	assert.True(t, doc.AmbiguousCreate)
+
+	f.sweep(f.node(nodeC))
+	assert.True(t, f.onlyDoc().Tombstoned())
+	_, held := f.providerHolds(doc.SnapshotID)
+	assert.False(t, held)
+}
+
+// AmbiguousCreate is on the document whenever the provider is called, and is
+// cleared only by a definite answer to a document with no earlier call left
+// unanswered.
+func TestAmbiguousCreate_SetBeforeEveryCall(t *testing.T) {
+	const reaped = "snap-ambiguousreap1"
+	cases := []struct {
+		name string
+		// reaper seeds a pending document with the flag clear and sweeps it,
+		// instead of a CreateSnapshot.
+		reaper  bool
+		pending bool              // the memory provider answers pending
+		onCall  func(n int) error // nil error: the memory provider answers
+		crash   createStep
+		want    bool
+		wantErr bool
+	}{
+		{name: "request answered completed", want: false},
+		{name: "request answered pending", pending: true, want: false},
+		{name: "request first call unanswered, repeat completed", onCall: firstFails(errAmbiguous), want: true},
+		{name: "request crashed after sending", crash: stepAnswered, want: true, wantErr: true},
+		{name: "request repeat refused for good", onCall: func(n int) error {
+			if n == 1 {
+				return errAmbiguous
+			}
+			return ebsprovider.ErrAlreadyExists
+		}, want: true, wantErr: true},
+		{name: "reaper repeat answered", reaper: true, want: false},
+		{name: "reaper repeat unanswered", reaper: true, onCall: firstFails(errAmbiguous), want: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := f.node(nodeA)
+			f.mem.AnswerSnapshotsPending(tc.pending)
+			var flagged []bool
+			f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
+				doc, _ := f.doc(req.SnapshotID)
+				flagged = append(flagged, doc.AmbiguousCreate)
+				if tc.onCall != nil {
+					if err := tc.onCall(n); err != nil {
+						return true, nil, err
+					}
+				}
+				return false, nil, nil
+			}
+			if tc.crash != 0 {
+				svc.crashAt = func(step createStep) bool { return step == tc.crash }
+			}
+
+			var id string
+			if tc.reaper {
+				id = reaped
+				f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+				f.sweep(f.node(nodeB))
+			} else {
+				_, err := f.create(svc)
+				assert.Equal(t, tc.wantErr, err != nil, "err %v", err)
+				id = f.onlyDoc().SnapshotID
+			}
+
+			require.NotEmpty(t, flagged)
+			for i, set := range flagged {
+				assert.True(t, set, "call %d was sent with AmbiguousCreate clear", i+1)
+			}
+			doc, ok := f.doc(id)
+			require.True(t, ok)
+			assert.Equal(t, tc.want, doc.AmbiguousCreate)
+		})
+	}
+}
+
+// firstFails answers the first call with err and lets the rest through.
+func firstFails(err error) func(int) error {
+	return func(n int) error {
+		if n == 1 {
+			return err
+		}
+		return nil
+	}
+}
+
+// The original call of a snapshot that a repeat completed and the customer
+// then deleted is accepted late: the snapshot it recreates is deleted under
+// the tombstone once it settles, and the tombstone is dropped after 7 days.
+func TestTombstone_DeletesALateAcceptedSnapshotAndExpires(t *testing.T) {
+	f := newPendingFixture(t)
+	svc := f.node(nodeA)
+	reaper := f.node(nodeB)
+	now := time.Now()
+	reaper.now = func() time.Time { return now }
+	var late ebsprovider.CreateSnapshotRequest
+	f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
+		if n <= 2 {
+			late = req
+			return true, nil, errAmbiguous
+		}
+		return false, nil, nil
+	}
+	_, err := f.create(svc)
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	id := f.onlyDoc().SnapshotID
+
+	// The reaper's repeat completes it, and the customer deletes it.
+	f.sweep(reaper)
+	doc := f.onlyDoc()
+	require.Equal(t, ebsmetadata.SnapshotStateCompleted, doc.State)
+	require.True(t, doc.AmbiguousCreate, "the request's calls were never answered")
+	require.NoError(t, f.deleteSnapshot(svc, id))
+	f.sweep(reaper)
+	require.True(t, f.onlyDoc().Tombstoned())
+	_, held := f.providerHolds(id)
+	require.False(t, held)
+
+	// The request's original call is accepted now, and is still in progress.
+	f.mem.AnswerSnapshotsPending(true)
+	_, err = f.mem.CreateSnapshot(t.Context(), late)
+	require.NoError(t, err)
+	f.sweep(reaper)
+	_, held = f.providerHolds(id)
+	assert.True(t, held, "a snapshot still pending is not deleted")
+	require.NoError(t, f.mem.SettleSnapshot(id, ebsprovider.SnapshotStateCompleted))
+	f.sweep(reaper)
+	_, held = f.providerHolds(id)
+	assert.False(t, held, "the late snapshot is deleted under the tombstone")
+	assert.True(t, f.onlyDoc().Tombstoned())
+	f.assertHidden(svc, id)
+	assert.Equal(t, 3, f.provider.createCalls(), "the request's two calls and one repeat; nothing repeats a tombstone's create")
+
+	now = now.Add(tombstoneRetention - time.Minute)
+	f.sweep(reaper)
+	assert.Len(t, f.docs(), 1, "the tombstone is kept for its retention")
+	now = now.Add(2 * time.Minute)
+	f.sweep(reaper)
+	assert.Empty(t, f.docs(), "the tombstone is dropped after its retention")
+	assert.Empty(t, f.refsOf(pendingVolumeID))
+}
+
+// The AMI pass settles each pending AMI from its snapshot's document.
+func TestPendingSnapshotReaper_SettlesPendingAMIs(t *testing.T) {
+	const snapID = "snap-amisettle00001"
+	cases := []struct {
+		name     string
+		amiState string
+		snap     *ebsmetadata.Snapshot // nil: no snapshot document
+		want     string
+	}{
+		{name: "snapshot completed", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted}, want: ebsmetadata.AMIStateAvailable},
+		{name: "snapshot errored", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: snapshotStateError}, want: ebsmetadata.AMIStateFailed},
+		{name: "snapshot document missing", amiState: ebsmetadata.AMIStatePending, want: ebsmetadata.AMIStateFailed},
+		{name: "snapshot pending", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: snapshotStatePending}, want: ebsmetadata.AMIStatePending},
+		{name: "AMI already available", amiState: ebsmetadata.AMIStateAvailable, want: ebsmetadata.AMIStateAvailable},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			if tc.snap != nil {
+				tc.snap.SnapshotID = snapID
+				f.seedDoc(*tc.snap, true)
+				if tc.snap.State == snapshotStatePending {
+					f.mem.AnswerSnapshotsPending(true)
+					providerSnapshot(f, snapID)
+				}
+			}
+			const amiID = "ami-0bbbbbbbbbbbbbbb1"
+			require.NoError(t, f.metadata.PutAMI(t.Context(), ebsmetadata.AMI{
+				ImageID: amiID, SnapshotID: snapID, ImageOwnerAlias: testAccountID, State: tc.amiState,
+			}))
+
+			f.sweep(f.node(nodeB))
+
+			ami, err := f.metadata.GetAMI(t.Context(), amiID)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, ami.State)
+		})
+	}
+}
+
+func TestCopySnapshot_RefusesASourceThatIsNotCompleted(t *testing.T) {
+	for _, state := range []string{snapshotStatePending, "error"} {
+		t.Run(state, func(t *testing.T) {
+			svc, _ := providerSnapshotService(t, "vol-copygate")
+			require.NoError(t, svc.MetadataStore().PutSnapshot(t.Context(), ebsmetadata.Snapshot{
+				SnapshotID: "snap-copygate00001", VolumeID: "vol-copygate", VolumeSize: 4, State: state, OwnerID: testAccountID,
+			}))
+
+			_, err := svc.CopySnapshot(t.Context(), &ec2.CopySnapshotInput{SourceSnapshotId: aws.String("snap-copygate00001")}, testAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, awserrors.ErrorIncorrectState, code)
+			snaps, err := svc.MetadataStore().ListAllSnapshots(t.Context())
+			require.NoError(t, err)
+			assert.Len(t, snaps, 1, "a refused copy writes no snapshot document")
+		})
+	}
+}

spinifex/domains/ec2/snapshot/pending.go

@@ -0,0 +1,113 @@
+package snapshot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// The snapshot document states CreateSnapshot and the reaper move between.
+const (
+	snapshotStatePending = string(ebsprovider.SnapshotStatePending)
+	snapshotStateError   = string(ebsprovider.SnapshotStateError)
+)
+
+// definiteRefusal reports a CreateSnapshot error that is the provider's answer
+// that it holds no snapshot for the call. Anything else, a timeout, a lost
+// connection or unavailable, may follow an accepted call.
+func definiteRefusal(err error) bool {
+	return errors.Is(err, ebsprovider.ErrInvalidArgument) || errors.Is(err, ebsprovider.ErrNotFound) ||
+		errors.Is(err, ebsprovider.ErrAlreadyExists) || errors.Is(err, ebsprovider.ErrVolumeInUse)
+}
+
+// refusalSettlesError reports a refusal no later call can change: the source
+// volume is gone, or the ID is another volume's snapshot. Only these make a
+// pending document error once an earlier call may still be accepted.
+func refusalSettlesError(err error) bool {
+	return errors.Is(err, ebsprovider.ErrNotFound) || errors.Is(err, ebsprovider.ErrAlreadyExists)
+}
+
+// applyProviderAnswer moves a document to what the provider answered: completed
+// at 100%, or pending with the provider's handle for the reaper to follow. The
+// call was answered, so AmbiguousCreate stays only if an earlier one was not.
+func applyProviderAnswer(created *ebsprovider.Snapshot, earlierUnanswered bool) func(*ebsmetadata.Snapshot) {
+	return func(doc *ebsmetadata.Snapshot) {
+		doc.AmbiguousCreate = earlierUnanswered
+		if created.Handle != "" {
+			doc.ProviderHandle = created.Handle
+		}
+		if created.State == ebsprovider.SnapshotStateCompleted {
+			doc.State = ebsmetadata.SnapshotStateCompleted
+			doc.Progress = "100%"
+		}
+	}
+}
+
+// repeatOutcome is the write a CreateSnapshot answer after an unanswered call
+// makes: answered, the provider's answer; a refusal no later call can change,
+// error; anything else, nil, leaving the document pending.
+func repeatOutcome(created *ebsprovider.Snapshot, err error, earlierUnanswered bool) (mutate func(*ebsmetadata.Snapshot), answered bool) {
+	switch {
+	case err == nil && created != nil:
+		return applyProviderAnswer(created, earlierUnanswered), true
+	case refusalSettlesError(err):
+		return func(d *ebsmetadata.Snapshot) {
+			markSnapshotError(d)
+			d.AmbiguousCreate = earlierUnanswered
+		}, false
+	}
+	return nil, false
+}
+
+func markSnapshotCompleted(doc *ebsmetadata.Snapshot) {
+	doc.State = ebsmetadata.SnapshotStateCompleted
+	doc.Progress = "100%"
+}
+
+func markSnapshotError(doc *ebsmetadata.Snapshot) {
+	doc.State = snapshotStateError
+}
+
+// updatePending rereads a snapshot document and applies mutate only while it
+// is still pending, so a delete or settlement since the caller read it is not
+// undone. It returns the document as read or written, zero when it is gone.
+func (s *SnapshotServiceImpl) updatePending(ctx context.Context, accountID, snapshotID string, mutate func(*ebsmetadata.Snapshot)) (ebsmetadata.Snapshot, bool, error) {
+	return s.rewrite(ctx, accountID, snapshotID, func(doc *ebsmetadata.Snapshot) bool {
+		// A marked document is the reaper's to delete, never to settle.
+		if doc.State != snapshotStatePending || doc.Hidden() {
+			return false
+		}
+		mutate(doc)
+		return true
+	})
+}
+
+// rewrite rereads snapshotID's document and writes it back when mutate asks
+// to. It returns the document as read or written and whether it wrote; one
+// that is gone is zero, and mutate never sees it.
+func (s *SnapshotServiceImpl) rewrite(ctx context.Context, accountID, snapshotID string, mutate func(*ebsmetadata.Snapshot) bool) (ebsmetadata.Snapshot, bool, error) {
+	doc, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return ebsmetadata.Snapshot{}, false, nil
+		}
+		return ebsmetadata.Snapshot{}, false, fmt.Errorf("read snapshot %s: %w", snapshotID, err)
+	}
+	if !mutate(&doc) {
+		return doc, false, nil
+	}
+	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
+		return ebsmetadata.Snapshot{}, false, fmt.Errorf("write snapshot %s: %w", snapshotID, err)
+	}
+	return doc, true, nil
+}
+
+// writeContext bounds a write made after the provider answered, which has to
+// land even when the request that asked has been cancelled.
+func writeContext(ctx context.Context) (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.WithoutCancel(ctx), snapshotCleanupTimeout)
+}

spinifex/domains/ec2/snapshot/pending_reaper.go

@@ -0,0 +1,464 @@
+package snapshot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// PendingSnapshotReaper settles every pending snapshot document from the
+// provider. CreateSnapshot returns once the provider answers, and a snapshot
+// that settles later, or a request that crashed or failed ambiguously, leaves
+// its document pending; this sweep is where that work converges (ADR-0003:S8).
+//
+// It deletes nothing a customer did not ask to delete (ADR-0003:S7): only a
+// snapshot whose document DeleteSnapshot marked, its reference and document,
+// and a late-accepted snapshot under the tombstone that delete left.
+type PendingSnapshotReaper struct {
+	svc *SnapshotServiceImpl
+}
+
+// tombstoneRetention is how long a deleted AmbiguousCreate snapshot's document
+// stays, so a create accepted that late is still found and deleted.
+const tombstoneRetention = 7 * 24 * time.Hour
+
+// reapStep is a point in the reaper's work on one document, for a test to
+// interrupt it at.
+type reapStep int
+
+const (
+	reapReread reapStep = iota + 1
+	reapProviderDeleted
+	reapRefRemoved
+)
+
+// NewPendingSnapshotReaper builds the reaper over this service's documents,
+// provider and claims.
+func (s *SnapshotServiceImpl) NewPendingSnapshotReaper() *PendingSnapshotReaper {
+	return &PendingSnapshotReaper{svc: s}
+}
+
+func (r *PendingSnapshotReaper) Class() string { return "pending-snapshot" }
+
+// Sweep reads every snapshot document and settles each pending one it can,
+// carries out each accepted delete, and rereads each tombstone; then it settles
+// each pending AMI from its snapshot. It reports how many documents it wrote.
+func (r *PendingSnapshotReaper) Sweep(ctx context.Context) (int, error) {
+	if r.svc.provider == nil {
+		return 0, nil
+	}
+	docs, err := r.svc.metadata.ListAllSnapshotsStrict(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list snapshot documents: %w", err)
+	}
+	written := 0
+	for _, doc := range docs {
+		if err := ctx.Err(); err != nil {
+			return written, err
+		}
+		var wrote bool
+		switch {
+		case doc.Tombstoned():
+			wrote = r.sweepTombstone(ctx, doc)
+		case doc.DeleteRequested, doc.State == snapshotStatePending:
+			wrote = r.settle(ctx, doc)
+		default:
+			continue
+		}
+		if wrote {
+			written++
+		}
+	}
+	amis, err := r.settleAMIs(ctx)
+	return written + amis, err
+}
+
+// interrupted reports a test stopping the reaper at step for snapshotID.
+func (r *PendingSnapshotReaper) interrupted(step reapStep, snapshotID string) bool {
+	return r.svc.reapAt != nil && r.svc.reapAt(step, snapshotID)
+}
+
+// providerSnapshot reads snapshotID from the provider. found is false when the
+// provider holds no snapshot by that ID.
+func (r *PendingSnapshotReaper) providerSnapshot(ctx context.Context, doc ebsmetadata.Snapshot) (ebsprovider.SnapshotRef, bool, error) {
+	ref, found, err := ebsprovider.ReadSnapshot(ctx, r.svc.provider, doc.SnapshotID)
+	if err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: provider read failed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "accountId", doc.OwnerID, "err", err)
+	}
+	return ref, found, err
+}
+
+// settle follows the provider's state for doc, carries out its accepted
+// delete, or repeats its CreateSnapshot when the provider holds no snapshot by
+// that ID. It reports whether it wrote.
+func (r *PendingSnapshotReaper) settle(ctx context.Context, doc ebsmetadata.Snapshot) bool {
+	// A marked document is only cleaned up, from a provider read under its claim.
+	if doc.DeleteRequested {
+		return r.underClaim(ctx, doc, nil)
+	}
+	ref, found, err := r.providerSnapshot(ctx, doc)
+	if err != nil {
+		return false
+	}
+	if found {
+		return r.follow(ctx, doc, ref)
+	}
+	return r.underClaim(ctx, doc, func(current ebsmetadata.Snapshot) bool { return r.repeat(ctx, current) })
+}
+
+// providerSettled reports a provider snapshot that is completed or errored,
+// the states in which a provider deletes it.
+func providerSettled(ref ebsprovider.SnapshotRef) bool {
+	return ref.State == ebsprovider.SnapshotStateCompleted || ref.State == ebsprovider.SnapshotStateError
+}
+
+// withClaim runs fn under id's claim, which take acquires, and reports false
+// without running it when the claim is not taken.
+func withClaim(ctx context.Context, id string, take func(context.Context, string) (*heldClaim, error), fn func() bool) bool {
+	held, err := take(ctx, id)
+	if err != nil {
+		level := slog.LevelWarn
+		if errors.Is(err, errClaimHeld) {
+			level = slog.LevelInfo
+		}
+		slog.Log(ctx, level, "pending-snapshot: claim not taken, left for the next sweep", "id", id, "err", err)
+		return false
+	}
+	defer held.release(ctx)
+	return fn()
+}
+
+// underClaim takes doc's claim, rereads its document and runs act on it while
+// it is unmarked and pending. A marked document, however it was read before,
+// is cleaned up instead.
+func (r *PendingSnapshotReaper) underClaim(ctx context.Context, doc ebsmetadata.Snapshot, act func(ebsmetadata.Snapshot) bool) bool {
+	return withClaim(ctx, doc.SnapshotID, r.svc.claims.take, func() bool {
+		// The claim may have been taken after the creating request rolled back or
+		// settled the document, so it is read again before anything is written.
+		current, err := r.svc.metadata.GetSnapshot(ctx, doc.OwnerID, doc.SnapshotID)
+		if err != nil || current.Tombstoned() {
+			return false
+		}
+		if r.interrupted(reapReread, doc.SnapshotID) {
+			return false
+		}
+		if current.DeleteRequested {
+			return r.cleanUpIfSettled(ctx, current)
+		}
+		if current.State != snapshotStatePending {
+			return false
+		}
+		return act(current)
+	})
+}
+
+// cleanUpIfSettled reads doc's snapshot from the provider and carries out its
+// accepted delete once the provider no longer holds it pending.
+func (r *PendingSnapshotReaper) cleanUpIfSettled(ctx context.Context, doc ebsmetadata.Snapshot) bool {
+	ref, found, err := r.providerSnapshot(ctx, doc)
+	if err != nil || (found && !providerSettled(ref)) {
+		return false
+	}
+	return r.cleanUp(ctx, doc, ref, found)
+}
+
+// deleteAtProvider deletes snapshotID at the provider; one already gone there
+// is deleted.
+func (r *PendingSnapshotReaper) deleteAtProvider(ctx context.Context, snapshotID, handle string) error {
+	err := r.svc.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{
+		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: handle,
+	})
+	if errors.Is(err, ebsprovider.ErrNotFound) {
+		return nil
+	}
+	return err
+}
+
+// follow writes the state the provider reports for a snapshot it holds.
+func (r *PendingSnapshotReaper) follow(ctx context.Context, doc ebsmetadata.Snapshot, ref ebsprovider.SnapshotRef) bool {
+	var mutate func(*ebsmetadata.Snapshot)
+	switch ref.State {
+	case ebsprovider.SnapshotStateCompleted:
+		mutate = func(d *ebsmetadata.Snapshot) {
+			markSnapshotCompleted(d)
+			if d.ProviderHandle == "" {
+				d.ProviderHandle = ref.Handle
+			}
+		}
+	case ebsprovider.SnapshotStateError:
+		mutate = markSnapshotError
+	case ebsprovider.SnapshotStatePending:
+		return false
+	default:
+		slog.WarnContext(ctx, "pending-snapshot: provider reported no state for a held snapshot, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "state", ref.State)
+		return false
+	}
+	return r.underClaim(ctx, doc, func(current ebsmetadata.Snapshot) bool { return r.write(ctx, current, mutate) })
+}
+
+// repeat asks the provider again, under the snapshot's claim, for a pending
+// snapshot it does not hold. Absence alone settles nothing, because a call the
+// creating request sent may still be accepted; the repeat returns that snapshot or makes it.
+func (r *PendingSnapshotReaper) repeat(ctx context.Context, current ebsmetadata.Snapshot) bool {
+	handle, ok := r.volumeHandle(ctx, current)
+	if !ok {
+		return false
+	}
+	// The flag goes on before the call, so a crash once it is sent leaves it.
+	earlierUnanswered := current.AmbiguousCreate
+	if !earlierUnanswered {
+		if _, ok, err := r.svc.updatePending(ctx, current.OwnerID, current.SnapshotID, func(d *ebsmetadata.Snapshot) {
+			d.AmbiguousCreate = true
+		}); err != nil || !ok {
+			slog.WarnContext(ctx, "pending-snapshot: could not record the repeat before sending it, retrying next sweep",
+				"snapshotId", current.SnapshotID, "err", err)
+			return false
+		}
+	}
+	created, err := r.svc.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+		Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: current.SnapshotID, VolumeID: current.VolumeID, VolumeHandle: handle,
+	})
+	mutate, answered := repeatOutcome(created, err, earlierUnanswered)
+	switch {
+	case answered:
+		return r.write(ctx, current, mutate)
+	case mutate != nil:
+		slog.WarnContext(ctx, "pending-snapshot: provider refused the repeat for good, marking the snapshot error",
+			"snapshotId", current.SnapshotID, "volumeId", current.VolumeID, "err", err)
+		return r.write(ctx, current, mutate)
+	default:
+		slog.WarnContext(ctx, "pending-snapshot: repeat not answered, retrying next sweep",
+			"snapshotId", current.SnapshotID, "volumeId", current.VolumeID, "err", err)
+		return false
+	}
+}
+
+// cleanUp carries out an accepted delete, under the snapshot's claim: the
+// provider's delete, then the reference, then the document, or a tombstone if
+// a create on it went unanswered. Each step must succeed before the next.
+func (r *PendingSnapshotReaper) cleanUp(ctx context.Context, doc ebsmetadata.Snapshot, ref ebsprovider.SnapshotRef, found bool) bool {
+	// A marked snapshot the provider does not hold is never repeated: the
+	// customer asked for it gone, and a tombstone catches a late acceptance.
+	if found {
+		handle := ref.Handle
+		if handle == "" {
+			handle = doc.ProviderHandle
+		}
+		if err := r.deleteAtProvider(ctx, doc.SnapshotID, handle); err != nil {
+			slog.WarnContext(ctx, "pending-snapshot: provider delete of a deleted snapshot failed, retrying next sweep",
+				"snapshotId", doc.SnapshotID, "err", err)
+			return false
+		}
+		if r.interrupted(reapProviderDeleted, doc.SnapshotID) {
+			return false
+		}
+	}
+	// The document goes last: it is the only record this sweep discovers work
+	// from, and a reference left without it would block DeleteVolume for good.
+	if err := r.svc.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: reference of a deleted snapshot not removed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+		return false
+	}
+	if r.interrupted(reapRefRemoved, doc.SnapshotID) {
+		return false
+	}
+	if doc.AmbiguousCreate {
+		doc.TombstonedAt = r.svc.clock().UTC()
+		if err := r.svc.metadata.PutSnapshot(ctx, doc); err != nil {
+			slog.WarnContext(ctx, "pending-snapshot: tombstone of a deleted snapshot not written, retrying next sweep",
+				"snapshotId", doc.SnapshotID, "err", err)
+			return false
+		}
+		slog.InfoContext(ctx, "pending-snapshot: deleted, keeping a tombstone for a create that may still be accepted",
+			"snapshotId", doc.SnapshotID)
+		return true
+	}
+	if err := r.svc.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.WarnContext(ctx, "pending-snapshot: document of a deleted snapshot not removed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "err", err)
+		return false
+	}
+	slog.InfoContext(ctx, "pending-snapshot: deleted", "snapshotId", doc.SnapshotID)
+	return true
+}
+
+// sweepTombstone reads the provider for a tombstone's ID and deletes a snapshot a late
+// create made there once it is no longer pending. A tombstone past its
+// retention, with nothing pending under it, is dropped.
+func (r *PendingSnapshotReaper) sweepTombstone(ctx context.Context, doc ebsmetadata.Snapshot) bool {
+	return withClaim(ctx, doc.SnapshotID, r.svc.claims.take, func() bool {
+		ref, found, err := r.providerSnapshot(ctx, doc)
+		if err != nil {
+			return false
+		}
+		if found {
+			if !providerSettled(ref) {
+				return false
+			}
+			if err := r.deleteAtProvider(ctx, doc.SnapshotID, ref.Handle); err != nil {
+				slog.WarnContext(ctx, "pending-snapshot: late-accepted snapshot of a deleted ID not deleted, retrying next sweep",
+					"snapshotId", doc.SnapshotID, "err", err)
+				return false
+			}
+			slog.WarnContext(ctx, "pending-snapshot: deleted a snapshot accepted after the customer deleted it",
+				"snapshotId", doc.SnapshotID)
+		}
+		age := r.svc.clock().Sub(doc.TombstonedAt)
+		if age < tombstoneRetention {
+			return found
+		}
+		if err := r.svc.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+			slog.WarnContext(ctx, "pending-snapshot: expired tombstone not dropped, retrying next sweep",
+				"snapshotId", doc.SnapshotID, "err", err)
+			return found
+		}
+		slog.InfoContext(ctx, "pending-snapshot: dropped an expired tombstone", "snapshotId", doc.SnapshotID, "age_ms", otelsetup.Millis(age))
+		return true
+	})
+}
+
+// settleAMIs moves each pending AMI to its snapshot's state: completed makes it
+// available, and error or a missing or deleted snapshot makes it failed.
+func (r *PendingSnapshotReaper) settleAMIs(ctx context.Context) (int, error) {
+	amis, err := r.svc.metadata.ListAMIs(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list AMI documents: %w", err)
+	}
+	written := 0
+	for _, ami := range amis {
+		if err := ctx.Err(); err != nil {
+			return written, err
+		}
+		if ami.State != ebsmetadata.AMIStatePending || ami.SnapshotID == "" {
+			continue
+		}
+		state, ok := r.amiStateFromSnapshot(ctx, ami)
+		if !ok {
+			continue
+		}
+		if r.svc.settlePendingAMI(ctx, ami.ImageID, state, r.svc.claims.take) {
+			written++
+		}
+	}
+	return written, nil
+}
+
+// amiStateFromSnapshot is the state ami's snapshot document gives it, and
+// false while that snapshot is pending or cannot be read.
+func (r *PendingSnapshotReaper) amiStateFromSnapshot(ctx context.Context, ami ebsmetadata.AMI) (string, bool) {
+	owner := awsidentifiers.GlobalAccountID
+	if awsidentifiers.IsAccountID(ami.ImageOwnerAlias) {
+		owner = ami.ImageOwnerAlias
+	}
+	snap, err := r.svc.metadata.GetSnapshot(ctx, owner, ami.SnapshotID)
+	switch {
+	case objectstore.IsNoSuchKeyError(err):
+		return ebsmetadata.AMIStateFailed, true
+	case err != nil:
+		slog.WarnContext(ctx, "pending-snapshot: snapshot of a pending AMI unreadable, retrying next sweep",
+			"imageId", ami.ImageID, "snapshotId", ami.SnapshotID, "err", err)
+		return "", false
+	case snap.Hidden() || snap.State == snapshotStateError:
+		return ebsmetadata.AMIStateFailed, true
+	case snap.State == ebsmetadata.SnapshotStateCompleted:
+		return ebsmetadata.AMIStateAvailable, true
+	default:
+		return "", false
+	}
+}
+
+// volumeHandle is the provider handle of doc's source volume, which the
+// repeat sends as the request did. A volume document that is gone gives no
+// handle, and the provider answers for the volume; one unreadable is retried.
+func (r *PendingSnapshotReaper) volumeHandle(ctx context.Context, doc ebsmetadata.Snapshot) (string, bool) {
+	volume, err := r.svc.metadata.GetVolume(ctx, doc.OwnerID, doc.VolumeID)
+	if err == nil {
+		return volume.ProviderHandle, true
+	}
+	if objectstore.IsNoSuchKeyError(err) {
+		return "", true
+	}
+	slog.WarnContext(ctx, "pending-snapshot: source volume document unreadable, retrying next sweep",
+		"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+	return "", false
+}
+
+// write applies mutate to doc's current document while it is still pending,
+// under a claim the caller holds. A request that crashed between its document
+// and its reference left the reference missing, so it is restored first.
+func (r *PendingSnapshotReaper) write(ctx context.Context, doc ebsmetadata.Snapshot, mutate func(*ebsmetadata.Snapshot)) bool {
+	if err := r.svc.ensureSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: volume reference not restored, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+		return false
+	}
+	defer r.dropRefIfDeleted(ctx, doc)
+	written, ok, err := r.svc.updatePending(ctx, doc.OwnerID, doc.SnapshotID, mutate)
+	if err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: document write failed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "accountId", doc.OwnerID, "err", err)
+		return false
+	}
+	if ok && written.State != snapshotStatePending {
+		slog.InfoContext(ctx, "pending-snapshot: settled", "snapshotId", doc.SnapshotID, "state", written.State)
+	}
+	return ok
+}
+
+// dropRefIfDeleted removes doc's reference again when its document is gone: a
+// rollback or DeleteSnapshot may have removed both before write restored it.
+func (r *PendingSnapshotReaper) dropRefIfDeleted(ctx context.Context, doc ebsmetadata.Snapshot) {
+	_, err := r.svc.metadata.GetSnapshot(ctx, doc.OwnerID, doc.SnapshotID)
+	if !objectstore.IsNoSuchKeyError(err) {
+		return
+	}
+	if err := r.svc.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: reference of a deleted snapshot not removed",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+	}
+}
+
+// ReleaseNodeClaims releases every snapshot and image claim that names this
+// node and no request of this process holds, whatever its document's state. A
+// daemon runs it as it starts and before it serves requests: none survived.
+func (s *SnapshotServiceImpl) ReleaseNodeClaims(ctx context.Context) (int, error) {
+	docs, err := s.metadata.ListAllSnapshotsStrict(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list snapshot documents: %w", err)
+	}
+	amis, err := s.metadata.ListAMIsStrict(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list AMI documents: %w", err)
+	}
+	ids := make([]string, 0, len(docs)+len(amis))
+	for _, doc := range docs {
+		ids = append(ids, doc.SnapshotID)
+	}
+	for _, ami := range amis {
+		ids = append(ids, ami.ImageID)
+	}
+	released := 0
+	var errs []error
+	for _, id := range ids {
+		ok, err := s.claims.releaseOwn(ctx, id)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		if ok {
+			released++
+		}
+	}
+	return released, errors.Join(errs...)
+}

spinifex/domains/ec2/snapshot/service_impl.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -52,6 +53,32 @@ type SnapshotServiceImpl struct {
 	mutex    sync.RWMutex
 	metadata *ebsmetadata.Store
 	provider ebsprovider.EBSProvider
+	claims   *Claims
+
+	// crashAt stops CreateSnapshot after a step, as a crash there would. Tests only.
+	crashAt func(createStep) bool
+	// reapAt interrupts the reaper at a step for snapshotID. Tests only.
+	reapAt func(step reapStep, snapshotID string) bool
+	// now is the reaper's clock for tombstones; nil is time.Now. Tests only.
+	now func() time.Time
+	// claimRetryDelay overrides the wait between claim attempts. Tests only.
+	claimRetryDelay time.Duration
+	// claimRefused observes each refused claim attempt of a writer. Tests only.
+	claimRefused func(snapshotID string)
+}
+
+// clock is the time the reaper judges a tombstone's age by.
+func (s *SnapshotServiceImpl) clock() time.Time {
+	if s.now != nil {
+		return s.now()
+	}
+	return time.Now()
+}
+
+// SetClaims wires the snapshot claims shared by CreateSnapshot and the
+// pending-snapshot reaper. Without it they exclude only within this process.
+func (s *SnapshotServiceImpl) SetClaims(claims *Claims) {
+	s.claims = claims
 }
 
 // SetEBSProvider injects the provider boundary for snapshot operations.
@@ -102,6 +129,7 @@ func NewSnapshotServiceImplWithNATS(ctx context.Context, cfg *config.Config, nat
 		natsConn: natsConn,
 		snapRefs: refs,
 		metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket),
+		claims:   localClaims(),
 	}, kv, nil
 }
 
@@ -113,6 +141,7 @@ func NewSnapshotServiceImplWithStore(cfg *config.Config, store objectstore.Objec
 		store:    store,
 		natsConn: natsConn,
 		metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket),
+		claims:   localClaims(),
 	}
 	if len(snapshotKV) > 0 {
 		svc.snapRefs = kvstore.Over[[]string](nil, snapshotKV[0], snapshotRefsConfig())
@@ -135,7 +164,8 @@ func snapshotRefsConfig() kvstore.Config {
 }
 
 // getSnapshotConfig reads the caller's own snapshot document. A snapshot
-// outside the caller's prefix is absent, so ownership needs no second check.
+// outside the caller's prefix is absent, so ownership needs no second check,
+// and so is one whose delete was accepted.
 func (s *SnapshotServiceImpl) getSnapshotConfig(ctx context.Context, accountID, snapshotID string) (ebsmetadata.Snapshot, error) {
 	cfg, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
 	if err != nil {
@@ -144,6 +174,9 @@ func (s *SnapshotServiceImpl) getSnapshotConfig(ctx context.Context, accountID,
 		}
 		return ebsmetadata.Snapshot{}, err
 	}
+	if cfg.Hidden() {
+		return ebsmetadata.Snapshot{}, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
 	return cfg, nil
 }
 
@@ -203,47 +236,288 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
-	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
-	})
-	if err != nil || created == nil {
-		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, ProviderCreateError(volumeID, err)
-	}
-	now := time.Now()
-	snapshotCfg := ebsmetadata.Snapshot{
-		SnapshotID: snapshotID, VolumeID: volumeID,
+	doc := pendingDocument(volume, snapshotID, accountID, aws.StringValue(input.Description))
+	doc.Tags = awstags.Extract(input.TagSpecifications, "snapshot")
+	return s.createSnapshot(ctx, doc, providerCreateRequest(volume, snapshotID), createHooks{})
+}
+
+// pendingDocument is the document a snapshot of volume is recorded as before
+// the provider is called. AmbiguousCreate is set because that call is next.
+func pendingDocument(volume ebsmetadata.Volume, snapshotID, accountID, description string) ebsmetadata.Snapshot {
+	return ebsmetadata.Snapshot{
+		SnapshotID: snapshotID, VolumeID: volume.VolumeID,
 		VolumeSize: safecast.Uint64ToInt64(volume.CapacityGiB),
-		State:      string(created.State), Progress: "100%", StartTime: now,
-		Encrypted: volume.Encrypted,
-		OwnerID:   accountID, AvailabilityZone: volume.AvailabilityZone,
-		Tags:           awstags.Extract(input.TagSpecifications, "snapshot"),
-		ProviderHandle: created.Handle,
+		State:      snapshotStatePending, Progress: "0%", StartTime: time.Now(),
+		Description: description, Encrypted: volume.Encrypted,
+		OwnerID: accountID, AvailabilityZone: volume.AvailabilityZone,
+		AmbiguousCreate: true,
 	}
-	if snapshotCfg.State == "" {
-		snapshotCfg.State = "completed"
+}
+
+func providerCreateRequest(volume ebsmetadata.Volume, snapshotID string) ebsprovider.CreateSnapshotRequest {
+	return ebsprovider.CreateSnapshotRequest{
+		Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: snapshotID, VolumeID: volume.VolumeID, VolumeHandle: volume.ProviderHandle,
 	}
-	if input.Description != nil {
-		snapshotCfg.Description = *input.Description
+}
+
+// ImageSnapshotHooks are the AMI document's writes around an image's snapshot.
+// Recorded runs after the snapshot's document and reference and before the
+// provider call; its error rolls the snapshot back.
+type ImageSnapshotHooks struct {
+	Recorded func(ctx context.Context) error
+	// Refused removes the AMI document when the provider refused the first call.
+	Refused func(ctx context.Context)
+	// Completed runs once the snapshot's document is written completed.
+	Completed func(ctx context.Context)
+}
+
+// ImageSnapshotCreator takes the snapshot behind a CreateImage through the
+// same document, reference, claim and provider path as CreateSnapshot.
+type ImageSnapshotCreator interface {
+	CreateImageSnapshot(ctx context.Context, volume ebsmetadata.Volume, snapshotID, accountID, description string, hooks ImageSnapshotHooks) (*ec2.Snapshot, error)
+}
+
+var _ ImageSnapshotCreator = (*SnapshotServiceImpl)(nil)
+
+// ImageClaims serialises the settling of a pending AMI with DeregisterImage:
+// each holds the image's claim, in the snapshot claim group keyed by image ID.
+type ImageClaims interface {
+	LockImage(ctx context.Context, imageID string) (unlock func(), err error)
+	SettlePendingAMI(ctx context.Context, imageID, state string)
+}
+
+var _ ImageClaims = (*SnapshotServiceImpl)(nil)
+
+// LockImage takes imageID's claim for a request, as writeUnderClaim takes a
+// snapshot's; one still held after the retries is a retryable IncorrectState.
+func (s *SnapshotServiceImpl) LockImage(ctx context.Context, imageID string) (func(), error) {
+	held, err := s.takeClaimWithRetry(ctx, "image", imageID)
+	if err != nil {
+		return nil, err
+	}
+	return func() { held.release(ctx) }, nil
+}
+
+// SettlePendingAMI moves imageID's AMI to state for a request, retrying a held
+// claim as LockImage does; an AMI it leaves pending is settled by the reaper.
+func (s *SnapshotServiceImpl) SettlePendingAMI(ctx context.Context, imageID, state string) {
+	s.settlePendingAMI(ctx, imageID, state, func(ctx context.Context, id string) (*heldClaim, error) {
+		return s.takeClaimWithRetry(ctx, "image", id)
+	})
+}
+
+// settlePendingAMI writes state to imageID's AMI under the claim take returns,
+// which DeregisterImage also takes, and only while a reread finds it pending.
+func (s *SnapshotServiceImpl) settlePendingAMI(ctx context.Context, imageID, state string, take func(context.Context, string) (*heldClaim, error)) bool {
+	return withClaim(ctx, imageID, take, func() bool {
+		current, err := s.metadata.GetAMI(ctx, imageID)
+		if err != nil || current.State != ebsmetadata.AMIStatePending {
+			return false
+		}
+		current.State = state
+		if err := s.metadata.PutAMI(ctx, current); err != nil {
+			slog.WarnContext(ctx, "pending AMI's state not written, left for the reaper", "imageId", imageID, "err", err)
+			return false
+		}
+		slog.InfoContext(ctx, "pending AMI settled", "imageId", imageID, "snapshotId", current.SnapshotID, "state", state)
+		return true
+	})
+}
+
+// CreateImageSnapshot snapshots an image's root volume as CreateSnapshot does,
+// running hooks to write the AMI document beside the snapshot's.
+func (s *SnapshotServiceImpl) CreateImageSnapshot(ctx context.Context, volume ebsmetadata.Volume, snapshotID, accountID, description string, hooks ImageSnapshotHooks) (*ec2.Snapshot, error) {
+	if s.provider == nil {
+		slog.ErrorContext(ctx, "no EBS provider configured", "op", "CreateImageSnapshot")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	doc := pendingDocument(volume, snapshotID, accountID, description)
+	return s.createSnapshot(ctx, doc, providerCreateRequest(volume, snapshotID), createHooks(hooks))
+}
+
+// createHooks are the writes a CreateImage adds around createSnapshot's.
+type createHooks ImageSnapshotHooks
+
+// createStep is a point in createSnapshot after which a crash leaves its
+// writes so far, for a test to stop the request at.
+type createStep int
+
+const (
+	stepClaimed createStep = iota + 1
+	stepRecorded
+	stepReferenced
+	stepAnswered
+	stepSettled
+	stepRollbackMarked
+	stepRollbackUnreferenced
+)
+
+// createAttempts bounds createSnapshot's provider calls: the first, and one
+// repeat of a call that may have been accepted.
+const createAttempts = 2
+
+// errCreateCrashed is what createSnapshot returns when a test stops it at a step.
+var errCreateCrashed = errors.New("snapshot create stopped at a test crash point")
+
+// createSnapshot records doc pending and then asks the provider for it, so a
+// provider snapshot never exists without a document. Only a refusal of the
+// first call removes the document; anything else leaves it for the reaper.
+func (s *SnapshotServiceImpl) createSnapshot(ctx context.Context, doc ebsmetadata.Snapshot, req ebsprovider.CreateSnapshotRequest, hooks createHooks) (*ec2.Snapshot, error) {
+	snapshotID, volumeID, accountID := doc.SnapshotID, doc.VolumeID, doc.OwnerID
+
+	held, err := s.claims.lock(ctx, snapshotID)
+	if err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot could not take the snapshot claim", "snapshotId", snapshotID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	crashed := false
+	defer func() {
+		if !crashed {
+			held.release(ctx)
+		}
+	}()
+	crash := func(step createStep) bool {
+		crashed = s.crashAt != nil && s.crashAt(step)
+		return crashed
+	}
+	if crash(stepClaimed) {
+		return nil, errCreateCrashed
 	}
 
-	// Track the volume→snapshot dependency in KV before persisting to S3.
-	// This ensures we never have an untracked snapshot in S3.
+	// The document comes first because it is what the reaper discovers work
+	// from; a reference with no document would block DeleteVolume for good.
+	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to write the pending snapshot document", "snapshotId", snapshotID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if crash(stepRecorded) {
+		return nil, errCreateCrashed
+	}
 	if err := s.addSnapshotRef(ctx, volumeID, snapshotID); err != nil {
 		slog.ErrorContext(ctx, "CreateSnapshot failed to add snapshot ref to KV", "snapshotId", snapshotID, "volumeId", volumeID, "err", err)
-		_ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle})
+		if s.rollBackCreate(ctx, doc, crash) {
+			return nil, errCreateCrashed
+		}
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if err := s.metadata.PutSnapshot(ctx, snapshotCfg); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot failed to write config", "snapshotId", snapshotID, "err", err)
-		_ = s.removeSnapshotRefForCleanup(ctx, volumeID, snapshotID)
-		_ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle})
-		return nil, errors.New(awserrors.ErrorServerInternal)
+	if crash(stepReferenced) {
+		return nil, errCreateCrashed
+	}
+	if hooks.Recorded != nil {
+		if err := hooks.Recorded(ctx); err != nil {
+			slog.ErrorContext(ctx, "CreateSnapshot failed to record the image beside its snapshot", "snapshotId", snapshotID, "err", err)
+			if s.rollBackCreate(ctx, doc, crash) {
+				return nil, errCreateCrashed
+			}
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		}
 	}
 
-	slog.InfoContext(ctx, "CreateSnapshot completed", "snapshotId", snapshotID, "volumeId", volumeID)
+	// The document carries AmbiguousCreate from its first write, so a crash
+	// once a call is sent leaves the flag set. A call that may have been
+	// accepted is repeated once: the provider answers it with what it holds.
+	var created *ebsprovider.Snapshot
+	earlierUnanswered := false
+	for range createAttempts {
+		created, err = s.provider.CreateSnapshot(ctx, req)
+		if crash(stepAnswered) {
+			return nil, errCreateCrashed
+		}
+		if (err == nil && created != nil) || earlierUnanswered {
+			break
+		}
+		if definiteRefusal(err) {
+			// The request holds the claim, so no other call for this ID exists that
+			// a provider could still accept: the document can go.
+			slog.WarnContext(ctx, "CreateSnapshot refused by the provider", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+			if hooks.Refused != nil {
+				hooks.Refused(ctx)
+			}
+			if s.rollBackCreate(ctx, doc, crash) {
+				return nil, errCreateCrashed
+			}
+			return nil, ProviderCreateError(volumeID, err)
+		}
+		slog.WarnContext(ctx, "CreateSnapshot provider call failed ambiguously, repeating it", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		earlierUnanswered = true
+	}
 
-	return snapshotConfigToEC2(snapshotCfg), nil
+	mutate, answered := repeatOutcome(created, err, earlierUnanswered)
+	switch {
+	case answered:
+		return s.answerCreate(ctx, doc, mutate, hooks, crash)
+	case mutate != nil:
+		slog.WarnContext(ctx, "CreateSnapshot repeat refused for good, marking the snapshot error", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		wctx, cancel := writeContext(ctx)
+		defer cancel()
+		if _, _, werr := s.updatePending(wctx, accountID, snapshotID, mutate); werr != nil {
+			slog.ErrorContext(ctx, "CreateSnapshot failed to mark the snapshot error", "snapshotId", snapshotID, "err", werr)
+		}
+		return nil, ProviderCreateError(volumeID, err)
+	case definiteRefusal(err):
+		// The first call may still be accepted once this refusal clears.
+		slog.WarnContext(ctx, "CreateSnapshot repeat refused, leaving the snapshot pending", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, ProviderCreateError(volumeID, err)
+	}
+	slog.ErrorContext(ctx, "CreateSnapshot provider call failed ambiguously twice, leaving the snapshot pending", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
+// answerCreate writes the provider's answer onto the pending document and
+// returns the snapshot as written. The answer fixed the contents, so a failed
+// write still reports success; the reaper settles the document.
+func (s *SnapshotServiceImpl) answerCreate(ctx context.Context, doc ebsmetadata.Snapshot, answer func(*ebsmetadata.Snapshot),
+	hooks createHooks, crash func(createStep) bool) (*ec2.Snapshot, error) {
+	wctx, cancel := writeContext(ctx)
+	defer cancel()
+	written, ok, err := s.updatePending(wctx, doc.OwnerID, doc.SnapshotID, answer)
+	completed := ok && written.State == ebsmetadata.SnapshotStateCompleted
+	switch {
+	case err != nil:
+		slog.ErrorContext(ctx, "CreateSnapshot failed to record the provider's answer", "snapshotId", doc.SnapshotID, "err", err)
+		written = doc
+	case !ok && written.SnapshotID == "":
+		// Deleted while the provider answered; report what the answer was.
+		written = doc
+		answer(&written)
+	}
+	if crash(stepSettled) {
+		return nil, errCreateCrashed
+	}
+	if completed && hooks.Completed != nil {
+		hooks.Completed(wctx)
+	}
+	slog.InfoContext(ctx, "CreateSnapshot completed", "snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "state", written.State)
+	return snapshotConfigToEC2(written), nil
+}
+
+// rollBackCreate removes a snapshot the provider refused or never saw, in the
+// reaper's order: the document is marked deleted with no call outstanding, then
+// the reference and the document go. It reports a test crash.
+func (s *SnapshotServiceImpl) rollBackCreate(ctx context.Context, doc ebsmetadata.Snapshot, crash func(createStep) bool) bool {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), snapshotCleanupTimeout)
+	defer cancel()
+	// A marked document a failure below leaves is the reaper's to delete, and it
+	// never repeats the create of one, so the refusal the caller saw stands.
+	doc.DeleteRequested, doc.AmbiguousCreate = true, false
+	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to mark a refused snapshot's document deleted", "snapshotId", doc.SnapshotID, "err", err)
+	}
+	if crash(stepRollbackMarked) {
+		return true
+	}
+	if err := s.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to remove a refused snapshot's ref, leaving its document for the reaper",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+		return false
+	}
+	if crash(stepRollbackUnreferenced) {
+		return true
+	}
+	if err := s.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to remove a refused snapshot's document, leaving it for the reaper", "snapshotId", doc.SnapshotID, "err", err)
+	}
+	return false
 }
 
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
@@ -316,6 +590,9 @@ func (s *SnapshotServiceImpl) describeSnapshots(ctx context.Context, input *ec2.
 
 	var snapshots []*ec2.Snapshot
 	for _, cfg := range configs {
+		if cfg.Hidden() {
+			continue
+		}
 		if len(parsedFilters) > 0 && !snapshotMatchesFilters(cfg, parsedFilters) {
 			continue
 		}
@@ -413,7 +690,7 @@ func (s *SnapshotServiceImpl) describeSnapshotsByIDs(ctx context.Context, input
 
 		// The read was keyed by the ID inside the caller's prefix, so the
 		// document is theirs and is the one named. Both stay as assertions.
-		if result.Document.SnapshotID != snapshotIDs[i] || result.Document.OwnerID != accountID {
+		if result.Document.SnapshotID != snapshotIDs[i] || result.Document.OwnerID != accountID || result.Document.Hidden() {
 			return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
 		}
 
@@ -649,6 +926,20 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 		slog.ErrorContext(ctx, "no EBS provider configured", "op", "DeleteSnapshot")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
+
+	// A provider may refuse to delete a snapshot still in progress, and an
+	// unanswered create may still be accepted, so either is marked for the reaper.
+	if deferDelete(cfg) {
+		marked, current, err := s.requestDelete(ctx, cfg)
+		if err != nil {
+			return nil, err
+		}
+		if marked {
+			slog.InfoContext(ctx, "DeleteSnapshot accepted; the snapshot is deleted once it settles", "snapshotId", snapshotID)
+			return &ec2.DeleteSnapshotOutput{}, nil
+		}
+		cfg = current
+	}
 	if err := s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: cfg.ProviderHandle}); err != nil {
 		slog.ErrorContext(ctx, "DeleteSnapshot provider deletion failed", "snapshotId", snapshotID, "err", err)
 		return nil, providerDeleteError(snapshotID, err)
@@ -673,6 +964,104 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 	return &ec2.DeleteSnapshotOutput{}, nil
 }
 
+// deferDelete reports a snapshot DeleteSnapshot marks rather than deletes.
+func deferDelete(doc ebsmetadata.Snapshot) bool {
+	return doc.State == snapshotStatePending || doc.AmbiguousCreate
+}
+
+// requestDelete marks doc DeleteRequested under its claim. A document that
+// settled with no call outstanding since it was read is returned unmarked,
+// for the caller to delete as any other.
+func (s *SnapshotServiceImpl) requestDelete(ctx context.Context, doc ebsmetadata.Snapshot) (bool, ebsmetadata.Snapshot, error) {
+	marked := false
+	var current ebsmetadata.Snapshot
+	err := s.writeUnderClaim(ctx, doc.OwnerID, doc.SnapshotID, func(d *ebsmetadata.Snapshot) (bool, error) {
+		if d.Hidden() {
+			return false, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+		}
+		current = *d
+		if !deferDelete(*d) {
+			return false, nil
+		}
+		d.DeleteRequested = true
+		marked = true
+		return true, nil
+	})
+	if err != nil {
+		return false, ebsmetadata.Snapshot{}, err
+	}
+	if current.SnapshotID == "" {
+		return false, ebsmetadata.Snapshot{}, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
+	return marked, current, nil
+}
+
+// Contention on a snapshot's claim is bounded: three retries over two seconds.
+const (
+	claimRetries    = 3
+	claimRetryDelay = 2 * time.Second / claimRetries
+)
+
+// writeUnderClaim rereads snapshotID's document under its claim and writes it
+// when mutate asks to, so no write lands between another writer's reread and
+// its write. A document that is gone is not mutated, and mutate never sees it.
+func (s *SnapshotServiceImpl) writeUnderClaim(ctx context.Context, accountID, snapshotID string,
+	mutate func(*ebsmetadata.Snapshot) (bool, error)) error {
+	held, err := s.takeClaimWithRetry(ctx, "snapshot", snapshotID)
+	if err != nil {
+		return err
+	}
+	defer held.release(ctx)
+	var mutateErr error
+	_, _, err = s.rewrite(ctx, accountID, snapshotID, func(doc *ebsmetadata.Snapshot) bool {
+		var write bool
+		write, mutateErr = mutate(doc)
+		return write && mutateErr == nil
+	})
+	if mutateErr != nil {
+		return mutateErr
+	}
+	if err != nil {
+		return awserrors.Errorf(awserrors.ErrorServerInternal, "%v", err)
+	}
+	return nil
+}
+
+// takeClaimWithRetry takes the claim on resource id, of kind, as the
+// reaper does, retrying a holder that is live; one still held after the
+// retries is IncorrectState, which the caller retries.
+func (s *SnapshotServiceImpl) takeClaimWithRetry(ctx context.Context, kind, id string) (*heldClaim, error) {
+	delay := claimRetryDelay
+	if s.claimRetryDelay > 0 {
+		delay = s.claimRetryDelay
+	}
+	for attempt := 0; ; attempt++ {
+		held, err := s.claims.take(ctx, id)
+		if err == nil {
+			return held, nil
+		}
+		if !errors.Is(err, errClaimHeld) {
+			slog.ErrorContext(ctx, "Snapshot claim could not be taken", "kind", kind, "id", id, "err", err)
+			return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "take the claim of %s %s: %v", kind, id, err)
+		}
+		if s.claimRefused != nil {
+			s.claimRefused(id)
+		}
+		if attempt == claimRetries {
+			slog.InfoContext(ctx, "Snapshot claim still held after retries", "kind", kind, "id", id, "err", err)
+			return nil, awserrors.Errorf(awserrors.ErrorIncorrectState,
+				"The %s '%s' is being updated. Retry the request.", kind, id)
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "take the claim of %s %s: %v", kind, id, ctx.Err())
+		case <-timer.C:
+		}
+	}
+}
+
 // CopySnapshot duplicates a snapshot under a new ID owned by the caller.
 // Served only through the provider: the copy needs its own snapshot metadata
 // on the backend, and on an encrypted volume that metadata is AEAD-sealed with
@@ -703,6 +1092,9 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS
 		slog.WarnContext(ctx, "CopySnapshot: account does not own source snapshot", "snapshotId", sourceSnapshotID, "accountID", accountID, "ownerID", sourceCfg.OwnerID)
 		return nil, errors.New(awserrors.ErrorUnauthorizedOperation)
 	}
+	if !sourceCfg.Ready() {
+		return nil, sourceCfg.NotReadyError()
+	}
 
 	// Without a provider the only reachable path writes the control-plane
 	// config alone. That copy described and created volumes fine, then failed
@@ -719,6 +1111,12 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS
 		DestinationSnapshotID: newSnapshotID,
 		VolumeID:              sourceCfg.VolumeID,
 	})
+	// A provider without SnapshotCopy refuses as unsupported, which no retry
+	// changes, so the caller is told the operation is unsupported here.
+	if errors.Is(err, ebsprovider.ErrUnsupportedCapability) {
+		slog.WarnContext(ctx, "CopySnapshot is not supported by the EBS provider", "sourceSnapshotId", sourceSnapshotID)
+		return nil, errors.New(awserrors.ErrorUnsupportedOperation)
+	}
 	if err != nil || copied == nil {
 		slog.ErrorContext(ctx, "CopySnapshot: provider copy failed", "sourceSnapshotId", sourceSnapshotID, "newSnapshotId", newSnapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
@@ -783,6 +1181,25 @@ func (s *SnapshotServiceImpl) addSnapshotRef(ctx context.Context, volumeID, snap
 	return nil
 }
 
+// ensureSnapshotRef adds snapshotID to the volume's snapshot list unless it
+// is already there, restoring a reference a crashed request did not write.
+func (s *SnapshotServiceImpl) ensureSnapshotRef(ctx context.Context, volumeID, snapshotID string) error {
+	if s.snapRefs == nil {
+		return nil
+	}
+	err := s.snapRefs.Upsert(ctx, volumeID, func(snapshots *[]string) (bool, error) {
+		if slices.Contains(*snapshots, snapshotID) {
+			return false, nil
+		}
+		*snapshots = append(*snapshots, snapshotID)
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("ensureSnapshotRef: failed to record %s against volume %s: %w", snapshotID, volumeID, err)
+	}
+	return nil
+}
+
 // removeSnapshotRefForCleanup removes a reference with a bounded context that
 // survives request cancellation after the snapshot operation has committed.
 func (s *SnapshotServiceImpl) removeSnapshotRefForCleanup(ctx context.Context, volumeID, snapshotID string) error {
@@ -914,10 +1331,31 @@ func (s *SnapshotServiceImpl) mirrorSnapshotTags(resources []*string, accountID
 			}
 			return err
 		}
-		if cfg.Tags == nil {
-			cfg.Tags = map[string]string{}
+		if cfg.Hidden() {
+			continue
 		}
-		mut(cfg.Tags)
+		apply := func(d *ebsmetadata.Snapshot) {
+			if d.Tags == nil {
+				d.Tags = map[string]string{}
+			}
+			mut(d.Tags)
+		}
+		// A document the reaper or a DeleteSnapshot may still write is written
+		// under its claim, so the tags cannot undo their write or it theirs.
+		if deferDelete(cfg) {
+			err := s.writeUnderClaim(context.Background(), accountID, *res, func(d *ebsmetadata.Snapshot) (bool, error) {
+				if d.Hidden() {
+					return false, nil
+				}
+				apply(d)
+				return true, nil
+			})
+			if err != nil {
+				return err
+			}
+			continue
+		}
+		apply(&cfg)
 		if err := s.metadata.PutSnapshot(context.Background(), cfg); err != nil {
 			return err
 		}

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1246,10 +1246,9 @@ func (s *snapshotMetadataFailingObjectStore) PutObject(ctx context.Context, inpu
 	return s.ObjectStore.PutObject(ctx, input)
 }
 
-// TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure covers
-// CreateSnapshot's rollback path: when persisting the snapshot config fails
-// after the provider has already created the snapshot, CreateSnapshot must
-// delete the just-created provider snapshot rather than orphan it.
+// TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure: the pending
+// document is written before the provider is called, so a failed write leaves
+// no provider snapshot behind to orphan.
 func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 	store := &snapshotMetadataFailingObjectStore{ObjectStore: objectstore.NewMemoryObjectStore()}
 	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
@@ -1273,13 +1272,10 @@ func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 
 	snapshotID := strings.TrimSuffix(path.Base(store.attemptedKey), ".json")
 	_, err = provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback"})
-	require.NoError(t, err, "the source volume must be unaffected by the snapshot rollback")
+	require.NoError(t, err, "the source volume must be unaffected by the failed create")
 
-	// Recreating the same snapshot ID against a different source volume only
-	// succeeds if the original snapshot no longer exists (MemoryProvider
-	// returns already_exists for a conflicting source volume on a live
-	// snapshot ID), proving the rollback actually deleted it rather than
-	// merely leaving it orphaned but reachable.
+	// The ID is free on another source volume only if the provider never took
+	// it: MemoryProvider refuses a held ID on a different volume.
 	_, err = provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback-check", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 * 1024 * 1024 * 1024},
 	})
@@ -1287,7 +1283,7 @@ func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 	_, err = provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: "vol-rollback-check",
 	})
-	require.NoError(t, err, "rollback must have deleted the just-created provider snapshot")
+	require.NoError(t, err, "a failed document write must leave no provider snapshot")
 }
 
 // providerSnapshotService builds a snapshot service backed by the memory
@@ -1297,7 +1293,7 @@ func providerSnapshotService(t *testing.T, volumeID string) (*SnapshotServiceImp
 	store := objectstore.NewMemoryObjectStore()
 	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
 	svc := NewSnapshotServiceImplWithStore(cfg, store, nil)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{SnapshotCopy: true})
 	svc.SetEBSProvider(provider)
 	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
 		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4, State: "available",
@@ -1417,6 +1413,34 @@ func TestCopySnapshot_Provider_ForeignSnapshotRefused(t *testing.T) {
 	require.True(t, ok, "the source must survive a refused copy")
 }
 
+// A provider that cannot copy in place answers UnsupportedOperation, and the
+// refusal leaves no copy behind in the control plane.
+func TestCopySnapshot_Provider_WithoutSnapshotCopyIsUnsupported(t *testing.T) {
+	ctx := context.Background()
+	svc, provider := providerSnapshotService(t, "vol-copyunsup")
+
+	src, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-copyunsup")}, testAccountID)
+	require.NoError(t, err)
+
+	svc.SetEBSProvider(&noCopyProvider{EBSProvider: provider})
+	_, err = svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{SourceSnapshotId: src.SnapshotId}, testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorUnsupportedOperation, err.Error())
+
+	listed, err := svc.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Len(t, listed.Snapshots, 1, "only the source snapshot should exist")
+}
+
+// noCopyProvider refuses CopySnapshot as a provider without SnapshotCopy does.
+type noCopyProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *noCopyProvider) CopySnapshot(context.Context, ebsprovider.CopySnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, ebsprovider.ErrUnsupportedCapability
+}
+
 // copyFailingProvider fails only CopySnapshot, leaving every other operation
 // intact so the rollback path is what the test exercises.
 type copyFailingProvider struct {
@@ -1547,17 +1571,17 @@ func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
 	assert.Contains(t, message, "Detach")
 }
 
-// A provider that cannot answer now, such as one refusing a volume another
-// node holds, is retry-later for the client; any other failure is internal.
+// A failure that is not a refusal may follow an accepted call, so after its
+// repeat fails the same way the request is internal and the snapshot is left
+// pending for the reaper rather than removed.
 func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
 	cases := []struct {
-		name     string
-		err      error
-		wantCode string
+		name string
+		err  error
 	}{
-		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
-		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
-		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
+		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}},
+		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}},
+		{"transport failure", errors.New("connection reset")},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -1567,7 +1591,11 @@ func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
 
 			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
 			require.Error(t, err)
-			assert.Equal(t, tc.wantCode, err.Error())
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+			docs, err := svc.metadata.ListAllSnapshotsStrict(context.Background())
+			require.NoError(t, err)
+			require.Len(t, docs, 1)
+			assert.Equal(t, "pending", docs[0].State)
 		})
 	}
 }

spinifex/domains/ec2/volume/provider_snapshot_source_test.go

@@ -51,6 +51,7 @@ func writeSnapshotMetadata(t *testing.T, store objectstore.ObjectStore, snapshot
 	t.Helper()
 	require.NoError(t, ebsmetadata.NewStore(store, "test-bucket").PutSnapshot(context.Background(), ebsmetadata.Snapshot{
 		SnapshotID: snapshotID, VolumeID: sourceVolumeID, VolumeSize: sizeGiB, OwnerID: ownerID,
+		State: ebsmetadata.SnapshotStateCompleted,
 	}))
 }
 
@@ -95,6 +96,25 @@ func TestCreateVolume_Provider_WithoutSnapshotSendsNoSource(t *testing.T) {
 	assert.Empty(t, provider.lastRequest.SourceSnapshotVolumeID)
 }
 
+// TestCreateVolume_Provider_SendsVolumeType locks that the provider is told the
+// volume type, which is how a provider with performance tiers picks one.
+func TestCreateVolume_Provider_SendsVolumeType(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	cfg := &config.Config{AZ: providerTestAZ, Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
+	svc := NewVolumeServiceImplWithStore(cfg, store, nil)
+	provider := newSnapshotSourceProvider()
+	svc.SetEBSProvider(provider)
+
+	_, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{
+		Size: aws.Int64(8), AvailabilityZone: aws.String(providerTestAZ), VolumeType: aws.String("GP3"),
+	}, "000000000001")
+	require.NoError(t, err)
+
+	params, err := ebsprovider.ParseVolumeParameters(provider.lastRequest.Parameters)
+	require.NoError(t, err)
+	assert.Equal(t, ebspolicy.VolumeTypeGP3, params.VolumeType)
+}
+
 // fixedAMILoader resolves one AMI to a snapshot on a known source volume.
 type fixedAMILoader struct {
 	meta           ebsmetadata.AMI

spinifex/domains/ec2/volume/service_impl.go

@@ -229,10 +229,15 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 		// The ownership check is the key: a snapshot outside the caller's prefix
 		// is not found, so the endpoint never confirms another account's IDs.
 		snapMeta, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
-		if err != nil {
+		if err != nil || snapMeta.Hidden() {
 			slog.ErrorContext(ctx, "CreateVolume: snapshot not found", "snapshotId", snapshotID, "err", err)
 			return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
 		}
+		// A pending snapshot's contents are not readable yet, and an errored
+		// one's never are.
+		if !snapMeta.Ready() {
+			return nil, snapMeta.NotReadyError()
+		}
 		sourceVolumeName = snapMeta.VolumeID
 		snapshotSizeGiB = snapMeta.VolumeSize
 	}
@@ -289,6 +294,12 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 		return nil, err
 	}
 
+	params, err := ebsprovider.NewVolumeParameters(volumeType)
+	if err != nil {
+		slog.ErrorContext(ctx, "CreateVolume: encode provider parameters failed", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	// SourceVolumeName must travel with the snapshot ID: a clone resolves its
 	// base blocks against the source volume's prefix, and the provider rejects
 	// a snapshot source without it.
@@ -299,6 +310,7 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 		AvailabilityZone:       *input.AvailabilityZone,
 		SourceSnapshotID:       snapshotID,
 		SourceSnapshotVolumeID: sourceVolumeName,
+		Parameters:             params,
 	})
 	if err != nil {
 		slog.ErrorContext(ctx, "CreateVolume: provider allocation failed", "volumeId", volumeID, "err", err)

spinifex/domains/ec2/volume/service_impl_test.go

@@ -743,6 +743,7 @@ func putTestSnapshotMetadata(t *testing.T, store *objectstore.MemoryObjectStore,
 		VolumeID:   "vol-source",
 		VolumeSize: sizeGiB,
 		OwnerID:    ownerID,
+		State:      ebsmetadata.SnapshotStateCompleted,
 	}))
 }
 

spinifex/domains/ec2/volume/snapshot_readiness_test.go

@@ -0,0 +1,48 @@
+package volume
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCreateVolume_FromSnapshot_RefusesASnapshotThatIsNotCompleted(t *testing.T) {
+	tests := []struct {
+		name     string
+		snapshot ebsmetadata.Snapshot
+		wantCode string
+	}{
+		{name: "pending", snapshot: ebsmetadata.Snapshot{State: "pending"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "error", snapshot: ebsmetadata.Snapshot{State: "error"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "deleted while pending", snapshot: ebsmetadata.Snapshot{State: "pending", DeleteRequested: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+		{name: "deleted after completing", snapshot: ebsmetadata.Snapshot{State: "completed", DeleteRequested: true, AmbiguousCreate: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			store := objectstore.NewMemoryObjectStore()
+			svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+			metadata := ebsmetadata.NewStore(store, "test-bucket")
+			snap := tc.snapshot
+			snap.SnapshotID, snap.VolumeID, snap.VolumeSize, snap.OwnerID = "snap-gate000001", "vol-source", 8, testVolAccountID
+			require.NoError(t, metadata.PutSnapshot(context.Background(), snap))
+
+			_, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{
+				AvailabilityZone: aws.String("ap-southeast-2a"),
+				SnapshotId:       aws.String(snap.SnapshotID),
+			}, testVolAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			volumes, err := metadata.ListAllVolumes(context.Background())
+			require.NoError(t, err)
+			assert.Empty(t, volumes, "a refused CreateVolume writes no volume")
+		})
+	}
+}

spinifex/domains/lambda/awsapi/get.go

@@ -56,27 +56,6 @@ func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (functi
 	return rec, revision, err
 }
 
-// render opens the record's environment and renders its configuration; an
-// environment that cannot be opened fails rather than being left out.
-func (d Deps) render(ctx context.Context, rec function.Record, revision uint64) (*lambda.FunctionConfiguration, error) {
-	var env map[string]string
-	if rec.EnvCipher != "" {
-		if d.AEAD == nil {
-			return nil, serviceError("environment decryption is unavailable")
-		}
-		var err error
-		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
-			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return nil, serviceError("could not read the function environment")
-		}
-	}
-	cfg, err := configuration(rec, revision, env)
-	if err != nil {
-		return nil, serviceError("could not build the function configuration")
-	}
-	return cfg, nil
-}
-
 // loadConfiguration resolves the requested $LATEST function and renders its
 // configuration; it is a metadata read and never touches the code store.
 func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (function.Record, *lambda.FunctionConfiguration, error) {
@@ -85,9 +64,19 @@ func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (fun
 		return function.Record{}, nil, err
 	}
 
-	cfg, err := d.render(ctx, rec, revision)
+	var env map[string]string
+	if rec.EnvCipher != "" {
+		if d.AEAD == nil {
+			return function.Record{}, nil, serviceError("environment decryption is unavailable")
+		}
+		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
+			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
+			return function.Record{}, nil, serviceError("could not read the function environment")
+		}
+	}
+	cfg, err := configuration(rec, revision, env)
 	if err != nil {
-		return function.Record{}, nil, err
+		return function.Record{}, nil, serviceError("could not build the function configuration")
 	}
 	return rec, cfg, nil
 }

spinifex/domains/lambda/awsapi/handler_test.go

@@ -60,14 +60,6 @@ type localClient struct {
 	configErr error
 	// configCalls counts UpdateConfiguration requests sent to the owner.
 	configCalls int
-	// deleteCalls counts Delete requests sent to the owner.
-	deleteCalls int
-	// deleteErr fails Delete before the owner sees it; loseDeleteReply
-	// reports a transport error after the owner committed it.
-	deleteErr       error
-	loseDeleteReply bool
-	// listErr fails List before the owner sees it.
-	listErr error
 }
 
 var _ function.Client = (*localClient)(nil)
@@ -177,42 +169,6 @@ func (c *localClient) UpdateConfiguration(ctx context.Context, account, region s
 	return r.Record, r.Revision, nil
 }
 
-func (c *localClient) List(ctx context.Context, account, region, marker string, maxItems int) ([]function.ListedFunction, string, error) {
-	c.mu.Lock()
-	fail := c.listErr
-	c.mu.Unlock()
-	if fail != nil {
-		return nil, "", fail
-	}
-	r, err := c.svc.List(ctx, &function.ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, account)
-	if err != nil {
-		return nil, "", err
-	}
-	return r.Functions, r.NextMarker, nil
-}
-
-func (c *localClient) Delete(ctx context.Context, account, region string, rec function.Record, rev uint64) error {
-	c.mu.Lock()
-	hook, fail, lose := c.beforeCommit, c.deleteErr, c.loseDeleteReply
-	c.beforeCommit = nil
-	c.deleteCalls++
-	c.mu.Unlock()
-	if fail != nil {
-		return fail
-	}
-	if hook != nil {
-		hook()
-	}
-	if _, err := c.svc.Delete(ctx, &function.DeleteRequest{Region: region, FunctionName: rec.FunctionName,
-		FunctionID: rec.FunctionID, Revision: rev}, account); err != nil {
-		return err
-	}
-	if lose {
-		return errors.New("nats: timeout")
-	}
-	return nil
-}
-
 // memStore is an in-memory code store; objects is keyed by Ref.
 type memStore struct {
 	mu      sync.Mutex
@@ -908,7 +864,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "DeleteFunction", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListFunctions", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
 	assert.Equal(t, "UpdateFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "DeleteEventSourceMapping", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/event-source-mappings/x", nil)))
@@ -917,10 +873,7 @@ func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
 	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "UpdateFunctionCode", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/code", nil)))
-	assert.Equal(t, "DeleteFunction", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
-	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions", nil)))
-	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/", nil)))
-	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x/configuration", nil)))
+	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
 }
 
 func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {

spinifex/domains/lambda/awsapi/invoke.go

@@ -57,13 +57,13 @@ func (r *RefusalError) Error() string { return "invocation refused before dispat
 
 func (r *RefusalError) Unwrap() error { return r.Err }
 
-// functionTarget is the function and qualifier a request names, once
+// invokeTarget is the function and qualifier an Invoke request names, once
 // every deterministic check on the request has passed.
-type functionTarget struct{ Name, Qualifier string }
+type invokeTarget struct{ Name, Qualifier string }
 
 // validateInvoke runs every check that needs no stored state: options, size,
 // payload JSON, function name and qualifier. Nothing here reads metadata.
-func validateInvoke(c *call) (*functionTarget, error) {
+func validateInvoke(c *call) (*invokeTarget, error) {
 	if err := checkInvokeOptions(c.Header); err != nil {
 		return nil, err
 	}
@@ -76,12 +76,6 @@ func validateInvoke(c *call) (*functionTarget, error) {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidRequestContentException,
 			"Could not parse request body into json")
 	}
-	return qualifiedTarget(c)
-}
-
-// qualifiedTarget resolves the function name and the qualifier from either
-// its suffix or the Qualifier query, which c.Query must hold strictly parsed.
-func qualifiedTarget(c *call) (*functionTarget, error) {
 	if len(c.Params[0]) > maxFunctionNameLen {
 		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must have length less than or equal to 256")
 	}
@@ -107,7 +101,7 @@ func qualifiedTarget(c *call) (*functionTarget, error) {
 		}
 		q = query[0]
 	}
-	return &functionTarget{Name: name, Qualifier: q}, nil
+	return &invokeTarget{Name: name, Qualifier: q}, nil
 }
 
 // invoke implements Invoke for RequestResponse on $LATEST. serve has already

spinifex/domains/lambda/awsapi/listdelete.go

@@ -1,107 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"log/slog"
-	"net/http"
-	"strconv"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/lambda"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// singleQuery returns the one value of a query member; present but repeated
-// or empty is refused.
-func singleQuery(c *call, member, field string) (string, bool, error) {
-	v, ok := c.Query[member]
-	if !ok {
-		return "", false, nil
-	}
-	if len(v) != 1 || v[0] == "" {
-		return "", true, invalid("1 validation error detected: Value at '%s' failed to satisfy constraint: Member must have length greater than or equal to 1", field)
-	}
-	return v[0], true, nil
-}
-
-// listFunctions implements ListFunctions over $LATEST. Pages hold at most
-// function.MaxListPage functions, and may be short or empty while a
-// NextMarker remains, since each owner request also bounds its scan.
-func (d Deps) listFunctions(ctx context.Context, c *call) (reply, error) {
-	if c.Query.Has("MasterRegion") {
-		return reply{}, unsupported("MasterRegion")
-	}
-	if v, ok, err := singleQuery(c, "FunctionVersion", "functionVersion"); err != nil {
-		return reply{}, err
-	} else if ok && v != lambda.FunctionVersionAll {
-		return reply{}, invalid("1 validation error detected: Value '%s' at 'functionVersion' failed to satisfy constraint: Member must satisfy enum value set: [ALL]", v)
-	}
-	maxItems := function.MaxListPage
-	if v, ok, err := singleQuery(c, "MaxItems", "maxItems"); err != nil {
-		return reply{}, err
-	} else if ok {
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 1 || n > maxListItems {
-			return reply{}, invalid("MaxItems must be between 1 and %d", maxListItems)
-		}
-		maxItems = min(n, function.MaxListPage)
-	}
-	marker, _, err := singleQuery(c, "Marker", "marker")
-	if err != nil {
-		return reply{}, err
-	}
-
-	listed, next, err := d.Meta.List(ctx, c.AccountID, c.Region, marker, maxItems)
-	if err != nil {
-		if _, coded := awserrors.ResolveErrorCode(err); coded {
-			return reply{}, err
-		}
-		slog.ErrorContext(ctx, "Lambda: list owner request failed", "err", err)
-		return reply{}, serviceError("the functions could not be listed; retry the request")
-	}
-	out := &lambda.ListFunctionsOutput{Functions: make([]*lambda.FunctionConfiguration, 0, len(listed))}
-	for _, l := range listed {
-		cfg, err := d.render(ctx, l.Record, l.Revision)
-		if err != nil {
-			return reply{}, err
-		}
-		// ListFunctions returns a subset of the configuration: AWS leaves the
-		// state and last-update members to GetFunction.
-		cfg.State, cfg.StateReason, cfg.StateReasonCode = nil, nil, nil
-		cfg.LastUpdateStatus, cfg.LastUpdateStatusReason, cfg.LastUpdateStatusReasonCode = nil, nil, nil
-		out.Functions = append(out.Functions, cfg)
-	}
-	if next != "" {
-		out.NextMarker = aws.String(next)
-	}
-	return reply{Status: http.StatusOK, Body: out}, nil
-}
-
-// deleteFunction implements DeleteFunction for a whole function. Any explicit
-// qualifier is refused before anything is read, so a request scoped to a
-// version can never delete the function.
-func (d Deps) deleteFunction(ctx context.Context, c *call) (reply, error) {
-	if c.Delete == nil {
-		return reply{}, serviceError("delete request was not validated")
-	}
-	if q := c.Delete.Qualifier; q != "" {
-		if q == "$LATEST" {
-			return reply{}, invalid("$LATEST version cannot be deleted without deleting the function.")
-		}
-		return reply{}, notFound(c.Region, c.AccountID, c.Delete.Name+":"+q)
-	}
-	rec, revision, err := d.lookupTarget(ctx, c, c.Delete.Name, "")
-	if err != nil {
-		return reply{}, err
-	}
-	if err := d.Meta.Delete(ctx, c.AccountID, c.Region, rec, revision); err != nil {
-		// A lost reply may follow the unbinding; the caller reads the function to learn which.
-		if _, coded := awserrors.ResolveErrorCode(err); coded {
-			return reply{}, err
-		}
-		slog.ErrorContext(ctx, "Lambda: delete owner request failed", "function_id", rec.FunctionID, "err", err)
-		return reply{}, serviceError("the function could not be deleted; retry the request")
-	}
-	return reply{Status: http.StatusNoContent}, nil
-}

spinifex/domains/lambda/awsapi/listdelete_test.go

@@ -1,385 +0,0 @@
-package awsapi_test
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/service/lambda"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	listPath  = "/2015-03-31/functions"
-	helloPath = "/2015-03-31/functions/hello"
-	helloARN  = "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
-)
-
-// listAll walks ListFunctions from path, following NextMarker, and returns the
-// pages it read.
-func listAll(t *testing.T, e *env, path string) []lambda.ListFunctionsOutput {
-	t.Helper()
-	var pages []lambda.ListFunctionsOutput
-	marker := ""
-	for range 100 {
-		p := path
-		if marker != "" {
-			p += "&Marker=" + url.QueryEscape(marker)
-		}
-		rec, err := e.do(http.MethodGet, p, nil)
-		require.NoError(t, err)
-		require.Equal(t, http.StatusOK, rec.Code)
-		var out lambda.ListFunctionsOutput
-		require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
-		pages = append(pages, out)
-		if out.NextMarker == nil {
-			return pages
-		}
-		marker = *out.NextMarker
-	}
-	t.Fatal("the walk does not end")
-	return nil
-}
-
-func listedNames(pages []lambda.ListFunctionsOutput) []string {
-	var names []string
-	for _, p := range pages {
-		for _, f := range p.Functions {
-			names = append(names, *f.FunctionName)
-		}
-	}
-	return names
-}
-
-func createActive(t *testing.T, e *env, names ...string) {
-	t.Helper()
-	for _, name := range names {
-		_, err := e.do(http.MethodPost, listPath, createBody(name, nil))
-		require.NoError(t, err)
-	}
-	e.activate(t)
-}
-
-func TestListFunctions_WalksEveryFunctionAcrossPagesOnBothPaths(t *testing.T) {
-	e := newUpdateEnv(t)
-	createActive(t, e, "second", "third")
-
-	pages := listAll(t, e, listPath+"?MaxItems=1")
-	assert.Len(t, pages, 3, "a page that fills on the last function has no NextMarker")
-	assert.ElementsMatch(t, []string{"hello", "second", "third"}, listedNames(pages))
-	slash := listAll(t, e, listPath+"/?MaxItems=1")
-	assert.Equal(t, listedNames(pages), listedNames(slash))
-
-	rec, err := e.do(http.MethodGet, listPath, nil)
-	require.NoError(t, err)
-	assert.NotContains(t, rec.Body.String(), "NextMarker")
-}
-
-// ListFunctions renders each function as GetFunctionConfiguration does, with
-// its environment opened, less the state and last-update members.
-func TestListFunctions_RendersTheConfigurationSubset(t *testing.T) {
-	e := newUpdateEnv(t)
-	rec, err := e.do(http.MethodGet, helloPath+"/configuration", nil)
-	require.NoError(t, err)
-	want := decodeConfig(t, rec.Body.Bytes())
-	want.State, want.StateReason, want.StateReasonCode = nil, nil, nil
-	want.LastUpdateStatus, want.LastUpdateStatusReason, want.LastUpdateStatusReasonCode = nil, nil, nil
-
-	rec, err = e.do(http.MethodGet, listPath, nil)
-	require.NoError(t, err)
-	for _, member := range []string{`"State"`, `"StateReason`, `"LastUpdateStatus`} {
-		assert.NotContains(t, rec.Body.String(), member)
-	}
-	var out lambda.ListFunctionsOutput
-	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
-	require.Len(t, out.Functions, 1)
-	assert.Equal(t, want, *out.Functions[0])
-	assert.Equal(t, "prod", *out.Functions[0].Environment.Variables["MODE"])
-}
-
-func TestListFunctions_EmptyAccountReturnsAnEmptyList(t *testing.T) {
-	e := newEnv(t)
-	rec, err := e.do(http.MethodGet, listPath+"?FunctionVersion=ALL", nil)
-	require.NoError(t, err)
-	assert.JSONEq(t, `{"Functions":[]}`, rec.Body.String())
-}
-
-func TestListFunctions_PageHoldsAtMostFifty(t *testing.T) {
-	e := newEnv(t)
-	for i := range function.MaxListPage + 1 {
-		_, err := e.do(http.MethodPost, listPath, createBody(fmt.Sprintf("fn-%02d", i), nil))
-		require.NoError(t, err)
-	}
-	pages := listAll(t, e, listPath+"?MaxItems=10000")
-	require.Len(t, pages, 2)
-	assert.Len(t, pages[0].Functions, function.MaxListPage)
-	assert.Len(t, listedNames(pages), function.MaxListPage+1)
-}
-
-// Two accounts holding the same function names see only their own functions,
-// with their own environments, on first and continuation pages, and neither
-// can continue the other's walk.
-func TestListFunctions_AccountsAreIsolated(t *testing.T) {
-	e := newEnv(t)
-	const other = "210987654321"
-	e.deps.Roles = roleSet{roleARN: true, "arn:aws:iam::" + other + ":role/lambda-exec": true}
-	names := map[string][]string{
-		gwtest.AccountID: {"shared-1", "only-a", "shared-2", "shared-3"},
-		other:            {"shared-2", "shared-1", "only-b", "shared-3"},
-	}
-	for _, acct := range []string{gwtest.AccountID, other} {
-		e.h.AccountID = acct
-		for _, name := range names[acct] {
-			rec, err := e.do(http.MethodPost, listPath, createBody(name, func(b map[string]any) {
-				b["Role"] = "arn:aws:iam::" + acct + ":role/lambda-exec"
-				b["Environment"] = map[string]any{"Variables": map[string]string{"OWNER": acct}}
-			}))
-			require.NoError(t, err)
-			require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String())
-		}
-	}
-	e.activate(t)
-
-	markers := map[string]string{}
-	for _, acct := range []string{gwtest.AccountID, other} {
-		e.h.AccountID = acct
-		pages := listAll(t, e, listPath+"?MaxItems=2")
-		require.Len(t, pages, 2, "the walk has a continuation page")
-		markers[acct] = *pages[0].NextMarker
-		assert.Equal(t, names[acct], listedNames(pages))
-		for _, p := range pages {
-			for _, f := range p.Functions {
-				assert.Equal(t, "arn:aws:lambda:"+gwtest.Region+":"+acct+":function:"+*f.FunctionName, *f.FunctionArn)
-				assert.Equal(t, "arn:aws:iam::"+acct+":role/lambda-exec", *f.Role)
-				assert.Equal(t, acct, *f.Environment.Variables["OWNER"])
-			}
-		}
-	}
-	for acct, foreign := range map[string]string{gwtest.AccountID: markers[other], other: markers[gwtest.AccountID]} {
-		e.h.AccountID = acct
-		_, err := e.do(http.MethodGet, listPath+"?Marker="+url.QueryEscape(foreign), nil)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}
-
-func TestListFunctions_ParameterValidation(t *testing.T) {
-	e := newUpdateEnv(t)
-	for name, query := range map[string]string{
-		"MaxItems zero":           "MaxItems=0",
-		"MaxItems over 10000":     "MaxItems=10001",
-		"MaxItems not a number":   "MaxItems=x",
-		"MaxItems repeated":       "MaxItems=1&MaxItems=2",
-		"empty Marker":            "Marker=",
-		"malformed Marker":        "Marker=not-a-marker",
-		"MasterRegion":            "MasterRegion=us-east-1",
-		"unknown FunctionVersion": "FunctionVersion=LATEST",
-		"malformed query":         "MaxItems=%zz",
-	} {
-		t.Run(name, func(t *testing.T) {
-			_, err := e.do(http.MethodGet, listPath+"?"+query, nil)
-			testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-		})
-	}
-}
-
-func TestListFunctions_AuthorizesOnEveryResource(t *testing.T) {
-	e := newUpdateEnv(t)
-	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", helloARN))
-	_, err := e.do(http.MethodGet, listPath, nil)
-	gwtest.AssertDenied(t, err)
-	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", "*"))
-	_, err = e.do(http.MethodGet, listPath, nil)
-	require.NoError(t, err)
-}
-
-func TestListFunctions_OwnerFailuresAreRetryable(t *testing.T) {
-	e := newUpdateEnv(t)
-	e.meta.listErr = fmt.Errorf("NATS request to %s: %w", function.SubjectList, nats.ErrNoResponders)
-	_, err := e.do(http.MethodGet, listPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.False(t, awserrors.IsTerminal(err))
-
-	e.meta.listErr = nil
-	e.deps.AEAD = nil
-	_, err = e.do(http.MethodGet, listPath, nil)
-	// An environment that cannot be opened fails the page.
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-}
-
-// deletion reads the retained obligation for functionID.
-func (e *env) deletion(t *testing.T, functionID string) (function.Deletion, bool) {
-	t.Helper()
-	entry, err := e.kv(t).Get(t.Context(), function.KVDeletedKey(gwtest.Region, functionID))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return function.Deletion{}, false
-	}
-	require.NoError(t, err)
-	var d function.Deletion
-	require.NoError(t, json.Unmarshal(entry.Value(), &d))
-	return d, true
-}
-
-func TestDeleteFunction_RemovesTheFunctionAndRetainsItsRecordsAndCode(t *testing.T) {
-	e := newUpdateEnv(t)
-	before, _ := e.record(t, "hello")
-	copies := e.store.count()
-	// Deletion reads neither the code store nor the environment.
-	codes, aead := e.deps.Codes, e.deps.AEAD
-	e.deps.Codes, e.deps.AEAD = nil, nil
-
-	rec, err := e.do(http.MethodDelete, helloPath, nil)
-	require.NoError(t, err)
-	assert.Equal(t, http.StatusNoContent, rec.Code)
-	assert.Empty(t, rec.Body.Bytes())
-
-	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	assert.Empty(t, listedNames(listAll(t, e, listPath+"?MaxItems=1")))
-	_, err = e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-
-	d, ok := e.deletion(t, before.FunctionID)
-	require.True(t, ok)
-	assert.Equal(t, "hello", d.FunctionName)
-	assert.Contains(t, e.uploads(t), before.FunctionID)
-	assert.Equal(t, copies, e.store.count(), "no code copy is removed")
-
-	e.deps.Codes, e.deps.AEAD = codes, aead
-	_, err = e.do(http.MethodPost, listPath, createBody("hello", nil))
-	require.NoError(t, err)
-	again, _ := e.record(t, "hello")
-	assert.NotEqual(t, before.FunctionID, again.FunctionID)
-	_, ok = e.deletion(t, before.FunctionID)
-	assert.True(t, ok, "the old obligation survives the name's reuse")
-}
-
-// Every explicit qualifier, in the query or the name, and every query that
-// cannot be read exactly, is refused before the owner is asked to delete.
-func TestDeleteFunction_RefusesEveryQualifierWithoutMutation(t *testing.T) {
-	cases := map[string]struct {
-		path string
-		code string
-	}{
-		"query $LATEST":         {helloPath + "?Qualifier=%24LATEST", awserrors.ErrorInvalidParameterValueException},
-		"query version":         {helloPath + "?Qualifier=1", awserrors.ErrorResourceNotFoundException},
-		"query alias":           {helloPath + "?Qualifier=live", awserrors.ErrorResourceNotFoundException},
-		"empty query":           {helloPath + "?Qualifier=", awserrors.ErrorInvalidParameterValueException},
-		"bare query key":        {helloPath + "?Qualifier", awserrors.ErrorInvalidParameterValueException},
-		"repeated query":        {helloPath + "?Qualifier=1&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
-		"malformed escape":      {helloPath + "?Qualifier=%zz", awserrors.ErrorInvalidParameterValueException},
-		"malformed other pair":  {helloPath + "?x=%zz&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
-		"semicolon":             {helloPath + "?Qualifier=1;x=2", awserrors.ErrorInvalidParameterValueException},
-		"invalid pattern":       {helloPath + "?Qualifier=a.b", awserrors.ErrorInvalidParameterValueException},
-		"name suffix $LATEST":   {helloPath + ":%24LATEST", awserrors.ErrorInvalidParameterValueException},
-		"name suffix version":   {helloPath + ":1", awserrors.ErrorResourceNotFoundException},
-		"ARN suffix version":    {"/2015-03-31/functions/" + url.PathEscape(helloARN+":1"), awserrors.ErrorResourceNotFoundException},
-		"empty name suffix":     {helloPath + ":", awserrors.ErrorInvalidParameterValueException},
-		"mismatched qualifiers": {helloPath + ":1?Qualifier=2", awserrors.ErrorInvalidParameterValueException},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			e := newUpdateEnv(t)
-			before, beforeRev := e.record(t, "hello")
-			_, err := e.do(http.MethodDelete, tc.path, nil)
-			testutil.RequireAWSCode(t, err, tc.code)
-			assert.Zero(t, e.meta.deleteCalls)
-			requireConfigUnchanged(t, e, before, beforeRev)
-			_, ok := e.deletion(t, before.FunctionID)
-			assert.False(t, ok)
-		})
-	}
-}
-
-func TestDeleteFunction_Authorization(t *testing.T) {
-	allowAll := gwtest.Statement("Allow", "*", "*")
-	t.Run("denied on the function ARN", func(t *testing.T) {
-		e := newUpdateEnv(t)
-		e.h = gwtest.Scoped(allowAll, gwtest.Statement("Deny", "lambda:DeleteFunction", helloARN))
-		before, beforeRev := e.record(t, "hello")
-		_, err := e.do(http.MethodDelete, helloPath, nil)
-		gwtest.AssertDenied(t, err)
-		requireConfigUnchanged(t, e, before, beforeRev)
-	})
-	t.Run("another function's grant does not reach it", func(t *testing.T) {
-		e := newUpdateEnv(t)
-		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN+"-other"))
-		_, err := e.do(http.MethodDelete, helloPath, nil)
-		gwtest.AssertDenied(t, err)
-	})
-	t.Run("allowed on the function ARN", func(t *testing.T) {
-		e := newUpdateEnv(t)
-		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN))
-		rec, err := e.do(http.MethodDelete, helloPath, nil)
-		require.NoError(t, err)
-		assert.Equal(t, http.StatusNoContent, rec.Code)
-	})
-}
-
-func TestDeleteFunction_LostReplyIsAnErrorButTheDeleteStands(t *testing.T) {
-	e := newUpdateEnv(t)
-	e.meta.loseDeleteReply = true
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.False(t, awserrors.IsTerminal(err))
-	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-}
-
-func TestDeleteFunction_OwnerWithoutTheSubjectIsRetryableAndChangesNothing(t *testing.T) {
-	e := newUpdateEnv(t)
-	before, beforeRev := e.record(t, "hello")
-	e.meta.deleteErr = fmt.Errorf("NATS request to %s: %w", function.SubjectDelete, nats.ErrNoResponders)
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.False(t, awserrors.IsTerminal(err))
-	requireConfigUnchanged(t, e, before, beforeRev)
-}
-
-func TestDeleteFunction_UpdateBetweenReadAndDeleteWins(t *testing.T) {
-	e := newConfigEnv(t)
-	e.meta.beforeCommit = func() {
-		_, err := e.do(http.MethodPut, configPath, map[string]any{"Handler": "handler.winner"})
-		assert.NoError(t, err)
-	}
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-	after, _ := e.record(t, "hello")
-	assert.Equal(t, "handler.winner", after.Handler)
-}
-
-func TestDeleteFunction_LeavesMappingsInPlace(t *testing.T) {
-	e, store, _ := newMappingEnv(t)
-	rec, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
-	require.NoError(t, err)
-	uuid, _ := decodeMapping(t, rec.Body.Bytes())["UUID"].(string)
-	before, beforeRev, err := store.Get(t.Context(), uuid)
-	require.NoError(t, err)
-
-	_, err = e.do(http.MethodDelete, helloPath, nil)
-	require.NoError(t, err)
-	after, afterRev, err := store.Get(t.Context(), uuid)
-	require.NoError(t, err)
-	assert.Equal(t, beforeRev, afterRev)
-	assert.Equal(t, before, after)
-	assert.Equal(t, eventsource.StateEnabled, after.State)
-}
-
-func TestDeleteFunction_LaterInvokeIsNotFound(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	require.NoError(t, err)
-	_, err = e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	assert.Empty(t, ex.calls)
-}

spinifex/domains/lambda/awsapi/registration.go

@@ -149,20 +149,6 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		}
 	case "GetEventSourceMapping", "DeleteEventSourceMapping":
 		resource = mappingARN(inv.Region, inv.AccountID, params[0])
-	case "ListFunctions":
-		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
-			return invalid("The request query string could not be parsed.")
-		}
-	case "DeleteFunction":
-		// A dropped Qualifier would widen the request to the whole function,
-		// so the query is parsed strictly and the target resolved before anything else.
-		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
-			return invalid("The request query string could not be parsed.")
-		}
-		if c.Delete, err = qualifiedTarget(c); err != nil {
-			return err
-		}
-		resource = qualifiedFunctionARN(inv.Region, inv.AccountID, c.Delete.Name, c.Delete.Qualifier)
 	case "Invoke":
 		// Invoke validates deterministically before authorizing, and its
 		// resource comes from that same validated target. URL.Query drops
@@ -248,11 +234,6 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }
 
 func writeJSON(w http.ResponseWriter, out reply) {
-	if out.Status == http.StatusNoContent {
-		maps.Copy(w.Header(), out.Header)
-		w.WriteHeader(out.Status)
-		return
-	}
 	var body []byte
 	var err error
 	if raw, ok := out.Body.(rawJSON); ok {

spinifex/domains/lambda/awsapi/routes.go

@@ -22,10 +22,7 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 	// Invoke is Invoke's validated target, resolved ahead of the
 	// authorization gate whose resource it names.
-	Invoke *functionTarget
-	// Delete is DeleteFunction's validated target, resolved from the strictly
-	// parsed query ahead of the authorization gate.
-	Delete *functionTarget
+	Invoke *invokeTarget
 	// UpdateConfig is the decoded UpdateFunctionConfiguration body, read
 	// ahead of the PassRole check on a new role.
 	UpdateConfig *lambda.UpdateFunctionConfigurationInput
@@ -52,18 +49,6 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.createFunction(ctx, c)
 		}},
-	{Method: "GET", Pattern: "/2015-03-31/functions", Action: "ListFunctions",
-		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
-			return d.listFunctions(ctx, c)
-		}},
-	{Method: "GET", Pattern: "/2015-03-31/functions/", Action: "ListFunctions",
-		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
-			return d.listFunctions(ctx, c)
-		}},
-	{Method: "DELETE", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "DeleteFunction",
-		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
-			return d.deleteFunction(ctx, c)
-		}},
 	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "GetFunction",
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)

spinifex/domains/lambda/function/client.go

@@ -26,10 +26,6 @@ type Client interface {
 	CodeUpdated(ctx context.Context, accountID, region string, rec Record, revision uint64, codeRevisionID, sha256Hex string, size int64) (Record, uint64, error)
 	// UpdateConfiguration commits cfg for the function at revision and returns the updated record and revision.
 	UpdateConfiguration(ctx context.Context, accountID, region string, rec Record, revision uint64, cfg Configuration) (Record, uint64, error)
-	// List returns up to maxItems of the Region's functions after marker and the next marker, empty at the end.
-	List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error)
-	// Delete unbinds the function read at revision, retaining its cleanup obligation.
-	Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error
 }
 
 // NATSClient is the Client over NATS request/reply to the daemon.
@@ -102,19 +98,3 @@ func (c *NATSClient) UpdateConfiguration(ctx context.Context, accountID, region
 	}
 	return resp.Record, resp.Revision, nil
 }
-
-func (c *NATSClient) List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error) {
-	resp, err := natsmsg.NATSRequest[ListResponse](ctx, c.conn, SubjectList,
-		&ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, metaRequestTimeout, accountID)
-	if err != nil {
-		return nil, "", err
-	}
-	return resp.Functions, resp.NextMarker, nil
-}
-
-func (c *NATSClient) Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error {
-	_, err := natsmsg.NATSRequest[DeleteResponse](ctx, c.conn, SubjectDelete,
-		&DeleteRequest{Region: region, FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, Revision: revision},
-		metaRequestTimeout, accountID)
-	return err
-}

spinifex/domains/lambda/function/delete.go

@@ -1,87 +0,0 @@
-package function
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Delete records the function's cleanup obligation, then unbinds its name by
-// compare-and-swap at the revision the caller read. Code copies and upload
-// records are retained; nothing here removes them.
-func (s *MetaService) Delete(ctx context.Context, req *DeleteRequest, accountID string) (*DeleteResponse, error) {
-	if !regionRE.MatchString(req.Region) {
-		return nil, badRegion()
-	}
-	if !ValidName(req.FunctionName) || !idRE.MatchString(req.FunctionID) || accountID == "" {
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "delete request is incomplete")
-	}
-	kv, err := s.bucket(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-	key := KVFunctionKey(req.Region, req.FunctionName)
-	entry, err := kv.Get(ctx, key)
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return nil, awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s", req.FunctionName)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("lambda: get function record: %w", err)
-	}
-	rec, err := decode(entry.Value())
-	if err != nil {
-		return nil, err
-	}
-	if entry.Revision() != req.Revision || rec.FunctionID != req.FunctionID {
-		return nil, conflict("Function was modified by another request: %s", req.FunctionName)
-	}
-	if err := s.recordDeletion(ctx, kv, accountID, rec); err != nil {
-		return nil, err
-	}
-	if err := kv.Delete(ctx, key, jetstream.LastRevision(req.Revision)); err != nil {
-		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, conflict("Function was modified by another request: %s", req.FunctionName)
-		}
-		return nil, fmt.Errorf("lambda: delete function record: %w", err)
-	}
-	return &DeleteResponse{}, nil
-}
-
-// recordDeletion writes the obligation for rec. One already present, from an
-// earlier attempt whose outcome was uncertain, is accepted only for the same
-// function; anything else refuses the delete.
-func (s *MetaService) recordDeletion(ctx context.Context, kv jetstream.KeyValue, accountID string, rec Record) error {
-	want := Deletion{
-		FormatVersion: FormatVersion, AccountID: accountID, Region: rec.Region,
-		FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, RecordedAt: s.now().UTC(),
-	}
-	data, err := json.Marshal(want)
-	if err != nil {
-		return err
-	}
-	key := KVDeletedKey(rec.Region, rec.FunctionID)
-	_, err = kv.Create(ctx, key, data)
-	if err == nil {
-		return nil
-	}
-	if !errors.Is(err, jetstream.ErrKeyExists) {
-		return fmt.Errorf("lambda: record function deletion: %w", err)
-	}
-	entry, err := kv.Get(ctx, key)
-	if err != nil {
-		return fmt.Errorf("lambda: read function deletion: %w", err)
-	}
-	var have Deletion
-	if err := json.Unmarshal(entry.Value(), &have); err != nil {
-		return fmt.Errorf("lambda: function deletion record is unreadable: %w", err)
-	}
-	if have.FormatVersion != FormatVersion || have.AccountID != accountID || have.Region != rec.Region ||
-		have.FunctionName != rec.FunctionName || have.FunctionID != rec.FunctionID {
-		return fmt.Errorf("lambda: function deletion record %s names another function", key)
-	}
-	return nil
-}

spinifex/domains/lambda/function/delete_test.go

@@ -1,296 +0,0 @@
-package function
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// serveAs creates name with stored code under functionID and activates it.
-func serveAs(t *testing.T, svc *MetaService, name, functionID string) *GetResponse {
-	t.Helper()
-	rev := mustCreate(t, svc, name, functionID)
-	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: name, Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-	require.NoError(t, err)
-	svc.SetReadiness(func(context.Context, Record) error { return nil })
-	require.NoError(t, svc.ActivatePending(t.Context()))
-	got := mustGet(t, svc, name)
-	require.Equal(t, StateActive, got.Record.State)
-	return got
-}
-
-func deleteReq(got *GetResponse) *DeleteRequest {
-	return &DeleteRequest{Region: testRegion, FunctionName: got.Record.FunctionName, FunctionID: got.Record.FunctionID, Revision: got.Revision}
-}
-
-// deletion reads the retained obligation for functionID, or reports it absent.
-func deletion(t *testing.T, svc *MetaService, functionID string) (Deletion, bool) {
-	t.Helper()
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	entry, err := kv.Get(t.Context(), KVDeletedKey(testRegion, functionID))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return Deletion{}, false
-	}
-	require.NoError(t, err)
-	var d Deletion
-	require.NoError(t, json.Unmarshal(entry.Value(), &d))
-	return d, true
-}
-
-func requireRetained(t *testing.T, svc *MetaService, name, functionID string) {
-	t.Helper()
-	d, ok := deletion(t, svc, functionID)
-	require.True(t, ok, "deletion obligation for %s is missing", functionID)
-	assert.Equal(t, Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion,
-		FunctionName: name, FunctionID: functionID, RecordedAt: d.RecordedAt}, d)
-	_, ok = upload(t, svc, functionID)
-	assert.True(t, ok, "upload record for %s was removed", functionID)
-}
-
-func TestDelete_UnbindsTheNameAndRetainsTheObligationAndUploads(t *testing.T) {
-	svc, c := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
-	require.NoError(t, err)
-
-	assert.False(t, mustGet(t, svc, "hello").Found)
-	requireRetained(t, svc, "hello", "f1")
-	d, _ := deletion(t, svc, "f1")
-	assert.Equal(t, c.now(), d.RecordedAt)
-	up, _ := upload(t, svc, "f1")
-	assert.Equal(t, UploadAttached, up.State, "the upload record is left exactly as it was")
-}
-
-func TestDelete_MissingFunctionIsNotFound(t *testing.T) {
-	svc, _ := newTestService(t)
-	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: 1}, testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	_, ok := deletion(t, svc, "f1")
-	assert.False(t, ok)
-}
-
-func TestDelete_StaleRevisionOrAnotherFunctionIsRefusedWithoutAnObligation(t *testing.T) {
-	svc, _ := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	for name, req := range map[string]*DeleteRequest{
-		"stale revision": {Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: got.Revision - 1},
-		"other function": {Region: testRegion, FunctionName: "hello", FunctionID: "f0", Revision: got.Revision},
-	} {
-		_, err := svc.Delete(t.Context(), req, testAccount)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-		assert.Equal(t, got.Revision, mustGet(t, svc, "hello").Revision, name)
-		_, ok := deletion(t, svc, req.FunctionID)
-		assert.False(t, ok, name)
-	}
-}
-
-func TestDelete_RefusesMalformedRequests(t *testing.T) {
-	svc, _ := newTestService(t)
-	for name, req := range map[string]*DeleteRequest{
-		"bad region": {Region: "Bad.Region", FunctionName: "hello", FunctionID: "f1", Revision: 1},
-		"bad name":   {Region: testRegion, FunctionName: "a.b", FunctionID: "f1", Revision: 1},
-		"bad id":     {Region: testRegion, FunctionName: "hello", FunctionID: "f.1", Revision: 1},
-	} {
-		_, err := svc.Delete(t.Context(), req, testAccount)
-		require.Error(t, err, name)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}
-
-// The delete passes its read, then pauses before writing while an update
-// commits; the conditional delete loses and the function stays bound. The
-// obligation it left proves nothing was deleted.
-func TestDelete_InterleavedUpdateLosesTheConditionalDelete(t *testing.T) {
-	svc, c := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	paused, release := make(chan struct{}), make(chan struct{})
-	var calls atomic.Int32
-	svc.now = func() time.Time {
-		if calls.Add(1) == 1 {
-			close(paused)
-			<-release
-		}
-		return c.now()
-	}
-	deleteErr := make(chan error, 1)
-	go func() {
-		_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
-		deleteErr <- err
-	}()
-	<-paused
-	cfg := Configuration{Role: got.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
-	won, err := svc.UpdateConfiguration(t.Context(), configReq(got.Revision, cfg), testAccount)
-	require.NoError(t, err)
-	close(release)
-	testutil.RequireAWSCode(t, <-deleteErr, awserrors.ErrorResourceConflictException)
-
-	after := mustGet(t, svc, "hello")
-	require.True(t, after.Found)
-	assert.Equal(t, won.Revision, after.Revision)
-	assert.Equal(t, "f1", after.Record.FunctionID)
-	_, ok := deletion(t, svc, "f1")
-	assert.True(t, ok, "the obligation from the losing attempt is retained")
-}
-
-func TestDelete_ObligationNamingAnotherFunctionRefusesTheDelete(t *testing.T) {
-	svc, _ := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	data, err := json.Marshal(Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion, FunctionName: "other", FunctionID: "f1"})
-	require.NoError(t, err)
-	_, err = kv.Create(t.Context(), KVDeletedKey(testRegion, "f1"), data)
-	require.NoError(t, err)
-
-	_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
-	require.Error(t, err)
-	_, coded := awserrors.ResolveErrorCode(err)
-	assert.False(t, coded, "an inconsistent obligation is an internal fault, retried by the caller")
-	assert.True(t, mustGet(t, svc, "hello").Found)
-}
-
-// A delete of a create still storing its code wins; the late CodeStored and
-// RollbackCreate of that create change nothing, and its upload stays owed.
-func TestDelete_PendingCreateRacingDeletionChangesNothingLater(t *testing.T) {
-	svc, _ := newTestService(t)
-	rev := mustCreate(t, svc, "hello", "f1")
-	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: rev}, testAccount)
-	require.NoError(t, err)
-	recreated := serveAs(t, svc, "hello", "f2")
-
-	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-	require.Error(t, err)
-	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
-	require.Error(t, err)
-
-	got := mustGet(t, svc, "hello")
-	assert.Equal(t, recreated.Revision, got.Revision)
-	assert.Equal(t, "f2", got.Record.FunctionID)
-	requireOwed(t, svc, "f1")
-	requireRetained(t, svc, "hello", "f1")
-}
-
-func TestDelete_SameNameRecreationKeepsEachObligationApart(t *testing.T) {
-	svc, _ := newTestService(t)
-	first := serveAs(t, svc, "hello", "f1")
-	_, err := svc.Delete(t.Context(), deleteReq(first), testAccount)
-	require.NoError(t, err)
-	second := serveAs(t, svc, "hello", "f2")
-	_, ok := deletion(t, svc, "f2")
-	assert.False(t, ok, "a recreated function has no obligation of its own")
-
-	_, err = svc.Delete(t.Context(), deleteReq(second), testAccount)
-	require.NoError(t, err)
-	requireRetained(t, svc, "hello", "f1")
-	requireRetained(t, svc, "hello", "f2")
-}
-
-// faultKV fails one write to key: before it commits, or after, losing the
-// acknowledgement.
-type faultKV struct {
-	jetstream.KeyValue
-
-	key   string
-	after bool
-	fired atomic.Bool
-}
-
-var errInjected = errors.New("nats: timeout")
-
-func (f *faultKV) trip(key string) bool { return key == f.key && f.fired.CompareAndSwap(false, true) }
-
-func (f *faultKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
-	if !f.trip(key) {
-		return f.KeyValue.Create(ctx, key, value, opts...)
-	}
-	if f.after {
-		if _, err := f.KeyValue.Create(ctx, key, value, opts...); err != nil {
-			return 0, err
-		}
-	}
-	return 0, errInjected
-}
-
-func (f *faultKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
-	if !f.trip(key) {
-		return f.KeyValue.Delete(ctx, key, opts...)
-	}
-	if f.after {
-		if err := f.KeyValue.Delete(ctx, key, opts...); err != nil {
-			return err
-		}
-	}
-	return errInjected
-}
-
-// Each write of a delete is interrupted before it commits, or commits with its
-// acknowledgement lost. A fresh owner then finds the binding and obligation the
-// stream actually holds, a retry converges, and a recreated name stays apart.
-func TestDelete_InterruptionAndLostAcknowledgementAtEachWrite(t *testing.T) {
-	cases := []struct {
-		name       string
-		key        string
-		after      bool
-		bound      bool
-		obligation bool
-	}{
-		{"obligation write interrupted", KVDeletedKey(testRegion, "f1"), false, true, false},
-		{"obligation acknowledgement lost", KVDeletedKey(testRegion, "f1"), true, true, true},
-		{"name delete interrupted", KVFunctionKey(testRegion, "hello"), false, true, true},
-		{"name delete acknowledgement lost", KVFunctionKey(testRegion, "hello"), true, false, true},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			_, nc, _ := testutil.StartTestJetStream(t)
-			js := testutil.NewJetStream(t, nc)
-			svc := NewMetaService(js)
-			got := serveAs(t, svc, "hello", "f1")
-			kv, err := svc.bucket(t.Context(), testAccount)
-			require.NoError(t, err)
-			svc.mu.Lock()
-			svc.buckets[testAccount] = &faultKV{KeyValue: kv, key: tc.key, after: tc.after}
-			svc.mu.Unlock()
-
-			_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
-			require.ErrorIs(t, err, errInjected)
-			_, coded := awserrors.ResolveErrorCode(err)
-			assert.False(t, coded, "an uncertain write is a retryable fault")
-
-			reopened := NewMetaService(js)
-			now := mustGet(t, reopened, "hello")
-			assert.Equal(t, tc.bound, now.Found, "binding")
-			if tc.bound {
-				assert.Equal(t, got.Revision, now.Revision)
-			}
-			_, ok := deletion(t, reopened, "f1")
-			assert.Equal(t, tc.obligation, ok, "obligation")
-			_, ok = upload(t, reopened, "f1")
-			assert.True(t, ok, "upload record retained")
-
-			_, err = reopened.Delete(t.Context(), deleteReq(got), testAccount)
-			if tc.bound {
-				require.NoError(t, err, "the retry completes the delete")
-			} else {
-				testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-			}
-			assert.False(t, mustGet(t, reopened, "hello").Found)
-			requireRetained(t, reopened, "hello", "f1")
-
-			recreated := serveAs(t, reopened, "hello", "f2")
-			assert.Equal(t, "f2", recreated.Record.FunctionID)
-			requireRetained(t, reopened, "hello", "f1")
-			_, ok = deletion(t, reopened, "f2")
-			assert.False(t, ok)
-		})
-	}
-}

spinifex/domains/lambda/function/list.go

@@ -1,141 +0,0 @@
-package function
-
-import (
-	"context"
-	"encoding/base64"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"math"
-	"strings"
-
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// List bounds: a page returns at most MaxListPage functions and examines at
-// most maxListScan stream sequences, whatever each one holds.
-const (
-	MaxListPage = 50
-	maxListScan = 200
-)
-
-// listCursor is the decoded marker: the last stream sequence a page examined,
-// bound to the account and Region it was issued for.
-type listCursor struct {
-	Version   int    `json:"v"`
-	AccountID string `json:"a"`
-	Region    string `json:"r"`
-	After     uint64 `json:"s"`
-}
-
-func encodeCursor(c listCursor) (string, error) {
-	data, err := json.Marshal(c)
-	if err != nil {
-		return "", err
-	}
-	return base64.RawURLEncoding.EncodeToString(data), nil
-}
-
-func decodeCursor(marker, accountID, region string) (uint64, error) {
-	bad := awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "Invalid marker: %s", marker)
-	data, err := base64.RawURLEncoding.DecodeString(marker)
-	if err != nil {
-		return 0, bad
-	}
-	var c listCursor
-	if err := json.Unmarshal(data, &c); err != nil {
-		return 0, bad
-	}
-	if c.Version != 1 || c.AccountID != accountID || c.Region != region || c.After == 0 || c.After == math.MaxUint64 {
-		return 0, bad
-	}
-	return c.After, nil
-}
-
-// List returns the Region's functions in stream order after the marker. Each
-// step is one leader get of an exact sequence, bounded by a fast stream-state
-// read, so a page costs at most its scan bound whatever the bucket holds.
-// NextMarker is set unless the page reached the stream end it read.
-func (s *MetaService) List(ctx context.Context, req *ListRequest, accountID string) (*ListResponse, error) {
-	if !regionRE.MatchString(req.Region) {
-		return nil, badRegion()
-	}
-	if accountID == "" || req.MaxItems < 1 || req.MaxItems > MaxListPage {
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "list request is incomplete")
-	}
-	var after uint64
-	if req.Marker != "" {
-		var err error
-		if after, err = decodeCursor(req.Marker, accountID, req.Region); err != nil {
-			return nil, err
-		}
-	}
-	kv, err := s.bucket(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-	stream := "KV_" + kv.Bucket()
-	// An unfiltered stream info without deleted details reads counters only.
-	info, err := s.js.Stream(ctx, stream)
-	if err != nil {
-		return nil, fmt.Errorf("lambda: list function records: %w", err)
-	}
-	state := info.CachedInfo().State
-	if state.FirstSeq > 0 {
-		after = max(after, state.FirstSeq-1)
-	}
-	keyPrefix := "$KV." + kv.Bucket() + "." + kvFunctionKeyPrefix + req.Region + "."
-	out := &ListResponse{Functions: []ListedFunction{}}
-	for gets := 0; gets < maxListScan && after < state.LastSeq; gets++ {
-		seq := after + 1
-		m, err := kvstore.LeaderMsgGet(ctx, s.js, stream, kvstore.MsgGetRequest{Seq: seq})
-		if errors.Is(err, jetstream.ErrMsgNotFound) {
-			after = seq
-			continue
-		}
-		if err != nil {
-			return nil, fmt.Errorf("lambda: list function records: %w", err)
-		}
-		if m.Sequence != seq {
-			return nil, fmt.Errorf("lambda: list function records: %w", jetstream.ErrInvalidJetStreamResponse)
-		}
-		if !strings.HasPrefix(m.Subject, keyPrefix) || removed(m.Header) {
-			after = seq
-			continue
-		}
-		if len(out.Functions) == req.MaxItems {
-			// A full page reached a function it cannot return; the marker
-			// stays before it.
-			break
-		}
-		rec, err := decode(m.Data)
-		if err != nil {
-			return nil, err
-		}
-		out.Functions = append(out.Functions, ListedFunction{Record: rec, Revision: seq})
-		after = seq
-	}
-	if after >= state.LastSeq {
-		return out, nil
-	}
-	if out.NextMarker, err = encodeCursor(listCursor{Version: 1, AccountID: accountID, Region: req.Region, After: after}); err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// removed reports a key's delete or purge marker, as the KV Get of nats.go
-// reads one.
-func removed(h nats.Header) bool {
-	if op := h.Get("KV-Operation"); op != "" {
-		return op == "DEL" || op == "PURGE"
-	}
-	switch h.Get(jetstream.MarkerReasonHeader) {
-	case "MaxAge", "Purge", "Remove":
-		return true
-	}
-	return false
-}

spinifex/domains/lambda/function/list_test.go

@@ -1,325 +0,0 @@
-package function
-
-import (
-	"fmt"
-	"math"
-	"testing"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func listPage(t *testing.T, svc *MetaService, region, marker string, maxItems int) *ListResponse {
-	t.Helper()
-	resp, err := svc.List(t.Context(), &ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, testAccount)
-	require.NoError(t, err)
-	return resp
-}
-
-// walk follows NextMarker to the end and returns the names in order and the
-// number of pages.
-func walk(t *testing.T, svc *MetaService, region string, maxItems int) ([]string, int) {
-	t.Helper()
-	var names []string
-	marker, pages := "", 0
-	for {
-		resp := listPage(t, svc, region, marker, maxItems)
-		pages++
-		require.LessOrEqual(t, len(resp.Functions), maxItems)
-		for _, f := range resp.Functions {
-			names = append(names, f.Record.FunctionName)
-		}
-		if resp.NextMarker == "" {
-			return names, pages
-		}
-		marker = resp.NextMarker
-		require.Less(t, pages, 1000, "the walk does not end")
-	}
-}
-
-func TestList_EmptyAccountHasNoFunctionsAndNoMarker(t *testing.T) {
-	svc, _ := newTestService(t)
-	resp := listPage(t, svc, testRegion, "", MaxListPage)
-	assert.Empty(t, resp.Functions)
-	assert.NotNil(t, resp.Functions)
-	assert.Empty(t, resp.NextMarker)
-}
-
-func TestList_PagesReturnEveryFunctionOnceWithItsRevision(t *testing.T) {
-	svc, _ := newTestService(t)
-	want := make([]string, 0, 7)
-	for i := range 7 {
-		name := fmt.Sprintf("fn-%d", i)
-		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
-		want = append(want, name)
-	}
-	names, pages := walk(t, svc, testRegion, 2)
-	assert.Equal(t, want, names, "stream order, each once")
-	assert.Equal(t, 4, pages)
-
-	page := listPage(t, svc, testRegion, "", 1)
-	require.Len(t, page.Functions, 1)
-	got := mustGet(t, svc, "fn-0")
-	assert.Equal(t, got.Revision, page.Functions[0].Revision)
-	assert.Equal(t, got.Record, page.Functions[0].Record)
-}
-
-// A page that fills on the last function looks ahead, finds the walk over,
-// and returns no marker; one that fills before it keeps the next one.
-func TestList_FullPageAtTheEndHasNoMarker(t *testing.T) {
-	svc, _ := newTestService(t)
-	for i := range 4 {
-		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
-	}
-	_, pages := walk(t, svc, testRegion, 2)
-	assert.Equal(t, 2, pages)
-	assert.Empty(t, listPage(t, svc, testRegion, "", 4).NextMarker)
-
-	first := listPage(t, svc, testRegion, "", 3)
-	require.NotEmpty(t, first.NextMarker)
-	rest := listPage(t, svc, testRegion, first.NextMarker, 3)
-	require.Len(t, rest.Functions, 1, "the record the full page looked at is not skipped")
-	assert.Equal(t, "fn-3", rest.Functions[0].Record.FunctionName)
-	assert.Empty(t, rest.NextMarker)
-}
-
-func TestList_SkipsDeletedFunctionsUploadsObligationsAndOtherRegions(t *testing.T) {
-	svc, _ := newTestService(t)
-	gone := serveAs(t, svc, "gone", "f0")
-	mustCreate(t, svc, "kept", "f1")
-	_, err := svc.Delete(t.Context(), deleteReq(gone), testAccount)
-	require.NoError(t, err)
-	other := createReq("elsewhere", "f2")
-	other.Region = "us-east-1"
-	_, err = svc.Create(t.Context(), other, testAccount)
-	require.NoError(t, err)
-
-	names, _ := walk(t, svc, testRegion, MaxListPage)
-	assert.Equal(t, []string{"kept"}, names)
-	names, _ = walk(t, svc, "us-east-1", MaxListPage)
-	assert.Equal(t, []string{"elsewhere"}, names)
-}
-
-// cursorAfter is the last sequence a marker says its page examined.
-func cursorAfter(t *testing.T, marker string) uint64 {
-	t.Helper()
-	after, err := decodeCursor(marker, testAccount, testRegion)
-	require.NoError(t, err)
-	return after
-}
-
-// walkFrom follows NextMarker from marker to the end, checking that no page
-// examines more than maxListScan sequences, and returns the names it listed.
-func walkFrom(t *testing.T, svc *MetaService, marker string, maxItems int) ([]string, int) {
-	t.Helper()
-	var names []string
-	var after uint64
-	if marker != "" {
-		after = cursorAfter(t, marker)
-	}
-	for pages := 1; ; pages++ {
-		resp := listPage(t, svc, testRegion, marker, maxItems)
-		require.LessOrEqual(t, len(resp.Functions), maxItems)
-		for _, f := range resp.Functions {
-			names = append(names, f.Record.FunctionName)
-		}
-		if resp.NextMarker == "" {
-			return names, pages
-		}
-		next := cursorAfter(t, resp.NextMarker)
-		require.LessOrEqual(t, next-after, uint64(maxListScan), "a page examined more than its scan bound")
-		require.True(t, next > after || len(resp.Functions) > 0, "a page made no progress")
-		marker, after = resp.NextMarker, next
-		require.Less(t, pages, 1000, "the walk does not end")
-	}
-}
-
-// tombstones appends n deleted functions to the stream.
-func tombstones(t *testing.T, svc *MetaService, n int) {
-	t.Helper()
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	for i := range n {
-		name := fmt.Sprintf("tomb-%d", i)
-		mustCreate(t, svc, name, fmt.Sprintf("t%d", i))
-		require.NoError(t, kv.Delete(t.Context(), KVFunctionKey(testRegion, name)))
-	}
-}
-
-// More delete markers than one request may examine give empty pages that
-// still continue, and the walk reaches the live function after them.
-func TestList_ScanBoundGivesShortPagesThatContinue(t *testing.T) {
-	svc, _ := newTestService(t)
-	tombstones(t, svc, maxListScan+5)
-	mustCreate(t, svc, "live", "f1")
-
-	first := listPage(t, svc, testRegion, "", MaxListPage)
-	assert.Empty(t, first.Functions)
-	require.NotEmpty(t, first.NextMarker)
-	assert.Equal(t, uint64(maxListScan), cursorAfter(t, first.NextMarker))
-	names, _ := walkFrom(t, svc, first.NextMarker, MaxListPage)
-	assert.Equal(t, []string{"live"}, names)
-}
-
-// Messages that are not this Region's function records count toward the scan
-// bound like any other sequence: uploads, obligations, other keys and other
-// Regions' functions.
-func TestList_ExcludedRecordsCountTowardTheScanBound(t *testing.T) {
-	svc, _ := newTestService(t)
-	mustCreate(t, svc, "first", "f1")
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	for i := range maxListScan + 30 {
-		var key string
-		switch i % 4 {
-		case 0:
-			key = KVFunctionKey("us-east-1", fmt.Sprintf("x-%d", i))
-		case 1:
-			key = fmt.Sprintf("upload.%s.f9.%d", testRegion, i)
-		case 2:
-			key = KVDeletedKey(testRegion, fmt.Sprintf("d%d", i))
-		default:
-			key = fmt.Sprintf("other.%d", i)
-		}
-		_, err := kv.Put(t.Context(), key, []byte("{}"))
-		require.NoError(t, err)
-	}
-	mustCreate(t, svc, "last", "f2")
-
-	names, pages := walkFrom(t, svc, "", MaxListPage)
-	assert.Equal(t, []string{"first", "last"}, names)
-	assert.GreaterOrEqual(t, pages, 2)
-}
-
-// A key rewritten many times leaves gaps behind its latest revision; missing
-// sequences count toward the bound and the walk crosses them.
-func TestList_SparseSequencesCountTowardTheScanBound(t *testing.T) {
-	svc, _ := newTestService(t)
-	mustCreate(t, svc, "first", "f1")
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	for range 2*maxListScan + 7 {
-		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
-		require.NoError(t, err)
-	}
-	mustCreate(t, svc, "middle", "f2")
-	for range maxListScan {
-		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
-		require.NoError(t, err)
-	}
-	mustCreate(t, svc, "last", "f3")
-
-	page := listPage(t, svc, testRegion, "", MaxListPage)
-	require.Len(t, page.Functions, 1)
-	assert.Equal(t, uint64(maxListScan), cursorAfter(t, page.NextMarker))
-	for _, size := range []int{1, MaxListPage} {
-		names, _ := walkFrom(t, svc, "", size)
-		assert.Equal(t, []string{"first", "middle", "last"}, names, "page size %d", size)
-	}
-}
-
-// A large live set is returned once each, in stream order, in pages that
-// never exceed the requested size.
-func TestList_LargeLiveSetPagesEveryFunctionOnce(t *testing.T) {
-	svc, _ := newTestService(t)
-	want := make([]string, 0, 3*MaxListPage+7)
-	for i := range cap(want) {
-		name := fmt.Sprintf("fn-%03d", i)
-		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
-		want = append(want, name)
-	}
-	for _, size := range []int{1, 7, MaxListPage} {
-		names, _ := walkFrom(t, svc, "", size)
-		assert.Equal(t, want, names, "page size %d", size)
-	}
-}
-
-// A full page looking ahead through delete markers to the end of the stream
-// returns no marker; when the scan bound runs out first, it continues and a
-// later page finds the function beyond them.
-func TestList_LookAheadThroughDeleteMarkers(t *testing.T) {
-	t.Run("reaches the end", func(t *testing.T) {
-		svc, _ := newTestService(t)
-		mustCreate(t, svc, "live", "f1")
-		tombstones(t, svc, 3)
-		page := listPage(t, svc, testRegion, "", 1)
-		require.Len(t, page.Functions, 1)
-		assert.Empty(t, page.NextMarker)
-	})
-	t.Run("scan bound exhausted", func(t *testing.T) {
-		svc, _ := newTestService(t)
-		mustCreate(t, svc, "live", "f1")
-		tombstones(t, svc, maxListScan)
-		mustCreate(t, svc, "beyond", "f2")
-		page := listPage(t, svc, testRegion, "", 1)
-		require.Len(t, page.Functions, 1)
-		assert.Equal(t, "live", page.Functions[0].Record.FunctionName)
-		require.NotEmpty(t, page.NextMarker, "the bound ran out before the walk was known to end")
-		names, _ := walkFrom(t, svc, page.NextMarker, 1)
-		assert.Equal(t, []string{"beyond"}, names)
-	})
-}
-
-// An update moves a function to a later sequence, so this walk, begun before
-// the update, meets it again after the function it had not yet reached.
-func TestList_UpdateDuringWalkMovesFunctionLater(t *testing.T) {
-	svc, _ := newTestService(t)
-	first := serveAs(t, svc, "a", "f1")
-	serveAs(t, svc, "b", "f2")
-	page := listPage(t, svc, testRegion, "", 1)
-	require.Equal(t, "a", page.Functions[0].Record.FunctionName)
-	cfg := Configuration{Role: first.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
-	_, err := svc.UpdateConfiguration(t.Context(), &ConfigurationUpdatedRequest{Region: testRegion, FunctionName: "a",
-		FunctionID: "f1", Revision: first.Revision, Configuration: cfg}, testAccount)
-	require.NoError(t, err)
-
-	var rest []string
-	for marker := page.NextMarker; marker != ""; {
-		p := listPage(t, svc, testRegion, marker, 1)
-		for _, f := range p.Functions {
-			rest = append(rest, f.Record.FunctionName)
-		}
-		marker = p.NextMarker
-	}
-	assert.Equal(t, []string{"b", "a"}, rest)
-}
-
-func TestList_RefusesForeignMalformedAndExhaustedMarkers(t *testing.T) {
-	svc, _ := newTestService(t)
-	for i := range 3 {
-		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
-	}
-	marker := listPage(t, svc, testRegion, "", 1).NextMarker
-	require.NotEmpty(t, marker)
-
-	foreign, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: marker, MaxItems: 1}, "000000000002")
-	assert.Nil(t, foreign)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	_, err = svc.List(t.Context(), &ListRequest{Region: "us-east-1", Marker: marker, MaxItems: 1}, testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-
-	exhausted, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion, After: math.MaxUint64})
-	require.NoError(t, err)
-	zero, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion})
-	require.NoError(t, err)
-	future, err := encodeCursor(listCursor{Version: 2, AccountID: testAccount, Region: testRegion, After: 1})
-	require.NoError(t, err)
-	for _, bad := range []string{"not base64!", "e30", exhausted, zero, future} {
-		_, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: bad, MaxItems: 1}, testAccount)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}
-
-func TestList_RefusesMalformedRequests(t *testing.T) {
-	svc, _ := newTestService(t)
-	for _, req := range []*ListRequest{
-		{Region: "Bad.Region", MaxItems: 1},
-		{Region: testRegion, MaxItems: 0},
-		{Region: testRegion, MaxItems: MaxListPage + 1},
-	} {
-		_, err := svc.List(t.Context(), req, testAccount)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}

spinifex/domains/lambda/function/record.go

@@ -63,7 +63,6 @@ const (
 	KVBucketAccountHistory = 1
 	kvFunctionKeyPrefix    = "fn."
 	kvUploadKeyPrefix      = "upload."
-	kvDeletedKeyPrefix     = "deleted."
 )
 
 // KVUploadKey returns the key of the record that tracks one code upload. The
@@ -112,10 +111,6 @@ const (
 	SubjectCodeUpdated     = "lambda.function.code-updated"
 	// SubjectConfigurationUpdated commits new settings for an Active function.
 	SubjectConfigurationUpdated = "lambda.function.configuration-updated"
-	// SubjectList pages the function records of one Region.
-	SubjectList = "lambda.function.list"
-	// SubjectDelete unbinds a function name and retains its cleanup obligation.
-	SubjectDelete = "lambda.function.delete"
 )
 
 // CreateRequest carries the record the gateway built; the daemon owns the
@@ -223,53 +218,3 @@ type ConfigurationUpdatedResponse struct {
 	Record   Record `json:"record"`
 	Revision uint64 `json:"revision"`
 }
-
-// KVDeletedKey returns the key of the obligation a deleted function leaves
-// behind. A recreated function receives a new random ID, and an obligation
-// whose identity does not match the deleted function is refused.
-func KVDeletedKey(region, functionID string) string {
-	return kvDeletedKeyPrefix + region + "." + functionID
-}
-
-// Deletion is the retained cleanup obligation of a deleted function: its code
-// copies and upload records are kept until a later owner proves them unused.
-// It is written before the name is unbound, so it alone proves no deletion.
-type Deletion struct {
-	FormatVersion int       `json:"formatVersion"`
-	AccountID     string    `json:"accountId"`
-	Region        string    `json:"region"`
-	FunctionName  string    `json:"functionName"`
-	FunctionID    string    `json:"functionId"`
-	RecordedAt    time.Time `json:"recordedAt"`
-}
-
-// ListRequest asks for one page of a Region's functions after Marker, which
-// is empty for the first page or a NextMarker the owner returned.
-type ListRequest struct {
-	Region   string `json:"region"`
-	Marker   string `json:"marker,omitempty"`
-	MaxItems int    `json:"maxItems"`
-}
-
-// ListedFunction is one function record and its revision.
-type ListedFunction struct {
-	Record   Record `json:"record"`
-	Revision uint64 `json:"revision"`
-}
-
-// ListResponse is one page; NextMarker is empty once the walk has ended.
-type ListResponse struct {
-	Functions  []ListedFunction `json:"functions"`
-	NextMarker string           `json:"nextMarker,omitempty"`
-}
-
-// DeleteRequest unbinds the function the caller read at Revision.
-type DeleteRequest struct {
-	Region       string `json:"region"`
-	FunctionName string `json:"functionName"`
-	FunctionID   string `json:"functionId"`
-	Revision     uint64 `json:"revision"`
-}
-
-// DeleteResponse is empty: success means the name was unbound.
-type DeleteResponse struct{}

spinifex/domains/network/external/ocinet/wiring_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/external/ocinet"
 	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
 )
@@ -25,21 +25,21 @@ func TestNewClientSelectsTheCredentialThePoolNames(t *testing.T) {
 	}{
 		{
 			name:          "instance principal",
-			auth:          external.OCIAuthInstancePrincipal,
+			auth:          config.OCIAuthInstancePrincipal,
 			wantPrincipal: true,
 		},
 		{
 			// A pool configured for an instance principal still must not fall
 			// back to a key file when one happens to be configured as well.
 			name:          "instance principal with a config file also set",
-			auth:          external.OCIAuthInstancePrincipal,
+			auth:          config.OCIAuthInstancePrincipal,
 			configFile:    "/etc/spinifex/oci/config",
 			profile:       "spinifex",
 			wantPrincipal: true,
 		},
 		{
 			name:       "config file",
-			auth:       external.OCIAuthConfigFile,
+			auth:       config.OCIAuthConfigFile,
 			configFile: "/etc/spinifex/oci/config",
 			profile:    "spinifex",
 		},
@@ -94,6 +94,6 @@ func TestNewClientReturnsTheConstructorError(t *testing.T) {
 	)
 	defer restore()
 
-	_, err := ocinet.NewClient(ocinet.PoolWithAuth(external.OCIAuthConfigFile, "/missing", "spinifex"))
+	_, err := ocinet.NewClient(ocinet.PoolWithAuth(config.OCIAuthConfigFile, "/missing", "spinifex"))
 	require.ErrorIs(t, err, wantErr)
 }

spinifex/domains/network/external/types.go

@@ -1,5 +1,7 @@
 package external
 
+import "github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+
 // ExternalPoolConfig describes one external IP pool wired to a VPC.
 // Mirrors [external_pools] in spinifex.toml.
 type ExternalPoolConfig struct {
@@ -32,8 +34,8 @@ type ExternalPoolConfig struct {
 	OCIVNICIface     string
 	OCISubnetID      string
 	OCIPublicIPPool  string
-	// OCIAuth selects how the provider authenticates: OCIAuthInstancePrincipal
-	// for the instance's own certificate, or OCIAuthConfigFile (the default) for
+	// OCIAuth selects how the provider authenticates: config.OCIAuthInstancePrincipal
+	// for the instance's own certificate, or config.OCIAuthConfigFile (the default) for
 	// an API key, which OCIConfigFile and OCIConfigProfile then locate.
 	OCIAuth          string
 	OCIConfigFile    string
@@ -77,7 +79,7 @@ func (p *ExternalPoolConfig) IsOCI() bool {
 // UsesInstancePrincipal reports whether the OCI provider authenticates as the
 // instance rather than from an API-key config file.
 func (p *ExternalPoolConfig) UsesInstancePrincipal() bool {
-	return p != nil && p.OCIAuth == OCIAuthInstancePrincipal
+	return p != nil && p.OCIAuth == config.OCIAuthInstancePrincipal
 }
 
 // IsExoscale reports whether the pool sources IPs from Exoscale Elastic IPs.
@@ -123,14 +125,6 @@ const (
 	// between servers. A routed Elastic IP arrives addressed to the host's MAC.
 	SourceLatitude = "latitude"
 
-	// OCIAuthConfigFile authenticates from an API-key config file. The default,
-	// because it is what an operator already has from `oci setup config`.
-	OCIAuthConfigFile = "config_file"
-	// OCIAuthInstancePrincipal authenticates as the instance, from the
-	// certificate its metadata service serves. No key material on any node and
-	// nothing to rotate, at the cost of a dynamic group and an IAM policy.
-	OCIAuthInstancePrincipal = "instance_principal"
-
 	// DHCPMACDerived leases with deterministic per-client-id 02:xx MACs.
 	DHCPMACDerived = "derived"
 	// DHCPMACInterface leases with the bind interface's own MAC; the

spinifex/domains/rds/instance/backup_sweep.go

@@ -193,7 +193,7 @@ func (s *Service) loadAutomatedBackups(ctx context.Context, kv *kvstore.Bucket,
 			continue
 		}
 		var snapshot DBSnapshotRecord
-		found, err = getJSON(ctx, kv, DBSnapshotKey(entry.DBSnapshotIdentifier), &snapshot)
+		rev, found, err := getJSONRevision(ctx, kv, DBSnapshotKey(entry.DBSnapshotIdentifier), &snapshot)
 		if err != nil {
 			return nil, err
 		}
@@ -206,7 +206,7 @@ func (s *Service) loadAutomatedBackups(ctx context.Context, kv *kvstore.Bucket,
 		// The snapshot record is the authority on age, not the index key: the key is
 		// a naming convention, and the record is what a restore reads.
 		entry.CreatedAt = snapshot.CreatedAt
-		entries = append(entries, automatedBackup{AutomatedBackupRecord: entry, stamp: stamp, snapshot: &snapshot})
+		entries = append(entries, automatedBackup{AutomatedBackupRecord: entry, stamp: stamp, snapshot: &snapshot, revision: rev})
 	}
 	return entries, nil
 }
@@ -217,6 +217,7 @@ type automatedBackup struct {
 
 	stamp    string
 	snapshot *DBSnapshotRecord
+	revision uint64
 }
 
 // The newest snapshot retention must not remove. Only an available one counts: a
@@ -242,7 +243,7 @@ func newestAvailableBackup(entries []automatedBackup) string {
 // entry stays so it is not lost from the index.
 func (s *Service) deleteAutomatedBackup(ctx context.Context, kv *kvstore.Bucket,
 	accountID string, entry automatedBackup) (bool, error) {
-	err := s.removeDBSnapshot(ctx, kv, accountID, entry.snapshot)
+	err := s.removeDBSnapshot(ctx, kv, accountID, entry.snapshot, entry.revision)
 	switch {
 	// Both readings of "not now": a snapshot an instance restored from it is still
 	// reading, and one still being cut. Neither is a failure — the next pass

spinifex/domains/rds/instance/delete.go

@@ -19,6 +19,10 @@ import (
 
 var errFinalSnapshotInProgress = errors.New("rds: final DB snapshot creation is still in progress")
 
+// errFinalSnapshotFailed holds a deletion in deleting with its data volume kept:
+// the final snapshot's EC2 snapshot ended in error, so the volume is the only copy.
+var errFinalSnapshotFailed = errors.New("rds: the final DB snapshot failed")
+
 // errModifyInProgress defers a teardown until a modify that already holds the
 // lease, and may be growing the data volume, has let go of it.
 var errModifyInProgress = errors.New("rds: a modification of the DB instance is still in progress")
@@ -103,8 +107,10 @@ func (s *Service) DeleteDBInstance(ctx context.Context, input *rds.DeleteDBInsta
 	teardownCtx, cancel := context.WithDeadline(ctx, deleteClaimEnds(rec))
 	defer cancel()
 	if err := s.teardownDBInstance(teardownCtx, kv, accountID, rec, reservation.creator); err != nil {
-		// The reconciler finishes the delete once the modify lets go.
-		if errors.Is(err, errModifyInProgress) {
+		// The reconciler finishes the delete once the modify lets go, or once
+		// the final snapshot settles.
+		if errors.Is(err, errModifyInProgress) || errors.Is(err, errFinalSnapshotInProgress) ||
+			errors.Is(err, errFinalSnapshotFailed) {
 			return &rds.DeleteDBInstanceOutput{DBInstance: s.projectDBInstance(rec)}, nil
 		}
 		return nil, err
@@ -204,7 +210,9 @@ func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, ac
 
 	// Asked to stop cleanly even though the VM is about to go: it is what makes
 	// the final snapshot a clean checkpoint rather than one needing WAL replay.
-	if rec.FinalSnapshotIdentifier != "" {
+	// Once the final snapshot is cut the VM is already gone, so a resumed
+	// teardown waiting on that snapshot has no engine left to stop.
+	if rec.FinalSnapshotIdentifier != "" && !s.finalSnapshotCut(ctx, kv, rec) {
 		s.stopEngineOrRecordFallback(ctx, accountID, rec, "deleting")
 	}
 
@@ -433,30 +441,28 @@ func (s *Service) takeFinalSnapshot(ctx context.Context, kv *kvstore.Bucket, acc
 		return awserrors.Errorf(awserrors.ErrorDBSnapshotAlreadyExists,
 			"DB snapshot %s already exists", rec.FinalSnapshotIdentifier)
 	}
-	if record.Status == SnapshotStatusAvailable && record.SnapshotID != "" {
+	switch {
+	case record.Status == SnapshotStatusAvailable && record.SnapshotID != "":
 		return nil
-	}
-	if record.Status != SnapshotStatusCreating {
+	case record.Status == SnapshotStatusFailed:
+		return fmt.Errorf("%w: %s", errFinalSnapshotFailed, rec.FinalSnapshotIdentifier)
+	case record.Status != SnapshotStatusCreating:
 		return awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
 			"DB snapshot %s is %s; expected %s", rec.FinalSnapshotIdentifier, record.Status, SnapshotStatusCreating)
+	case record.SnapshotID != "":
+		// Already cut and still settling, which reconcileSnapshots records.
+		return errFinalSnapshotInProgress
 	}
 
 	if !creator {
-		snapshotID, err := s.findEC2SnapshotFor(ctx, accountID, rec.FinalSnapshotIdentifier)
+		snapshotID, state, err := s.findEC2SnapshotFor(ctx, accountID, rec.FinalSnapshotIdentifier)
 		if err != nil {
 			return err
 		}
 		if snapshotID == "" {
 			return errFinalSnapshotInProgress
 		}
-		completed, err := s.completeFinalSnapshot(ctx, kv, rec, snapshotID)
-		if err != nil {
-			return err
-		}
-		if completed {
-			s.recordFinalSnapshotCreated(ctx, accountID, rec, snapshotID)
-		}
-		return nil
+		return s.settleFinalSnapshot(ctx, kv, accountID, rec, snapshotID, state)
 	}
 
 	// System-owned, like the volume it is taken from: the customer addresses it
@@ -483,68 +489,119 @@ func (s *Service) takeFinalSnapshot(ctx context.Context, kv *kvstore.Bucket, acc
 		return fmt.Errorf("rds: take the final snapshot of %s: empty snapshot id", rec.DBInstanceIdentifier)
 	}
 	snapshotID := aws.StringValue(snapshot.SnapshotId)
-	completed, err := s.completeFinalSnapshot(ctx, kv, rec, snapshotID)
-	if err != nil {
+	if err := s.settleFinalSnapshot(ctx, kv, accountID, rec, snapshotID, aws.StringValue(snapshot.State)); err != nil {
 		return fmt.Errorf("rds: record the final snapshot of %s: %w", rec.DBInstanceIdentifier, err)
 	}
-	if completed {
-		s.recordFinalSnapshotCreated(ctx, accountID, rec, snapshotID)
-	}
 	return nil
 }
 
-// Completes only the snapshot fields under CAS, preserving tags changed while
-// the EC2 snapshot was being cut.
+// Records the final snapshot's EC2 snapshot and its state, reporting a pending
+// one as errFinalSnapshotInProgress and an errored or gone one as errFinalSnapshotFailed.
+func (s *Service) settleFinalSnapshot(ctx context.Context, kv *kvstore.Bucket, accountID string,
+	rec *DBInstanceRecord, snapshotID, state string) error {
+	written, err := s.completeFinalSnapshot(ctx, kv, rec, snapshotID, state)
+	if err != nil {
+		return err
+	}
+	if written != nil {
+		s.recordSnapshotSettled(ctx, accountID, written, state)
+	}
+	switch status, _ := rdsSnapshotStatus(state); status {
+	case SnapshotStatusCreating:
+		return errFinalSnapshotInProgress
+	case SnapshotStatusFailed:
+		return fmt.Errorf("%w: %s", errFinalSnapshotFailed, rec.FinalSnapshotIdentifier)
+	}
+	return nil
+}
+
+// Whether the final snapshot's record already names its EC2 snapshot. A read
+// that fails answers false, so the engine stop it guards still runs.
+func (s *Service) finalSnapshotCut(ctx context.Context, kv *kvstore.Bucket, rec *DBInstanceRecord) bool {
+	var record DBSnapshotRecord
+	found, err := getJSON(ctx, kv, DBSnapshotKey(rec.FinalSnapshotIdentifier), &record)
+	return err == nil && found && record.SnapshotID != "" && finalSnapshotIsOurs(&record, rec)
+}
+
+// How long a deletion has spent tearing down, which transitionTimeout bounds:
+// from the later of its start and its final snapshot's creation, so a retaken
+// snapshot starts afresh, less the time that snapshot was pending.
+func (s *Service) deletionElapsed(ctx context.Context, kv *kvstore.Bucket, rec *DBInstanceRecord) (time.Duration, error) {
+	start := transitionStarted(rec)
+	if rec.FinalSnapshotIdentifier == "" {
+		return time.Since(start), nil
+	}
+	var record DBSnapshotRecord
+	found, err := getJSON(ctx, kv, DBSnapshotKey(rec.FinalSnapshotIdentifier), &record)
+	if err != nil {
+		return 0, fmt.Errorf("rds: read the final snapshot %s: %w", rec.FinalSnapshotIdentifier, err)
+	}
+	if !found || !finalSnapshotIsOurs(&record, rec) {
+		return time.Since(start), nil
+	}
+	if record.CreatedAt.After(start) {
+		start = record.CreatedAt
+	}
+	settled := time.Now()
+	if record.SettledAt != nil {
+		settled = *record.SettledAt
+	}
+	return time.Since(start) - max(settled.Sub(record.CreatedAt), 0), nil
+}
+
+// Writes only the snapshot fields under CAS, preserving tags changed while the
+// EC2 snapshot was being cut. It returns the record it wrote, nil when the
+// record already said as much.
 func (s *Service) completeFinalSnapshot(ctx context.Context, kv *kvstore.Bucket,
-	source *DBInstanceRecord, snapshotID string) (bool, error) {
+	source *DBInstanceRecord, snapshotID, state string) (*DBSnapshotRecord, error) {
 	key := DBSnapshotKey(source.FinalSnapshotIdentifier)
 	for range tagWriteAttempts {
 		var record DBSnapshotRecord
 		rev, found, err := getJSONRevision(ctx, kv, key, &record)
 		if err != nil {
-			return false, err
+			return nil, err
 		}
 		if !found {
-			return false, fmt.Errorf("final snapshot reservation %s is missing", source.FinalSnapshotIdentifier)
+			return nil, fmt.Errorf("final snapshot reservation %s is missing", source.FinalSnapshotIdentifier)
 		}
 		if !finalSnapshotIsOurs(&record, source) {
-			return false, awserrors.Errorf(awserrors.ErrorDBSnapshotAlreadyExists,
+			return nil, awserrors.Errorf(awserrors.ErrorDBSnapshotAlreadyExists,
 				"DB snapshot %s already exists", source.FinalSnapshotIdentifier)
 		}
-		if record.Status == SnapshotStatusAvailable {
-			if record.SnapshotID != snapshotID {
-				return false, fmt.Errorf("final snapshot %s already points to %s, not %s",
-					source.FinalSnapshotIdentifier, record.SnapshotID, snapshotID)
-			}
-			return false, nil
+		if record.SnapshotID != "" && record.SnapshotID != snapshotID {
+			return nil, fmt.Errorf("final snapshot %s already points to %s, not %s",
+				source.FinalSnapshotIdentifier, record.SnapshotID, snapshotID)
+		}
+		want, known := rdsSnapshotStatus(state)
+		if !known {
+			return nil, fmt.Errorf("final snapshot %s: EC2 snapshot %s reports state %q",
+				source.FinalSnapshotIdentifier, snapshotID, state)
+		}
+		if record.Status == want && record.SnapshotID == snapshotID {
+			return nil, nil
 		}
 		if record.Status != SnapshotStatusCreating {
-			return false, awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
+			return nil, awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
 				"DB snapshot %s is %s; expected %s", source.FinalSnapshotIdentifier,
 				record.Status, SnapshotStatusCreating)
 		}
 
 		record.SnapshotID = snapshotID
-		record.Status = SnapshotStatusAvailable
+		record.Status = want
+		if want != SnapshotStatusCreating {
+			now := time.Now().UTC()
+			record.SettledAt = &now
+		}
 		if err := updateJSON(ctx, kv, key, rev, &record); err == nil {
-			return true, nil
+			return &record, nil
 		} else if !errors.Is(err, kvstore.ErrConflict) {
-			return false, err
+			return nil, err
 		}
 	}
-	return false, fmt.Errorf("rds: completing final snapshot %s contended after %d attempts",
+	return nil, fmt.Errorf("rds: completing final snapshot %s contended after %d attempts",
 		source.FinalSnapshotIdentifier, tagWriteAttempts)
 }
 
-func (s *Service) recordFinalSnapshotCreated(ctx context.Context, accountID string,
-	rec *DBInstanceRecord, snapshotID string) {
-	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.FinalSnapshotIdentifier,
-		"Final DB snapshot created.", EventCategoryBackup, EventCategoryCreation)
-	slog.InfoContext(ctx, "rds: final DB snapshot created",
-		"dbSnapshot", rec.FinalSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier,
-		"snapshotId", snapshotID)
-}
-
 // A viperblock snapshot references its source volume's chunk files rather than
 // copying them, so the volume cannot be deleted while any snapshot survives —
 // including the final one just taken. It is retained instead, recorded with the

spinifex/domains/rds/instance/lifecycle_test.go

@@ -130,6 +130,15 @@ type fakeSnapshots struct {
 	// The tags each created snapshot carries, which is what a tag-filtered
 	// describe resolves a DB snapshot identifier through.
 	tagged map[string]map[string]string
+	// The state a snapshot reports; one absent from it is completed.
+	states map[string]string
+}
+
+func (f *fakeSnapshots) stateOf(id string) string {
+	if state, ok := f.states[id]; ok {
+		return state
+	}
+	return ec2.SnapshotStateCompleted
 }
 
 var _ snapshotProvider = (*fakeSnapshots)(nil)
@@ -156,7 +165,7 @@ func (f *fakeSnapshots) CreateSnapshot(ctx context.Context, input *ec2.CreateSna
 	// A snapshot pins the chunks of the volume it was taken from, so taking one
 	// is also what makes that volume undeletable.
 	f.holding = append(f.holding, id)
-	return &ec2.Snapshot{SnapshotId: aws.String(id), VolumeId: input.VolumeId}, nil
+	return &ec2.Snapshot{SnapshotId: aws.String(id), VolumeId: input.VolumeId, State: aws.String(f.stateOf(id))}, nil
 }
 
 // A tag filter is answered from the recorded tags; anything else reports every
@@ -170,7 +179,7 @@ func (f *fakeSnapshots) DescribeSnapshots(_ context.Context, input *ec2.Describe
 		if !f.matchesFilters(id, input) {
 			continue
 		}
-		out.Snapshots = append(out.Snapshots, &ec2.Snapshot{SnapshotId: aws.String(id)})
+		out.Snapshots = append(out.Snapshots, &ec2.Snapshot{SnapshotId: aws.String(id), State: aws.String(f.stateOf(id))})
 	}
 	if f.afterDescribe != nil {
 		f.afterDescribe()

spinifex/domains/rds/instance/pending_snapshot_test.go

@@ -0,0 +1,667 @@
+package instance
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/rds"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const pendingFinalSnapshotID = "orders-db-final"
+
+// ec2Snapshots is the real EC2 snapshot service over the memory provider, so
+// a snapshot can answer pending and settle later as an OCI backup does. It
+// records every snapshot delete it is asked for, and can fail the volume
+// describe a teardown's release reads.
+type ec2Snapshots struct {
+	*snapshot.SnapshotServiceImpl
+
+	mem *ebsprovider.MemoryProvider
+
+	mu          sync.Mutex
+	deleted     []string
+	describeErr error
+	// strictErr fails the strict describe a reconciler reads a snapshot's state by.
+	strictErr error
+}
+
+var _ snapshotProvider = (*ec2Snapshots)(nil)
+
+func (e *ec2Snapshots) DescribeSnapshots(ctx context.Context, input *ec2.DescribeSnapshotsInput, accountID string) (*ec2.DescribeSnapshotsOutput, error) {
+	e.mu.Lock()
+	err := e.describeErr
+	e.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	return e.SnapshotServiceImpl.DescribeSnapshots(ctx, input, accountID)
+}
+
+func (e *ec2Snapshots) DescribeSnapshotsStrict(ctx context.Context, input *ec2.DescribeSnapshotsInput, accountID string) (*ec2.DescribeSnapshotsOutput, error) {
+	e.mu.Lock()
+	err := e.strictErr
+	e.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	return e.SnapshotServiceImpl.DescribeSnapshotsStrict(ctx, input, accountID)
+}
+
+func (e *ec2Snapshots) DeleteSnapshot(ctx context.Context, input *ec2.DeleteSnapshotInput, accountID string) (*ec2.DeleteSnapshotOutput, error) {
+	e.mu.Lock()
+	e.deleted = append(e.deleted, aws.StringValue(input.SnapshotId))
+	e.mu.Unlock()
+	return e.SnapshotServiceImpl.DeleteSnapshot(ctx, input, accountID)
+}
+
+// withEC2Snapshots replaces the harness's snapshot fake with the real service,
+// holding the instance's data volume as the system account's.
+func withEC2Snapshots(t *testing.T, h *lifecycleHarness) *ec2Snapshots {
+	t.Helper()
+	store := objectstore.NewMemoryObjectStore()
+	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
+	require.NoError(t, ebsmetadata.NewStore(store, cfg.Predastore.Bucket).PutVolume(t.Context(), ebsmetadata.Volume{
+		VolumeID: "vol-rdsdata01", TenantID: awsidentifiers.GlobalAccountID, CapacityGiB: 20, State: "in-use",
+		AvailabilityZone: "us-east-1a", ProviderHandle: "memory://volume/vol-rdsdata01",
+	}))
+	mem := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	_, err := mem.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rdsdata01",
+		CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 20 << 30},
+	})
+	require.NoError(t, err)
+	svc := snapshot.NewSnapshotServiceImplWithStore(cfg, store, nil)
+	svc.SetEBSProvider(mem)
+	e := &ec2Snapshots{SnapshotServiceImpl: svc, mem: mem}
+	h.svc.deps.Snapshots = e
+	return e
+}
+
+// settle finishes the provider's backup and lets the EC2 reaper record it.
+func (e *ec2Snapshots) settle(t *testing.T, snapshotID string, state ebsprovider.SnapshotState) {
+	t.Helper()
+	require.NoError(t, e.mem.SettleSnapshot(snapshotID, state))
+	_, err := e.NewPendingSnapshotReaper().Sweep(t.Context())
+	require.NoError(t, err)
+}
+
+func (e *ec2Snapshots) deletes() []string {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return append([]string(nil), e.deleted...)
+}
+
+func (h *lifecycleHarness) rewriteSnapshotRecord(t *testing.T, id string, edit func(*DBSnapshotRecord)) {
+	t.Helper()
+	kv, err := h.svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+	rec, found := h.snapshotRecord(t, id)
+	require.True(t, found)
+	edit(&rec)
+	require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(id), &rec))
+}
+
+func eventsOf(t *testing.T, h *lifecycleHarness, sourceType, id string) []string {
+	t.Helper()
+	kv, err := h.svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+	var ring eventRing
+	found, err := getJSON(t.Context(), kv, EventRingKey(sourceType, id), &ring)
+	require.NoError(t, err)
+	if !found {
+		return nil
+	}
+	messages := make([]string, 0, len(ring.Events))
+	for _, event := range ring.Events {
+		messages = append(messages, event.Message)
+	}
+	return messages
+}
+
+func countContaining(messages []string, part string) int {
+	n := 0
+	for _, m := range messages {
+		if strings.Contains(m, part) {
+			n++
+		}
+	}
+	return n
+}
+
+// A DB snapshot whose EC2 snapshot is pending stays creating, past the resolve
+// timeout and across a reconciler restart, and settles with the EC2 snapshot:
+// completed makes it available, error makes it failed with an event.
+func TestReconciler_SettlesACreatingDBSnapshotFromItsEC2Snapshot(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name string
+		// adopt seeds the record a dead worker left, naming no EC2 snapshot,
+		// and the EC2 snapshot it cut, instead of a CreateDBSnapshot.
+		adopt      bool
+		ambiguous  bool // the provider's first reply is lost after it accepts
+		settleTo   ebsprovider.SnapshotState
+		wantStatus string
+	}{
+		{name: "completed", settleTo: ebsprovider.SnapshotStateCompleted, wantStatus: SnapshotStatusAvailable},
+		{name: "error", settleTo: ebsprovider.SnapshotStateError, wantStatus: SnapshotStatusFailed},
+		{name: "completed after a lost reply", ambiguous: true, settleTo: ebsprovider.SnapshotStateCompleted,
+			wantStatus: SnapshotStatusAvailable},
+		{name: "adopted, completed", adopt: true, settleTo: ebsprovider.SnapshotStateCompleted, wantStatus: SnapshotStatusAvailable},
+		{name: "adopted, error", adopt: true, settleTo: ebsprovider.SnapshotStateError, wantStatus: SnapshotStatusFailed},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			h := newLifecycleHarness(t, false)
+			e := withEC2Snapshots(t, h)
+			e.mem.AnswerSnapshotsPending(true)
+			if tc.ambiguous {
+				e.mem.FailCreateSnapshotsAfterAccept(1, nil)
+			}
+			rec := availableRecord()
+			seedInstance(t, h.svc, rec)
+
+			if tc.adopt {
+				kv, err := h.svc.bucket(t.Context(), testAccountID)
+				require.NoError(t, err)
+				stale := newDBSnapshotRecord(testAccountID, &rec, &validatedSnapshot{DBSnapshotIdentifier: testSnapshotID})
+				stale.CreatedAt = time.Now().UTC().Add(-2 * snapshotResolveTimeout)
+				require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(testSnapshotID), &stale))
+				_, _, _, err = h.svc.snapshotDataVolume(t.Context(), testAccountID, &rec, StatusStopped, testSnapshotID)
+				require.NoError(t, err)
+			} else {
+				out, err := h.svc.CreateDBSnapshot(t.Context(), &rds.CreateDBSnapshotInput{
+					DBSnapshotIdentifier: aws.String(testSnapshotID), DBInstanceIdentifier: aws.String(testDBID),
+				}, testAccountID)
+				require.NoError(t, err)
+				assert.Equal(t, SnapshotStatusCreating, aws.StringValue(out.DBSnapshot.Status))
+				assert.Equal(t, StatusAvailable, h.record(t).Status, "the instance leaves backing-up once CreateSnapshot returns")
+			}
+
+			require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
+			stored, found := h.snapshotRecord(t, testSnapshotID)
+			require.True(t, found)
+			assert.Equal(t, SnapshotStatusCreating, stored.Status)
+			require.NotEmpty(t, stored.SnapshotID, "the record names its EC2 snapshot")
+			assert.Equal(t, tc.adopt, stored.CrashConsistent)
+			ec2ID := stored.SnapshotID
+
+			// Past the resolve timeout, and on a reconciler that never saw the record.
+			h.rewriteSnapshotRecord(t, testSnapshotID, func(r *DBSnapshotRecord) {
+				r.CreatedAt = time.Now().UTC().Add(-2 * snapshotResolveTimeout)
+			})
+			restarted := testReconciler(t, h.svc, "node-b")
+			revisit, err := restarted.reconcileOnce(t.Context())
+			require.NoError(t, err)
+			assert.LessOrEqual(t, revisit, reconcileInterval, "a pending EC2 snapshot is read again every pass")
+			stored, _ = h.snapshotRecord(t, testSnapshotID)
+			assert.Equal(t, SnapshotStatusCreating, stored.Status, "a pending EC2 snapshot keeps the DB snapshot creating")
+			assert.Equal(t, ec2ID, stored.SnapshotID)
+
+			e.settle(t, ec2ID, tc.settleTo)
+			require.NoError(t, onePass(t, restarted))
+			require.NoError(t, onePass(t, restarted))
+
+			stored, found = h.snapshotRecord(t, testSnapshotID)
+			require.True(t, found, "a settled DB snapshot is never withdrawn")
+			assert.Equal(t, tc.wantStatus, stored.Status)
+			assert.Equal(t, ec2ID, stored.SnapshotID)
+			assert.NotNil(t, stored.SettledAt)
+			events := eventsOf(t, h, EventSourceTypeDBSnapshot, testSnapshotID)
+			if tc.wantStatus == SnapshotStatusFailed {
+				assert.Equal(t, 1, countContaining(events, "ended in error"), "one failure event: %v", events)
+			} else {
+				assert.Equal(t, 1, countContaining(events, "DB snapshot created."), "one creation event: %v", events)
+			}
+			assert.Empty(t, e.deletes())
+		})
+	}
+}
+
+// Two lost replies are an EC2 InternalError, and the DB snapshot record that
+// only described that call is discarded rather than left creating.
+func TestCreateDBSnapshot_DiscardsTheRecordWhenEC2AnswersInternalError(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	e := withEC2Snapshots(t, h)
+	e.mem.AnswerSnapshotsPending(true)
+	e.mem.FailCreateSnapshotsAfterAccept(2, nil)
+	seedInstance(t, h.svc, availableRecord())
+
+	_, err := h.svc.CreateDBSnapshot(t.Context(), &rds.CreateDBSnapshotInput{
+		DBSnapshotIdentifier: aws.String(testSnapshotID), DBInstanceIdentifier: aws.String(testDBID),
+	}, testAccountID)
+	require.Error(t, err)
+
+	_, found := h.snapshotRecord(t, testSnapshotID)
+	assert.False(t, found)
+	assert.Equal(t, StatusAvailable, h.record(t).Status)
+}
+
+// startPendingFinalSnapshot deletes the instance with a final snapshot the
+// provider answers pending, then backdates the delete past its handler's claim
+// and transitionTimeout, as a backup that outlasts both does.
+func startPendingFinalSnapshot(t *testing.T, h *lifecycleHarness, e *ec2Snapshots, ago time.Duration) string {
+	t.Helper()
+	e.mem.AnswerSnapshotsPending(true)
+	seedInstance(t, h.svc, availableRecord())
+
+	out, err := h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
+		DBInstanceIdentifier: aws.String(testDBID), FinalDBSnapshotIdentifier: aws.String(pendingFinalSnapshotID),
+	}, testAccountID)
+	require.NoError(t, err, "a pending final snapshot is not a failed delete")
+	assert.Equal(t, string(StatusDeleting), aws.StringValue(out.DBInstance.DBInstanceStatus))
+
+	rec := h.record(t)
+	started := time.Now().UTC().Add(-ago)
+	rec.TransitionStartedAt = &started
+	seedInstance(t, h.svc, rec)
+	h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(r *DBSnapshotRecord) { r.CreatedAt = started.Add(time.Minute) })
+
+	final, found := h.snapshotRecord(t, pendingFinalSnapshotID)
+	require.True(t, found)
+	assert.Equal(t, SnapshotStatusCreating, final.Status)
+	require.NotEmpty(t, final.SnapshotID)
+	return final.SnapshotID
+}
+
+// A deletion stays in deleting while its final snapshot is pending, however
+// long that takes, and the wait does not count toward transitionTimeout once
+// the snapshot completes; the time after it still does.
+func TestReconciler_HoldsADeletionWhileItsFinalSnapshotIsPending(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	e := withEC2Snapshots(t, h)
+	ec2ID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
+	stopsBefore := len(h.agent.received())
+
+	r := testReconciler(t, h.svc, "node-a")
+	require.NoError(t, onePass(t, r))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-b")))
+
+	assert.Equal(t, StatusDeleting, h.record(t).Status, "transitionTimeout does not run while the final snapshot is pending")
+	assert.Len(t, h.agent.received(), stopsBefore, "a teardown waiting on its cut snapshot does not stop the engine again")
+	assert.Empty(t, h.volumes.deleted)
+
+	// Completes, but the volume release then fails: the three hours waited do
+	// not count, so the deletion is retried rather than failed.
+	e.mu.Lock()
+	e.describeErr = errors.New("snapshot metadata temporarily unavailable")
+	e.mu.Unlock()
+	e.settle(t, ec2ID, ebsprovider.SnapshotStateCompleted)
+	require.NoError(t, onePass(t, r))
+
+	final, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
+	assert.Equal(t, SnapshotStatusAvailable, final.Status)
+	assert.Equal(t, StatusDeleting, h.record(t).Status, "the wait on the final snapshot is excluded from transitionTimeout")
+
+	// Still failing transitionTimeout after it settled, the deletion fails.
+	h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(r *DBSnapshotRecord) {
+		settled := r.SettledAt.Add(-transitionTimeout - time.Minute)
+		r.SettledAt = &settled
+	})
+	require.NoError(t, onePass(t, r))
+	assert.Equal(t, StatusFailed, h.record(t).Status)
+	assert.Empty(t, h.volumes.deleted)
+
+	// Once the release can be read, the retained volume is recorded and the
+	// instance goes, as for a final snapshot that completed at once.
+	e.mu.Lock()
+	e.describeErr = nil
+	e.mu.Unlock()
+	rec := h.record(t)
+	rec.Status = StatusDeleting
+	now := time.Now().UTC().Add(-deleteTimeout - time.Minute)
+	rec.TransitionStartedAt = &now
+	seedInstance(t, h.svc, rec)
+	require.NoError(t, onePass(t, r))
+	assert.False(t, h.recordExists(t, testDBID))
+	retained, found := h.retainedVolume(t, "vol-rdsdata01")
+	require.True(t, found)
+	assert.Equal(t, []string{ec2ID}, retained.Snapshots)
+	assert.Empty(t, h.volumes.deleted)
+	assert.Equal(t, 1, countContaining(eventsOf(t, h, EventSourceTypeDBSnapshot, pendingFinalSnapshotID), "Final DB snapshot created."))
+}
+
+// A pending final snapshot whose state cannot be read for a while does not
+// fail the deletion past transitionTimeout: the time it has been pending is
+// excluded before it settles, and the deletion finishes once it completes.
+func TestReconciler_PendingFinalSnapshotReadFailureDoesNotFailTheDeletion(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	e := withEC2Snapshots(t, h)
+	ec2ID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
+	r := testReconciler(t, h.svc, "node-a")
+
+	e.mu.Lock()
+	e.strictErr = errors.New("snapshot metadata temporarily unavailable")
+	e.mu.Unlock()
+	require.ErrorContains(t, onePass(t, r), "temporarily unavailable", "the creating DB snapshot's settle reports the read")
+	assert.Equal(t, StatusDeleting, h.record(t).Status, "a pending backup's wait is excluded even when its state cannot be read")
+	assert.Empty(t, h.volumes.deleted)
+
+	e.mu.Lock()
+	e.strictErr = nil
+	e.mu.Unlock()
+	require.NoError(t, onePass(t, r))
+	assert.Equal(t, StatusDeleting, h.record(t).Status, "still pending, so still waiting")
+
+	// One pass records the snapshot available, and the next finishes the deletion.
+	e.settle(t, ec2ID, ebsprovider.SnapshotStateCompleted)
+	require.NoError(t, onePass(t, r))
+	require.NoError(t, onePass(t, r))
+	assert.False(t, h.recordExists(t, testDBID), "the deletion recovers once the backup completes")
+	final, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
+	assert.Equal(t, SnapshotStatusAvailable, final.Status)
+}
+
+// A final snapshot whose EC2 snapshot errors or is gone keeps the instance in
+// deleting, with one event naming the failed snapshot, however many passes follow.
+func TestReconciler_KeepsADeletionWhoseFinalSnapshotFailed(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name    string
+		breakIt func(t *testing.T, e *ec2Snapshots, ec2ID string)
+	}{
+		{name: "ended in error", breakIt: func(t *testing.T, e *ec2Snapshots, ec2ID string) {
+			e.settle(t, ec2ID, ebsprovider.SnapshotStateError)
+		}},
+		{name: "gone", breakIt: func(t *testing.T, e *ec2Snapshots, ec2ID string) {
+			require.NoError(t, e.MetadataStore().DeleteSnapshot(t.Context(), awsidentifiers.GlobalAccountID, ec2ID))
+		}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			h := newLifecycleHarness(t, false)
+			e := withEC2Snapshots(t, h)
+			ec2ID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
+
+			tc.breakIt(t, e, ec2ID)
+			r := testReconciler(t, h.svc, "node-a")
+			for range 3 {
+				require.NoError(t, onePass(t, r))
+			}
+
+			assert.Equal(t, StatusDeleting, h.record(t).Status)
+			final, found := h.snapshotRecord(t, pendingFinalSnapshotID)
+			require.True(t, found)
+			assert.Equal(t, SnapshotStatusFailed, final.Status)
+			instanceEvents := eventsOf(t, h, EventSourceTypeDBInstance, testDBID)
+			assert.Equal(t, 1, countContaining(instanceEvents, "The final DB snapshot "+pendingFinalSnapshotID+" failed"), "%v", instanceEvents)
+			assert.Equal(t, 1, countContaining(eventsOf(t, h, EventSourceTypeDBSnapshot, pendingFinalSnapshotID), ec2ID))
+			assert.Empty(t, h.volumes.deleted, "the data volume is the only copy left")
+			_, retained := h.retainedVolume(t, "vol-rdsdata01")
+			assert.False(t, retained, "the teardown stops before it decides the volume's fate")
+
+			// A retried DeleteDBInstance reports the deletion and changes nothing.
+			_, err := h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
+				DBInstanceIdentifier: aws.String(testDBID), FinalDBSnapshotIdentifier: aws.String(pendingFinalSnapshotID),
+			}, testAccountID)
+			require.NoError(t, err)
+			assert.Empty(t, h.volumes.deleted)
+		})
+	}
+}
+
+// Verifies: ADR-0003:S7.
+//
+// TestS7_RDSDeletionNeverDeletesTheDataVolumeWithoutACompletedFinalSnapshot
+// drives a deletion whose final snapshot is pending or errored through every
+// pass past every timeout, and requires no volume or snapshot was deleted.
+func TestS7_RDSDeletionNeverDeletesTheDataVolumeWithoutACompletedFinalSnapshot(t *testing.T) {
+	t.Parallel()
+	for _, settleTo := range []ebsprovider.SnapshotState{"", ebsprovider.SnapshotStateError} {
+		name := "pending"
+		if settleTo != "" {
+			name = string(settleTo)
+		}
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			h := newLifecycleHarness(t, false)
+			e := withEC2Snapshots(t, h)
+			ec2ID := startPendingFinalSnapshot(t, h, e, 24*time.Hour)
+			if settleTo != "" {
+				e.settle(t, ec2ID, settleTo)
+			}
+			for _, holder := range []string{"node-a", "node-b", "node-a"} {
+				require.NoError(t, onePass(t, testReconciler(t, h.svc, holder)))
+			}
+
+			assert.Empty(t, h.volumes.deleted, "ADR-0003:S7: the data volume was deleted with no completed final snapshot")
+			assert.Empty(t, e.deletes(), "ADR-0003:S7: an EC2 snapshot was deleted")
+			_, err := e.mem.GetVolume(t.Context(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rdsdata01"})
+			assert.NoError(t, err, "ADR-0003:S7: the data volume is gone at the provider")
+			_, held := e.mem.Snapshot(ec2ID)
+			assert.True(t, held, "ADR-0003:S7: the final snapshot is gone at the provider")
+			assert.True(t, h.recordExists(t, testDBID), "ADR-0003:S7: the DB instance record is gone")
+		})
+	}
+}
+
+// A creating record whose EC2 snapshot is gone fails with an event, once: no
+// pass can make that snapshot again, so creating would wait on it forever.
+func TestReconciler_FailsACreatingDBSnapshotWhoseEC2SnapshotIsGone(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	kv, err := h.svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+	rec := availableRecord()
+	named := newDBSnapshotRecord(testAccountID, &rec, &validatedSnapshot{DBSnapshotIdentifier: testSnapshotID})
+	named.SnapshotID = "snap-named01"
+	require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(testSnapshotID), &named))
+
+	r := testReconciler(t, h.svc, "node-a")
+	for range 3 {
+		require.NoError(t, onePass(t, r))
+	}
+
+	stored, found := h.snapshotRecord(t, testSnapshotID)
+	require.True(t, found, "a failed DB snapshot is kept for the customer to see and delete")
+	assert.Equal(t, SnapshotStatusFailed, stored.Status)
+	assert.NotNil(t, stored.SettledAt)
+	events := eventsOf(t, h, EventSourceTypeDBSnapshot, testSnapshotID)
+	assert.Equal(t, 1, countContaining(events, "snap-named01 no longer exists"), "one failure event: %v", events)
+}
+
+// A creating record naming an EC2 snapshot that reports no state, or cannot
+// be read, is left creating: neither says the data is restorable or lost.
+func TestReconciler_LeavesACreatingDBSnapshotItCannotSettle(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name    string
+		setup   func(f *fakeSnapshots)
+		wantErr bool
+	}{
+		{name: "the EC2 snapshot reports no state", setup: func(f *fakeSnapshots) {
+			f.holding = []string{"snap-named01"}
+			f.states = map[string]string{"snap-named01": ""}
+		}},
+		{name: "the EC2 snapshot cannot be read", setup: func(f *fakeSnapshots) {
+			f.describeErr = errors.New("snapshot metadata temporarily unavailable")
+		}, wantErr: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			h := newLifecycleHarness(t, false)
+			if tc.setup != nil {
+				tc.setup(h.snaps)
+			}
+			kv, err := h.svc.bucket(t.Context(), testAccountID)
+			require.NoError(t, err)
+			rec := availableRecord()
+			named := newDBSnapshotRecord(testAccountID, &rec, &validatedSnapshot{DBSnapshotIdentifier: testSnapshotID})
+			named.SnapshotID = "snap-named01"
+			named.CreatedAt = time.Now().UTC().Add(-2 * snapshotResolveTimeout)
+			require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(testSnapshotID), &named))
+
+			err = onePass(t, testReconciler(t, h.svc, "node-a"))
+			if tc.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			stored, found := h.snapshotRecord(t, testSnapshotID)
+			require.True(t, found)
+			assert.Equal(t, SnapshotStatusCreating, stored.Status)
+			assert.Equal(t, "snap-named01", stored.SnapshotID)
+		})
+	}
+}
+
+// The state the creating handler's own CreateSnapshot answers decides the
+// final snapshot: error holds the deletion with the volume kept, and a state
+// it cannot read fails the request with the volume kept.
+func TestDeleteDBInstance_FinalSnapshotStateAtCreate(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name       string
+		state      string
+		wantErr    bool
+		wantStatus string
+	}{
+		{name: "error", state: ec2.SnapshotStateError, wantStatus: SnapshotStatusFailed},
+		{name: "no state", state: "", wantErr: true, wantStatus: SnapshotStatusCreating},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			h := newLifecycleHarness(t, false)
+			h.snaps.states = map[string]string{"snap-0001": tc.state}
+			seedInstance(t, h.svc, availableRecord())
+
+			_, err := h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
+				DBInstanceIdentifier: aws.String(testDBID), FinalDBSnapshotIdentifier: aws.String(pendingFinalSnapshotID),
+			}, testAccountID)
+
+			if tc.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			final, found := h.snapshotRecord(t, pendingFinalSnapshotID)
+			require.True(t, found)
+			assert.Equal(t, tc.wantStatus, final.Status)
+			assert.Equal(t, StatusDeleting, h.record(t).Status)
+			assert.Empty(t, h.volumes.deleted)
+
+			if tc.wantStatus == SnapshotStatusCreating {
+				rec := h.record(t)
+				h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(r *DBSnapshotRecord) {
+					r.CreatedAt = time.Now().UTC().Add(-2 * time.Hour)
+				})
+				elapsed, err := h.svc.deletionElapsed(t.Context(), mustBucket(t, h), &rec)
+				require.NoError(t, err)
+				assert.Less(t, elapsed, time.Minute, "an unsettled final snapshot excludes the time since it was reserved")
+			}
+		})
+	}
+}
+
+func mustBucket(t *testing.T, h *lifecycleHarness) *kvstore.Bucket {
+	t.Helper()
+	kv, err := h.svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+	return kv
+}
+
+// A failed final snapshot can be deleted, as on AWS, and the deletion then
+// takes it again: the data volume stays until a final snapshot is available,
+// and the failed backup's hours do not count against the retake's timeout.
+func TestDeleteDBSnapshot_AFailedFinalSnapshotIsDeletedAndTakenAgain(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	e := withEC2Snapshots(t, h)
+	failedID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
+	del := &rds.DeleteDBSnapshotInput{DBSnapshotIdentifier: aws.String(pendingFinalSnapshotID)}
+	r := testReconciler(t, h.svc, "node-a")
+
+	_, err := h.svc.DeleteDBSnapshot(t.Context(), del, testAccountID)
+	require.True(t, awserrors.IsErrorCode(err, awserrors.ErrorDBSnapshotInvalidState), "a creating DB snapshot is refused: %v", err)
+
+	e.settle(t, failedID, ebsprovider.SnapshotStateError)
+	require.NoError(t, onePass(t, r))
+	failed, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
+	require.Equal(t, SnapshotStatusFailed, failed.Status)
+
+	_, err = h.svc.DeleteDBSnapshot(t.Context(), del, testAccountID)
+	require.NoError(t, err)
+	assert.Contains(t, e.deletes(), failedID)
+
+	require.NoError(t, onePass(t, r))
+	retake, found := h.snapshotRecord(t, pendingFinalSnapshotID)
+	require.True(t, found, "the deletion reserves its final snapshot again")
+	assert.Equal(t, SnapshotStatusCreating, retake.Status)
+	require.NotEmpty(t, retake.SnapshotID)
+	assert.NotEqual(t, failedID, retake.SnapshotID)
+	assert.Empty(t, h.volumes.deleted)
+
+	// Completes, but the volume release then fails: the retake starts its own
+	// timeout, so the deletion is retried rather than failed.
+	e.mu.Lock()
+	e.describeErr = errors.New("snapshot metadata temporarily unavailable")
+	e.mu.Unlock()
+	e.settle(t, retake.SnapshotID, ebsprovider.SnapshotStateCompleted)
+	require.NoError(t, onePass(t, r))
+	require.NoError(t, onePass(t, r))
+	final, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
+	require.Equal(t, SnapshotStatusAvailable, final.Status)
+	assert.Equal(t, StatusDeleting, h.record(t).Status, "the failed backup's wait was counted against the retake")
+	assert.Empty(t, h.volumes.deleted)
+
+	e.mu.Lock()
+	e.describeErr = nil
+	e.mu.Unlock()
+	require.NoError(t, onePass(t, r))
+	assert.False(t, h.recordExists(t, testDBID))
+}
+
+// A delete removes only the DB snapshot it read: the same snapshot rewritten
+// since, by a tag write, is still deleted, and a delayed delete of a failed
+// final snapshot leaves the reservation the deletion has since retaken.
+func TestRemoveDBSnapshot_DeletesOnlyTheSnapshotItRead(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	e := withEC2Snapshots(t, h)
+	failedID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
+	e.settle(t, failedID, ebsprovider.SnapshotStateError)
+	r := testReconciler(t, h.svc, "node-a")
+	require.NoError(t, onePass(t, r))
+	kv := mustBucket(t, h)
+
+	stale, staleRev, err := h.svc.getDBSnapshot(t.Context(), kv, pendingFinalSnapshotID)
+	require.NoError(t, err)
+	require.Equal(t, SnapshotStatusFailed, stale.Status)
+	h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(*DBSnapshotRecord) {})
+	require.NoError(t, h.svc.removeDBSnapshot(t.Context(), kv, testAccountID, stale, staleRev))
+	_, found := h.snapshotRecord(t, pendingFinalSnapshotID)
+	require.False(t, found, "a rewrite of the same snapshot does not stop its delete")
+
+	require.NoError(t, onePass(t, r))
+	retake, found := h.snapshotRecord(t, pendingFinalSnapshotID)
+	require.True(t, found)
+
+	err = h.svc.removeDBSnapshot(t.Context(), kv, testAccountID, stale, staleRev)
+	require.True(t, awserrors.IsErrorCode(err, awserrors.ErrorDBSnapshotInvalidState), "%v", err)
+	kept, found := h.snapshotRecord(t, pendingFinalSnapshotID)
+	require.True(t, found, "a delayed delete removed the retaken reservation")
+	assert.Equal(t, retake.SnapshotID, kept.SnapshotID)
+	assert.Empty(t, h.volumes.deleted)
+}

spinifex/domains/rds/instance/reconciler.go

@@ -597,15 +597,19 @@ func (r *Reconciler) reconcileDeleting(ctx context.Context, kv *kvstore.Bucket,
 		return nil
 	}
 	err := r.svc.teardownDBInstance(ctx, kv, accountID, rec, false)
-	if err == nil || errors.Is(err, errFinalSnapshotInProgress) || errors.Is(err, errModifyInProgress) {
+	// A failed final snapshot keeps the instance, and its data volume, in
+	// deleting: failing the delete would let a retry skip the only copy.
+	if err == nil || errors.Is(err, errFinalSnapshotInProgress) || errors.Is(err, errFinalSnapshotFailed) ||
+		errors.Is(err, errModifyInProgress) {
 		return nil
 	}
-	if time.Since(transitionStarted(rec)) > transitionTimeout {
+	elapsed, waitErr := r.svc.deletionElapsed(ctx, kv, rec)
+	if waitErr == nil && elapsed > transitionTimeout {
 		return r.transition(ctx, kv, rev, rec, StatusFailed,
 			fmt.Sprintf("the DB instance could not be deleted within %s: %v", transitionTimeout, err))
 	}
 	slog.WarnContext(ctx, "rds reconciler: resuming a delete failed; retrying next pass",
-		"dbInstance", rec.DBInstanceIdentifier, "err", err)
+		"dbInstance", rec.DBInstanceIdentifier, "err", errors.Join(err, waitErr))
 	return nil
 }
 
@@ -675,6 +679,15 @@ func (r *Reconciler) reconcileSnapshots(ctx context.Context, kv *kvstore.Bucket,
 		if !found || rec.Status != SnapshotStatusCreating {
 			continue
 		}
+		// A record naming its EC2 snapshot waits on that snapshot settling, which
+		// no KV write announces, so it is read every pass.
+		if rec.SnapshotID != "" {
+			revisit = reconciler.Earliest(revisit, reconcileInterval)
+			if err := r.settleCreatingSnapshot(ctx, kv, rev, accountID, &rec); err != nil {
+				failures = append(failures, fmt.Errorf("settle DB snapshot %s: %w", id, err))
+			}
+			continue
+		}
 		// Not due yet, so ask to be back when it is rather than leaving a dead
 		// worker's record to the resync.
 		if due := time.Until(rec.CreatedAt.Add(snapshotResolveTimeout)); due > 0 {
@@ -693,7 +706,7 @@ func (r *Reconciler) reconcileSnapshots(ctx context.Context, kv *kvstore.Bucket,
 // the name forever while naming nothing a customer can restore.
 func (r *Reconciler) resolveCreatingSnapshot(ctx context.Context, kv *kvstore.Bucket, rev uint64,
 	accountID string, rec *DBSnapshotRecord) error {
-	snapshotID, err := r.svc.findEC2SnapshotFor(ctx, accountID, rec.DBSnapshotIdentifier)
+	snapshotID, state, err := r.svc.findEC2SnapshotFor(ctx, accountID, rec.DBSnapshotIdentifier)
 	if err != nil {
 		return err
 	}
@@ -716,30 +729,146 @@ func (r *Reconciler) resolveCreatingSnapshot(ctx context.Context, kv *kvstore.Bu
 	}
 
 	rec.SnapshotID = snapshotID
-	rec.Status = SnapshotStatusAvailable
 	// The worker died before it could report whether the quiesce held, so the
 	// conservative reading is the one that never overstates the snapshot.
 	rec.CrashConsistent = true
+	slog.InfoContext(ctx, "rds reconciler: adopted the EC2 snapshot of an unfinished DB snapshot",
+		"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", snapshotID, "accountId", accountID, "state", state)
+	return r.svc.applyEC2SnapshotState(ctx, kv, rev, accountID, rec, state, true)
+}
+
+// Settles a creating record that names its EC2 snapshot from that snapshot's
+// state; a pending one is left for the next pass.
+func (r *Reconciler) settleCreatingSnapshot(ctx context.Context, kv *kvstore.Bucket, rev uint64,
+	accountID string, rec *DBSnapshotRecord) error {
+	state, err := r.svc.ec2SnapshotState(ctx, rec.SnapshotID)
+	if err != nil {
+		return err
+	}
+	return r.svc.applyEC2SnapshotState(ctx, kv, rev, accountID, rec, state, false)
+}
+
+// ec2SnapshotGone is the state ec2SnapshotState reports for an EC2 snapshot
+// whose document is gone. Nothing recreates one, so it settles the record.
+const ec2SnapshotGone = "gone"
+
+// rdsSnapshotStatus is the DB snapshot status an EC2 snapshot's state gives:
+// pending is creating, completed available, and error or gone failed. known is
+// false for any other state, which says neither restorable nor lost.
+func rdsSnapshotStatus(ec2State string) (status string, known bool) {
+	switch ec2State {
+	case ec2.SnapshotStatePending:
+		return SnapshotStatusCreating, true
+	case ec2.SnapshotStateCompleted:
+		return SnapshotStatusAvailable, true
+	case ec2.SnapshotStateError, ec2SnapshotGone:
+		return SnapshotStatusFailed, true
+	}
+	return "", false
+}
+
+// Moves a creating record, read at rev, to what its EC2 snapshot's state says,
+// writing a pending one only when adopted says the record has changed. A state
+// rdsSnapshotStatus does not know leaves it.
+func (s *Service) applyEC2SnapshotState(ctx context.Context, kv *kvstore.Bucket, rev uint64,
+	accountID string, rec *DBSnapshotRecord, state string, adopted bool) error {
+	status, known := rdsSnapshotStatus(state)
+	switch {
+	case !known:
+		slog.WarnContext(ctx, "rds reconciler: the EC2 snapshot of a creating DB snapshot reports an unknown state; leaving it creating",
+			"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", rec.SnapshotID, "accountId", accountID, "state", state)
+		return nil
+	case status == SnapshotStatusCreating:
+		if !adopted {
+			return nil
+		}
+	default:
+		now := time.Now().UTC()
+		rec.Status = status
+		rec.SettledAt = &now
+	}
 	if err := updateJSON(ctx, kv, DBSnapshotKey(rec.DBSnapshotIdentifier), rev, rec); err != nil {
 		if errors.Is(err, kvstore.ErrConflict) {
 			return nil
 		}
 		return err
 	}
-	r.svc.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.DBSnapshotIdentifier,
-		"DB snapshot created.", EventCategoryBackup, EventCategoryCreation)
-	slog.InfoContext(ctx, "rds reconciler: adopted the EC2 snapshot of an unfinished DB snapshot",
-		"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", snapshotID, "accountId", accountID)
+	s.recordSnapshotSettled(ctx, accountID, rec, state)
 	return nil
 }
 
-// The EC2 snapshot an account-scoped DB snapshot identifier owns, empty when
-// none was cut. Strict lookup is required because a partial metadata scan cannot
-// prove absence and must never cause reconciliation to withdraw the RDS record.
-func (s *Service) findEC2SnapshotFor(ctx context.Context, accountID, dbSnapshotIdentifier string) (string, error) {
+// Reports a DB snapshot record just written available or failed; one still
+// creating reports nothing. Every settlement of a DB snapshot comes through here.
+func (s *Service) recordSnapshotSettled(ctx context.Context, accountID string, rec *DBSnapshotRecord, state string) {
+	switch rec.Status {
+	case SnapshotStatusAvailable:
+		message := "DB snapshot created."
+		if rec.FinalSnapshot {
+			message = "Final DB snapshot created."
+		}
+		s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.DBSnapshotIdentifier,
+			message, EventCategoryBackup, EventCategoryCreation)
+		slog.InfoContext(ctx, "rds: DB snapshot created",
+			"dbSnapshot", rec.DBSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier, "snapshotId", rec.SnapshotID,
+			"snapshotType", rec.SnapshotType, "crashConsistent", rec.CrashConsistent, "finalSnapshot", rec.FinalSnapshot)
+	case SnapshotStatusFailed:
+		s.recordDBSnapshotFailed(ctx, accountID, rec, state)
+	}
+}
+
+// Reports a DB snapshot whose EC2 snapshot settled in error or is gone, and
+// for a final snapshot the DB instance its deletion now holds in deleting.
+func (s *Service) recordDBSnapshotFailed(ctx context.Context, accountID string, rec *DBSnapshotRecord, state string) {
+	why := " ended in error."
+	if state == ec2SnapshotGone {
+		why = " no longer exists."
+	}
+	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.DBSnapshotIdentifier,
+		"The DB snapshot failed: its EC2 snapshot "+rec.SnapshotID+why,
+		EventCategoryFailure, EventCategoryBackup)
+	if rec.FinalSnapshot {
+		s.RecordEvent(ctx, accountID, EventSourceTypeDBInstance, rec.DBInstanceIdentifier,
+			"The final DB snapshot "+rec.DBSnapshotIdentifier+" failed; the DB instance stays in deleting and its data volume is kept.",
+			EventCategoryFailure, EventCategoryDeletion)
+	}
+	slog.WarnContext(ctx, "rds: a DB snapshot's EC2 snapshot ended in error or is gone",
+		"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", rec.SnapshotID, "accountId", accountID,
+		"finalSnapshot", rec.FinalSnapshot, "state", state)
+}
+
+// The state of an EC2 snapshot, ec2SnapshotGone when it is gone. The strict
+// describe fails on unreadable metadata, so a partial read is never absence.
+func (s *Service) ec2SnapshotState(ctx context.Context, snapshotID string) (string, error) {
 	if s.deps.Snapshots == nil {
 		return "", errors.New("rds: no snapshot service configured")
 	}
+	out, err := s.deps.Snapshots.DescribeSnapshotsStrict(ctx, &ec2.DescribeSnapshotsInput{
+		SnapshotIds: aws.StringSlice([]string{snapshotID}),
+	}, awsidentifiers.GlobalAccountID)
+	if err != nil {
+		if awserrors.IsNotFound(err) {
+			return ec2SnapshotGone, nil
+		}
+		return "", fmt.Errorf("rds: read the EC2 snapshot %s: %w", snapshotID, err)
+	}
+	if out == nil {
+		return ec2SnapshotGone, nil
+	}
+	for _, snapshot := range out.Snapshots {
+		if aws.StringValue(snapshot.SnapshotId) == snapshotID {
+			return aws.StringValue(snapshot.State), nil
+		}
+	}
+	return ec2SnapshotGone, nil
+}
+
+// The EC2 snapshot an account-scoped DB snapshot identifier owns and its state,
+// empty when none was cut. Strict lookup is required because a partial metadata
+// scan cannot prove absence and must never cause reconciliation to withdraw the RDS record.
+func (s *Service) findEC2SnapshotFor(ctx context.Context, accountID, dbSnapshotIdentifier string) (snapshotID, state string, err error) {
+	if s.deps.Snapshots == nil {
+		return "", "", errors.New("rds: no snapshot service configured")
+	}
 	out, err := s.deps.Snapshots.DescribeSnapshotsStrict(ctx, &ec2.DescribeSnapshotsInput{
 		Filters: []*ec2.Filter{
 			{
@@ -754,28 +883,28 @@ func (s *Service) findEC2SnapshotFor(ctx context.Context, accountID, dbSnapshotI
 	}, awsidentifiers.GlobalAccountID)
 	if err != nil {
 		if awserrors.IsNotFound(err) {
-			return "", nil
+			return "", "", nil
 		}
-		return "", fmt.Errorf("rds: find the EC2 snapshot of %s in account %s: %w",
+		return "", "", fmt.Errorf("rds: find the EC2 snapshot of %s in account %s: %w",
 			dbSnapshotIdentifier, accountID, err)
 	}
 	if out == nil {
-		return "", nil
+		return "", "", nil
 	}
 
-	var found string
+	var found, foundState string
 	for _, snapshot := range out.Snapshots {
 		id := aws.StringValue(snapshot.SnapshotId)
 		if id == "" {
 			continue
 		}
 		if found != "" {
-			return "", fmt.Errorf("rds: DB snapshot %s in account %s has multiple EC2 snapshots (%s and %s)",
+			return "", "", fmt.Errorf("rds: DB snapshot %s in account %s has multiple EC2 snapshots (%s and %s)",
 				dbSnapshotIdentifier, accountID, found, id)
 		}
-		found = id
+		found, foundState = id, aws.StringValue(snapshot.State)
 	}
-	return found, nil
+	return found, foundState, nil
 }
 
 // When the transition began. A record written by an older control plane carries

spinifex/domains/rds/instance/records.go

@@ -271,6 +271,8 @@ const (
 	// in between leaves a reconcilable trace rather than an orphaned EC2 snapshot.
 	SnapshotStatusCreating  = "creating"
 	SnapshotStatusAvailable = "available"
+	// The EC2 snapshot settled in error, so the record holds no restorable data.
+	SnapshotStatusFailed = "failed"
 )
 
 // SnapshotOperation is the snapshot operation holding a DB instance, written under the same CAS that
@@ -334,6 +336,9 @@ type DBSnapshotRecord struct {
 	Tags map[string]string `json:"tags,omitempty"`
 
 	CreatedAt time.Time `json:"createdAt"`
+	// When the EC2 snapshot settled to available or failed. A deletion waiting
+	// on its final snapshot excludes the wait between this and CreatedAt.
+	SettledAt *time.Time `json:"settledAt,omitempty"`
 }
 
 var _ TaggedRecord = (*DBSnapshotRecord)(nil)

spinifex/domains/rds/instance/snapshot.go

@@ -114,7 +114,7 @@ func (s *Service) snapshotDBInstance(ctx context.Context, kv *kvstore.Bucket, re
 		return nil, err
 	}
 
-	snapshotID, crashConsistent, err := s.snapshotDataVolume(ctx, accountID, rec, resume, req.DBSnapshotIdentifier)
+	snapshotID, state, crashConsistent, err := s.snapshotDataVolume(ctx, accountID, rec, resume, req.DBSnapshotIdentifier)
 	if err != nil {
 		// The record only ever described a snapshot that now does not exist, so it
 		// goes with it rather than being left for the reconciler to puzzle over.
@@ -123,26 +123,35 @@ func (s *Service) snapshotDBInstance(ctx context.Context, kv *kvstore.Bucket, re
 	}
 
 	record.SnapshotID = snapshotID
-	record.Status = SnapshotStatusAvailable
 	record.CrashConsistent = crashConsistent
+	// A pending EC2 snapshot's contents are already fixed, so the quiesce is
+	// over; the record stays creating until the reconciler sees it settle.
+	if status, _ := rdsSnapshotStatus(state); status == SnapshotStatusCreating {
+		if err := putJSON(ctx, kv, key, &record); err != nil {
+			return nil, err
+		}
+		slog.InfoContext(ctx, "rds: DB snapshot cut; waiting for its EC2 snapshot to complete",
+			"dbSnapshot", req.DBSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier,
+			"snapshotType", record.SnapshotType, "snapshotId", snapshotID, "crashConsistent", crashConsistent)
+		return &record, nil
+	}
+	now := time.Now().UTC()
+	record.Status = SnapshotStatusAvailable
+	record.SettledAt = &now
 	if err := putJSON(ctx, kv, key, &record); err != nil {
 		return nil, err
 	}
 
-	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, req.DBSnapshotIdentifier,
-		"DB snapshot created.", EventCategoryBackup, EventCategoryCreation)
-	slog.InfoContext(ctx, "rds: DB snapshot created",
-		"dbSnapshot", req.DBSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier,
-		"snapshotType", record.SnapshotType, "snapshotId", snapshotID, "crashConsistent", crashConsistent)
+	s.recordSnapshotSettled(ctx, accountID, &record, state)
 
 	return &record, nil
 }
 
 // Quiesces the engine, snapshots the data volume, and releases the engine again
-// however the snapshot went. Reports the EC2 snapshot and whether the engine was
-// still writing when it was taken.
+// however the snapshot went. Reports the EC2 snapshot, its state, and whether
+// the engine was still writing when it was taken.
 func (s *Service) snapshotDataVolume(ctx context.Context, accountID string, rec *DBInstanceRecord,
-	resume Status, dbSnapshotIdentifier string) (snapshotID string, crashConsistent bool, err error) {
+	resume Status, dbSnapshotIdentifier string) (snapshotID, state string, crashConsistent bool, err error) {
 	// A stopped instance has no engine to hold: its datadir was sealed by the
 	// graceful stop, which is the checkpoint a quiesce would be forcing.
 	if resume == StatusAvailable {
@@ -176,12 +185,12 @@ func (s *Service) snapshotDataVolume(ctx context.Context, accountID string, rec
 		}},
 	}, awsidentifiers.GlobalAccountID)
 	if err != nil {
-		return "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: %w", rec.DBInstanceIdentifier, err)
+		return "", "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: %w", rec.DBInstanceIdentifier, err)
 	}
 	if snapshot == nil || aws.StringValue(snapshot.SnapshotId) == "" {
-		return "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: empty snapshot id", rec.DBInstanceIdentifier)
+		return "", "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: empty snapshot id", rec.DBInstanceIdentifier)
 	}
-	return aws.StringValue(snapshot.SnapshotId), crashConsistent, nil
+	return aws.StringValue(snapshot.SnapshotId), aws.StringValue(snapshot.State), crashConsistent, nil
 }
 
 // The half of the warning that holds for every engine. What restoring such a
@@ -309,11 +318,11 @@ func (s *Service) DeleteDBSnapshot(ctx context.Context, input *rds.DeleteDBSnaps
 	if err != nil {
 		return nil, err
 	}
-	rec, _, err := s.getDBSnapshot(ctx, kv, id)
+	rec, rev, err := s.getDBSnapshot(ctx, kv, id)
 	if err != nil {
 		return nil, err
 	}
-	if err := s.removeDBSnapshot(ctx, kv, accountID, rec); err != nil {
+	if err := s.removeDBSnapshot(ctx, kv, accountID, rec, rev); err != nil {
 		return nil, err
 	}
 	// AWS answers with the snapshot as it last stood, the way a delete of a DB
@@ -325,16 +334,16 @@ func (s *Service) DeleteDBSnapshot(ctx context.Context, input *rds.DeleteDBSnaps
 // data volume behind it. Shared with the retention sweep, which cannot go
 // through DeleteDBSnapshot: that rejects the rds: namespace automated snapshots
 // live in, so a customer can never delete one by hand.
-func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, accountID string, rec *DBSnapshotRecord) error {
+func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, accountID string, rec *DBSnapshotRecord, rev uint64) error {
 	if s.deps.Snapshots == nil {
 		return errors.New("rds: no snapshot service configured")
 	}
 	id := rec.DBSnapshotIdentifier
-	// A snapshot still being taken has no data to remove and an in-flight writer
-	// that would recreate the record; the reconciler resolves it either way.
-	if rec.Status != SnapshotStatusAvailable {
+	// A snapshot still being taken has an in-flight writer that would recreate
+	// the record; the reconciler settles it available or failed, as AWS's are.
+	if rec.Status != SnapshotStatusAvailable && rec.Status != SnapshotStatusFailed {
 		return awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
-			"DB snapshot %s is %s; it must be %s to be deleted", id, rec.Status, SnapshotStatusAvailable)
+			"DB snapshot %s is %s; it must be %s or %s to be deleted", id, rec.Status, SnapshotStatusAvailable, SnapshotStatusFailed)
 	}
 
 	if err := s.deleteEC2Snapshot(ctx, kv, accountID, rec); err != nil {
@@ -347,8 +356,8 @@ func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, acco
 	}
 	// Last: while it exists the snapshot is still nameable, and a retry re-runs
 	// the steps above, each of which tolerates work it has already done.
-	if err := kv.Delete(ctx, DBSnapshotKey(id)); err != nil {
-		return fmt.Errorf("rds: delete the DB snapshot record for %s: %w", id, err)
+	if err := deleteDBSnapshotRecord(ctx, kv, rec, rev); err != nil {
+		return err
 	}
 
 	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, id, "DB snapshot deleted.", EventCategoryDeletion)
@@ -356,6 +365,35 @@ func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, acco
 	return nil
 }
 
+// Deletes the record read at rev, or the same snapshot rewritten since, such as
+// by a tag change. A record of the same name for another snapshot, such as a
+// retaken final snapshot's reservation, is left alone.
+func deleteDBSnapshotRecord(ctx context.Context, kv *kvstore.Bucket, rec *DBSnapshotRecord, rev uint64) error {
+	id := rec.DBSnapshotIdentifier
+	for range tagWriteAttempts {
+		err := kv.CompareAndDelete(ctx, DBSnapshotKey(id), rev)
+		if !errors.Is(err, kvstore.ErrConflict) {
+			if err != nil {
+				return fmt.Errorf("rds: delete the DB snapshot record for %s: %w", id, err)
+			}
+			return nil
+		}
+		var current DBSnapshotRecord
+		var found bool
+		if rev, found, err = getJSONRevision(ctx, kv, DBSnapshotKey(id), &current); err != nil {
+			return err
+		}
+		if !found {
+			return nil
+		}
+		if current.SnapshotID != rec.SnapshotID || !current.CreatedAt.Equal(rec.CreatedAt) {
+			return awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
+				"DB snapshot %s was replaced while it was being deleted", id)
+		}
+	}
+	return fmt.Errorf("rds: deleting the DB snapshot record for %s contended after %d attempts", id, tagWriteAttempts)
+}
+
 // A snapshot with a volume still reading through it cannot go, which is exactly
 // the case where an instance restored from it is still running. That is
 // translated into an RDS fault naming the instance rather than surfaced as the

spinifex/domains/sqs/storevolume/prepare.go

@@ -48,9 +48,10 @@ type Preparer struct {
 	Provider   Publisher
 	NodeID     string
 	ScratchDir string
-	// WriteImage copies a raw image onto an NBD export. The composition root
-	// supplies it, since a domain does not import the runtime copy helper.
-	WriteImage func(ctx context.Context, sourcePath, nbdURI string, progress io.Writer) error
+	// WriteImage copies a raw image onto the published export, refusing a kind
+	// it cannot write. The composition root supplies it, since a domain does
+	// not import the runtime copy helper.
+	WriteImage func(ctx context.Context, sourcePath string, export ebsprovider.VolumeExport, progress io.Writer) error
 
 	lookPath  func(string) (string, error)
 	runMkfs   func(ctx context.Context, mkfs, path string) error
@@ -163,8 +164,9 @@ func (e *prepError) Error() string { return e.err.Error() }
 // fill publishes the volume, copies the image and unpublishes it. Its error
 // says whether the export is known to be closed.
 func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
+	// A block export has no version 1 form, so publish as the VM path does.
 	published, err := p.Provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
+		Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
 	})
 	if err != nil {
 		// These refusals may mean another export exists, which an unpublish
@@ -181,10 +183,10 @@ func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
 	}
 
 	var writeErr error
-	if published == nil || published.Export.Kind != ebsprovider.ExportKindNBD || published.Export.NBD == nil {
-		writeErr = errors.New("volume was not published as an nbd export")
+	if published == nil {
+		writeErr = errors.New("publish returned no export")
 	} else {
-		writeErr = p.WriteImage(ctx, image, published.Export.NBD.NBDURI, nil)
+		writeErr = p.WriteImage(ctx, image, published.Export, nil)
 	}
 
 	if err := p.unpublish(ctx, volumeID); err != nil {

spinifex/domains/sqs/storevolume/prepare_test.go

@@ -1,6 +1,6 @@
 package storevolume
 
-//test:in-package — the host tools, mkfs and NBD copy are swapped for fakes
+//test:in-package — the host tools, mkfs and image copy are swapped for fakes
 // through unexported fields so no test formats or writes a real device.
 
 import (
@@ -32,6 +32,7 @@ const (
 	testVolume  = "vol-0123456789abcdef0"
 	testNode    = "node-a"
 	testURI     = "nbd://127.0.0.1:10809"
+	testDevice  = "/dev/sdx"
 )
 
 // harness records every call, in order, across the fakes.
@@ -99,8 +100,14 @@ func (h *harness) PublishVolume(_ context.Context, req ebsprovider.PublishVolume
 		kind = ebsprovider.ExportKindNBD
 	}
 	export := ebsprovider.VolumeExport{Kind: kind}
-	if kind == ebsprovider.ExportKindNBD {
+	switch kind {
+	case ebsprovider.ExportKindNBD:
 		export.NBD = &ebsprovider.NBDExport{NBDURI: testURI}
+	case ebsprovider.ExportKindBlock:
+		if req.SchemaVersion < ebsprovider.DescriptorSchemaVersion {
+			return nil, fmt.Errorf("%w: a block export has no version %d form", ebsprovider.ErrUnsupportedVersion, req.SchemaVersion)
+		}
+		export.Block = &ebsprovider.BlockExport{DevicePath: testDevice, SizeBytes: 1 << 30}
 	}
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }
@@ -133,8 +140,15 @@ func (h *harness) preparer(t *testing.T) *Preparer {
 			h.record("mkfs " + filepath.Base(path))
 			return h.mkfsErr
 		},
-		WriteImage: func(_ context.Context, src, uri string, _ io.Writer) error {
-			h.record("write " + uri)
+		WriteImage: func(_ context.Context, src string, export ebsprovider.VolumeExport, _ io.Writer) error {
+			switch {
+			case export.Kind == ebsprovider.ExportKindNBD && export.NBD != nil:
+				h.record("write " + export.NBD.NBDURI)
+			case export.Kind == ebsprovider.ExportKindBlock && export.Block != nil:
+				h.record("write " + export.Block.DevicePath)
+			default:
+				return fmt.Errorf("no writer for a %q export", export.Kind)
+			}
 			h.writtenImage = src
 			return h.writeErr
 		},
@@ -265,12 +279,22 @@ func TestPrepare_FailedCopyUnpublishesThenDiscards(t *testing.T) {
 	}, h.calls)
 }
 
-func TestPrepare_NonNBDExportIsNotWrittenAndIsDiscarded(t *testing.T) {
-	h := &harness{exportKind: "iscsi"}
-	_, err := prepare(t, h)
-	require.ErrorContains(t, err, "not published as an nbd export")
-	assert.NotContains(t, h.calls, "write "+testURI)
-	assert.Equal(t, "discard "+testVolume, h.calls[len(h.calls)-1])
+// A block export has no version 1 form, so the block path works only because
+// preparation publishes at the latest version.
+func TestPrepare_BlockExportIsWrittenAtItsDevicePathAndReleased(t *testing.T) {
+	h := &harness{exportKind: ebsprovider.ExportKindBlock}
+	id, err := prepare(t, h)
+	require.NoError(t, err)
+	assert.Equal(t, testVolume, id)
+	assert.Equal(t, []string{
+		"mkfs ext4.raw",
+		"create 000000000000 1GiB",
+		"publish " + testVolume + " on " + testNode,
+		"write " + testDevice,
+		"unpublish " + testVolume + " on " + testNode,
+		"complete " + testVolume,
+	}, h.calls)
+	assert.Equal(t, "available", h.state)
 }
 
 // An in-use or already-exists refusal may mean another export of the volume
@@ -359,7 +383,7 @@ func TestPrepare_StuckUnpublishIsBounded(t *testing.T) {
 	p := h.preparer(t)
 	p.cleanupTimeout = 50 * time.Millisecond
 	ctx, cancel := context.WithCancel(context.Background())
-	p.WriteImage = func(context.Context, string, string, io.Writer) error {
+	p.WriteImage = func(context.Context, string, ebsprovider.VolumeExport, io.Writer) error {
 		cancel()
 		return context.Canceled
 	}
@@ -391,7 +415,7 @@ func TestPrepare_CancelledCopyStillUnpublishes(t *testing.T) {
 	h := &harness{}
 	ctx, cancel := context.WithCancel(context.Background())
 	p := h.preparer(t)
-	p.WriteImage = func(context.Context, string, string, io.Writer) error {
+	p.WriteImage = func(context.Context, string, ebsprovider.VolumeExport, io.Writer) error {
 		cancel()
 		return context.Canceled
 	}

spinifex/ingress/aws/pipeline/lambda_invoke_wire_test.go

@@ -59,14 +59,6 @@ func (activeMeta) UpdateConfiguration(context.Context, string, string, function.
 	return function.Record{}, 0, errors.New("unused")
 }
 
-func (activeMeta) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
-	return nil, "", errors.New("unused")
-}
-
-func (activeMeta) Delete(context.Context, string, string, function.Record, uint64) error {
-	return errors.New("unused")
-}
-
 // countingExec records how many invocations reached it.
 type countingExec struct{ calls atomic.Int32 }
 

spinifex/operator/admin/admin.go

@@ -122,6 +122,18 @@ type ConfigSettings struct {
 	BootstrapCidr       string
 	BootstrapSubnetCidr string
 
+	// EBSProvider is rendered as [nodes.X.ebs] provider; empty renders the
+	// default. The OCI fields fill [nodes.X.ebs.oci] for ocibsd.
+	EBSProvider           string
+	OCICompartmentID      string
+	OCIAvailabilityDomain string
+	OCIClusterID          string
+	// OCIAuth, OCIConfigFile and OCIConfigProfile are this node's credential,
+	// rendered only when set.
+	OCIAuth          string
+	OCIConfigFile    string
+	OCIConfigProfile string
+
 	// GPUPassthrough enables VFIO GPU passthrough in the daemon config.
 	// Sets gpu_passthrough = true under [nodes.<node>.daemon].
 	GPUPassthrough bool

spinifex/operator/admin/image_import.go

@@ -5,9 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/imagewrite"
 )
 
 // ImportOpts describes one raw disk image to import into a new volume.
@@ -15,7 +16,7 @@ type ImportOpts struct {
 	// VolumeID names the volume to create. For an AMI import this is the
 	// image ID, so the snapshot lands at SnapPrefix(imageID).
 	VolumeID string
-	// NodeID is the node whose provider serves the NBD export written to. It
+	// NodeID is the node whose provider serves the export written to. It
 	// must be the node this process runs on: the export is local to it.
 	NodeID string
 	// SizeBytes is the volume's capacity, which the caller rounds to a whole
@@ -31,10 +32,17 @@ type ImportOpts struct {
 	// Progress receives the importer's own output (a percentage counter).
 	// Nil discards it.
 	Progress io.Writer
+	// SnapshotTimeout bounds the wait for a snapshot the provider answers
+	// pending to complete. Zero leaves only ctx to bound it.
+	SnapshotTimeout time.Duration
 }
 
-// ImportImage writes a local raw disk image into a new provider volume over
-// NBD, then optionally snapshots it. Everything here goes through the provider
+// snapshotPollInterval is how often a pending snapshot is read again; a test
+// shortens it so the wait does not sleep.
+var snapshotPollInterval = 5 * time.Second
+
+// ImportImage writes a local raw disk image into a new provider volume through
+// the export the provider publishes, then optionally snapshots it. Everything here goes through the provider
 // contract, so an import works against any conforming provider rather than
 // only the one whose storage library this process happens to link.
 //
@@ -58,8 +66,9 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 		return fmt.Errorf("import: create volume %s: %w", opts.VolumeID, err)
 	}
 
+	// A block export has no version 1 form, so publish as the VM path does.
 	published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned: ebsprovider.NewLatestVersioned(),
 		VolumeID:  opts.VolumeID,
 		NodeID:    opts.NodeID,
 	})
@@ -67,12 +76,7 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 		return fmt.Errorf("import: publish volume %s: %w", opts.VolumeID, err)
 	}
 
-	var writeErr error
-	if published.Export.Kind == ebsprovider.ExportKindNBD && published.Export.NBD != nil {
-		writeErr = writeImage(ctx, opts.SourcePath, published.Export.NBD.NBDURI, opts.Progress)
-	} else {
-		writeErr = fmt.Errorf("volume is published as a %q export; image import writes only through nbd", published.Export.Kind)
-	}
+	writeErr := writeImage(ctx, opts.SourcePath, published.Export, opts.Progress)
 
 	// Unpublish regardless: a volume left published cannot be snapshotted or
 	// attached, and on the failure path it would strand the export too.
@@ -91,18 +95,63 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 	}
 
 	if opts.Snapshot {
-		if _, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+		snapshotID := SnapPrefix(opts.VolumeID)
+		created, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
 			Versioned:  ebsprovider.NewVersioned(),
-			SnapshotID: SnapPrefix(opts.VolumeID),
+			SnapshotID: snapshotID,
 			VolumeID:   opts.VolumeID,
-		}); err != nil {
+		})
+		if err != nil {
 			return fmt.Errorf("import: snapshot volume %s: %w", opts.VolumeID, err)
 		}
+		if created != nil && created.State == ebsprovider.SnapshotStatePending {
+			if err := waitForSnapshot(ctx, provider, snapshotID, opts.SnapshotTimeout); err != nil {
+				return fmt.Errorf("import: snapshot volume %s: %w", opts.VolumeID, err)
+			}
+		}
 	}
 
 	return nil
 }
 
+// waitForSnapshot reads a pending snapshot every snapshotPollInterval until it
+// is completed, failing when it errors, is gone, or timeout passes. A read that
+// fails is retried, and reported if the timeout passes first.
+func waitForSnapshot(ctx context.Context, provider ebsprovider.EBSProvider, snapshotID string, timeout time.Duration) error {
+	if timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
+	ticker := time.NewTicker(snapshotPollInterval)
+	defer ticker.Stop()
+	var lastErr error
+	for {
+		select {
+		case <-ctx.Done():
+			if lastErr != nil {
+				return fmt.Errorf("snapshot %s did not complete: %w (last read failed: %w)", snapshotID, ctx.Err(), lastErr)
+			}
+			return fmt.Errorf("snapshot %s did not complete: %w", snapshotID, ctx.Err())
+		case <-ticker.C:
+		}
+		ref, found, err := ebsprovider.ReadSnapshot(ctx, provider, snapshotID)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if !found {
+			return fmt.Errorf("snapshot %s is no longer held by the provider", snapshotID)
+		}
+		switch ref.State {
+		case ebsprovider.SnapshotStateCompleted:
+			return nil
+		case ebsprovider.SnapshotStateError:
+			return fmt.Errorf("snapshot %s ended in error", snapshotID)
+		}
+	}
+}
+
 // writeImage is the bulk copy step, indirected so a test can drive the verb
-// sequence around it without a live NBD export to write to.
-var writeImage = nbd.WriteRawImage
+// sequence around it without a live export to write to.
+var writeImage = imagewrite.Writer{}.Write

spinifex/operator/admin/image_import_test.go

@@ -3,10 +3,15 @@ package admin
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/imagewrite"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,6 +30,21 @@ type recordingProvider struct {
 	// publishExport, when its Kind is set, replaces the nbd export built
 	// from publishURI.
 	publishExport ebsprovider.VolumeExport
+
+	// onList runs before read n (from 1) of the snapshot; a non-nil error
+	// fails that read.
+	onList func(n int) error
+	lists  int
+}
+
+func (p *recordingProvider) ListSnapshots(ctx context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
+	p.lists++
+	if p.onList != nil {
+		if err := p.onList(p.lists); err != nil {
+			return nil, err
+		}
+	}
+	return p.MemoryProvider.ListSnapshots(ctx, req)
 }
 
 func newRecordingProvider() *recordingProvider {
@@ -39,6 +59,9 @@ func (p *recordingProvider) CreateVolume(ctx context.Context, req ebsprovider.Cr
 func (p *recordingProvider) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
 	p.calls = append(p.calls, "publish")
 	p.publishes = append(p.publishes, req)
+	if p.publishExport.Kind == ebsprovider.ExportKindBlock && req.SchemaVersion < ebsprovider.DescriptorSchemaVersion {
+		return nil, fmt.Errorf("%w: a block export has no version %d form", ebsprovider.ErrUnsupportedVersion, req.SchemaVersion)
+	}
 	if _, err := p.MemoryProvider.PublishVolume(ctx, req); err != nil {
 		return nil, err
 	}
@@ -65,9 +88,9 @@ func (p *recordingProvider) CreateSnapshot(ctx context.Context, req ebsprovider.
 func stubWriteImage(t *testing.T, err error, seen *string) {
 	t.Helper()
 	original := writeImage
-	writeImage = func(_ context.Context, _, nbdURI string, _ io.Writer) error {
-		if seen != nil {
-			*seen = nbdURI
+	writeImage = func(_ context.Context, _ string, export ebsprovider.VolumeExport, _ io.Writer) error {
+		if seen != nil && export.NBD != nil {
+			*seen = export.NBD.NBDURI
 		}
 		return err
 	}
@@ -124,30 +147,54 @@ func TestImportImage_WriteFailureStillUnpublishes(t *testing.T) {
 		"a failed copy must not be snapshotted, and must still be unpublished")
 }
 
-// TestImportImage_BlockExportIsRefused covers a provider that publishes a host
-// block device: the import writes only through NBD, so it must fail without
-// copying and still release the export.
-func TestImportImage_BlockExportIsRefused(t *testing.T) {
-	var wroteTo string
-	stubWriteImage(t, nil, &wroteTo)
+// blockImport sets up an import against a provider publishing a host block
+// device, with the shared writer running a fake qemu-img.
+func blockImport(t *testing.T, imageBytes int64, deviceBytes uint64) (*recordingProvider, ImportOpts, *[][]string) {
+	t.Helper()
 	provider := newRecordingProvider()
 	provider.publishExport = ebsprovider.VolumeExport{
 		Kind:  ebsprovider.ExportKindBlock,
-		Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: 1 << 30},
+		Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: deviceBytes},
 	}
+	image := filepath.Join(t.TempDir(), "image.raw")
+	require.NoError(t, os.WriteFile(image, nil, 0o600))
+	require.NoError(t, os.Truncate(image, imageBytes))
 
-	err := ImportImage(context.Background(), provider, ImportOpts{
-		VolumeID:   "ami-import005",
-		NodeID:     "node-a",
-		SizeBytes:  1 << 30,
-		SourcePath: "/dev/null",
-		Snapshot:   true,
-	})
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), `"block" export`)
-	assert.Empty(t, wroteTo, "a block export must not reach the NBD writer")
+	var ran [][]string
+	original := writeImage
+	writeImage = imagewrite.Writer{QemuImg: func(_ context.Context, _ io.Writer, args ...string) error {
+		ran = append(ran, args)
+		return nil
+	}}.Write
+	t.Cleanup(func() { writeImage = original })
+	return provider, ImportOpts{VolumeID: "ami-import005", NodeID: "node-a", SizeBytes: 1 << 30, SourcePath: image, Snapshot: true}, &ran
+}
+
+// A block export has no version 1 form, so only a publish at the latest
+// version reaches the device at all.
+func TestImportImage_BlockExportIsWrittenAtItsDevicePath(t *testing.T) {
+	provider, opts, ran := blockImport(t, 1<<20, 1<<30)
+
+	require.NoError(t, ImportImage(context.Background(), provider, opts))
+
+	require.Len(t, provider.publishes, 1)
+	assert.Equal(t, ebsprovider.SchemaVersion, provider.publishes[0].SchemaVersion)
+	require.Len(t, *ran, 1)
+	assert.Equal(t, "/dev/sdx", (*ran)[0][len((*ran)[0])-1], "the copy must target the device the provider published")
+	assert.Equal(t, []string{"create", "publish", "unpublish", "snapshot"}, provider.calls)
+	vol, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: opts.VolumeID})
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.VolumeStateAvailable, vol.State)
+}
+
+func TestImportImage_ImageLargerThanTheBlockExportIsRefusedAndUnpublished(t *testing.T) {
+	provider, opts, ran := blockImport(t, 1<<20+1, 1<<20)
+
+	err := ImportImage(context.Background(), provider, opts)
+	require.ErrorContains(t, err, "larger than")
+	assert.Empty(t, *ran, "nothing may be written to a device too small for the image")
 	assert.Equal(t, []string{"create", "publish", "unpublish"}, provider.calls,
-		"a refused export must not be snapshotted, and must still be unpublished")
+		"a refused write must not be snapshotted, and must still be unpublished")
 }
 
 // TestImportImage_NoSnapshotWhenNotAsked covers the weights-style import that
@@ -184,3 +231,87 @@ func TestImportImage_RejectsIncompleteOpts(t *testing.T) {
 		})
 	}
 }
+
+// A snapshot the provider answers pending is read until it completes, and the
+// import fails if it errors, disappears, or outlasts SnapshotTimeout.
+func TestImportImage_WaitsForAPendingSnapshot(t *testing.T) {
+	original := snapshotPollInterval
+	snapshotPollInterval = time.Millisecond
+	t.Cleanup(func() { snapshotPollInterval = original })
+	snapshotID := SnapPrefix("ami-import001")
+
+	cases := []struct {
+		name      string
+		pending   bool
+		onList    func(p *recordingProvider) func(n int) error
+		wantErr   string // empty: the import succeeds
+		wantLists int    // -1: at least one
+	}{
+		{name: "answered completed is not read", wantLists: 0},
+		{name: "completes on the second read", pending: true, wantLists: 2,
+			onList: func(p *recordingProvider) func(int) error {
+				return func(n int) error {
+					if n == 2 {
+						return p.SettleSnapshot(snapshotID, ebsprovider.SnapshotStateCompleted)
+					}
+					return nil
+				}
+			}},
+		{name: "a failed read is retried", pending: true, wantLists: 2,
+			onList: func(p *recordingProvider) func(int) error {
+				return func(n int) error {
+					if n == 1 {
+						return ebsprovider.ErrUnavailable
+					}
+					return p.SettleSnapshot(snapshotID, ebsprovider.SnapshotStateCompleted)
+				}
+			}},
+		{name: "ends in error", pending: true, wantErr: "ended in error", wantLists: 1,
+			onList: func(p *recordingProvider) func(int) error {
+				return func(int) error { return p.SettleSnapshot(snapshotID, ebsprovider.SnapshotStateError) }
+			}},
+		{name: "disappears", pending: true, wantErr: "no longer held", wantLists: 1,
+			onList: func(p *recordingProvider) func(int) error {
+				return func(int) error {
+					return p.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{
+						Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID,
+					})
+				}
+			}},
+		{name: "outlasts the timeout", pending: true, wantErr: context.DeadlineExceeded.Error(), wantLists: -1},
+		{name: "outlasts the timeout while reads fail", pending: true, wantErr: "last read failed", wantLists: -1,
+			onList: func(*recordingProvider) func(int) error {
+				return func(int) error { return ebsprovider.ErrUnavailable }
+			}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			stubWriteImage(t, nil, nil)
+			provider := newRecordingProvider()
+			provider.AnswerSnapshotsPending(tc.pending)
+			if tc.onList != nil {
+				provider.onList = tc.onList(provider)
+			}
+
+			err := ImportImage(context.Background(), provider, ImportOpts{
+				VolumeID: "ami-import001", NodeID: "node-a", SizeBytes: 1 << 30,
+				SourcePath: "/tmp/image.raw", Snapshot: true, SnapshotTimeout: 50 * time.Millisecond,
+			})
+
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				snap, ok := provider.Snapshot(snapshotID)
+				require.True(t, ok)
+				assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
+			} else {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tc.wantErr)
+			}
+			if tc.wantLists < 0 {
+				assert.Positive(t, provider.lists)
+			} else {
+				assert.Equal(t, tc.wantLists, provider.lists)
+			}
+		})
+	}
+}

spinifex/operator/cli/admin.go

@@ -480,8 +480,9 @@ func init() {
 
 const bytesPerGiB = 1024 * 1024 * 1024
 
-// imageImportTimeout bounds each provider call the import makes. Generous
-// because the snapshot at the end runs over the whole imported volume.
+// imageImportTimeout bounds each provider call the import makes, and the wait
+// for a snapshot the provider answers pending. Generous because the snapshot
+// at the end runs over the whole imported volume.
 const imageImportTimeout = 30 * time.Minute
 
 // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes.
@@ -895,6 +896,7 @@ func registerExtractedImage(image imagecatalog.Images, amiNameOverride, imagePat
 		SourcePath:       extractedImagePath,
 		Snapshot:         true,
 		Progress:         os.Stdout,
+		SnapshotTimeout:  imageImportTimeout,
 	})
 	if err != nil {
 		return "", fmt.Errorf("import image: %w", err)
@@ -1658,6 +1660,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(0)
 	}
 
+	// Resolved once and written to every node (ADR-0018:S3), before anything
+	// is written, so a refused OCI credential leaves the node untouched.
+	ebsChoice, err := resolveInitEBS(context.Background(), spinifexTomlPath, node, productionEBSResolver())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+
 	if spxRoot == "" {
 		spxRoot = DefaultDataDir()
 	}
@@ -1846,6 +1856,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		// external networking is disabled.
 		networkConfig := &formation.NetworkConfig{
 			IPSecEnabled: ipsecEnabled,
+			EBS:          formationEBS(ebsChoice.EBSConfig),
 		}
 		if externalMode != "" {
 			bootstrapVpcId := awsidentifiers.GenerateResourceID("vpc")
@@ -1865,7 +1876,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey,
 			masterKey, viperblockKey, natsToken, clusterName,
 			configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email,
-			port, nodes, formationTimeoutStr, tokenTTLStr, services, nameserver, networkConfig, northstarCreds)
+			port, nodes, formationTimeoutStr, tokenTTLStr, services, nameserver, networkConfig, northstarCreds, ebsChoice)
 		return
 	}
 
@@ -2027,6 +2038,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}
 
+	ebsChoice.apply(&configSettings)
+
 	// Generate config files
 	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, predastoreNodesStr != ""); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
@@ -2034,6 +2047,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 
 	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)
+	exitOnActivationError(activateEBSProvider(context.Background(), ebsChoice, productionOCIBSDActivation()))
 
 	skipHostDNS, _ := cmd.Flags().GetBool("skip-host-dns")
 	configureHostDNS(configSettings, skipHostDNS)
@@ -2075,7 +2089,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	masterKey, viperblockKey []byte, natsToken, clusterName,
 	configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email string,
 	port, expectedNodes int, formationTimeoutStr, tokenTTLStr string, services []string, nameserver bool, networkConfig *formation.NetworkConfig,
-	northstarCreds admin.NorthstarCredentials) {
+	northstarCreds admin.NorthstarCredentials, ebsChoice nodeEBS) {
 	formationTimeout, err := time.ParseDuration(formationTimeoutStr)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err)
@@ -2302,6 +2316,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	if networkConfig != nil {
 		applyNetworkConfig(&configSettings, networkConfig)
 	}
+	ebsChoice.apply(&configSettings)
 
 	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
@@ -2309,6 +2324,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	}
 
 	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)
+	exitOnActivationError(activateEBSProvider(context.Background(), ebsChoice, productionOCIBSDActivation()))
 
 	skipHostDNS, _ := cmd.Flags().GetBool("skip-host-dns")
 	configureHostDNS(configSettings, skipHostDNS)
@@ -2883,6 +2899,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	if statusResp.NetworkConfig != nil {
 		applyNetworkConfig(&configSettings, statusResp.NetworkConfig)
 	}
+	// The provider is the cluster's, from formation; the credential is this
+	// node's, and is checked before its config is written.
+	ebsChoice, err := resolveJoinEBS(context.Background(), spinifexTomlPath, node, statusResp.NetworkConfig, productionEBSResolver())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+	ebsChoice.apply(&configSettings)
 
 	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
@@ -2890,6 +2914,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 
 	finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP)
+	exitOnActivationError(activateEBSProvider(context.Background(), ebsChoice, productionOCIBSDActivation()))
 
 	skipHostDNS, _ := cmd.Flags().GetBool("skip-host-dns")
 	configureHostDNS(configSettings, skipHostDNS)

spinifex/operator/cli/admin_ebs.go

@@ -0,0 +1,405 @@
+package cli
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	toml "github.com/pelletier/go-toml/v2"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/operator/admin"
+	"github.com/mulgadc/spinifex/spinifex/operator/host/systemd"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	ebsocibsd "github.com/mulgadc/spinifex/spinifex/providers/ebs/ocibsd"
+	"github.com/mulgadc/spinifex/spinifex/runtime/formation"
+	ocibsdrole "github.com/mulgadc/spinifex/spinifex/runtime/roles/ocibsd"
+)
+
+// ebsProviderEnv names the provider for an install with no existing config,
+// in place of detection.
+const ebsProviderEnv = "SPINIFEX_EBS_PROVIDER"
+
+const (
+	// imdsDetectTimeout bounds the metadata read detection makes, so an
+	// install off OCI pays at most this for it.
+	imdsDetectTimeout = time.Second
+	// credentialCheckTimeout bounds the OCI read the credential check makes.
+	credentialCheckTimeout = 30 * time.Second
+)
+
+// Where a node's provider came from, printed by init and join so a run can
+// tell a detected provider from a configured one.
+const (
+	ebsFromConfig    = "from the existing config"
+	ebsFromEnv       = "from " + ebsProviderEnv
+	ebsFromDetection = "detected"
+	ebsFromFormation = "from formation"
+)
+
+// nodeEBS is one node's EBS choice: the cluster-wide part formation carries
+// to joining nodes, and this node's own OCI credential.
+type nodeEBS struct {
+	formation.EBSConfig
+
+	Auth          string
+	ConfigFile    string
+	ConfigProfile string
+	// Source is one of the ebsFrom* values; Detail says what detection saw.
+	Source string
+	Detail string
+}
+
+// describe is the line init and join print for the choice.
+func (e nodeEBS) describe() string {
+	provider := (config.EBSConfig{Provider: e.Provider}).ResolvedProvider()
+	if e.Detail != "" {
+		return fmt.Sprintf("EBS provider: %s (%s: %s)", provider, e.Source, e.Detail)
+	}
+	return fmt.Sprintf("EBS provider: %s (%s)", provider, e.Source)
+}
+
+// apply copies the choice into the settings spinifex.toml is rendered from.
+func (e nodeEBS) apply(s *admin.ConfigSettings) {
+	s.EBSProvider = e.Provider
+	s.OCICompartmentID = e.OCICompartmentID
+	s.OCIAvailabilityDomain = e.OCIAvailabilityDomain
+	s.OCIClusterID = e.ClusterID
+	s.OCIAuth = e.Auth
+	s.OCIConfigFile = e.ConfigFile
+	s.OCIConfigProfile = e.ConfigProfile
+}
+
+// ebsResolver holds the host and cloud seams provider resolution reads.
+type ebsResolver struct {
+	getenv   func(string) string
+	instance func(ctx context.Context) (oci.InstanceMetadata, error)
+	// newClient builds the OCI client the credential check reads through.
+	newClient func(auth, configFile, configProfile string) (oci.BlockClient, error)
+}
+
+func productionEBSResolver() ebsResolver {
+	md := &oci.MetadataClient{BaseURL: oci.IMDSBaseURL, HTTP: &http.Client{Timeout: imdsDetectTimeout}}
+	return ebsResolver{
+		getenv:   os.Getenv,
+		instance: md.Instance,
+		newClient: func(auth, configFile, configProfile string) (oci.BlockClient, error) {
+			return ocibsdrole.BlockClientFor(&ocibsdrole.Config{Auth: auth, ConfigFile: configFile, ConfigProfile: configProfile})
+		},
+	}
+}
+
+// existingEBS is a node's [nodes.<node>.ebs] in a spinifex.toml already on
+// disk, kept as written: an empty Provider is the default, viperblockd.
+type existingEBS struct {
+	Provider string `toml:"provider"`
+	OCI      struct {
+		CompartmentID      string `toml:"compartment_id"`
+		AvailabilityDomain string `toml:"availability_domain"`
+		ClusterID          string `toml:"cluster_id"`
+		Auth               string `toml:"auth"`
+		ConfigFile         string `toml:"config_file"`
+		ConfigProfile      string `toml:"config_profile"`
+	} `toml:"oci"`
+}
+
+// readExistingEBS returns node's EBS table from tomlPath, nil when there is
+// no file or it has no [nodes.<node>]. A file that cannot be read is an
+// error: the provider it names must never be guessed.
+func readExistingEBS(tomlPath, node string) (*existingEBS, error) {
+	raw, err := os.ReadFile(tomlPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read the existing config %s: %w", tomlPath, err)
+	}
+	var cfg struct {
+		Nodes map[string]struct {
+			EBS existingEBS `toml:"ebs"`
+		} `toml:"nodes"`
+	}
+	if err := toml.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("parse the existing config %s, whose EBS provider this node keeps: %w", tomlPath, err)
+	}
+	n, ok := cfg.Nodes[node]
+	if !ok {
+		return nil, nil
+	}
+	return &n.EBS, nil
+}
+
+// isOCIVM reports whether md is an OCI virtual machine; bare-metal shapes
+// stay on viperblockd until a bare-metal iSCSI run is proven.
+func isOCIVM(md oci.InstanceMetadata) bool {
+	return md.Shape != "" && !strings.HasPrefix(md.Shape, "BM.")
+}
+
+// detect reads the OCI instance metadata within imdsDetectTimeout; ok is
+// false when nothing answers, which is every host off OCI.
+func (r ebsResolver) detect(ctx context.Context) (oci.InstanceMetadata, bool) {
+	ctx, cancel := context.WithTimeout(ctx, imdsDetectTimeout)
+	defer cancel()
+	md, err := r.instance(ctx)
+	if err != nil {
+		return oci.InstanceMetadata{}, false
+	}
+	return md, true
+}
+
+// resolveProvider picks init's provider: an existing node config's, then
+// ebsProviderEnv, then detection. md is set when detection read it.
+func (r ebsResolver) resolveProvider(ctx context.Context, existing *existingEBS) (nodeEBS, *oci.InstanceMetadata) {
+	switch {
+	case existing != nil:
+		return nodeEBS{
+			EBSConfig: formation.EBSConfig{Provider: (config.EBSConfig{Provider: existing.Provider}).ResolvedProvider()},
+			Source:    ebsFromConfig,
+		}, nil
+	case r.getenv(ebsProviderEnv) != "":
+		return nodeEBS{EBSConfig: formation.EBSConfig{Provider: r.getenv(ebsProviderEnv)}, Source: ebsFromEnv}, nil
+	}
+	md, ok := r.detect(ctx)
+	switch {
+	case !ok:
+		return nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderViperblockd}, Source: ebsFromDetection, Detail: "no OCI instance metadata"}, nil
+	case !isOCIVM(md):
+		return nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderViperblockd}, Source: ebsFromDetection, Detail: "OCI bare metal " + md.Shape}, nil
+	}
+	return nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderOCIBSD}, Source: ebsFromDetection, Detail: "OCI virtual machine " + md.Shape}, &md
+}
+
+// fillOCIBSD completes an ocibsd choice: the existing config's values win,
+// then the instance metadata, and auth defaults to instance_principal.
+// md is read here when the caller has none and a value is missing.
+func (r ebsResolver) fillOCIBSD(ctx context.Context, e *nodeEBS, existing *existingEBS, md *oci.InstanceMetadata) error {
+	var prior existingEBS
+	if existing != nil {
+		prior = *existing
+	}
+	if e.OCICompartmentID == "" {
+		e.OCICompartmentID = prior.OCI.CompartmentID
+	}
+	if e.OCIAvailabilityDomain == "" {
+		e.OCIAvailabilityDomain = prior.OCI.AvailabilityDomain
+	}
+	if e.OCICompartmentID == "" || e.OCIAvailabilityDomain == "" {
+		if md == nil {
+			got, ok := r.detect(ctx)
+			if !ok {
+				return fmt.Errorf("EBS provider %s (%s) needs this node's OCI instance metadata for its compartment and availability domain, and none answered", config.EBSProviderOCIBSD, e.Source)
+			}
+			md = &got
+		}
+		if e.OCICompartmentID == "" {
+			e.OCICompartmentID = md.CompartmentID
+		}
+		if e.OCIAvailabilityDomain == "" {
+			e.OCIAvailabilityDomain = md.AvailabilityDomain
+		}
+	}
+	e.Auth, e.ConfigFile, e.ConfigProfile = prior.OCI.Auth, prior.OCI.ConfigFile, prior.OCI.ConfigProfile
+	if e.Auth == "" && e.ConfigFile == "" && e.ConfigProfile == "" {
+		e.Auth = config.OCIAuthInstancePrincipal
+	}
+	if e.ClusterID == "" {
+		e.ClusterID = prior.OCI.ClusterID
+	}
+	if e.ClusterID == "" {
+		id, err := generateClusterID()
+		if err != nil {
+			return err
+		}
+		e.ClusterID = id
+	}
+	return nil
+}
+
+// checkCredential makes the read ocibsd makes at start with this node's
+// credential, and never falls back to another provider when it is refused.
+func (r ebsResolver) checkCredential(ctx context.Context, e nodeEBS) error {
+	client, err := r.newClient(e.Auth, e.ConfigFile, e.ConfigProfile)
+	if err != nil {
+		return fmt.Errorf("ocibsd: build the OCI client for auth %q: %w", e.Auth, err)
+	}
+	ctx, cancel := context.WithTimeout(ctx, credentialCheckTimeout)
+	defer cancel()
+	return ebsocibsd.CheckCredential(ctx, client, e.OCICompartmentID, e.OCIAvailabilityDomain, e.ClusterID)
+}
+
+// resolveInitEBS resolves init's provider once, for every node (ADR-0018:S3),
+// and for ocibsd completes its table and checks the credential.
+func resolveInitEBS(ctx context.Context, tomlPath, node string, r ebsResolver) (nodeEBS, error) {
+	existing, err := readExistingEBS(tomlPath, node)
+	if err != nil {
+		return nodeEBS{}, err
+	}
+	e, md := r.resolveProvider(ctx, existing)
+	if !config.SupportedEBSProvider(e.Provider) {
+		return nodeEBS{}, fmt.Errorf("EBS provider %q (%s) is unsupported; use %q, %q or %q", e.Provider, e.Source, config.EBSProviderViperblockd, config.EBSProviderQEMUNBD, config.EBSProviderOCIBSD)
+	}
+	fmt.Println(e.describe())
+	if e.Provider != config.EBSProviderOCIBSD {
+		return e, nil
+	}
+	if err := r.fillOCIBSD(ctx, &e, existing, md); err != nil {
+		return nodeEBS{}, err
+	}
+	if err := r.checkCredential(ctx, e); err != nil {
+		return nodeEBS{}, err
+	}
+	return e, nil
+}
+
+// resolveJoinEBS takes the provider from formation; for ocibsd this node's
+// own credential comes from its existing config, else instance_principal,
+// and is checked before its config is written.
+func resolveJoinEBS(ctx context.Context, tomlPath, node string, nc *formation.NetworkConfig, r ebsResolver) (nodeEBS, error) {
+	e := nodeEBS{Source: ebsFromFormation}
+	if nc != nil && nc.EBS != nil {
+		e.EBSConfig = *nc.EBS
+	}
+	fmt.Println(e.describe())
+	if e.Provider != config.EBSProviderOCIBSD {
+		return e, nil
+	}
+	if e.OCICompartmentID == "" || e.OCIAvailabilityDomain == "" || e.ClusterID == "" {
+		return nodeEBS{}, errors.New("formation named ocibsd without its compartment, availability domain and cluster ID")
+	}
+	existing, err := readExistingEBS(tomlPath, node)
+	if err != nil {
+		return nodeEBS{}, err
+	}
+	if err := r.fillOCIBSD(ctx, &e, existing, nil); err != nil {
+		return nodeEBS{}, err
+	}
+	if err := r.checkCredential(ctx, e); err != nil {
+		return nodeEBS{}, err
+	}
+	return e, nil
+}
+
+// formationEBS is the EBS object sent to joining nodes: nil for the default
+// provider, so a viperblock cluster's formation payload has no ebs object.
+func formationEBS(c formation.EBSConfig) *formation.EBSConfig {
+	if (config.EBSConfig{Provider: c.Provider}).ResolvedProvider() == config.EBSProviderViperblockd {
+		return nil
+	}
+	return &c
+}
+
+// generateClusterID returns a random 128-bit value in hex.
+func generateClusterID() (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("generate the cluster ID: %w", err)
+	}
+	return hex.EncodeToString(b[:]), nil
+}
+
+// ocibsdStageDir is where setup.sh stages the tarball's ocibsd/ files on an
+// OCI virtual machine.
+const ocibsdStageDir = "/usr/local/share/spinifex/ocibsd"
+
+// ocibsdFiles maps each staged file to where activation installs it.
+var ocibsdFiles = []struct{ staged, installed string }{
+	{"spinifex-ocibsd.service", "etc/systemd/system/spinifex-ocibsd.service"},
+	{"99-spinifex-ocibsd.rules", "etc/udev/rules.d/99-spinifex-ocibsd.rules"},
+	{"spinifex-daemon.service.d/ocibsd.conf", "etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf"},
+}
+
+// viperblockUnit is masked on an ocibsd node, so exactly one service answers
+// ebs.provider.v1.* there.
+const viperblockUnit = "etc/systemd/system/spinifex-viperblock.service"
+
+// ocibsdActivation installs the staged ocibsd files under Root and runs the
+// systemd and udev steps through Runner.
+type ocibsdActivation struct {
+	Root   string
+	Stage  string
+	Runner ebsocibsd.CommandRunner
+}
+
+func productionOCIBSDActivation() ocibsdActivation {
+	return ocibsdActivation{Root: "/", Stage: ocibsdStageDir, Runner: ebsocibsd.ExecRunner{}}
+}
+
+// activateEBSProvider readies an ocibsd node before spinifex.target starts;
+// any other provider's node is left as setup.sh installed it.
+func activateEBSProvider(ctx context.Context, e nodeEBS, a ocibsdActivation) error {
+	if e.Provider != config.EBSProviderOCIBSD {
+		return nil
+	}
+	for _, f := range ocibsdFiles {
+		if err := installStaged(filepath.Join(a.Stage, f.staged), filepath.Join(a.Root, f.installed)); err != nil {
+			return err
+		}
+	}
+	if err := maskUnit(filepath.Join(a.Root, viperblockUnit)); err != nil {
+		return err
+	}
+	for _, cmd := range [][]string{
+		{"systemctl", "enable", "spinifex-ocibsd.service"},
+		{"systemctl", "daemon-reload"},
+		{"udevadm", "control", "--reload-rules"},
+	} {
+		if _, err := a.Runner.Run(ctx, cmd[0], cmd[1:]...); err != nil {
+			return fmt.Errorf("activate ocibsd: %w", err)
+		}
+	}
+	fmt.Println("✅ ocibsd installed: spinifex-ocibsd.service enabled, spinifex-viperblock.service masked")
+	return nil
+}
+
+// installStaged copies src to dst with mode 0644, through a temporary file
+// so a unit is never read half-written.
+func installStaged(src, dst string) error {
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return fmt.Errorf("activate ocibsd: read the staged %s (setup.sh stages it on an OCI virtual machine): %w", src, err)
+	}
+	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { //nolint:gosec // systemd and udev directories are world-readable
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	tmp := dst + ".spx-tmp"
+	if err := os.WriteFile(tmp, data, 0o644); err != nil { //nolint:gosec // unit files and udev rules are world-readable, as setup.sh installs them
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	if err := os.Rename(tmp, dst); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	return nil
+}
+
+// maskUnit links path to /dev/null. systemctl mask refuses a real unit file
+// in /etc, which is where setup.sh installs the viperblock unit.
+func maskUnit(path string) error {
+	if systemd.IsMasked(path) {
+		return nil
+	}
+	if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("activate ocibsd: mask %s: %w", path, err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // the systemd unit directory is world-readable
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	if err := os.Symlink(os.DevNull, path); err != nil {
+		return fmt.Errorf("activate ocibsd: mask %s: %w", path, err)
+	}
+	return nil
+}
+
+func exitOnActivationError(err error) {
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+}

spinifex/operator/cli/admin_ebs_test.go

@@ -0,0 +1,530 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/operator/admin"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/runtime/formation"
+)
+
+const (
+	testOCICompartment = "ocid1.compartment.oc1..test"
+	testOCIAD          = "Uocm:US-ASHBURN-AD-1"
+	testClusterID      = "5f0c3a9e1b7d42c68e2a4f1d9b3c7e05"
+)
+
+var errRefused = errors.New("oci ListVolumes: Authorization failed or requested resource not found (NotAuthorizedOrNotFound, http 404)")
+
+// fakeResolver is an ebsResolver over a staged environment, instance
+// metadata and OCI client, counting the metadata reads it answers.
+type fakeResolver struct {
+	env       map[string]string
+	md        *oci.InstanceMetadata
+	client    *oci.BlockFake
+	imdsReads int
+	clientFor []string
+}
+
+func newFakeResolver() *fakeResolver {
+	return &fakeResolver{env: map[string]string{}, client: oci.NewBlockFake()}
+}
+
+func (f *fakeResolver) onOCI(shape string) *fakeResolver {
+	f.md = &oci.InstanceMetadata{ID: "ocid1.instance.oc1..node", CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, Shape: shape}
+	return f
+}
+
+func (f *fakeResolver) resolver() ebsResolver {
+	return ebsResolver{
+		getenv: func(k string) string { return f.env[k] },
+		instance: func(ctx context.Context) (oci.InstanceMetadata, error) {
+			f.imdsReads++
+			if _, ok := ctx.Deadline(); !ok {
+				return oci.InstanceMetadata{}, errors.New("the metadata read has no timeout")
+			}
+			if f.md == nil {
+				return oci.InstanceMetadata{}, errors.New("dial tcp 169.254.169.254:80: i/o timeout")
+			}
+			return *f.md, nil
+		},
+		newClient: func(auth, file, profile string) (oci.BlockClient, error) {
+			f.clientFor = append(f.clientFor, strings.Join([]string{auth, file, profile}, "|"))
+			return f.client, nil
+		},
+	}
+}
+
+// writeExisting writes a spinifex.toml holding only node1's EBS tables.
+func writeExisting(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, os.WriteFile(path, []byte(body), 0o600))
+	return path
+}
+
+func absentConfig(t *testing.T) string {
+	return filepath.Join(t.TempDir(), "spinifex.toml")
+}
+
+func TestResolveInitEBS_Precedence(t *testing.T) {
+	cases := []struct {
+		name     string
+		existing string
+		env      string
+		shape    string
+		want     string
+		source   string
+	}{
+		{name: "an existing config that omits the provider keeps viperblockd over the environment and detection", existing: "[nodes.node1.daemon]\nhost = \"x\"\n", env: config.EBSProviderOCIBSD, shape: "VM.Standard.E6.Flex", want: config.EBSProviderViperblockd, source: ebsFromConfig},
+		{name: "an existing config's provider wins over the environment", existing: "[nodes.node1.ebs]\nprovider = \"qemunbd\"\n", env: config.EBSProviderViperblockd, shape: "VM.Standard.E6.Flex", want: config.EBSProviderQEMUNBD, source: ebsFromConfig},
+		{name: "the environment wins over detection", env: config.EBSProviderViperblockd, shape: "VM.Standard.E6.Flex", want: config.EBSProviderViperblockd, source: ebsFromEnv},
+		{name: "detection on an OCI virtual machine", shape: "VM.Standard.E6.Flex", want: config.EBSProviderOCIBSD, source: ebsFromDetection},
+		{name: "detection on an OCI bare-metal shape", shape: "BM.Standard.E2.64", want: config.EBSProviderViperblockd, source: ebsFromDetection},
+		{name: "detection off OCI", want: config.EBSProviderViperblockd, source: ebsFromDetection},
+		{name: "another node's table is not this node's config", existing: "[nodes.node2.ebs]\nprovider = \"qemunbd\"\n", want: config.EBSProviderViperblockd, source: ebsFromDetection},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newFakeResolver()
+			if tc.shape != "" {
+				f.onOCI(tc.shape)
+			}
+			if tc.env != "" {
+				f.env[ebsProviderEnv] = tc.env
+			}
+			path := absentConfig(t)
+			if tc.existing != "" {
+				path = writeExisting(t, tc.existing)
+			}
+			got, err := resolveInitEBS(context.Background(), path, "node1", f.resolver())
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got.Provider)
+			assert.Equal(t, tc.source, got.Source)
+			if tc.source != ebsFromDetection {
+				assert.Zero(t, f.imdsReads, "a configured provider never reads the metadata service")
+			}
+			if tc.want != config.EBSProviderOCIBSD {
+				assert.Equal(t, formation.EBSConfig{Provider: tc.want}, got.EBSConfig, "no OCI table or cluster ID off ocibsd")
+				assert.Empty(t, f.clientFor, "no OCI call off ocibsd")
+			}
+		})
+	}
+}
+
+func TestResolveInitEBS_DetectionFillsTheOCITable(t *testing.T) {
+	f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+	got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err)
+
+	assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+	assert.Equal(t, testOCICompartment, got.OCICompartmentID)
+	assert.Equal(t, testOCIAD, got.OCIAvailabilityDomain)
+	assert.Equal(t, "instance_principal", got.Auth)
+	assert.Empty(t, got.ConfigFile)
+	assert.Regexp(t, `^[0-9a-f]{32}$`, got.ClusterID, "128 random bits in hex")
+	assert.Equal(t, 1, f.imdsReads, "detection's read also fills the table")
+	assert.Equal(t, []string{"instance_principal||"}, f.clientFor, "the credential is checked as instance principal")
+	assert.Equal(t, "EBS provider: ocibsd (detected: OCI virtual machine VM.Standard.E6.Flex)", got.describe())
+
+	again, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err)
+	assert.NotEqual(t, got.ClusterID, again.ClusterID, "each new cluster gets its own ID")
+}
+
+// An existing config's [nodes.<node>.ebs.oci] values win over the metadata
+// and the instance_principal default, and its cluster ID is kept.
+func TestResolveInitEBS_ExistingOCIValuesWin(t *testing.T) {
+	f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+	path := writeExisting(t, `[nodes.node1.ebs]
+provider = "ocibsd"
+
+[nodes.node1.ebs.oci]
+compartment_id = "ocid1.compartment.oc1..mine"
+cluster_id = "`+testClusterID+`"
+auth = "config_file"
+config_file = "/etc/spinifex/oci/config"
+config_profile = "spinifex"
+`)
+	got, err := resolveInitEBS(context.Background(), path, "node1", f.resolver())
+	require.NoError(t, err)
+
+	assert.Equal(t, ebsFromConfig, got.Source)
+	assert.Equal(t, "ocid1.compartment.oc1..mine", got.OCICompartmentID)
+	assert.Equal(t, testOCIAD, got.OCIAvailabilityDomain, "a value the config omits comes from the metadata")
+	assert.Equal(t, testClusterID, got.ClusterID, "a re-init keeps the ID its volumes are tagged with")
+	assert.Equal(t, []string{"config_file|/etc/spinifex/oci/config|spinifex"}, f.clientFor)
+}
+
+func TestResolveInitEBS_OCIBSDFromTheEnvironmentNeedsTheMetadata(t *testing.T) {
+	f := newFakeResolver()
+	f.env[ebsProviderEnv] = config.EBSProviderOCIBSD
+	_, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.ErrorContains(t, err, "needs this node's OCI instance metadata")
+
+	f.onOCI("BM.Standard.E2.64")
+	got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err, "a named provider is the operator's choice, whatever the shape")
+	assert.Equal(t, ebsFromEnv, got.Source)
+	assert.Equal(t, testOCIAD, got.OCIAvailabilityDomain)
+}
+
+func TestResolveInitEBS_Refusals(t *testing.T) {
+	t.Run("an unknown provider in the environment", func(t *testing.T) {
+		f := newFakeResolver()
+		f.env[ebsProviderEnv] = "ceph"
+		_, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.ErrorContains(t, err, `EBS provider "ceph" (from SPINIFEX_EBS_PROVIDER) is unsupported`)
+	})
+	t.Run("an existing config that cannot be parsed", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		_, err := resolveInitEBS(context.Background(), writeExisting(t, "[nodes.node1.ebs\n"), "node1", f.resolver())
+		require.ErrorContains(t, err, "parse the existing config")
+		assert.Zero(t, f.imdsReads, "an unreadable config is not replaced by detection")
+	})
+}
+
+// The check is the read ocibsd makes at start: it stops init naming the
+// policy when refused, and an inspect-only credential passes it.
+func TestResolveInitEBS_CredentialCheck(t *testing.T) {
+	t.Run("refused", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		f.client.FailOp("ListVolumes", errRefused)
+		_, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.Error(t, err)
+		assert.ErrorIs(t, err, errRefused)
+		assert.ErrorContains(t, err, "manage volume-family in compartment id "+testOCICompartment)
+		assert.ErrorContains(t, err, "use instances in compartment id "+testOCICompartment)
+	})
+	t.Run("inspect only", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		for _, op := range []string{"CreateVolume", "AttachVolume", "CreateVolumeBackup"} {
+			f.client.FailOp(op, errRefused)
+		}
+		got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+		_, err = f.client.CreateVolume(context.Background(), oci.CreateVolumeInput{CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, SizeGB: oci.MinVolumeSizeGB})
+		assert.ErrorIs(t, err, errRefused, "the check made no create")
+		assert.Empty(t, f.client.Volumes())
+	})
+	t.Run("ok", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+	})
+}
+
+func TestResolveJoinEBS(t *testing.T) {
+	ocibsdFormation := &formation.NetworkConfig{EBS: &formation.EBSConfig{
+		Provider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD, ClusterID: testClusterID,
+	}}
+
+	t.Run("a viperblock cluster checks nothing", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		got, err := resolveJoinEBS(context.Background(), absentConfig(t), "node2", &formation.NetworkConfig{}, f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, nodeEBS{Source: ebsFromFormation}, got)
+		assert.Zero(t, f.imdsReads)
+		assert.Empty(t, f.clientFor)
+	})
+	t.Run("ocibsd defaults this node to its instance principal", func(t *testing.T) {
+		f := newFakeResolver()
+		got, err := resolveJoinEBS(context.Background(), absentConfig(t), "node2", ocibsdFormation, f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, *ocibsdFormation.EBS, got.EBSConfig, "the cluster's values come from formation unchanged")
+		assert.Equal(t, "instance_principal", got.Auth)
+		assert.Zero(t, f.imdsReads, "formation carries the compartment and AD")
+		assert.Equal(t, []string{"instance_principal||"}, f.clientFor)
+	})
+	t.Run("this node's existing credential wins", func(t *testing.T) {
+		f := newFakeResolver()
+		path := writeExisting(t, "[nodes.node2.ebs]\nprovider = \"ocibsd\"\n\n[nodes.node2.ebs.oci]\nauth = \"config_file\"\nconfig_file = \"/etc/spinifex/oci/config\"\ncluster_id = \"0000\"\n")
+		got, err := resolveJoinEBS(context.Background(), path, "node2", ocibsdFormation, f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, "config_file", got.Auth)
+		assert.Equal(t, "/etc/spinifex/oci/config", got.ConfigFile)
+		assert.Equal(t, testClusterID, got.ClusterID, "the cluster ID is the cluster's, not a stale local one")
+	})
+	t.Run("a refused credential stops the join", func(t *testing.T) {
+		f := newFakeResolver()
+		f.client.FailOp("ListVolumes", errRefused)
+		_, err := resolveJoinEBS(context.Background(), absentConfig(t), "node2", ocibsdFormation, f.resolver())
+		require.ErrorContains(t, err, "manage volume-family")
+	})
+}
+
+// fakeRunner records each command it is given.
+type fakeRunner struct{ calls []string }
+
+func (r *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) {
+	r.calls = append(r.calls, strings.Join(append([]string{name}, args...), " "))
+	return nil, nil
+}
+
+// stageOCIBSD stages the repository's build/ocibsd files as setup.sh does.
+func stageOCIBSD(t *testing.T) string {
+	t.Helper()
+	stage := filepath.Join(t.TempDir(), "ocibsd")
+	for _, f := range ocibsdFiles {
+		data, err := os.ReadFile(filepath.Join("..", "..", "..", "build", "ocibsd", f.staged))
+		require.NoError(t, err)
+		require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(stage, f.staged)), 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(stage, f.staged), data, 0o644))
+	}
+	return stage
+}
+
+func TestActivateEBSProvider_OCIBSD(t *testing.T) {
+	root := t.TempDir()
+	stage := stageOCIBSD(t)
+	unitDir := filepath.Join(root, "etc", "systemd", "system")
+	require.NoError(t, os.MkdirAll(unitDir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(unitDir, "spinifex-viperblock.service"), []byte("[Unit]\n"), 0o644))
+	runner := &fakeRunner{}
+
+	e := nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderOCIBSD}}
+	require.NoError(t, activateEBSProvider(context.Background(), e, ocibsdActivation{Root: root, Stage: stage, Runner: runner}))
+
+	for _, f := range ocibsdFiles {
+		want, err := os.ReadFile(filepath.Join(stage, f.staged))
+		require.NoError(t, err)
+		got, err := os.ReadFile(filepath.Join(root, f.installed))
+		require.NoError(t, err, f.installed)
+		assert.Equal(t, string(want), string(got), f.installed)
+	}
+	target, err := os.Readlink(filepath.Join(unitDir, "spinifex-viperblock.service"))
+	require.NoError(t, err, "the viperblock unit file is replaced by a mask")
+	assert.Equal(t, "/dev/null", target)
+	assert.Equal(t, []string{
+		"systemctl enable spinifex-ocibsd.service",
+		"systemctl daemon-reload",
+		"udevadm control --reload-rules",
+	}, runner.calls)
+
+	// A repeat, as an init --force makes, refreshes the files and keeps the mask.
+	runner.calls = nil
+	require.NoError(t, activateEBSProvider(context.Background(), e, ocibsdActivation{Root: root, Stage: stage, Runner: runner}))
+	target, err = os.Readlink(filepath.Join(unitDir, "spinifex-viperblock.service"))
+	require.NoError(t, err)
+	assert.Equal(t, "/dev/null", target)
+	assert.Len(t, runner.calls, 3)
+}
+
+func TestActivateEBSProvider_OtherProvidersAreUntouched(t *testing.T) {
+	for _, provider := range []string{"", config.EBSProviderViperblockd, config.EBSProviderQEMUNBD} {
+		root := t.TempDir()
+		runner := &fakeRunner{}
+		e := nodeEBS{EBSConfig: formation.EBSConfig{Provider: provider}}
+		require.NoError(t, activateEBSProvider(context.Background(), e, ocibsdActivation{Root: root, Stage: filepath.Join(root, "absent"), Runner: runner}))
+		assert.Empty(t, runner.calls, provider)
+		entries, err := os.ReadDir(root)
+		require.NoError(t, err)
+		assert.Empty(t, entries, provider)
+	}
+}
+
+func TestActivateEBSProvider_MissingStage(t *testing.T) {
+	runner := &fakeRunner{}
+	e := nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderOCIBSD}}
+	err := activateEBSProvider(context.Background(), e, ocibsdActivation{Root: t.TempDir(), Stage: filepath.Join(t.TempDir(), "absent"), Runner: runner})
+	require.ErrorContains(t, err, "setup.sh stages it on an OCI virtual machine")
+	assert.Empty(t, runner.calls, "nothing is enabled without its files")
+}
+
+// A viperblock cluster's formation payload is the JSON it was before the
+// provider choice existed: no ebs object at all.
+func TestFormationEBS_ViperblockPayloadIsUnchanged(t *testing.T) {
+	cases := []struct {
+		name, provider string
+	}{
+		{name: "viperblockd", provider: config.EBSProviderViperblockd},
+		{name: "an empty provider", provider: ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ebs := formationEBS(formation.EBSConfig{Provider: tc.provider})
+			assert.Nil(t, ebs)
+
+			minimal, err := json.Marshal(formation.NetworkConfig{IPSecEnabled: true, EBS: ebs})
+			require.NoError(t, err)
+			assert.Equal(t, `{"external_mode":"","ipsec_enabled":true}`, string(minimal))
+
+			external, err := json.Marshal(formation.NetworkConfig{
+				ExternalMode: "nat", BootstrapVpcId: "vpc-1", BootstrapCidr: "172.31.0.0/16", EBS: ebs,
+			})
+			require.NoError(t, err)
+			assert.Equal(t, `{"external_mode":"nat","ipsec_enabled":false,"bootstrap_vpc_id":"vpc-1","bootstrap_cidr":"172.31.0.0/16"}`, string(external))
+		})
+	}
+}
+
+// A non-default provider is sent with no cluster ID; only ocibsd has one.
+func TestFormationEBS_OtherProvidersAreSent(t *testing.T) {
+	f := newFakeResolver()
+	f.env[ebsProviderEnv] = config.EBSProviderQEMUNBD
+	qemu, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err)
+	wire, err := json.Marshal(formation.NetworkConfig{EBS: formationEBS(qemu.EBSConfig)})
+	require.NoError(t, err)
+	assert.Equal(t, `{"external_mode":"","ipsec_enabled":false,"ebs":{"provider":"qemunbd"}}`, string(wire))
+}
+
+// A joining node of a viperblock cluster receives no ebs object and writes
+// the default provider with no OCI table.
+func TestApplyNetworkConfig_ViperblockJoinerWritesTheDefault(t *testing.T) {
+	var received formation.NetworkConfig
+	require.NoError(t, json.Unmarshal([]byte(`{"external_mode":"","ipsec_enabled":false}`), &received))
+	assert.Nil(t, received.EBS)
+
+	settings := admin.ConfigSettings{}
+	applyNetworkConfig(&settings, &received)
+	choice, err := resolveJoinEBS(context.Background(), absentConfig(t), "node1", &received, newFakeResolver().resolver())
+	require.NoError(t, err)
+	choice.apply(&settings)
+	assert.Empty(t, settings.EBSProvider)
+	assert.Empty(t, settings.OCIClusterID)
+
+	got := renderEBS(t, settings)
+	assert.Equal(t, config.EBSProviderViperblockd, got.Provider)
+	assert.Equal(t, config.EBSOCIConfig{}, got.OCI)
+}
+
+// A joining node writes the provider and cluster ID the init node chose,
+// and its own instance-principal credential.
+func TestApplyNetworkConfig_PropagatesTheEBSProvider(t *testing.T) {
+	init, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", newFakeResolver().onOCI("VM.Standard.E6.Flex").resolver())
+	require.NoError(t, err)
+	wire, err := json.Marshal(formation.NetworkConfig{EBS: formationEBS(init.EBSConfig)})
+	require.NoError(t, err)
+	assert.Contains(t, string(wire), `"cluster_id":"`+init.ClusterID+`"`)
+	assert.NotContains(t, string(wire), "instance_principal", "a node's credential is its own")
+	var received formation.NetworkConfig
+	require.NoError(t, json.Unmarshal(wire, &received))
+	require.NotNil(t, received.EBS)
+
+	settings := admin.ConfigSettings{}
+	applyNetworkConfig(&settings, &received)
+	choice, err := resolveJoinEBS(context.Background(), absentConfig(t), "node1", &received, newFakeResolver().resolver())
+	require.NoError(t, err)
+	choice.apply(&settings)
+
+	got := renderEBS(t, settings)
+	assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+	assert.Equal(t, config.EBSOCIConfig{
+		CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, ClusterID: init.ClusterID, Auth: "instance_principal",
+	}, got.OCI, "the joining node's config carries the init node's ID")
+}
+
+// renderSpinifexToml renders the spinifex.toml template for one node.
+func renderSpinifexToml(t *testing.T, settings admin.ConfigSettings) string {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings.Node, settings.Az, settings.Region, settings.Port = "node1", "ap-southeast-2a", "ap-southeast-2", "4432"
+	settings.BindIP, settings.ConfigDir = "10.11.12.1", "/etc/spinifex"
+	settings.OVNNBAddr, settings.OVNSBAddr = "tcp:127.0.0.1:6641", "tcp:127.0.0.1:6642"
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+	return path
+}
+
+// renderEBS renders the template and loads node1's [ebs] back.
+func renderEBS(t *testing.T, settings admin.ConfigSettings) config.EBSConfig {
+	t.Helper()
+	path := renderSpinifexToml(t, settings)
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+	cfg, err := config.LoadConfig(path)
+	require.NoError(t, err)
+	return cfg.Nodes["node1"].EBS
+}
+
+func TestSpinifexTomlTemplate_EBSProvider(t *testing.T) {
+	cases := []struct {
+		name     string
+		settings admin.ConfigSettings
+		want     config.EBSConfig
+	}{
+		{name: "unset renders viperblockd", want: config.EBSConfig{Provider: config.EBSProviderViperblockd}},
+		{name: "qemunbd", settings: admin.ConfigSettings{EBSProvider: config.EBSProviderQEMUNBD}, want: config.EBSConfig{Provider: config.EBSProviderQEMUNBD}},
+		{
+			name: "ocibsd as the instance principal",
+			settings: admin.ConfigSettings{EBSProvider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD,
+				OCIClusterID: testClusterID, OCIAuth: "instance_principal"},
+			want: config.EBSConfig{Provider: config.EBSProviderOCIBSD, OCI: config.EBSOCIConfig{
+				CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, ClusterID: testClusterID, Auth: "instance_principal",
+			}},
+		},
+		{
+			name: "ocibsd with a config file",
+			settings: admin.ConfigSettings{EBSProvider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD,
+				OCIClusterID: testClusterID, OCIAuth: "config_file", OCIConfigFile: "/etc/spinifex/oci/config", OCIConfigProfile: "spinifex"},
+			want: config.EBSConfig{Provider: config.EBSProviderOCIBSD, OCI: config.EBSOCIConfig{
+				CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, ClusterID: testClusterID, Auth: "config_file", ConfigFile: "/etc/spinifex/oci/config", ConfigProfile: "spinifex",
+			}},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := renderEBS(t, tc.settings)
+			assert.Equal(t, tc.want.Provider, got.Provider)
+			assert.Equal(t, tc.want.OCI, got.OCI)
+		})
+	}
+}
+
+// A viperblock init renders the same spinifex.toml as one that never chose a
+// provider, so resolution and the cluster ID leave its config untouched.
+func TestSpinifexTomlTemplate_ViperblockInitIsUnchanged(t *testing.T) {
+	ebs, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", newFakeResolver().resolver())
+	require.NoError(t, err)
+	settings := admin.ConfigSettings{}
+	ebs.apply(&settings)
+
+	before, err := os.ReadFile(renderSpinifexToml(t, admin.ConfigSettings{}))
+	require.NoError(t, err)
+	after, err := os.ReadFile(renderSpinifexToml(t, settings))
+	require.NoError(t, err)
+	assert.Equal(t, string(before), string(after))
+	assert.NotContains(t, string(after), "cluster_id")
+	assert.NotContains(t, string(after), ".ebs.oci]")
+}
+
+// What init renders for ocibsd is what a re-init reads back as the existing
+// config, so a --force keeps the provider, the table and the cluster ID.
+func TestReadExistingEBS_RoundTrip(t *testing.T) {
+	path := renderSpinifexToml(t, admin.ConfigSettings{
+		EBSProvider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD, OCIClusterID: testClusterID, OCIAuth: "instance_principal",
+	})
+	f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+	f.env[ebsProviderEnv] = config.EBSProviderViperblockd
+	got, err := resolveInitEBS(context.Background(), path, "node1", f.resolver())
+	require.NoError(t, err)
+	assert.Equal(t, ebsFromConfig, got.Source)
+	assert.Equal(t, formation.EBSConfig{
+		Provider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD, ClusterID: testClusterID,
+	}, got.EBSConfig)
+	assert.Equal(t, "instance_principal", got.Auth)
+	assert.Zero(t, f.imdsReads, "a complete existing table needs no metadata")
+
+	viperblock, err := readExistingEBS(renderSpinifexToml(t, admin.ConfigSettings{}), "node1")
+	require.NoError(t, err)
+	require.NotNil(t, viperblock)
+	assert.Equal(t, config.EBSProviderViperblockd, viperblock.Provider)
+
+	absent, err := readExistingEBS(absentConfig(t), "node1")
+	require.NoError(t, err)
+	assert.Nil(t, absent)
+}

spinifex/operator/cli/admin_ochre.go

@@ -786,6 +786,7 @@ func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvi
 		SourcePath:       imagePath,
 		Snapshot:         true,
 		Progress:         os.Stdout,
+		SnapshotTimeout:  imageImportTimeout,
 	}); err != nil {
 		return "", fmt.Errorf("could not import weights volume %s: %w", volumeId, err)
 	}

spinifex/operator/cli/admin_upgrade.go

@@ -203,6 +203,8 @@ func reportUnitStatus(result systemd.Result) {
 			fmt.Printf("  %-38s %d → %d (stale, will replace)\n", s.Name+":", s.InstalledVersion, s.EmbeddedVersion)
 		case systemd.ActionConflict:
 			fmt.Printf("  %-38s %d (operator-modified, not touched — see `systemctl edit %s`)\n", s.Name+":", s.InstalledVersion, s.Name)
+		case systemd.ActionMasked:
+			fmt.Printf("  %-38s masked, not touched\n", s.Name+":")
 		}
 	}
 	if result.HasConflicts() {

spinifex/operator/cli/admin_upgrade_test.go

@@ -76,6 +76,7 @@ func TestReportUnitStatus_PrintsEachActionAndConflictFooter(t *testing.T) {
 		{Name: "install.service", Action: systemd.ActionInstall, EmbeddedVersion: 2},
 		{Name: "replace.service", Action: systemd.ActionReplace, InstalledVersion: 1, EmbeddedVersion: 2},
 		{Name: "conflict.service", Action: systemd.ActionConflict, InstalledVersion: 2},
+		{Name: "masked.service", Action: systemd.ActionMasked},
 	}}
 
 	out := captureStdout(t, func() { reportUnitStatus(result) })
@@ -85,6 +86,7 @@ func TestReportUnitStatus_PrintsEachActionAndConflictFooter(t *testing.T) {
 		"install.service:", "missing → will install (v2)",
 		"replace.service:", "1 → 2 (stale, will replace)",
 		"conflict.service:", "2 (operator-modified, not touched",
+		"masked.service:", "masked, not touched",
 		"Operator-modified units above are left as-is",
 	} {
 		assert.Contains(t, out, want)

spinifex/operator/cli/service.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/nats"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/northstar"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/ocibsd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/predastore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qemunbdd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qmpcollector"
@@ -249,6 +250,11 @@ var viperblockStartCmd = &cobra.Command{
 		if err != nil {
 			return fmt.Errorf("load config file: %w", err)
 		}
+		// A reinstalled unit can outlive its mask on a node with another
+		// provider, which would answer the same subjects.
+		if err := requireEBSProvider(clusterConfig, config.EBSProviderViperblockd); err != nil {
+			return err
+		}
 		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		natsHost := viper.GetString("nats-host")
@@ -379,6 +385,15 @@ var viperblockStartCmd = &cobra.Command{
 	},
 }
 
+// requireEBSProvider refuses to start a provider service on a node whose
+// configured provider is another one; an unset provider is viperblockd.
+func requireEBSProvider(clusterConfig *config.ClusterConfig, want string) error {
+	if got := clusterConfig.Nodes[clusterConfig.Node].EBS.ResolvedProvider(); got != want {
+		return fmt.Errorf("[nodes.%s.ebs] provider is %q, not %q", clusterConfig.Node, got, want)
+	}
+	return nil
+}
+
 var qemunbdCmd = &cobra.Command{
 	Use:   "qemunbd",
 	Short: "Manage the qemunbd service",
@@ -453,6 +468,68 @@ var qemunbdStartCmd = &cobra.Command{
 	},
 }
 
+var ocibsdCmd = &cobra.Command{
+	Use:   "ocibsd",
+	Short: "Manage the ocibsd service",
+}
+
+// ocibsd answers the same ebs.provider.v1.* subjects as viperblock and
+// qemunbd, so only the cluster's one provider may run against its NATS.
+var ocibsdStartCmd = &cobra.Command{
+	Use:           "start",
+	Short:         "Start the ocibsd service",
+	SilenceErrors: true,
+	SilenceUsage:  true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cfgFile := viper.GetString("config")
+		if cfgFile == "" {
+			return fmt.Errorf("config file is not set")
+		}
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		if err != nil {
+			return fmt.Errorf("load config file: %w", err)
+		}
+		if err := requireEBSProvider(clusterConfig, config.EBSProviderOCIBSD); err != nil {
+			return err
+		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
+		if natsHost := viper.GetString("nats-host"); natsHost != "" {
+			nodeConfig.NATS.Host = natsHost
+		}
+		baseDir := nodeConfig.Predastore.BaseDir
+		if override := viper.GetString("base-dir"); override != "" {
+			baseDir = override
+		}
+
+		debug := viper.GetBool("ocibsd-debug")
+		defer initTelemetry("ocibsd", debug)()
+
+		oci := nodeConfig.EBS.OCI
+		service, err := service.New("ocibsd", &ocibsd.Config{
+			BaseDir:            baseDir,
+			NatsHost:           nodeConfig.NATS.Host,
+			NatsToken:          nodeConfig.NATS.ACL.Token,
+			NatsCACert:         nodeConfig.NATS.CACert,
+			NodeName:           clusterConfig.Node,
+			AvailabilityZone:   nodeConfig.AZ,
+			ClusterID:          oci.ClusterID,
+			CompartmentID:      oci.CompartmentID,
+			AvailabilityDomain: oci.AvailabilityDomain,
+			Auth:               oci.Auth,
+			ConfigFile:         oci.ConfigFile,
+			ConfigProfile:      oci.ConfigProfile,
+			Debug:              debug,
+		})
+		if err != nil {
+			return fmt.Errorf("create ocibsd service: %w", err)
+		}
+		if _, err := service.Start(); err != nil {
+			return fmt.Errorf("start ocibsd service: %w", err)
+		}
+		return nil
+	},
+}
+
 // Repeat for nats.
 var natsStartCmd = &cobra.Command{
 	Use:           "start",
@@ -1147,6 +1224,14 @@ func init() {
 
 	qemunbdCmd.AddCommand(qemunbdStartCmd)
 
+	serviceCmd.AddCommand(ocibsdCmd)
+
+	ocibsdCmd.PersistentFlags().Bool("debug", false, "ocibsd (EBS) debug logging")
+	mustBindEnv("ocibsd-debug", "SPINIFEX_OCIBSD_DEBUG")
+	mustBindPFlag("ocibsd-debug", ocibsdCmd.PersistentFlags(), "debug")
+
+	ocibsdCmd.AddCommand(ocibsdStartCmd)
+
 	// Nats
 	serviceCmd.AddCommand(natsCmd)
 

spinifex/operator/cli/service_exit_test.go

@@ -188,6 +188,48 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
 			},
 			wantErr: "start viperblock service",
 		},
+		{
+			name: "viperblock start (explicit viperblockd provider passes the guard)",
+			cmd:  viperblockStartCmd,
+			setup: func(t *testing.T) {
+				resetGlobalViper(t)
+				toml := minimalClusterToml + "\n[nodes.node1.ebs]\nprovider = \"viperblockd\"\n"
+				viper.Set("config", writeSpinifexToml(t, toml))
+				viper.Set("plugin-path", t.TempDir())
+				viper.Set("base-dir", blockedDir(t))
+			},
+			wantErr: "start viperblock service",
+		},
+		{
+			name: "viperblock start (refused on a qemunbd node)",
+			cmd:  viperblockStartCmd,
+			setup: func(t *testing.T) {
+				resetGlobalViper(t)
+				toml := minimalClusterToml + "\n[nodes.node1.ebs]\nprovider = \"qemunbd\"\n"
+				viper.Set("config", writeSpinifexToml(t, toml))
+				viper.Set("plugin-path", t.TempDir())
+			},
+			wantErr: `[nodes.node1.ebs] provider is "qemunbd", not "viperblockd"`,
+		},
+		{
+			name: "viperblock start (refused on an ocibsd node)",
+			cmd:  viperblockStartCmd,
+			setup: func(t *testing.T) {
+				resetGlobalViper(t)
+				toml := minimalClusterToml + `
+[nodes.node1.ebs]
+provider = "ocibsd"
+
+[nodes.node1.ebs.oci]
+compartment_id = "ocid1.compartment.oc1..test"
+availability_domain = "Uocm:US-ASHBURN-AD-1"
+cluster_id = "5f0c3a9e1b7d42c68e2a4f1d9b3c7e05"
+`
+				viper.Set("config", writeSpinifexToml(t, toml))
+				viper.Set("plugin-path", t.TempDir())
+			},
+			wantErr: `[nodes.node1.ebs] provider is "ocibsd", not "viperblockd"`,
+		},
 		{name: "qemunbd start (missing config)", cmd: qemunbdStartCmd},
 		{
 			name: "qemunbd start (config file load failure)",

spinifex/operator/cli/templates/spinifex.toml

@@ -139,7 +139,7 @@ encryption_key_file = "{{.EncryptionKeyFile}}"
 # provider: every node in a cluster must name the same one, and the daemon
 # refuses to start on a provider other than the one the cluster recorded.
 # Changing it needs the cluster wiped and reformed.
-provider = "viperblockd"
+provider = "{{or .EBSProvider "viperblockd"}}"
 # Per-volume export tunables. Both are applied per nbdkit process and there is
 # one of those per volume, so the host cost of each is the value times the
 # volume count. Uncomment to override; a service restart plus a volume
@@ -152,6 +152,27 @@ provider = "viperblockd"
 # cache_size_mb: per-volume plaintext read cache. 0 disables it. Auxiliary -efi
 # volumes are uncached regardless.
 #cache_size_mb = 128
+{{- if eq .EBSProvider "ocibsd"}}
+
+[nodes.{{.Node}}.ebs.oci]
+# Every node names the same compartment and availability domain.
+compartment_id = "{{.OCICompartmentID}}"
+availability_domain = "{{.OCIAvailabilityDomain}}"
+# Generated once at init; tags every OCI volume and backup of this cluster.
+cluster_id = "{{.OCIClusterID}}"
+# auth: "config_file" (the default when unset) reads config_file and
+# config_profile, which default to ~/.oci/config and DEFAULT;
+# "instance_principal" uses the instance's own certificate and takes neither.
+{{- if .OCIAuth}}
+auth = "{{.OCIAuth}}"
+{{- end}}
+{{- if .OCIConfigFile}}
+config_file = "{{.OCIConfigFile}}"
+{{- end}}
+{{- if .OCIConfigProfile}}
+config_profile = "{{.OCIConfigProfile}}"
+{{- end}}
+{{- end}}
 
 [nodes.{{.Node}}.vpcd]
 ovn_nb_addr = "{{.OVNNBAddr}}"

spinifex/operator/host/systemd/reconcile.go

@@ -30,6 +30,7 @@ const (
 	ActionInstall  Action = "install"  // unit missing on disk
 	ActionReplace  Action = "replace"  // installed marker version is older than embedded
 	ActionConflict Action = "conflict" // marker version matches but content differs — operator-modified
+	ActionMasked   Action = "masked"   // unit is linked to /dev/null; left masked
 )
 
 // UnitStatus reports the reconcile decision for a single unit.
@@ -103,6 +104,12 @@ func sortedUnitNames() []string {
 	return slices.Sorted(maps.Keys(Units))
 }
 
+// IsMasked reports whether the unit file at path is masked, a link to /dev/null.
+func IsMasked(path string) bool {
+	target, err := os.Readlink(path)
+	return err == nil && target == os.DevNull
+}
+
 // Reconcile compares each embedded unit (Units, generated from
 // build/systemd) against the copy installed under root. Missing or
 // version-stale units are replaced after a timestamped backup; units whose
@@ -117,6 +124,13 @@ func Reconcile(root string, opts Options) (Result, error) {
 		embeddedVersion := unitVersion(embedded)
 		path := filepath.Join(root, name)
 
+		if IsMasked(path) {
+			result.Statuses = append(result.Statuses, UnitStatus{
+				Name: name, Action: ActionMasked, EmbeddedVersion: embeddedVersion,
+			})
+			continue
+		}
+
 		installed, err := os.ReadFile(path)
 		if os.IsNotExist(err) {
 			result.Statuses = append(result.Statuses, UnitStatus{

spinifex/operator/host/systemd/reconcile_test.go

@@ -521,3 +521,33 @@ func TestBackupUnit_UnreadableSourceReturnsError(t *testing.T) {
 		t.Fatal("backupUnit: want an error for a nonexistent source path, got nil")
 	}
 }
+
+func TestReconcile_MaskedUnitStaysMasked(t *testing.T) {
+	root := t.TempDir()
+	reloadCalls := stubDaemonReload(t)
+	const name = "spinifex-viperblock.service"
+	path := filepath.Join(root, name)
+	if err := os.Symlink(os.DevNull, path); err != nil {
+		t.Fatalf("mask: %v", err)
+	}
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionMasked || st.Applied || st.Backup != "" {
+		t.Errorf("status = %+v, want masked, not applied, no backup", st)
+	}
+	if target, err := os.Readlink(path); err != nil || target != os.DevNull {
+		t.Errorf("unit no longer masked: target %q, err %v", target, err)
+	}
+	backups, _ := filepath.Glob(path + ".pre-reconcile-*")
+	if len(backups) != 0 {
+		t.Errorf("backups written for a masked unit: %v", backups)
+	}
+	if reloadCalls() != 1 {
+		t.Errorf("daemon-reload calls = %d, want 1 for the other units", reloadCalls())
+	}
+}

spinifex/providers/cloud/oci/apiclient_test.go

@@ -29,17 +29,27 @@ type capture struct {
 	method string
 	path   string
 	query  url.Values
+	header http.Header
 	body   map[string]any
 }
 
-// testClient wires the real SDK-backed client to a handler. The signing key is
-// generated per test — the SDK refuses to build a client without one, and a
-// throwaway key is enough because nothing verifies the signature here.
+// testClient wires the real SDK-backed client to a handler.
 func testClient(t *testing.T, h http.HandlerFunc) (oci.Client, *[]capture) {
+	t.Helper()
+	provider, endpoint, hc, seen := testEndpoint(t, h)
+	c, err := oci.NewTestClient(provider, endpoint, hc)
+	require.NoError(t, err)
+	return c, seen
+}
+
+// testEndpoint serves h over TLS and returns a configuration provider for it.
+// The signing key is generated per test — the SDK refuses to build a client
+// without one, and nothing verifies the signature here.
+func testEndpoint(t *testing.T, h http.HandlerFunc) (common.ConfigurationProvider, string, *http.Client, *[]capture) {
 	t.Helper()
 	var seen []capture
 	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		c := capture{method: r.Method, path: r.URL.Path, query: r.URL.Query()}
+		c := capture{method: r.Method, path: r.URL.Path, query: r.URL.Query(), header: r.Header.Clone()}
 		if b, _ := io.ReadAll(r.Body); len(b) > 0 {
 			_ = json.Unmarshal(b, &c.body)
 		}
@@ -57,10 +67,7 @@ func testClient(t *testing.T, h http.HandlerFunc) (oci.Client, *[]capture) {
 	provider := common.NewRawConfigurationProvider(
 		"ocid1.tenancy.oc1..t", "ocid1.user.oc1..u", "ap-sydney-1",
 		"aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", string(pemKey), nil)
-
-	c, err := oci.NewTestClient(provider, strings.TrimPrefix(srv.URL, "https://"), srv.Client())
-	require.NoError(t, err)
-	return c, &seen
+	return provider, strings.TrimPrefix(srv.URL, "https://"), srv.Client(), &seen
 }
 
 // OCI pages list responses and a VNIC can hold 64 addresses. A client that

spinifex/providers/cloud/oci/block.go

@@ -0,0 +1,501 @@
+package oci
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"time"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/common/auth"
+	"github.com/oracle/oci-go-sdk/v65/core"
+)
+
+// BlockClient is the narrow block-storage surface an EBS provider needs: OCI
+// block volumes, their iSCSI attachments and their backups. It is separate
+// from Client so the address allocator and the storage provider stay apart.
+type BlockClient interface {
+	CreateVolume(ctx context.Context, in CreateVolumeInput) (BlockVolume, error)
+	GetVolume(ctx context.Context, volumeID string) (BlockVolume, error)
+	// ListVolumes lists the compartment, then keeps only volumes carrying
+	// every tag in the filter: OCI's list call has no tag filter of its own.
+	// Each volume is listed once, though OCI may list one twice.
+	ListVolumes(ctx context.Context, filter VolumeFilter) ([]BlockVolume, error)
+	UpdateVolume(ctx context.Context, volumeID string, in UpdateVolumeInput) (BlockVolume, error)
+	DeleteVolume(ctx context.Context, volumeID string) error
+
+	AttachVolume(ctx context.Context, in AttachVolumeInput) (VolumeAttachment, error)
+	GetVolumeAttachment(ctx context.Context, attachmentID string) (VolumeAttachment, error)
+	ListVolumeAttachments(ctx context.Context, filter AttachmentFilter) ([]VolumeAttachment, error)
+	DetachVolume(ctx context.Context, attachmentID string) error
+
+	CreateVolumeBackup(ctx context.Context, in CreateVolumeBackupInput) (VolumeBackup, error)
+	GetVolumeBackup(ctx context.Context, backupID string) (VolumeBackup, error)
+	// ListVolumeBackups filters and lists each backup once, as ListVolumes does.
+	ListVolumeBackups(ctx context.Context, filter BackupFilter) ([]VolumeBackup, error)
+	DeleteVolumeBackup(ctx context.Context, backupID string) error
+}
+
+// VolumeState is an OCI block volume's lifecycle state.
+type VolumeState string
+
+const (
+	VolumeProvisioning VolumeState = "PROVISIONING"
+	VolumeRestoring    VolumeState = "RESTORING"
+	VolumeAvailable    VolumeState = "AVAILABLE"
+	VolumeTerminating  VolumeState = "TERMINATING"
+	VolumeTerminated   VolumeState = "TERMINATED"
+	VolumeFaulty       VolumeState = "FAULTY"
+)
+
+// Gone reports whether the volume is deleted or being deleted. OCI keeps a
+// terminated volume readable and listed for a while after the delete.
+func (s VolumeState) Gone() bool { return s == VolumeTerminating || s == VolumeTerminated }
+
+// BackupState is an OCI volume backup's lifecycle state.
+type BackupState string
+
+const (
+	BackupRequestReceived BackupState = "REQUEST_RECEIVED"
+	BackupCreating        BackupState = "CREATING"
+	BackupAvailable       BackupState = "AVAILABLE"
+	BackupTerminating     BackupState = "TERMINATING"
+	BackupTerminated      BackupState = "TERMINATED"
+	BackupFaulty          BackupState = "FAULTY"
+)
+
+// Gone reports whether the backup is deleted or being deleted.
+func (s BackupState) Gone() bool { return s == BackupTerminating || s == BackupTerminated }
+
+// InProgress reports whether OCI is still taking the backup. OCI refuses to
+// delete the backup, or its source volume, while this is true.
+func (s BackupState) InProgress() bool { return s == BackupRequestReceived || s == BackupCreating }
+
+// AttachmentState is an OCI volume attachment's lifecycle state.
+type AttachmentState string
+
+const (
+	AttachmentAttaching AttachmentState = "ATTACHING"
+	AttachmentAttached  AttachmentState = "ATTACHED"
+	AttachmentDetaching AttachmentState = "DETACHING"
+	AttachmentDetached  AttachmentState = "DETACHED"
+)
+
+// MinVolumeSizeGB is the smallest block volume OCI creates.
+const MinVolumeSizeGB = 50
+
+// IQNPrefix starts the IQN of every OCI block volume attachment.
+const IQNPrefix = "iqn.2015-12.com.oracleiaas:"
+
+// BlockVolume is an OCI block volume. SourceBackupID is set on a volume
+// restored from a backup.
+type BlockVolume struct {
+	ID                 string
+	CompartmentID      string
+	AvailabilityDomain string
+	DisplayName        string
+	SizeGB             int64
+	VPUsPerGB          int64
+	State              VolumeState
+	SourceBackupID     string
+	FreeformTags       map[string]string
+	TimeCreated        time.Time
+}
+
+// Tags is the volume's freeform tags.
+func (v BlockVolume) Tags() map[string]string { return v.FreeformTags }
+
+// IsGone reports whether the volume is deleted or being deleted.
+func (v BlockVolume) IsGone() bool { return v.State.Gone() }
+
+// CreateVolumeInput creates a volume, empty or restored from SourceBackupID.
+// A zero VPUsPerGB takes OCI's default tier. RetryToken is sent as
+// opc-retry-token, so OCI answers a repeat with the first call's volume.
+type CreateVolumeInput struct {
+	CompartmentID      string
+	AvailabilityDomain string
+	DisplayName        string
+	SizeGB             int64
+	VPUsPerGB          int64
+	SourceBackupID     string
+	FreeformTags       map[string]string
+	RetryToken         string
+}
+
+// VolumeFilter narrows ListVolumes. CompartmentID is required; DisplayName
+// and AvailabilityDomain are matched by OCI and Tags by the client.
+type VolumeFilter struct {
+	CompartmentID      string
+	AvailabilityDomain string
+	DisplayName        string
+	Tags               map[string]string
+}
+
+// UpdateVolumeInput changes a volume. A zero SizeGB leaves the size alone and
+// a nil FreeformTags leaves the tags alone; a non-nil map replaces them all.
+type UpdateVolumeInput struct {
+	SizeGB       int64
+	FreeformTags map[string]string
+}
+
+// VolumeAttachment is a volume attached to an instance. IQN, IPv4 and Port
+// are the iSCSI target and are empty on an attachment of another type.
+type VolumeAttachment struct {
+	ID                 string
+	InstanceID         string
+	VolumeID           string
+	AvailabilityDomain string
+	State              AttachmentState
+	ReadOnly           bool
+	IQN                string
+	IPv4               string
+	Port               int
+}
+
+// AttachVolumeInput attaches VolumeID to InstanceID over iSCSI, never
+// shareable, so OCI refuses a second attachment of the volume anywhere.
+type AttachVolumeInput struct {
+	InstanceID  string
+	VolumeID    string
+	DisplayName string
+	ReadOnly    bool
+}
+
+// AttachmentFilter narrows ListVolumeAttachments. CompartmentID is required.
+type AttachmentFilter struct {
+	CompartmentID string
+	VolumeID      string
+	InstanceID    string
+}
+
+// VolumeBackup is an OCI volume backup. SizeGB is the source volume's size.
+type VolumeBackup struct {
+	ID            string
+	CompartmentID string
+	VolumeID      string
+	DisplayName   string
+	SizeGB        int64
+	UniqueSizeGB  int64
+	State         BackupState
+	FreeformTags  map[string]string
+	TimeCreated   time.Time
+}
+
+// Tags is the backup's freeform tags.
+func (b VolumeBackup) Tags() map[string]string { return b.FreeformTags }
+
+// IsGone reports whether the backup is deleted or being deleted.
+func (b VolumeBackup) IsGone() bool { return b.State.Gone() }
+
+// CreateVolumeBackupInput takes an incremental backup of VolumeID, in the
+// volume's compartment. RetryToken is sent as opc-retry-token.
+type CreateVolumeBackupInput struct {
+	VolumeID     string
+	DisplayName  string
+	FreeformTags map[string]string
+	RetryToken   string
+}
+
+// BackupFilter narrows ListVolumeBackups the way VolumeFilter does.
+type BackupFilter struct {
+	CompartmentID string
+	DisplayName   string
+	Tags          map[string]string
+}
+
+// blockAPIClient is the BlockClient over the OCI Go SDK.
+type blockAPIClient struct {
+	storage core.BlockstorageClient
+	compute core.ComputeClient
+}
+
+var _ BlockClient = (*blockAPIClient)(nil)
+
+// NewBlockClientWithProvider builds a BlockClient from an explicit
+// configuration provider.
+func NewBlockClientWithProvider(p common.ConfigurationProvider) (BlockClient, error) {
+	storage, err := core.NewBlockstorageClientWithConfigurationProvider(p)
+	if err != nil {
+		return nil, fmt.Errorf("oci block storage client: %w", err)
+	}
+	compute, err := core.NewComputeClientWithConfigurationProvider(p)
+	if err != nil {
+		return nil, fmt.Errorf("oci compute client: %w", err)
+	}
+	return &blockAPIClient{storage: storage, compute: compute}, nil
+}
+
+// NewInstancePrincipalBlockClient authenticates as the instance itself, as
+// NewInstancePrincipalClient does.
+func NewInstancePrincipalBlockClient() (BlockClient, error) {
+	provider, err := auth.InstancePrincipalConfigurationProvider()
+	if err != nil {
+		return nil, fmt.Errorf("oci instance principal: %w", err)
+	}
+	return NewBlockClientWithProvider(provider)
+}
+
+// NewConfigFileBlockClient authenticates from an API-key config file, with the
+// same defaults as NewConfigFileClient.
+func NewConfigFileBlockClient(path, profile string) (BlockClient, error) {
+	provider, err := configFileProvider(path, profile)
+	if err != nil {
+		return nil, err
+	}
+	return NewBlockClientWithProvider(provider)
+}
+
+func (c *blockAPIClient) CreateVolume(ctx context.Context, in CreateVolumeInput) (BlockVolume, error) {
+	details := core.CreateVolumeDetails{
+		CompartmentId:      &in.CompartmentID,
+		AvailabilityDomain: &in.AvailabilityDomain,
+		DisplayName:        optional(in.DisplayName),
+		SizeInGBs:          &in.SizeGB,
+		FreeformTags:       in.FreeformTags,
+	}
+	if in.VPUsPerGB > 0 {
+		details.VpusPerGB = &in.VPUsPerGB
+	}
+	if in.SourceBackupID != "" {
+		details.SourceDetails = core.VolumeSourceFromVolumeBackupDetails{Id: &in.SourceBackupID}
+	}
+	resp, err := c.storage.CreateVolume(ctx, core.CreateVolumeRequest{
+		CreateVolumeDetails: details,
+		OpcRetryToken:       optional(in.RetryToken),
+	})
+	if err != nil {
+		return BlockVolume{}, wrap("CreateVolume", err)
+	}
+	return toBlockVolume(resp.Volume), nil
+}
+
+func (c *blockAPIClient) GetVolume(ctx context.Context, volumeID string) (BlockVolume, error) {
+	resp, err := c.storage.GetVolume(ctx, core.GetVolumeRequest{VolumeId: &volumeID})
+	if err != nil {
+		return BlockVolume{}, wrap("GetVolume", err)
+	}
+	return toBlockVolume(resp.Volume), nil
+}
+
+func (c *blockAPIClient) ListVolumes(ctx context.Context, filter VolumeFilter) ([]BlockVolume, error) {
+	seen := map[string]bool{}
+	return listAll(func(page *string) ([]core.Volume, *string, error) {
+		resp, err := c.storage.ListVolumes(ctx, core.ListVolumesRequest{
+			CompartmentId:      &filter.CompartmentID,
+			AvailabilityDomain: optional(filter.AvailabilityDomain),
+			DisplayName:        optional(filter.DisplayName),
+			Page:               page,
+		})
+		return resp.Items, resp.OpcNextPage, wrap("ListVolumes", err)
+	}, func(item core.Volume) bool {
+		return hasTags(item.FreeformTags, filter.Tags) && firstSight(seen, item.Id)
+	}, infallible(toBlockVolume))
+}
+
+func (c *blockAPIClient) UpdateVolume(ctx context.Context, volumeID string, in UpdateVolumeInput) (BlockVolume, error) {
+	details := core.UpdateVolumeDetails{FreeformTags: in.FreeformTags}
+	if in.SizeGB > 0 {
+		details.SizeInGBs = &in.SizeGB
+	}
+	resp, err := c.storage.UpdateVolume(ctx, core.UpdateVolumeRequest{VolumeId: &volumeID, UpdateVolumeDetails: details})
+	if err != nil {
+		return BlockVolume{}, wrap("UpdateVolume", err)
+	}
+	return toBlockVolume(resp.Volume), nil
+}
+
+func (c *blockAPIClient) DeleteVolume(ctx context.Context, volumeID string) error {
+	_, err := c.storage.DeleteVolume(ctx, core.DeleteVolumeRequest{VolumeId: &volumeID})
+	return wrap("DeleteVolume", err)
+}
+
+func (c *blockAPIClient) AttachVolume(ctx context.Context, in AttachVolumeInput) (VolumeAttachment, error) {
+	notShareable := false
+	resp, err := c.compute.AttachVolume(ctx, core.AttachVolumeRequest{
+		AttachVolumeDetails: core.AttachIScsiVolumeDetails{
+			InstanceId:  &in.InstanceID,
+			VolumeId:    &in.VolumeID,
+			DisplayName: optional(in.DisplayName),
+			IsReadOnly:  &in.ReadOnly,
+			IsShareable: &notShareable,
+		},
+	})
+	if err != nil {
+		return VolumeAttachment{}, wrap("AttachVolume", err)
+	}
+	return toVolumeAttachment(resp.VolumeAttachment), nil
+}
+
+func (c *blockAPIClient) GetVolumeAttachment(ctx context.Context, attachmentID string) (VolumeAttachment, error) {
+	resp, err := c.compute.GetVolumeAttachment(ctx, core.GetVolumeAttachmentRequest{VolumeAttachmentId: &attachmentID})
+	if err != nil {
+		return VolumeAttachment{}, wrap("GetVolumeAttachment", err)
+	}
+	return toVolumeAttachment(resp.VolumeAttachment), nil
+}
+
+func (c *blockAPIClient) ListVolumeAttachments(ctx context.Context, filter AttachmentFilter) ([]VolumeAttachment, error) {
+	return listAll(func(page *string) ([]core.VolumeAttachment, *string, error) {
+		resp, err := c.compute.ListVolumeAttachments(ctx, core.ListVolumeAttachmentsRequest{
+			CompartmentId: &filter.CompartmentID,
+			VolumeId:      optional(filter.VolumeID),
+			InstanceId:    optional(filter.InstanceID),
+			Page:          page,
+		})
+		return resp.Items, resp.OpcNextPage, wrap("ListVolumeAttachments", err)
+	}, nil, infallible(toVolumeAttachment))
+}
+
+func (c *blockAPIClient) DetachVolume(ctx context.Context, attachmentID string) error {
+	_, err := c.compute.DetachVolume(ctx, core.DetachVolumeRequest{VolumeAttachmentId: &attachmentID})
+	return wrap("DetachVolume", err)
+}
+
+func (c *blockAPIClient) CreateVolumeBackup(ctx context.Context, in CreateVolumeBackupInput) (VolumeBackup, error) {
+	resp, err := c.storage.CreateVolumeBackup(ctx, core.CreateVolumeBackupRequest{
+		CreateVolumeBackupDetails: core.CreateVolumeBackupDetails{
+			VolumeId:     &in.VolumeID,
+			DisplayName:  optional(in.DisplayName),
+			FreeformTags: in.FreeformTags,
+			Type:         core.CreateVolumeBackupDetailsTypeIncremental,
+		},
+		OpcRetryToken: optional(in.RetryToken),
+	})
+	if err != nil {
+		return VolumeBackup{}, wrap("CreateVolumeBackup", err)
+	}
+	return toVolumeBackup(resp.VolumeBackup), nil
+}
+
+func (c *blockAPIClient) GetVolumeBackup(ctx context.Context, backupID string) (VolumeBackup, error) {
+	resp, err := c.storage.GetVolumeBackup(ctx, core.GetVolumeBackupRequest{VolumeBackupId: &backupID})
+	if err != nil {
+		return VolumeBackup{}, wrap("GetVolumeBackup", err)
+	}
+	return toVolumeBackup(resp.VolumeBackup), nil
+}
+
+func (c *blockAPIClient) ListVolumeBackups(ctx context.Context, filter BackupFilter) ([]VolumeBackup, error) {
+	seen := map[string]bool{}
+	return listAll(func(page *string) ([]core.VolumeBackup, *string, error) {
+		resp, err := c.storage.ListVolumeBackups(ctx, core.ListVolumeBackupsRequest{
+			CompartmentId: &filter.CompartmentID,
+			DisplayName:   optional(filter.DisplayName),
+			Page:          page,
+		})
+		return resp.Items, resp.OpcNextPage, wrap("ListVolumeBackups", err)
+	}, func(item core.VolumeBackup) bool {
+		return hasTags(item.FreeformTags, filter.Tags) && firstSight(seen, item.Id)
+	}, infallible(toVolumeBackup))
+}
+
+func (c *blockAPIClient) DeleteVolumeBackup(ctx context.Context, backupID string) error {
+	_, err := c.storage.DeleteVolumeBackup(ctx, core.DeleteVolumeBackupRequest{VolumeBackupId: &backupID})
+	return wrap("DeleteVolumeBackup", err)
+}
+
+func toBlockVolume(v core.Volume) BlockVolume {
+	out := BlockVolume{
+		ID:                 deref(v.Id),
+		CompartmentID:      deref(v.CompartmentId),
+		AvailabilityDomain: deref(v.AvailabilityDomain),
+		DisplayName:        deref(v.DisplayName),
+		SizeGB:             derefInt64(v.SizeInGBs),
+		VPUsPerGB:          derefInt64(v.VpusPerGB),
+		State:              VolumeState(v.LifecycleState),
+		FreeformTags:       maps.Clone(v.FreeformTags),
+	}
+	switch src := v.SourceDetails.(type) {
+	case core.VolumeSourceFromVolumeBackupDetails:
+		out.SourceBackupID = deref(src.Id)
+	case *core.VolumeSourceFromVolumeBackupDetails:
+		out.SourceBackupID = deref(src.Id)
+	}
+	if v.TimeCreated != nil {
+		out.TimeCreated = v.TimeCreated.Time
+	}
+	return out
+}
+
+func toVolumeAttachment(a core.VolumeAttachment) VolumeAttachment {
+	if a == nil {
+		return VolumeAttachment{}
+	}
+	out := VolumeAttachment{
+		ID:                 deref(a.GetId()),
+		InstanceID:         deref(a.GetInstanceId()),
+		VolumeID:           deref(a.GetVolumeId()),
+		AvailabilityDomain: deref(a.GetAvailabilityDomain()),
+		State:              AttachmentState(a.GetLifecycleState()),
+		ReadOnly:           a.GetIsReadOnly() != nil && *a.GetIsReadOnly(),
+	}
+	var iscsi *core.IScsiVolumeAttachment
+	switch v := a.(type) {
+	case core.IScsiVolumeAttachment:
+		iscsi = &v
+	case *core.IScsiVolumeAttachment:
+		iscsi = v
+	}
+	if iscsi != nil {
+		out.IQN = deref(iscsi.Iqn)
+		out.IPv4 = deref(iscsi.Ipv4)
+		if iscsi.Port != nil {
+			out.Port = *iscsi.Port
+		}
+	}
+	return out
+}
+
+func toVolumeBackup(b core.VolumeBackup) VolumeBackup {
+	out := VolumeBackup{
+		ID:            deref(b.Id),
+		CompartmentID: deref(b.CompartmentId),
+		VolumeID:      deref(b.VolumeId),
+		DisplayName:   deref(b.DisplayName),
+		SizeGB:        derefInt64(b.SizeInGBs),
+		UniqueSizeGB:  derefInt64(b.UniqueSizeInGBs),
+		State:         BackupState(b.LifecycleState),
+		FreeformTags:  maps.Clone(b.FreeformTags),
+	}
+	if b.TimeCreated != nil {
+		out.TimeCreated = b.TimeCreated.Time
+	}
+	return out
+}
+
+// hasTags reports whether tags holds every key and value in want.
+func hasTags(tags, want map[string]string) bool {
+	for k, v := range want {
+		if got, ok := tags[k]; !ok || got != v {
+			return false
+		}
+	}
+	return true
+}
+
+// firstSight reports whether id is new to seen, and records it. OCI's list
+// calls can return a resource twice while it changes state, as a new backup
+// leaving REQUEST_RECEIVED does.
+func firstSight(seen map[string]bool, id *string) bool {
+	key := deref(id)
+	if seen[key] {
+		return false
+	}
+	seen[key] = true
+	return true
+}
+
+// optional is nil for the empty string, which the SDK omits from a request.
+func optional(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return &s
+}
+
+func derefInt64(n *int64) int64 {
+	if n == nil {
+		return 0
+	}
+	return *n
+}

spinifex/providers/cloud/oci/block_apiclient_test.go

@@ -0,0 +1,293 @@
+package oci_test
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+)
+
+func testBlockClient(t *testing.T, h http.HandlerFunc) (oci.BlockClient, *[]capture) {
+	t.Helper()
+	provider, endpoint, hc, seen := testEndpoint(t, h)
+	c, err := oci.NewTestBlockClient(provider, endpoint, hc)
+	require.NoError(t, err)
+	return c, seen
+}
+
+const volumeJSON = `{"id":"ocid1.volume.oc1..v1","compartmentId":"c1","availabilityDomain":"AD-1",
+"displayName":"vol-1","sizeInGBs":50,"sizeInMBs":51200,"vpusPerGB":20,"lifecycleState":"PROVISIONING",
+"timeCreated":"2026-11-10T03:00:00Z","freeformTags":{"spinifex-volume-id":"vol-1"},
+"sourceDetails":{"type":"volumeBackup","id":"ocid1.volumebackup.oc1..b1"}}`
+
+// The retry token is what makes a create after a lost reply return the first
+// volume instead of a second one, so it has to reach OCI as the header, with
+// the tags and the restore source in the body.
+func TestBlockCreateVolumeSendsRetryTokenTagsAndSource(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(volumeJSON))
+	})
+
+	got, err := c.CreateVolume(context.Background(), oci.CreateVolumeInput{
+		CompartmentID: "c1", AvailabilityDomain: "AD-1", DisplayName: "vol-1",
+		SizeGB: 50, VPUsPerGB: 20, SourceBackupID: "ocid1.volumebackup.oc1..b1",
+		FreeformTags: map[string]string{"spinifex-volume-id": "vol-1"},
+		RetryToken:   "token-1",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, oci.BlockVolume{
+		ID: "ocid1.volume.oc1..v1", CompartmentID: "c1", AvailabilityDomain: "AD-1", DisplayName: "vol-1",
+		SizeGB: 50, VPUsPerGB: 20, State: oci.VolumeProvisioning, SourceBackupID: "ocid1.volumebackup.oc1..b1",
+		FreeformTags: map[string]string{"spinifex-volume-id": "vol-1"}, TimeCreated: got.TimeCreated,
+	}, got)
+	assert.False(t, got.TimeCreated.IsZero())
+
+	require.Len(t, *seen, 1)
+	req := (*seen)[0]
+	assert.Equal(t, http.MethodPost, req.method)
+	assert.Equal(t, "/20160918/volumes", req.path)
+	assert.Equal(t, "token-1", req.header.Get("Opc-Retry-Token"))
+	assert.Equal(t, "c1", req.body["compartmentId"])
+	assert.Equal(t, "AD-1", req.body["availabilityDomain"])
+	assert.Equal(t, "vol-1", req.body["displayName"])
+	assert.EqualValues(t, 50, req.body["sizeInGBs"])
+	assert.EqualValues(t, 20, req.body["vpusPerGB"])
+	assert.Equal(t, map[string]any{"spinifex-volume-id": "vol-1"}, req.body["freeformTags"])
+	assert.Equal(t, map[string]any{"type": "volumeBackup", "id": "ocid1.volumebackup.oc1..b1"}, req.body["sourceDetails"])
+}
+
+// A zero tier is OCI's Lower Cost tier, so the client must leave the field out
+// rather than send a zero nobody asked for.
+func TestBlockCreateVolumeOmitsUnsetTierAndSource(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"id":"v1","lifecycleState":"AVAILABLE"}`))
+	})
+
+	_, err := c.CreateVolume(context.Background(), oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50})
+	require.NoError(t, err)
+	body := (*seen)[0].body
+	assert.NotContains(t, body, "vpusPerGB")
+	assert.NotContains(t, body, "sourceDetails")
+	assert.NotContains(t, body, "displayName")
+}
+
+// OCI's list call has no tag filter, so the client filters every page; a
+// volume carrying a different cluster's tag must not be returned.
+func TestBlockListVolumesFiltersTagsAcrossPages(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("page") == "" {
+			w.Header().Set("Opc-Next-Page", "page2")
+			_, _ = w.Write([]byte(`[{"id":"v1","freeformTags":{"spinifex-cluster":"a"}},{"id":"v2","freeformTags":{"spinifex-cluster":"b"}}]`))
+			return
+		}
+		_, _ = w.Write([]byte(`[{"id":"v3","freeformTags":{"spinifex-cluster":"a","other":"x"}},{"id":"v4"}]`))
+	})
+
+	got, err := c.ListVolumes(context.Background(), oci.VolumeFilter{
+		CompartmentID: "c1", DisplayName: "vol-1", Tags: map[string]string{"spinifex-cluster": "a"},
+	})
+	require.NoError(t, err)
+	require.Len(t, got, 2)
+	assert.Equal(t, "v1", got[0].ID)
+	assert.Equal(t, "v3", got[1].ID)
+	require.Len(t, *seen, 2)
+	assert.Equal(t, "c1", (*seen)[0].query.Get("compartmentId"))
+	assert.Equal(t, "vol-1", (*seen)[0].query.Get("displayName"))
+	assert.False(t, (*seen)[0].query.Has("availabilityDomain"), "an unset filter must not be sent")
+}
+
+func TestBlockUpdateVolumeSendsOnlyWhatChanges(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"id":"v1","sizeInGBs":60,"lifecycleState":"AVAILABLE"}`))
+	})
+
+	got, err := c.UpdateVolume(context.Background(), "v1", oci.UpdateVolumeInput{FreeformTags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	assert.Equal(t, int64(60), got.SizeGB)
+	_, err = c.UpdateVolume(context.Background(), "v1", oci.UpdateVolumeInput{SizeGB: 60})
+	require.NoError(t, err)
+
+	require.Len(t, *seen, 2)
+	assert.Equal(t, http.MethodPut, (*seen)[0].method)
+	assert.Equal(t, "/20160918/volumes/v1", (*seen)[0].path)
+	assert.NotContains(t, (*seen)[0].body, "sizeInGBs", "a tag update must not resize the volume")
+	assert.Equal(t, map[string]any{"k": "v"}, (*seen)[0].body["freeformTags"])
+	assert.EqualValues(t, 60, (*seen)[1].body["sizeInGBs"])
+	assert.NotContains(t, (*seen)[1].body, "freeformTags", "a resize must not clear the tags")
+}
+
+// The attachment is never shareable: that is what makes OCI refuse a second
+// writer anywhere, which the provider's cluster-wide exclusion rests on.
+func TestBlockAttachVolumeIsISCSIAndNotShareable(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"attachmentType":"iscsi","id":"a1","instanceId":"i1","volumeId":"v1",
+"availabilityDomain":"AD-1","lifecycleState":"ATTACHING","isReadOnly":true,
+"iqn":"iqn.2015-12.com.oracleiaas:x","ipv4":"169.254.2.2","port":3260,"compartmentId":"c1","timeCreated":"2026-11-10T03:00:00Z"}`))
+	})
+
+	got, err := c.AttachVolume(context.Background(), oci.AttachVolumeInput{InstanceID: "i1", VolumeID: "v1", DisplayName: "vol-1", ReadOnly: true})
+	require.NoError(t, err)
+	assert.Equal(t, oci.VolumeAttachment{
+		ID: "a1", InstanceID: "i1", VolumeID: "v1", AvailabilityDomain: "AD-1", State: oci.AttachmentAttaching,
+		ReadOnly: true, IQN: "iqn.2015-12.com.oracleiaas:x", IPv4: "169.254.2.2", Port: 3260,
+	}, got)
+
+	body := (*seen)[0].body
+	assert.Equal(t, "/20160918/volumeAttachments", (*seen)[0].path)
+	assert.Equal(t, "iscsi", body["type"])
+	assert.Equal(t, true, body["isReadOnly"])
+	assert.Equal(t, false, body["isShareable"])
+	assert.Equal(t, "i1", body["instanceId"])
+	assert.Equal(t, "v1", body["volumeId"])
+}
+
+// A paravirtualized attachment has no iSCSI target, and must read back with
+// none rather than fail the whole list.
+func TestBlockListVolumeAttachmentsReadsEveryType(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`[
+{"attachmentType":"iscsi","id":"a1","instanceId":"i1","volumeId":"v1","availabilityDomain":"AD-1","lifecycleState":"ATTACHED","iqn":"iqn.x","ipv4":"169.254.2.2","port":3260,"compartmentId":"c1","timeCreated":"2026-11-10T03:00:00Z"},
+{"attachmentType":"paravirtualized","id":"a2","instanceId":"i2","volumeId":"v1","availabilityDomain":"AD-1","lifecycleState":"DETACHED","compartmentId":"c1","timeCreated":"2026-11-10T03:00:00Z"}]`))
+	})
+
+	got, err := c.ListVolumeAttachments(context.Background(), oci.AttachmentFilter{CompartmentID: "c1", VolumeID: "v1"})
+	require.NoError(t, err)
+	require.Len(t, got, 2)
+	assert.Equal(t, "iqn.x", got[0].IQN)
+	assert.Equal(t, 3260, got[0].Port)
+	assert.Equal(t, oci.AttachmentDetached, got[1].State)
+	assert.Empty(t, got[1].IQN)
+	assert.Equal(t, "v1", (*seen)[0].query.Get("volumeId"))
+	assert.False(t, (*seen)[0].query.Has("instanceId"))
+}
+
+func TestBlockCreateVolumeBackupIsIncrementalWithRetryToken(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"id":"b1","compartmentId":"c1","volumeId":"v1","displayName":"snap-1","sizeInGBs":50,
+"uniqueSizeInGBs":2,"lifecycleState":"CREATING","type":"INCREMENTAL","timeCreated":"2026-11-10T03:00:00Z","freeformTags":{"spinifex-snapshot-id":"snap-1"}}`))
+	})
+
+	got, err := c.CreateVolumeBackup(context.Background(), oci.CreateVolumeBackupInput{
+		VolumeID: "v1", DisplayName: "snap-1", FreeformTags: map[string]string{"spinifex-snapshot-id": "snap-1"}, RetryToken: "token-2",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, oci.BackupCreating, got.State)
+	assert.Equal(t, int64(2), got.UniqueSizeGB)
+	assert.Equal(t, "v1", got.VolumeID)
+
+	req := (*seen)[0]
+	assert.Equal(t, "/20160918/volumeBackups", req.path)
+	assert.Equal(t, "token-2", req.header.Get("Opc-Retry-Token"))
+	assert.Equal(t, "INCREMENTAL", req.body["type"])
+	assert.Equal(t, map[string]any{"spinifex-snapshot-id": "snap-1"}, req.body["freeformTags"])
+}
+
+func TestBlockListVolumeBackupsFiltersTags(t *testing.T) {
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`[{"id":"b1","freeformTags":{"spinifex-cluster":"a"}},{"id":"b2","freeformTags":{"spinifex-cluster":"b"}}]`))
+	})
+
+	got, err := c.ListVolumeBackups(context.Background(), oci.BackupFilter{CompartmentID: "c1", Tags: map[string]string{"spinifex-cluster": "a"}})
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "b1", got[0].ID)
+}
+
+// OCI listed one new backup twice in a live run, as it left REQUEST_RECEIVED,
+// and a provider counting the entries saw two backups for one snapshot.
+func TestBlockListsAnswerEachResourceOnce(t *testing.T) {
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("page") == "" {
+			w.Header().Set("Opc-Next-Page", "page2")
+			_, _ = w.Write([]byte(`[{"id":"x1","lifecycleState":"REQUEST_RECEIVED"},{"id":"x1","lifecycleState":"CREATING"},{"id":"x2"}]`))
+			return
+		}
+		_, _ = w.Write([]byte(`[{"id":"x2"},{"id":"x3"}]`))
+	})
+
+	backups, err := c.ListVolumeBackups(context.Background(), oci.BackupFilter{CompartmentID: "c1"})
+	require.NoError(t, err)
+	ids := make([]string, 0, len(backups))
+	for _, b := range backups {
+		ids = append(ids, b.ID)
+	}
+	assert.Equal(t, []string{"x1", "x2", "x3"}, ids)
+
+	volumes, err := c.ListVolumes(context.Background(), oci.VolumeFilter{CompartmentID: "c1"})
+	require.NoError(t, err)
+	ids = ids[:0]
+	for _, v := range volumes {
+		ids = append(ids, v.ID)
+	}
+	assert.Equal(t, []string{"x1", "x2", "x3"}, ids)
+}
+
+// The two 409s OCI answers while a backup is in progress are what the provider
+// maps to volume_in_use and unavailable, so they must stay classified.
+func TestBlockConflictsAreClassified(t *testing.T) {
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusConflict)
+		_, _ = w.Write([]byte(`{"code":"Conflict","message":"There is currently a backup operation in progress"}`))
+	})
+
+	for name, call := range map[string]func() error{
+		"DeleteVolume":       func() error { return c.DeleteVolume(context.Background(), "v1") },
+		"DeleteVolumeBackup": func() error { return c.DeleteVolumeBackup(context.Background(), "b1") },
+		"DetachVolume":       func() error { return c.DetachVolume(context.Background(), "a1") },
+	} {
+		t.Run(name, func(t *testing.T) {
+			err := call()
+			require.ErrorIs(t, err, oci.ErrConflict)
+			var api *oci.APIError
+			require.ErrorAs(t, err, &api)
+			assert.Equal(t, name, api.Op)
+		})
+	}
+}
+
+func TestBlockNotFoundAndLimitAreClassified(t *testing.T) {
+	var status int
+	var code string
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(`{"code":"` + code + `","message":"m"}`))
+	})
+
+	ctx := context.Background()
+	for name, tc := range map[string]struct {
+		status int
+		code   string
+		want   error
+		call   func() error
+	}{
+		"GetVolume": {http.StatusNotFound, "NotAuthorizedOrNotFound", oci.ErrNotFound, func() error {
+			_, err := c.GetVolume(ctx, "v1")
+			return err
+		}},
+		"GetVolumeBackup": {http.StatusNotFound, "NotAuthorizedOrNotFound", oci.ErrNotFound, func() error {
+			_, err := c.GetVolumeBackup(ctx, "b1")
+			return err
+		}},
+		"GetVolumeAttachment": {http.StatusNotFound, "NotAuthorizedOrNotFound", oci.ErrNotFound, func() error {
+			_, err := c.GetVolumeAttachment(ctx, "a1")
+			return err
+		}},
+		"AttachVolume": {http.StatusBadRequest, "LimitExceeded", oci.ErrLimitExceeded, func() error {
+			_, err := c.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: "v1"})
+			return err
+		}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			status, code = tc.status, tc.code
+			err := tc.call()
+			require.ErrorIs(t, err, tc.want)
+			var api *oci.APIError
+			require.ErrorAs(t, err, &api)
+			assert.Equal(t, name, api.Op)
+		})
+	}
+}

spinifex/providers/cloud/oci/block_fake.go

@@ -0,0 +1,492 @@
+package oci
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+)
+
+// MaxISCSIAttachmentsPerInstance is OCI's cap on iSCSI attachments to one
+// instance, the same on every VM and bare-metal shape.
+const MaxISCSIAttachmentsPerInstance = 32
+
+// ErrReplyLost is what BlockFake returns for a call LoseReply staged: the
+// call took effect in OCI but its answer never arrived.
+var ErrReplyLost = errors.New("oci fake: reply lost after the call took effect")
+
+// BlockFake is an in-memory BlockClient. It models the OCI behaviours an EBS
+// provider has to handle: the 50 GB floor, retry tokens, backups that stay
+// CREATING until a test settles them, and attachments that move by state.
+type BlockFake struct {
+	stagedFailures
+
+	mu sync.Mutex
+
+	// AttachAfter and DetachAfter are how many GetVolumeAttachment calls an
+	// attachment spends ATTACHING or DETACHING before it settles. Zero means
+	// it settles in the call that started it.
+	AttachAfter int
+	DetachAfter int
+
+	// AttachmentCap caps iSCSI attachments per instance; zero means
+	// MaxISCSIAttachmentsPerInstance.
+	AttachmentCap int
+
+	volumes     map[string]*BlockVolume
+	backups     map[string]*VolumeBackup
+	attachments map[string]*fakeAttachment
+	tokens      map[string]string // op and retry token -> resource OCID
+	tokenOf     map[string]string // resource OCID -> the retry token it was created with
+	loseReply   map[string]bool
+	seq         int
+}
+
+type fakeAttachment struct {
+	VolumeAttachment
+
+	countdown int
+}
+
+var _ BlockClient = (*BlockFake)(nil)
+
+// NewBlockFake returns an empty BlockFake whose attachments spend one read in
+// each transitional state, so a caller that does not wait is caught.
+func NewBlockFake() *BlockFake {
+	return &BlockFake{
+		AttachAfter: 1,
+		DetachAfter: 1,
+		volumes:     map[string]*BlockVolume{},
+		backups:     map[string]*VolumeBackup{},
+		attachments: map[string]*fakeAttachment{},
+		tokens:      map[string]string{},
+		tokenOf:     map[string]string{},
+		loseReply:   map[string]bool{},
+
+		stagedFailures: stagedFailures{FailWith: map[string]error{}},
+	}
+}
+
+// LoseReply makes the next call to op take effect and return ErrReplyLost.
+func (f *BlockFake) LoseReply(op string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.loseReply[op] = true
+}
+
+// SettleBackup moves a backup still in progress to state, as OCI does once
+// the backup finishes or fails.
+func (f *BlockFake) SettleBackup(backupID string, state BackupState) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	b, ok := f.backups[backupID]
+	if !ok {
+		return fmt.Errorf("oci fake: no backup %s", backupID)
+	}
+	if !b.State.InProgress() {
+		return fmt.Errorf("oci fake: backup %s is %s, not in progress", backupID, b.State)
+	}
+	b.State = state
+	return nil
+}
+
+// SeedAttachment inserts an attachment directly, for state OCI holds that no
+// call in the test made, such as another instance's attachment.
+func (f *BlockFake) SeedAttachment(a VolumeAttachment) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.attachments[a.ID] = &fakeAttachment{VolumeAttachment: a}
+}
+
+// Volumes returns every volume the fake holds, terminated ones included.
+func (f *BlockFake) Volumes() []BlockVolume {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]BlockVolume, 0, len(f.volumes))
+	for _, v := range f.volumes {
+		out = append(out, cloneVolume(*v))
+	}
+	return out
+}
+
+// Backups returns every backup the fake holds, terminated ones included.
+func (f *BlockFake) Backups() []VolumeBackup {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]VolumeBackup, 0, len(f.backups))
+	for _, b := range f.backups {
+		out = append(out, cloneBackup(*b))
+	}
+	return out
+}
+
+// RetryToken is the opc-retry-token the resource was created with.
+func (f *BlockFake) RetryToken(resourceID string) string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.tokenOf[resourceID]
+}
+
+func (f *BlockFake) CreateVolume(_ context.Context, in CreateVolumeInput) (BlockVolume, error) {
+	const op = "CreateVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return BlockVolume{}, err
+	}
+	if id, ok := f.replayed(op, in.RetryToken); ok {
+		return answerOf(f, op, cloneVolume(*f.volumes[id]))
+	}
+	if in.CompartmentID == "" || in.AvailabilityDomain == "" {
+		return BlockVolume{}, fakeError(op, 400, "InvalidParameter", "compartmentId and availabilityDomain are required")
+	}
+	if in.SizeGB < MinVolumeSizeGB {
+		return BlockVolume{}, fakeError(op, 400, "InvalidParameter", fmt.Sprintf("sizeInGBs must be at least %d", MinVolumeSizeGB))
+	}
+	if in.SourceBackupID != "" {
+		b, ok := f.backups[in.SourceBackupID]
+		if !ok || b.State.Gone() {
+			return BlockVolume{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume backup not found")
+		}
+		if b.State != BackupAvailable {
+			return BlockVolume{}, fakeError(op, 409, "Conflict", "volume backup is not available")
+		}
+		if in.SizeGB < b.SizeGB {
+			return BlockVolume{}, fakeError(op, 400, "InvalidParameter", "sizeInGBs is smaller than the backup")
+		}
+	}
+	vpus := in.VPUsPerGB
+	if vpus == 0 {
+		vpus = 10
+	}
+	id := f.newID("volume")
+	f.volumes[id] = &BlockVolume{
+		ID: id, CompartmentID: in.CompartmentID, AvailabilityDomain: in.AvailabilityDomain,
+		DisplayName: in.DisplayName, SizeGB: in.SizeGB, VPUsPerGB: vpus, State: VolumeAvailable,
+		SourceBackupID: in.SourceBackupID, FreeformTags: maps.Clone(in.FreeformTags), TimeCreated: time.Now().UTC(),
+	}
+	f.remember(op, in.RetryToken, id)
+	return answerOf(f, op, cloneVolume(*f.volumes[id]))
+}
+
+func (f *BlockFake) GetVolume(_ context.Context, volumeID string) (BlockVolume, error) {
+	const op = "GetVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return BlockVolume{}, err
+	}
+	v, ok := f.volumes[volumeID]
+	if !ok {
+		return BlockVolume{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	return cloneVolume(*v), nil
+}
+
+func (f *BlockFake) ListVolumes(_ context.Context, filter VolumeFilter) ([]BlockVolume, error) {
+	const op = "ListVolumes"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return nil, err
+	}
+	var out []BlockVolume
+	for _, v := range f.volumes {
+		if v.CompartmentID != filter.CompartmentID ||
+			(filter.AvailabilityDomain != "" && v.AvailabilityDomain != filter.AvailabilityDomain) ||
+			(filter.DisplayName != "" && v.DisplayName != filter.DisplayName) ||
+			!hasTags(v.FreeformTags, filter.Tags) {
+			continue
+		}
+		out = append(out, cloneVolume(*v))
+	}
+	return out, nil
+}
+
+func (f *BlockFake) UpdateVolume(_ context.Context, volumeID string, in UpdateVolumeInput) (BlockVolume, error) {
+	const op = "UpdateVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return BlockVolume{}, err
+	}
+	v, ok := f.volumes[volumeID]
+	if !ok || v.State.Gone() {
+		return BlockVolume{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	if in.SizeGB > 0 && in.SizeGB < v.SizeGB {
+		return BlockVolume{}, fakeError(op, 400, "InvalidParameter", "a volume cannot shrink")
+	}
+	if in.SizeGB > 0 {
+		v.SizeGB = in.SizeGB
+	}
+	if in.FreeformTags != nil {
+		v.FreeformTags = maps.Clone(in.FreeformTags)
+	}
+	return answerOf(f, op, cloneVolume(*v))
+}
+
+func (f *BlockFake) DeleteVolume(_ context.Context, volumeID string) error {
+	const op = "DeleteVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return err
+	}
+	v, ok := f.volumes[volumeID]
+	if !ok || v.State.Gone() {
+		return fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	for _, b := range f.backups {
+		if b.VolumeID == volumeID && b.State.InProgress() {
+			return fakeError(op, 409, "Conflict", "There is currently a backup operation in progress for Volume "+volumeID+".")
+		}
+	}
+	for _, a := range f.attachments {
+		if a.VolumeID == volumeID && a.State != AttachmentDetached {
+			return fakeError(op, 409, "Conflict", "volume is attached")
+		}
+	}
+	v.State = VolumeTerminated
+	_, err := answerOf(f, op, struct{}{})
+	return err
+}
+
+func (f *BlockFake) AttachVolume(_ context.Context, in AttachVolumeInput) (VolumeAttachment, error) {
+	const op = "AttachVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeAttachment{}, err
+	}
+	v, ok := f.volumes[in.VolumeID]
+	if !ok || v.State.Gone() {
+		return VolumeAttachment{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	if v.State != VolumeAvailable {
+		return VolumeAttachment{}, fakeError(op, 409, "Conflict", "volume is not available")
+	}
+	limit := f.AttachmentCap
+	if limit == 0 {
+		limit = MaxISCSIAttachmentsPerInstance
+	}
+	onInstance := 0
+	for _, a := range f.attachments {
+		if a.State == AttachmentDetached {
+			continue
+		}
+		if a.VolumeID == in.VolumeID {
+			return VolumeAttachment{}, fakeError(op, 409, "Conflict", "volume is already attached and is not shareable")
+		}
+		if a.InstanceID == in.InstanceID {
+			onInstance++
+		}
+	}
+	if onInstance >= limit {
+		return VolumeAttachment{}, fakeError(op, 400, "LimitExceeded", "the instance has reached its volume attachment limit")
+	}
+	id := f.newID("volumeattachment")
+	a := &fakeAttachment{
+		VolumeAttachment: VolumeAttachment{
+			ID: id, InstanceID: in.InstanceID, VolumeID: in.VolumeID, AvailabilityDomain: v.AvailabilityDomain,
+			State: AttachmentAttaching, ReadOnly: in.ReadOnly,
+			IQN: IQNPrefix + id, IPv4: fmt.Sprintf("169.254.2.%d", 2+f.seq%250), Port: 3260,
+		},
+		countdown: f.AttachAfter,
+	}
+	if a.countdown == 0 {
+		a.State = AttachmentAttached
+	}
+	f.attachments[id] = a
+	return answerOf(f, op, a.VolumeAttachment)
+}
+
+func (f *BlockFake) GetVolumeAttachment(_ context.Context, attachmentID string) (VolumeAttachment, error) {
+	const op = "GetVolumeAttachment"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeAttachment{}, err
+	}
+	a, ok := f.attachments[attachmentID]
+	if !ok {
+		return VolumeAttachment{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume attachment not found")
+	}
+	if a.State == AttachmentAttaching || a.State == AttachmentDetaching {
+		if a.countdown > 0 {
+			a.countdown--
+		}
+		if a.countdown == 0 && a.State == AttachmentAttaching {
+			a.State = AttachmentAttached
+		} else if a.countdown == 0 {
+			a.State = AttachmentDetached
+		}
+	}
+	return a.VolumeAttachment, nil
+}
+
+func (f *BlockFake) ListVolumeAttachments(_ context.Context, filter AttachmentFilter) ([]VolumeAttachment, error) {
+	const op = "ListVolumeAttachments"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return nil, err
+	}
+	var out []VolumeAttachment
+	for _, a := range f.attachments {
+		if (filter.VolumeID != "" && a.VolumeID != filter.VolumeID) ||
+			(filter.InstanceID != "" && a.InstanceID != filter.InstanceID) {
+			continue
+		}
+		out = append(out, a.VolumeAttachment)
+	}
+	slices.SortFunc(out, func(a, b VolumeAttachment) int { return strings.Compare(a.ID, b.ID) })
+	return out, nil
+}
+
+func (f *BlockFake) DetachVolume(_ context.Context, attachmentID string) error {
+	const op = "DetachVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return err
+	}
+	a, ok := f.attachments[attachmentID]
+	if !ok {
+		return fakeError(op, 404, "NotAuthorizedOrNotFound", "volume attachment not found")
+	}
+	if a.State == AttachmentDetaching || a.State == AttachmentDetached {
+		return fakeError(op, 409, "Conflict", "volume attachment is "+string(a.State))
+	}
+	a.State, a.countdown = AttachmentDetaching, f.DetachAfter
+	if a.countdown == 0 {
+		a.State = AttachmentDetached
+	}
+	_, err := answerOf(f, op, struct{}{})
+	return err
+}
+
+func (f *BlockFake) CreateVolumeBackup(_ context.Context, in CreateVolumeBackupInput) (VolumeBackup, error) {
+	const op = "CreateVolumeBackup"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeBackup{}, err
+	}
+	if id, ok := f.replayed(op, in.RetryToken); ok {
+		return answerOf(f, op, cloneBackup(*f.backups[id]))
+	}
+	v, ok := f.volumes[in.VolumeID]
+	if !ok || v.State.Gone() {
+		return VolumeBackup{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	id := f.newID("volumebackup")
+	f.backups[id] = &VolumeBackup{
+		ID: id, CompartmentID: v.CompartmentID, VolumeID: in.VolumeID, DisplayName: in.DisplayName,
+		SizeGB: v.SizeGB, State: BackupCreating, FreeformTags: maps.Clone(in.FreeformTags), TimeCreated: time.Now().UTC(),
+	}
+	f.remember(op, in.RetryToken, id)
+	return answerOf(f, op, cloneBackup(*f.backups[id]))
+}
+
+func (f *BlockFake) GetVolumeBackup(_ context.Context, backupID string) (VolumeBackup, error) {
+	const op = "GetVolumeBackup"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeBackup{}, err
+	}
+	b, ok := f.backups[backupID]
+	if !ok {
+		return VolumeBackup{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume backup not found")
+	}
+	return cloneBackup(*b), nil
+}
+
+func (f *BlockFake) ListVolumeBackups(_ context.Context, filter BackupFilter) ([]VolumeBackup, error) {
+	const op = "ListVolumeBackups"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return nil, err
+	}
+	var out []VolumeBackup
+	for _, b := range f.backups {
+		if b.CompartmentID != filter.CompartmentID ||
+			(filter.DisplayName != "" && b.DisplayName != filter.DisplayName) ||
+			!hasTags(b.FreeformTags, filter.Tags) {
+			continue
+		}
+		out = append(out, cloneBackup(*b))
+	}
+	return out, nil
+}
+
+func (f *BlockFake) DeleteVolumeBackup(_ context.Context, backupID string) error {
+	const op = "DeleteVolumeBackup"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return err
+	}
+	b, ok := f.backups[backupID]
+	if !ok || b.State.Gone() {
+		return fakeError(op, 404, "NotAuthorizedOrNotFound", "volume backup not found")
+	}
+	if b.State.InProgress() {
+		return fakeError(op, 409, "Conflict", "Cannot delete Volume Backup "+backupID+" while in state Creating.")
+	}
+	b.State = BackupTerminated
+	_, err := answerOf(f, op, struct{}{})
+	return err
+}
+
+// answerOf returns out, or ErrReplyLost once if LoseReply staged op.
+func answerOf[T any](f *BlockFake, op string, out T) (T, error) {
+	if f.loseReply[op] {
+		delete(f.loseReply, op)
+		var zero T
+		return zero, ErrReplyLost
+	}
+	return out, nil
+}
+
+// replayed finds the resource an earlier call with the same retry token made.
+func (f *BlockFake) replayed(op, token string) (string, bool) {
+	if token == "" {
+		return "", false
+	}
+	id, ok := f.tokens[op+"/"+token]
+	return id, ok
+}
+
+func (f *BlockFake) remember(op, token, id string) {
+	if token == "" {
+		return
+	}
+	f.tokens[op+"/"+token] = id
+	f.tokenOf[id] = token
+}
+
+func (f *BlockFake) newID(kind string) string {
+	f.seq++
+	return fmt.Sprintf("ocid1.%s.oc1..fake%06d", kind, f.seq)
+}
+
+func fakeError(op string, status int, code, message string) error {
+	return newAPIError(op, status, code, message, "fake")
+}
+
+func cloneVolume(v BlockVolume) BlockVolume {
+	v.FreeformTags = maps.Clone(v.FreeformTags)
+	return v
+}
+
+func cloneBackup(b VolumeBackup) VolumeBackup {
+	b.FreeformTags = maps.Clone(b.FreeformTags)
+	return b
+}

spinifex/providers/cloud/oci/block_fake_test.go

@@ -0,0 +1,193 @@
+package oci_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+)
+
+func fakeVolume(t *testing.T, f *oci.BlockFake, name string) oci.BlockVolume {
+	t.Helper()
+	v, err := f.CreateVolume(context.Background(), oci.CreateVolumeInput{
+		CompartmentID: "c1", AvailabilityDomain: "AD-1", DisplayName: name, SizeGB: oci.MinVolumeSizeGB,
+		FreeformTags: map[string]string{"spinifex-volume-id": name},
+	})
+	require.NoError(t, err)
+	return v
+}
+
+func TestBlockFakeVolumeFloorAndRetryToken(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+
+	_, err := f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: oci.MinVolumeSizeGB - 1})
+	require.Error(t, err, "OCI refuses a volume below its 50 GB floor")
+
+	in := oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 60, RetryToken: "t1"}
+	first, err := f.CreateVolume(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, int64(10), first.VPUsPerGB, "an unset tier is OCI's Balanced default")
+	again, err := f.CreateVolume(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, first.ID, again.ID, "a repeated retry token answers with the first volume")
+	assert.Equal(t, "t1", f.RetryToken(first.ID))
+	assert.Len(t, f.Volumes(), 1)
+}
+
+func TestBlockFakeLostReplyStillTakesEffect(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	f.LoseReply("CreateVolume")
+
+	_, err := f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50})
+	require.ErrorIs(t, err, oci.ErrReplyLost)
+	assert.Len(t, f.Volumes(), 1, "the volume exists although its creator never heard back")
+
+	f.FailOp("GetVolume", fmt.Errorf("boom"))
+	_, err = f.GetVolume(ctx, f.Volumes()[0].ID)
+	require.EqualError(t, err, "boom")
+	_, err = f.GetVolume(ctx, f.Volumes()[0].ID)
+	require.NoError(t, err, "a staged failure is used once")
+}
+
+// The experiment's two refusals: OCI will not delete a volume or a backup
+// while a backup of it is still being taken.
+func TestBlockFakeBackupInProgressRefusesDeletes(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+
+	b, err := f.CreateVolumeBackup(ctx, oci.CreateVolumeBackupInput{VolumeID: v.ID, DisplayName: "snap-1", RetryToken: "t2"})
+	require.NoError(t, err)
+	assert.Equal(t, oci.BackupCreating, b.State)
+	assert.Equal(t, v.SizeGB, b.SizeGB)
+	assert.Equal(t, "c1", b.CompartmentID)
+
+	require.ErrorIs(t, f.DeleteVolume(ctx, v.ID), oci.ErrConflict)
+	require.ErrorIs(t, f.DeleteVolumeBackup(ctx, b.ID), oci.ErrConflict)
+	_, err = f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50, SourceBackupID: b.ID})
+	require.ErrorIs(t, err, oci.ErrConflict, "a backup still being taken cannot be restored")
+
+	require.NoError(t, f.SettleBackup(b.ID, oci.BackupAvailable))
+	require.Error(t, f.SettleBackup(b.ID, oci.BackupFaulty), "a settled backup does not settle again")
+	require.Error(t, f.SettleBackup("absent", oci.BackupAvailable))
+
+	restored, err := f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50, SourceBackupID: b.ID})
+	require.NoError(t, err)
+	assert.Equal(t, b.ID, restored.SourceBackupID)
+
+	require.NoError(t, f.DeleteVolume(ctx, v.ID))
+	got, err := f.GetVolume(ctx, v.ID)
+	require.NoError(t, err, "OCI keeps a terminated volume readable")
+	assert.Equal(t, oci.VolumeTerminated, got.State)
+	require.ErrorIs(t, f.DeleteVolume(ctx, v.ID), oci.ErrNotFound)
+
+	require.NoError(t, f.DeleteVolumeBackup(ctx, b.ID))
+	require.ErrorIs(t, f.DeleteVolumeBackup(ctx, b.ID), oci.ErrNotFound)
+	_, err = f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50, SourceBackupID: b.ID})
+	require.ErrorIs(t, err, oci.ErrNotFound)
+}
+
+func TestBlockFakeAttachmentLifecycle(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+
+	a, err := f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: v.ID})
+	require.NoError(t, err)
+	assert.Equal(t, oci.AttachmentAttaching, a.State)
+	assert.NotEmpty(t, a.IQN)
+	assert.Equal(t, 3260, a.Port)
+
+	_, err = f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i2", VolumeID: v.ID})
+	require.ErrorIs(t, err, oci.ErrConflict, "a volume that is not shareable attaches once")
+	require.ErrorIs(t, f.DeleteVolume(ctx, v.ID), oci.ErrConflict, "an attached volume cannot be deleted")
+
+	got, err := f.GetVolumeAttachment(ctx, a.ID)
+	require.NoError(t, err)
+	assert.Equal(t, oci.AttachmentAttached, got.State)
+
+	require.NoError(t, f.DetachVolume(ctx, a.ID))
+	listed, err := f.ListVolumeAttachments(ctx, oci.AttachmentFilter{CompartmentID: "c1", VolumeID: v.ID})
+	require.NoError(t, err)
+	require.Len(t, listed, 1)
+	assert.Equal(t, oci.AttachmentDetaching, listed[0].State)
+	require.ErrorIs(t, f.DetachVolume(ctx, a.ID), oci.ErrConflict)
+
+	got, err = f.GetVolumeAttachment(ctx, a.ID)
+	require.NoError(t, err)
+	assert.Equal(t, oci.AttachmentDetached, got.State)
+	none, err := f.ListVolumeAttachments(ctx, oci.AttachmentFilter{CompartmentID: "c1", InstanceID: "i2"})
+	require.NoError(t, err)
+	assert.Empty(t, none)
+
+	_, err = f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i2", VolumeID: v.ID})
+	require.NoError(t, err, "a detached volume attaches again")
+}
+
+func TestBlockFakeAttachmentCap(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	f.AttachAfter = 0
+	for i := range oci.MaxISCSIAttachmentsPerInstance {
+		v := fakeVolume(t, f, fmt.Sprintf("vol-%d", i))
+		_, err := f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: v.ID})
+		require.NoError(t, err)
+	}
+	extra := fakeVolume(t, f, "vol-extra")
+	_, err := f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: extra.ID})
+	require.ErrorIs(t, err, oci.ErrLimitExceeded)
+	_, err = f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i2", VolumeID: extra.ID})
+	require.NoError(t, err, "the cap is per instance")
+}
+
+func TestBlockFakeListsFilterByCompartmentNameAndTag(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+	fakeVolume(t, f, "vol-2")
+	_, err := f.CreateVolumeBackup(ctx, oci.CreateVolumeBackupInput{VolumeID: v.ID, DisplayName: "snap-1", FreeformTags: map[string]string{"k": "a"}})
+	require.NoError(t, err)
+
+	vols, err := f.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: "c1", Tags: map[string]string{"spinifex-volume-id": "vol-1"}})
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	assert.Equal(t, v.ID, vols[0].ID)
+	vols, err = f.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: "c1", DisplayName: "vol-2", AvailabilityDomain: "AD-1"})
+	require.NoError(t, err)
+	assert.Len(t, vols, 1)
+	vols, err = f.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: "c2"})
+	require.NoError(t, err)
+	assert.Empty(t, vols)
+
+	backups, err := f.ListVolumeBackups(ctx, oci.BackupFilter{CompartmentID: "c1", Tags: map[string]string{"k": "a"}})
+	require.NoError(t, err)
+	assert.Len(t, backups, 1)
+	backups, err = f.ListVolumeBackups(ctx, oci.BackupFilter{CompartmentID: "c1", DisplayName: "snap-2"})
+	require.NoError(t, err)
+	assert.Empty(t, backups)
+}
+
+func TestBlockFakeUpdateVolume(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+
+	got, err := f.UpdateVolume(ctx, v.ID, oci.UpdateVolumeInput{FreeformTags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	assert.Equal(t, v.SizeGB, got.SizeGB)
+	assert.Equal(t, map[string]string{"k": "v"}, got.FreeformTags)
+	got, err = f.UpdateVolume(ctx, v.ID, oci.UpdateVolumeInput{SizeGB: 80})
+	require.NoError(t, err)
+	assert.Equal(t, int64(80), got.SizeGB)
+	assert.Equal(t, map[string]string{"k": "v"}, got.FreeformTags, "a resize keeps the tags")
+	_, err = f.UpdateVolume(ctx, v.ID, oci.UpdateVolumeInput{SizeGB: 60})
+	require.Error(t, err, "a volume cannot shrink")
+	_, err = f.UpdateVolume(ctx, "absent", oci.UpdateVolumeInput{SizeGB: 60})
+	require.ErrorIs(t, err, oci.ErrNotFound)
+}

spinifex/providers/cloud/oci/client.go

@@ -94,6 +94,16 @@ const DefaultConfigProfile = "DEFAULT"
 // The default, because it needs nothing from a tenancy admin. It costs a private
 // key on every node, which NewInstancePrincipalClient avoids.
 func NewConfigFileClient(path, profile string) (Client, error) {
+	provider, err := configFileProvider(path, profile)
+	if err != nil {
+		return nil, err
+	}
+	return NewClientWithProvider(provider)
+}
+
+// configFileProvider reads API-key credentials from path and profile, each
+// defaulting as the OCI CLI does.
+func configFileProvider(path, profile string) (common.ConfigurationProvider, error) {
 	if path == "" {
 		path = DefaultConfigFile
 	}
@@ -108,7 +118,7 @@ func NewConfigFileClient(path, profile string) (Client, error) {
 	if err != nil {
 		return nil, fmt.Errorf("oci config file %s (profile %s): %w", expanded, profile, err)
 	}
-	return NewClientWithProvider(provider)
+	return provider, nil
 }
 
 // expandHome resolves a leading ~ so operators can configure the conventional
@@ -176,54 +186,56 @@ func (c *apiClient) GetPrivateIP(ctx context.Context, privateIPID string) (Priva
 }
 
 func (c *apiClient) ListPrivateIPs(ctx context.Context, vnicID string) ([]PrivateIP, error) {
-	var out []PrivateIP
-	var page *string
-	for {
+	return listAll(func(page *string) ([]core.PrivateIp, *string, error) {
 		resp, err := c.net.ListPrivateIps(ctx, core.ListPrivateIpsRequest{VnicId: &vnicID, Page: page})
-		if err != nil {
-			return nil, wrap("ListPrivateIPs", err)
-		}
-		for _, item := range resp.Items {
-			ip, err := toPrivateIP(item)
-			if err != nil {
-				return nil, err
-			}
-			out = append(out, ip)
-		}
-		if resp.OpcNextPage == nil {
-			return out, nil
-		}
-		page = resp.OpcNextPage
-	}
+		return resp.Items, resp.OpcNextPage, wrap("ListPrivateIPs", err)
+	}, nil, toPrivateIP)
 }
 
 func (c *apiClient) ListReservedPublicIPs(ctx context.Context, compartmentID string) ([]PublicIP, error) {
-	var out []PublicIP
-	var page *string
-	for {
+	return listAll(func(page *string) ([]core.PublicIp, *string, error) {
 		resp, err := c.net.ListPublicIps(ctx, core.ListPublicIpsRequest{
 			CompartmentId: &compartmentID,
 			Scope:         core.ListPublicIpsScopeRegion,
 			Lifetime:      core.ListPublicIpsLifetimeReserved,
 			Page:          page,
 		})
+		return resp.Items, resp.OpcNextPage, wrap("ListReservedPublicIPs", err)
+	}, nil, toPublicIP)
+}
+
+// listAll reads every page fetch returns, from the first, and converts each
+// item keep accepts; a nil keep accepts every item.
+func listAll[I, T any](fetch func(page *string) ([]I, *string, error), keep func(I) bool, convert func(I) (T, error)) ([]T, error) {
+	var out []T
+	var page *string
+	for {
+		items, next, err := fetch(page)
 		if err != nil {
-			return nil, wrap("ListReservedPublicIPs", err)
+			return nil, err
 		}
-		for _, item := range resp.Items {
-			ip, err := toPublicIP(item)
+		for _, item := range items {
+			if keep != nil && !keep(item) {
+				continue
+			}
+			v, err := convert(item)
 			if err != nil {
 				return nil, err
 			}
-			out = append(out, ip)
+			out = append(out, v)
 		}
-		if resp.OpcNextPage == nil {
+		if next == nil {
 			return out, nil
 		}
-		page = resp.OpcNextPage
+		page = next
 	}
 }
 
+// infallible adapts a conversion that cannot fail to listAll.
+func infallible[I, T any](convert func(I) T) func(I) (T, error) {
+	return func(item I) (T, error) { return convert(item), nil }
+}
+
 func (c *apiClient) GetSubnet(ctx context.Context, subnetID string) (Subnet, error) {
 	resp, err := c.net.GetSubnet(ctx, core.GetSubnetRequest{SubnetId: &subnetID})
 	if err != nil {
@@ -361,21 +373,21 @@ func wrap(op string, err error) error {
 	if !ok {
 		return fmt.Errorf("oci %s: %w", op, err)
 	}
-	api := &APIError{
-		Op:         op,
-		Code:       svc.GetCode(),
-		Message:    svc.GetMessage(),
-		StatusCode: svc.GetHTTPStatusCode(),
-		RequestID:  svc.GetOpcRequestID(),
-	}
+	return newAPIError(op, svc.GetHTTPStatusCode(), svc.GetCode(), svc.GetMessage(), svc.GetOpcRequestID())
+}
+
+// newAPIError builds an APIError classified by its HTTP status, as wrap does
+// for the SDK's errors and the fakes do for the failures they model.
+func newAPIError(op string, status int, code, message, requestID string) *APIError {
+	api := &APIError{Op: op, Code: code, Message: message, StatusCode: status, RequestID: requestID}
 	switch {
-	case svc.GetHTTPStatusCode() == 404:
+	case status == 404:
 		api.kind = ErrNotFound
-	case svc.GetHTTPStatusCode() == 409, svc.GetHTTPStatusCode() == 412:
+	case status == 409, status == 412:
 		api.kind = ErrConflict
-	case svc.GetHTTPStatusCode() == 400 && svc.GetCode() == "LimitExceeded":
+	case status == 400 && code == "LimitExceeded":
 		api.kind = ErrLimitExceeded
-	case svc.GetHTTPStatusCode() == 429:
+	case status == 429:
 		api.kind = ErrLimitExceeded
 	}
 	return api

spinifex/providers/cloud/oci/export_test.go

@@ -1,6 +1,7 @@
 package oci
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
@@ -38,3 +39,19 @@ func NewTestClient(provider common.ConfigurationProvider, endpoint string, hc *h
 	c.HTTPClient = hc
 	return &apiClient{net: c}, nil
 }
+
+// NewTestBlockClient points the real SDK-backed BlockClient at a local
+// endpoint, for both the block storage and the compute half.
+func NewTestBlockClient(provider common.ConfigurationProvider, endpoint string, hc *http.Client) (BlockClient, error) {
+	client, err := NewBlockClientWithProvider(provider)
+	if err != nil {
+		return nil, err
+	}
+	c, ok := client.(*blockAPIClient)
+	if !ok {
+		return nil, fmt.Errorf("NewBlockClientWithProvider built a %T", client)
+	}
+	c.storage.Host, c.storage.HTTPClient = endpoint, hc
+	c.compute.Host, c.compute.HTTPClient = endpoint, hc
+	return c, nil
+}