test: forget stale host keys for reused OCI addresses before formation

This reverts commit 5adf4130e4.

.github/workflows/e2e-cloudvendor-nightly.yml

@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
     inputs:
       topologies:
-        description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
+        description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal vm-single-ocibsd vm-multi-ocibsd). One name runs one topology."
         required: false
       workbooks:
         description: "Space-separated workbooks (blank = the driver's default list)"
@@ -62,9 +62,10 @@ jobs:
     # a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
     # lifts the six-hour cap a hosted job carries.
     runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
-    # Three topologies including a bare-metal build, each with its own install,
-    # formation and workbook pass. The sweep still runs if this fires.
-    timeout-minutes: 300
+    # Six topologies including a bare-metal build, each with its own install,
+    # formation and workbook pass, so this is the hosted runner's cap. The sweep
+    # still runs if this fires, ocibsd volumes and backups included.
+    timeout-minutes: 360
     steps:
       # Set here rather than in a job-level env block: the runner context is not
       # available at job level, only from a step onwards, and a workflow that
@@ -236,8 +237,8 @@ jobs:
 
       # always(), and before the artifact upload: an OCI bare-metal host left
       # running by a cancelled or timed-out job is the most expensive way this
-      # workflow can fail. The sweep works from the Terraform state files, so it
-      # can only destroy what this run created.
+      # workflow can fail. It destroys by state file, then deletes OCI volumes and
+      # backups by the cluster tag each ocibsd topology recorded.
       - name: Sweep anything left behind
         if: always() && !inputs.keep_on_fail
         working-directory: spinifex/scripts/terraform/oci-spx

build/Dockerfile.distro

@@ -51,6 +51,8 @@ RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
 RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
 RUN cd spinifex/scripts && cp install-node.sh smoke-test.sh node-reset.sh uninstall-spx.sh /out/
 RUN mkdir -p /out/udev && cp spinifex/build/udev/* /out/udev/
+# Staged by setup.sh on an OCI VM; spx admin init or join installs it on ocibsd nodes.
+RUN cp -r spinifex/build/ocibsd /out/ocibsd
 RUN mkdir -p /out/lib && cp spinifex/scripts/lib/distro.sh /out/lib/distro.sh
 RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
 RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/

build/ocibsd/99-spinifex-ocibsd.rules

@@ -0,0 +1,4 @@
+# Give QEMU (group spinifex) read-write on the disks of OCI Block Volume
+# attachments, named by ID_PATH, which is also the /dev/disk/by-path link.
+# Partitions and every other block device keep their default ownership.
+SUBSYSTEM=="block", ACTION!="remove", ENV{DEVTYPE}=="disk", ENV{ID_PATH}=="ip-*-iscsi-iqn.2015-12.com.oracleiaas:*", GROUP="spinifex", MODE="0660"

build/ocibsd/spinifex-daemon.service.d/ocibsd.conf

@@ -0,0 +1,7 @@
+# ocibsd nodes only: QEMU, in the daemon's device cgroup, opens the SCSI disk
+# an iSCSI login creates, including one attached after the daemon started.
+[Unit]
+After=spinifex-ocibsd.service
+
+[Service]
+DeviceAllow=block-sd rw

build/ocibsd/spinifex-ocibsd.service

@@ -0,0 +1,67 @@
+# spinifex-unit-version: 1
+[Unit]
+Description=Spinifex ocibsd (OCI Block Volumes)
+PartOf=spinifex.target
+After=spinifex-nats.service iscsid.socket iscsid.service
+Wants=spinifex-nats.service iscsid.socket
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+# Root without capabilities: iscsid answers iscsiadm only for uid 0, and the
+# node records, lock and devices it touches are root-owned. iscsid does the
+# privileged session and module work.
+User=root
+Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/usr/local/bin/spx service ocibsd start
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+OOMScoreAdjust=-500
+MemoryHigh=10%
+MemoryMax=20%
+EnvironmentFile=/etc/spinifex/systemd.env
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/ocibsd/
+StateDirectory=spinifex/ocibsd
+
+# Filesystem access: iscsiadm writes node records and its lock
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=no
+DevicePolicy=closed
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem
+ReadWritePaths=/var/log/spinifex -/etc/iscsi -/var/lib/iscsi -/run/lock
+# read-only, not yes: a config_file credential may default to ~/.oci/config
+ProtectHome=read-only
+
+# Hardening
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+AmbientCapabilities=
+# The kernel hides an iSCSI session's credential attributes without CAP_SYS_ADMIN,
+# and iscsiadm then lists no session and logs out nothing.
+CapabilityBoundingSet=CAP_SYS_ADMIN
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
+[Install]
+WantedBy=spinifex.target

docs/oci-integration/README.md

@@ -41,6 +41,8 @@ resources:
 - [Harden it before production](#harden-it-before-production)
 - [Next steps](#next-steps)
 - [Removing the deployment](#removing-the-deployment)
+- [EBS on OCI Block Volumes](#ebs-on-oci-block-volumes)
+- [Testing the OCI Block Volumes provider live](#testing-the-oci-block-volumes-provider-live)
 - [Troubleshooting](#troubleshooting)
 
 ---
@@ -773,6 +775,116 @@ oci limits resource-availability get --service-name vcn \
     --limit-name reserved-public-ip-count --compartment-id "$TENANCY_OCID"
 ```
 
+## EBS on OCI Block Volumes
+
+On an OCI virtual machine, EBS volumes can be backed by OCI Block Volumes through the `ocibsd` provider instead of Viperblock. Each EBS volume is one OCI block volume, attached over iSCSI to the node that runs the instance using it.
+
+### How the provider is chosen
+
+There is no flag. `spx admin init` takes the provider from the first of these that gives one:
+
+1. The existing `/etc/spinifex/spinifex.toml`, whose provider the node keeps; a node table that omits it is `viperblockd`.
+2. The `SPINIFEX_EBS_PROVIDER` environment variable: `viperblockd`, `qemunbd` or `ocibsd`.
+3. The OCI instance metadata service (IMDS v2), read with a one-second timeout. An OCI VM shape selects `ocibsd`; a `BM.*` shape, or no answer, selects `viperblockd`.
+
+Init prints its choice and where it came from, for example `EBS provider: ocibsd (detected: OCI virtual machine <shape>)`. Joining nodes take the provider from formation rather than choosing their own. The choice is written once for every node and cannot be changed without wiping and re-forming the cluster.
+
+For `ocibsd`, init fills the compartment and availability domain from instance metadata unless the existing config names them, generates a cluster ID that tags every OCI volume and backup the cluster makes, and defaults `auth` to `instance_principal` when the existing config names no credential. It then reads the compartment's volumes with that credential, and stops with an error naming the needed policy, `manage volume-family` and `use instances`, if the read is refused. It never falls back to Viperblock.
+
+The settings land in the node's table in `spinifex.toml`:
+
+```toml
+[nodes.<node>.ebs]
+provider = "ocibsd"
+
+[nodes.<node>.ebs.oci]
+compartment_id = "<compartment-ocid>"
+availability_domain = "<availability-domain>"
+cluster_id = "<generated-cluster-id>"
+auth = "instance_principal"
+```
+
+On an OCI VM, `setup.sh` installs the iSCSI initiator (`open-iscsi` with apt, `iscsi-initiator-utils` with dnf) and stages the `ocibsd` files under `/usr/local/share/spinifex/ocibsd`. When the provider resolves to `ocibsd`, init and join install the `spinifex-ocibsd.service` unit, its udev rule and a `spinifex-daemon` drop-in, mask `spinifex-viperblock.service`, and enable `spinifex-ocibsd.service`, so exactly one service answers for EBS on the node.
+
+To keep Viperblock on an OCI VM, name it before init:
+
+```bash
+sudo env SPINIFEX_EBS_PROVIDER=viperblockd spx admin init <your usual flags>
+```
+
+`validate-topology.sh` does this for every topology except `vm-single-ocibsd` and `vm-multi-ocibsd`, which rely on detection and need `--distro`.
+
+### One availability domain per cluster
+
+Every volume is created in the availability domain the cluster recorded at init, whichever node handles the request. Config validation refuses nodes that name different compartments or availability domains, and `spinifex-ocibsd` refuses to start on a node whose instance metadata places it in another availability domain. Deploy every node of an `ocibsd` cluster into one availability domain.
+
+### IAM policy
+
+The node's credential needs two grants in the compartment, on top of the four the address allocator needs:
+
+```text
+Allow dynamic-group <dynamic-group-name> to manage volume-family in compartment id <compartment-ocid>
+Allow dynamic-group <dynamic-group-name> to use instances in compartment id <compartment-ocid>
+```
+
+`manage volume-family` covers volumes, backups and attachments, and `use instances` covers attaching them to the node. The instance-principal policy that `--setup-instance-principal` on an `ocibsd` topology (or `setup-identity.sh --ebs-provider ocibsd`) creates grants both, for a Dynamic Group matching `ALL {instance.compartment.id = '<compartment-ocid>'}`. Built for `viperblockd`, the default, it grants neither, and because the policy is shared by every deployment in the compartment, re-running `setup-identity.sh` without the flag removes them.
+
+For an API key, grant the same two verbs to the key user's group, alongside the four from [section 2.2](#grant-the-group-its-four-permissions):
+
+```text
+Allow group <group-name> to manage volume-family in compartment id <compartment-ocid>
+Allow group <group-name> to use instances in compartment id <compartment-ocid>
+```
+
+`terraform output -json api_key_policy_statements` prints all six for the group named by `api_key_group_name` (default `spinifex-nodes`) when `ebs_provider = "ocibsd"`, and only the four address grants otherwise. Because init defaults to the instance principal, an API-key node needs its credential named in `spinifex.toml` before init, which init then keeps:
+
+```toml
+[nodes.<node>.ebs]
+provider = "ocibsd"
+
+[nodes.<node>.ebs.oci]
+auth = "config_file"
+config_file = "/etc/spinifex/oci/config"
+config_profile = "spinifex"
+```
+
+`validate-topology.sh` writes this for you on an API-key run of an `ocibsd` topology, after the credential hook has installed the key.
+
+### The 50 GB floor
+
+OCI creates no block volume smaller than 50 GB, where OCI's GB is 2^30 bytes. The OCI volume is the requested size rounded up to a whole GB, and never less than 50 GB. Spinifex records the requested size in a `spinifex-capacity-bytes` tag, reports it as the EBS volume's size, and presents exactly that size to the guest, so a 1 GiB EBS volume is 1 GiB to the API and the guest. OCI bills for the 50 GB volume underneath it.
+
+## Testing the OCI Block Volumes provider live
+
+`providers/ebs/ocibsd/live_test.go` runs the EBS provider conformance suite, in process and over NATS, against a real compartment and one OCI instance.
+It also checks that OCI's GB is 2^30 bytes on the attached device, and that recreating a deleted volume under the same ID never answers with the deleted volume.
+The test runs on your workstation with the API-key profile from `~/.oci/config`, and drives `sudo iscsiadm` and the device check on the instance over SSH, so no credential is placed on the instance.
+Every subtest skips unless `SPINIFEX_OCIBSD_LIVE` is set.
+
+| Variable | Meaning |
+| --- | --- |
+| `SPINIFEX_OCIBSD_LIVE` | Any value runs the live tests |
+| `SPINIFEX_OCIBSD_COMPARTMENT_ID` | Compartment the volumes and backups are created in |
+| `SPINIFEX_OCIBSD_AVAILABILITY_DOMAIN` | The instance's availability domain, where every volume is created |
+| `SPINIFEX_OCIBSD_INSTANCE_ID` | OCID of the instance volumes are attached to |
+| `SPINIFEX_OCIBSD_SSH_HOST` | Address of that instance |
+| `SPINIFEX_OCIBSD_SSH_USER` | SSH user with passwordless `sudo`, `opc` when unset |
+| `SPINIFEX_OCIBSD_SSH_KEY` | Private key for that user |
+| `SPINIFEX_OCIBSD_SSH_KNOWN_HOSTS` | Known-hosts file, a fresh temporary one when unset |
+| `SPINIFEX_OCIBSD_CLUSTER_ID` | `spinifex-cluster` tag value for the run, random when unset and logged either way |
+| `SPINIFEX_OCIBSD_OCI_PROFILE` | OCI config profile, `DEFAULT` when unset |
+
+The instance needs `iscsi-initiator-utils` and the Oracle Cloud Agent "Block Volume Management" plugin disabled, so no agent logs sessions in or out behind the test.
+The profile needs `manage volume-family` and `use instances` in the compartment.
+
+```bash
+cd spinifex
+SPINIFEX_OCIBSD_LIVE=1 go test ./providers/ebs/ocibsd/ -run TestLive -count=1 -v -timeout 60m
+```
+
+The suite deletes what it creates, but a failed run can leave volumes and backups behind, and OCI bills for them.
+Sweep everything tagged `spinifex-cluster=<cluster ID>` in the compartment after every run.
+
 ## Troubleshooting
 
 The deploy stops at the first failure and names the log it wrote, all under `.validate-<topology>/`. Add `--keep-on-fail` to leave a failed deployment up so you can log in and look. **It keeps billing** until you run `--destroy-only`.

scripts/dev-install.sh

@@ -40,6 +40,7 @@ mkdir -p "$STAGING/scripts"
 cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
 mkdir -p "$STAGING/udev"
 cp "$PROJECT_ROOT/build/udev/"* "$STAGING/udev/"
+cp -r "$PROJECT_ROOT/build/ocibsd" "$STAGING/ocibsd"
 cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
 tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .
 

scripts/install-node.sh

@@ -50,6 +50,11 @@
 #   --yes                    Skip the confirmation prompt
 #   --dry-run                Print what would happen and touch nothing
 #
+# Environment:
+#   SPINIFEX_EBS_PROVIDER    Passed to spx admin init on the first host, naming
+#                            what backs EBS in place of its detection; joining
+#                            nodes take the provider from formation.
+#
 # THE HOST FIREWALL
 #
 # Nodes installed from the ISO boot armed, scoped to the only cluster member
@@ -159,7 +164,7 @@ while [[ $# -gt 0 ]]; do
         --no-firewall)  MANAGE_FIREWALL=false; shift ;;
         --yes|-y)   ASSUME_YES=true; shift ;;
         --dry-run)  DRY_RUN=true; shift ;;
-        -h|--help)  sed -n '2,67p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+        -h|--help)  sed -n '2,72p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
         -*)         echo "ERROR: unknown option: $1" >&2; exit 2 ;;
         *)          HOSTS+=("$1"); shift ;;
     esac
@@ -210,6 +215,18 @@ else
     done
 fi
 
+# init_env prints the env prefix that carries SPINIFEX_EBS_PROVIDER to init, or
+# nothing when it is unset. It is joined into a remote shell, so the value is
+# limited to a provider name's characters rather than quoted.
+init_env() {
+    case "$1" in
+        "") ;;
+        *[!a-z]*) echo "SPINIFEX_EBS_PROVIDER must be a provider name such as viperblockd, got: $1"; return 1 ;;
+        *) printf 'env SPINIFEX_EBS_PROVIDER=%s ' "$1" ;;
+    esac
+}
+INIT_ENV="$(init_env "${SPINIFEX_EBS_PROVIDER:-}")" || fail "$INIT_ENV"
+
 case "$IPSEC" in
     on)  IPSEC_ENABLED=true ;;
     off) IPSEC_ENABLED=false ;;
@@ -671,7 +688,7 @@ log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s
 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid ${INIT_ENV}sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
     exit 0" || fail "${HOSTS[0]}: could not start spx admin init"
 
 TOKEN=""

scripts/install-node_test.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
-# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
-# sudo's log (which holds the command it was given) never contains it, and the other values must
-# arrive as plain arguments even when they hold shell syntax.
+# Tests remote_join_cmd and init_env from install-node.sh: the join token reaches spx only on
+# stdin, so sudo's log never holds it; other values arrive as plain arguments even when they
+# hold shell syntax; and a SPINIFEX_EBS_PROVIDER that is not a provider name is refused.
 # preflight runs this with sh
 [ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
 set -euo pipefail
@@ -38,4 +38,15 @@ check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--t
 check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
 check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
 
+# init_env: its value is joined into a remote sh -c unquoted, so anything beyond
+# a provider name's characters is refused, not passed on.
+eval "$(sed -n '/^init_env()/,/^}/p' "$here/install-node.sh")"
+check "no provider sets nothing" "$(init_env "")" ""
+check "a provider is carried to init" "$(init_env viperblockd)" "env SPINIFEX_EBS_PROVIDER=viperblockd "
+check "a provider with shell syntax is refused" \
+    "$(init_env 'ocibsd;touch PWNED3' >/dev/null 2>&1 && echo accepted || echo refused)" refused
+check "a provider with a space is refused" \
+    "$(init_env 'ocibsd $(touch PWNED4)' >/dev/null 2>&1 && echo accepted || echo refused)" refused
+check "no refused value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
+
 [ "$fails" -eq 0 ]

scripts/setup.sh

@@ -17,7 +17,7 @@
 #   SETUP_STAGES               Comma-separated subset of stages to run:
 #                                deps, aws, dns, users, sudoers, firewall, timesync, files,
 #                                directories, env, systemd, sysctl, needrestart, modules,
-#                                logrotate, udev, fixown, migrations
+#                                logrotate, udev, ocibsd, fixown, migrations
 #                              Unset = run every stage appropriate for the current mode.
 
 set -e
@@ -43,6 +43,13 @@ FIREWALL_TRACK="${FIREWALL_DIR}/conntrack.nft"
 FIREWALL_MODE_FILE="${SPINIFEX_FIREWALL_MODE_FILE:-${FIREWALL_DIR}/mode}"
 FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"
 
+# The OCI instance metadata service spx admin init detects ocibsd by. v2 needs
+# the bearer header; it is a confused-deputy guard, not a credential.
+OCI_IMDS_SHAPE_URL="http://169.254.169.254/opc/v2/instance/shape"
+# Where the tarball's ocibsd/ is staged; init and join install it from here.
+OCIBSD_STAGE_DIR="/usr/local/share/spinifex/ocibsd"
+SPINIFEX_TOML="/etc/spinifex/spinifex.toml"
+
 # The daemon runs with ProtectKernelModules=yes, so every module it needs must
 # already be loaded when it starts.
 SPINIFEX_KERNEL_MODULES="openvswitch geneve vhost_net tun"
@@ -1351,6 +1358,7 @@ install_deps() {
             apt) install_apt_deps ;;
             dnf) install_dnf_deps ;;
         esac
+        install_iscsi_initiator "$pm"
     fi
 
     # Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
@@ -1385,6 +1393,53 @@ install_apt_deps() {
     fi
 }
 
+# is_oci_vm is spx admin init's detection: the OCI metadata service answers
+# within a second with a shape that is not bare metal. Asked once per run,
+# because off OCI each ask costs the full second.
+is_oci_vm() {
+    if [ -z "${OCI_VM:-}" ]; then
+        local shape=""
+        OCI_VM=no
+        if command -v curl >/dev/null 2>&1 \
+            && shape=$(curl -sf -m 1 -H 'Authorization: Bearer Oracle' "$OCI_IMDS_SHAPE_URL" 2>/dev/null); then
+            case "$shape" in
+                "" | BM.*) ;;
+                *) OCI_VM=yes ;;
+            esac
+        fi
+    fi
+    [ "$OCI_VM" = yes ]
+}
+
+# node_runs_ocibsd reports whether this node's existing config names ocibsd in
+# a [nodes.<name>.ebs] table, the provider being the same on every node.
+node_runs_ocibsd() {
+    local text
+    [ -e "$SPINIFEX_TOML" ] || return 1
+    if [ -r "$SPINIFEX_TOML" ]; then
+        text=$(cat "$SPINIFEX_TOML") || return 1
+    else
+        text=$($SUDO cat "$SPINIFEX_TOML") || return 1
+    fi
+    printf '%s\n' "$text" | awk '
+        /^[[:space:]]*\[/ { ebs = ($0 ~ /^[[:space:]]*\[nodes\.[^].]+\.ebs\][[:space:]]*(#.*)?$/); next }
+        ebs && /^[[:space:]]*provider[[:space:]]*=[[:space:]]*"ocibsd"/ { found = 1 }
+        END { exit !found }'
+}
+
+# The initiator ocibsd drives, on the only host it can back EBS on.
+install_iscsi_initiator() {
+    is_oci_vm || return 0
+    info "OCI virtual machine detected, installing the iSCSI initiator..."
+    case "$1" in
+        apt) DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq open-iscsi > /dev/null ;;
+        dnf) $SUDO dnf install -y -q iscsi-initiator-utils > /dev/null ;;
+    esac
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO systemctl enable --now iscsid.socket
+    fi
+}
+
 # The package table is the other place keyed on the OS ID: dnf alone does not
 # say which repositories or package names a host has.
 install_dnf_deps() {
@@ -2135,6 +2190,45 @@ install_udev() {
     info "udev rules installed"
 }
 
+# install_ocibsd stages the tarball's ocibsd/ on an OCI virtual machine, where
+# spx admin init or join installs it once the provider resolves to ocibsd. On
+# a node whose config already runs ocibsd it also reinstalls the staged files.
+install_ocibsd() {
+    stage "staging the ocibsd files"
+    local ocibsd_node=false f
+    node_runs_ocibsd && ocibsd_node=true
+    if ! $ocibsd_node && ! is_oci_vm; then
+        return 0
+    fi
+    if [ ! -d "$EXTRACT_DIR/ocibsd" ]; then
+        $ocibsd_node && fatal "This node runs ocibsd and the tarball has no ocibsd/ directory to reinstall it from"
+        warn "The tarball has no ocibsd/ directory, so OCI Block Volumes cannot back EBS on this node"
+        return 0
+    fi
+    $SUDO rm -rf "$OCIBSD_STAGE_DIR"
+    $SUDO install -d -m 0755 "$OCIBSD_STAGE_DIR/spinifex-daemon.service.d"
+    for f in spinifex-ocibsd.service 99-spinifex-ocibsd.rules spinifex-daemon.service.d/ocibsd.conf; do
+        $SUDO install -m 0644 "$EXTRACT_DIR/ocibsd/$f" "$OCIBSD_STAGE_DIR/$f"
+    done
+    info "  $OCIBSD_STAGE_DIR (staged)"
+    $ocibsd_node || return 0
+
+    $SUDO install -m 0644 "$OCIBSD_STAGE_DIR/spinifex-ocibsd.service" /etc/systemd/system/spinifex-ocibsd.service
+    $SUDO install -d -m 0755 /etc/systemd/system/spinifex-daemon.service.d
+    $SUDO install -m 0644 "$OCIBSD_STAGE_DIR/spinifex-daemon.service.d/ocibsd.conf" \
+        /etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf
+    $SUDO install -d /etc/udev/rules.d
+    $SUDO install -m 0644 "$OCIBSD_STAGE_DIR/99-spinifex-ocibsd.rules" /etc/udev/rules.d/99-spinifex-ocibsd.rules
+    # install_systemd has just written the viperblock unit over its mask.
+    $SUDO ln -sf /dev/null /etc/systemd/system/spinifex-viperblock.service
+    info "  ocibsd unit, device drop-in and udev rule reinstalled; spinifex-viperblock.service masked"
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO systemctl daemon-reload
+        $SUDO systemctl enable spinifex-ocibsd.service
+        $SUDO udevadm control --reload-rules
+    fi
+}
+
 # --- Upgrade handling ---
 handle_upgrade() {
     if $SUDO systemctl is-active --quiet spinifex.target 2>/dev/null; then
@@ -2273,12 +2367,12 @@ main() {
     detect_os
     detect_arch
 
-    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev.
+    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev, ocibsd.
     # Only download/extract when at least one such stage is enabled, and before
     # anything on the host changes, so the hand-off finds it untouched.
     if stage_enabled files || stage_enabled directories \
         || stage_enabled systemd || stage_enabled logrotate \
-        || stage_enabled udev; then
+        || stage_enabled udev || stage_enabled ocibsd; then
         download_spinifex
         hand_off_to_packaged_setup "${args[@]}"
     fi
@@ -2306,6 +2400,7 @@ main() {
     stage_enabled modules    && install_kernel_modules
     stage_enabled logrotate  && install_logrotate
     stage_enabled udev       && install_udev
+    stage_enabled ocibsd     && install_ocibsd
     stage_enabled swap       && setup_swap
 
     # Migrations are only safe on a live system (need a running NATS and a

scripts/setup_ocibsd_test.sh

@@ -0,0 +1,157 @@
+#!/bin/sh
+# setup_ocibsd_test.sh — setup.sh's systemd and udev stages never install the
+# tarball's ocibsd/ files; install_ocibsd stages them on an OCI VM, and
+# reinstalls them on upgrade only where the node's config already runs ocibsd.
+# A stub SUDO records every privileged command and runs the file ones under a
+# scratch root, so what each case leaves on disk is checked as well.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+SETUP="$SCRIPT_DIR/setup.sh"
+BUILD="$(dirname -- "$SCRIPT_DIR")/build"
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+# The tarball layout the release image and dev-install.sh assemble.
+EXTRACT="$TMP/extract"
+mkdir -p "$EXTRACT/systemd" "$EXTRACT/udev"
+cp "$BUILD"/systemd/* "$EXTRACT/systemd/"
+cp "$BUILD"/udev/* "$EXTRACT/udev/"
+cp -r "$BUILD/ocibsd" "$EXTRACT/ocibsd"
+
+# install, ln, rm and mkdir run with /etc and /usr/local/share/spinifex moved
+# under $ROOT; anything else is only recorded.
+cat > "$TMP/sudo" <<'EOF'
+#!/usr/bin/env bash
+echo "$*" >> "$CALLS"
+case "$1" in
+    install | ln | rm | mkdir) ;;
+    tee) cat > /dev/null; exit 0 ;;
+    *) exit 0 ;;
+esac
+args=()
+for a in "$@"; do
+    case "$a" in
+        /etc/* | /usr/local/share/spinifex/*) a="$ROOT$a" ;;
+    esac
+    args+=("$a")
+done
+exec "${args[@]}"
+EOF
+chmod +x "$TMP/sudo"
+
+# run_setup <case> <oci_vm yes|no> <config body or -> <functions...>
+# runs the named setup.sh functions in a fresh root for one case.
+run_setup() {
+    name="$1" vm="$2" body="$3"
+    shift 3
+    ROOT="$TMP/$name/root"
+    CALLS="$TMP/$name/calls"
+    mkdir -p "$ROOT/etc/systemd/system" "$ROOT/etc/spinifex" "$ROOT/usr/local/share/spinifex"
+    : > "$CALLS"
+    [ "$body" = - ] || printf '%s\n' "$body" > "$ROOT/etc/spinifex/spinifex.toml"
+    export ROOT CALLS
+    OUT=$(INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '
+        . "$1"; SUDO="$2"; EXTRACT_DIR="$3"; OCI_VM="$4"; SPINIFEX_TOML="$ROOT/etc/spinifex/spinifex.toml"
+        shift 4
+        for f in "$@"; do $f; done' _ "$SETUP" "$TMP/sudo" "$EXTRACT" "$vm" "$@" 2>&1) \
+        || fail "$name: $* failed: $OUT"
+}
+
+echo "-- the systemd and udev stages leave the tarball's ocibsd/ alone"
+run_setup units yes - install_systemd install_udev
+if grep -q " /etc/systemd/system/spinifex-viperblock.service$" "$CALLS" && grep -q " /etc/udev/rules.d/99-spinifex-vfio.rules$" "$CALLS"; then
+    pass "installs the systemd/ units and udev/ rules"
+else
+    fail "did not install the systemd/ and udev/ files: $(cat "$CALLS")"
+fi
+if grep -q "ocibsd" "$CALLS"; then
+    fail "installed an ocibsd/ file: $(grep ocibsd "$CALLS")"
+else
+    pass "installs nothing from ocibsd/"
+fi
+
+STAGED="usr/local/share/spinifex/ocibsd"
+FILES="spinifex-ocibsd.service 99-spinifex-ocibsd.rules spinifex-daemon.service.d/ocibsd.conf"
+installed_count() {
+    n=0
+    for p in etc/systemd/system/spinifex-ocibsd.service etc/udev/rules.d/99-spinifex-ocibsd.rules \
+        etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf; do
+        [ ! -e "$ROOT/$p" ] || n=$((n + 1))
+    done
+    echo "$n"
+}
+staged_ok() {
+    for f in $FILES; do
+        cmp -s "$EXTRACT/ocibsd/$f" "$ROOT/$STAGED/$f" || return 1
+    done
+}
+
+echo "-- a fresh install on an OCI VM stages ocibsd/ and installs none of it"
+run_setup fresh yes - install_systemd install_udev install_ocibsd
+if staged_ok; then pass "the three files are staged"; else fail "the stage does not match the tarball: $(ls -R "$ROOT/$STAGED" 2>&1)"; fi
+[ "$(installed_count)" = 0 ] && pass "nothing is installed" || fail "installed $(installed_count) ocibsd file(s) on a node init has not resolved"
+[ -L "$ROOT/etc/systemd/system/spinifex-viperblock.service" ] && fail "masked viperblock" || pass "viperblock is not masked"
+
+echo "-- an upgrade of an ocibsd node replaces the installed files with the new stage"
+OCIBSD_TOML='[nodes.node1.ebs]
+provider = "ocibsd"
+
+[nodes.node1.ebs.oci]
+auth = "instance_principal"'
+ROOT="$TMP/upgrade/root"
+mkdir -p "$ROOT/$STAGED/spinifex-daemon.service.d" "$ROOT/etc/systemd/system/spinifex-daemon.service.d" "$ROOT/etc/udev/rules.d"
+for f in $FILES; do echo "# the previous release" > "$ROOT/$STAGED/$f"; done
+echo "# the previous release" > "$ROOT/etc/systemd/system/spinifex-ocibsd.service"
+echo "# the previous release" > "$ROOT/etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf"
+echo "# the previous release" > "$ROOT/etc/udev/rules.d/99-spinifex-ocibsd.rules"
+ln -s /dev/null "$ROOT/etc/systemd/system/spinifex-viperblock.service"
+run_setup upgrade yes "$OCIBSD_TOML" install_systemd install_udev install_ocibsd
+if staged_ok; then pass "the stage holds the new files"; else fail "the stage kept the previous release"; fi
+if cmp -s "$EXTRACT/ocibsd/spinifex-ocibsd.service" "$ROOT/etc/systemd/system/spinifex-ocibsd.service" \
+    && cmp -s "$EXTRACT/ocibsd/spinifex-daemon.service.d/ocibsd.conf" "$ROOT/etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf" \
+    && cmp -s "$EXTRACT/ocibsd/99-spinifex-ocibsd.rules" "$ROOT/etc/udev/rules.d/99-spinifex-ocibsd.rules"; then
+    pass "the unit, drop-in and udev rule are the new ones"
+else
+    fail "an installed ocibsd file is not the new one"
+fi
+if [ "$(readlink "$ROOT/etc/systemd/system/spinifex-viperblock.service")" = /dev/null ]; then
+    pass "viperblock is masked again after the unit reinstall"
+else
+    fail "the reinstalled viperblock unit is not masked"
+fi
+grep -q '^systemctl daemon-reload$' "$CALLS" && grep -q '^udevadm control --reload-rules$' "$CALLS" \
+    && pass "systemd and udev are reloaded" || fail "no reload: $(cat "$CALLS")"
+
+echo "-- an ocibsd node is refreshed even when the metadata service does not answer"
+run_setup upgrade-no-imds no "$OCIBSD_TOML" install_ocibsd
+[ "$(installed_count)" = 3 ] && pass "the three files are installed" || fail "installed $(installed_count) of 3"
+
+echo "-- an upgrade of a viperblock node on an OCI VM installs none of them"
+run_setup viperblock yes '[nodes.node1.ebs]
+provider = "viperblockd"' install_systemd install_udev install_ocibsd
+[ "$(installed_count)" = 0 ] && pass "nothing is installed" || fail "installed $(installed_count) ocibsd file(s) on a viperblock node"
+[ -f "$ROOT/etc/systemd/system/spinifex-viperblock.service" ] && [ ! -L "$ROOT/etc/systemd/system/spinifex-viperblock.service" ] \
+    && pass "the viperblock unit is installed and not masked" || fail "the viperblock unit was masked"
+
+echo "-- off OCI nothing is staged"
+run_setup off-oci no '[nodes.node1.daemon]' install_ocibsd
+[ ! -e "$ROOT/$STAGED" ] && pass "no stage" || fail "staged ocibsd/ off OCI"
+
+echo "-- the initiator is installed on an OCI VM only"
+run_setup initiator-apt yes - "install_iscsi_initiator apt"
+grep -q 'apt-get install -y -qq open-iscsi' "$CALLS" && grep -q '^systemctl enable --now iscsid.socket$' "$CALLS" \
+    && pass "apt installs open-iscsi and enables iscsid.socket" || fail "apt: $(cat "$CALLS")"
+run_setup initiator-dnf yes - "install_iscsi_initiator dnf"
+grep -q 'dnf install -y -q iscsi-initiator-utils' "$CALLS" && pass "dnf installs iscsi-initiator-utils" || fail "dnf: $(cat "$CALLS")"
+run_setup initiator-off no - "install_iscsi_initiator apt"
+[ -s "$CALLS" ] && fail "installed an initiator off OCI: $(cat "$CALLS")" || pass "no initiator off OCI"
+
+if [ "$fails" -gt 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

scripts/terraform/oci-spx/README.md

@@ -230,11 +230,11 @@ Create them once per tenancy, then every deployment and rebuild afterwards uses
 
 **`adopt` references the dynamic group by nothing at all.** Its matching rule is `instance.compartment.id`, so it covers every instance in the compartment and names no OCID — which is why adopting needs no read on an identity resource and no tenancy rights. The cost is that a missing policy is invisible at apply time: the node forms, passes every health check, and then refuses every launch wanting a public address. The allocator gate in `validate-topology.sh` is what catches that, by requiring `ocinet credential authorised to allocate` in each node's `spinifex-daemon` journal.
 
-The policy grants four verbs in one compartment: `use vnics`, `manage private-ips`, `manage public-ips` and `use subnets`.
-That is exactly what allocating an external address does and nothing more.
+The policy grants `use vnics`, `manage private-ips`, `manage public-ips` and `use subnets` in one compartment for the allocator, and with `ebs_provider = "ocibsd"` also `manage volume-family` and `use instances` for the `ocibsd` EBS provider. `setup-identity.sh --ebs-provider ocibsd` creates it with them; the policy is shared by every deployment in the compartment, so running it without the flag removes them.
+The first four are exactly what allocating an external address does and nothing more; the last two are what creating, attaching, backing up and deleting a block volume does.
 `use subnets` looks unrelated and is not: `CreatePrivateIp` is checked against `SUBNET_ATTACH` and `CreatePrivateIp` is how an address is registered, so a policy without it authorises nothing and every allocation returns a 404.
 
-**An API key is the fallback**, and the default because it needs nothing from a tenancy admin. Grant that user only the operations Spinifex performs on addresses and nothing else; it is not a tenancy admin, and broader rights widen the blast radius of a node compromise for no benefit.
+**An API key is the fallback**, and the default because it needs nothing from a tenancy admin. Grant that user only the operations Spinifex performs on addresses and volumes and nothing else: `terraform output -json api_key_policy_statements` prints the same statements for the group named by `api_key_group_name` (default `spinifex-nodes`), for a tenancy admin to apply; it is not a tenancy admin, and broader rights widen the blast radius of a node compromise for no benefit.
 
 `spx-oci-config.sh` installs it on **every** node, as `/etc/spinifex/oci/{oci_api_key.pem,config}` — not a home directory, because the daemon's unit sets `ProtectHome=yes`. Both files are needed: `oci_config_file` names an ordinary OCI SDK config, so a node holding only the PEM forms and then fails every allocation. The profile it writes is `spinifex`, matching `oci_config_profile`:
 
@@ -275,6 +275,20 @@ The count that later gates read is the number of addresses the `hosts_file` outp
 
 **A clean teardown can still leave public IPs behind, and they count against the tenancy.** The addresses Spinifex allocates for guests are created by the running node, so they are in no Terraform state and `destroy` neither sees nor removes them. The node's own reconcile collects detached ones every ten minutes, which is exactly the sweep a destroy takes away. Twenty accumulated in `spxbm` over four runs and exhausted the tenancy's 50 reserved-public-IP limit, at which point every later launch in every compartment failed. [Check for leftover public IPs afterwards](../../../docs/oci-integration/README.md#check-for-leftover-public-ips-afterwards) has the query and the rule for reading it.
 
+### The ocibsd topologies
+
+`vm-single-ocibsd` and `vm-multi-ocibsd` are `vm-single` and `vm-multi` with EBS backed by OCI Block Volumes. They build with `ebs_provider = "ocibsd"`, which disables the Block Volume Management plugin and attaches each node's data volume paravirtualized, because every iSCSI session on the node is `ocibsd`'s to manage. They need `--distro`, because a published release predates the `ocibsd` files that `setup.sh` stages, and an external pool, because an API-key run gives `ocibsd` the pool's credential.
+
+The ocibsd topologies name no EBS provider. `spx admin init` detects that it is on an OCI virtual machine, reads the compartment, availability domain and region from the instance metadata, and generates the cluster's `cluster_id`. Every other topology sets `SPINIFEX_EBS_PROVIDER=viperblockd`, so its viperblock baseline is not replaced by detection on OCI. The run checks the `EBS provider:` line init prints: `detected` on the ocibsd topologies under the instance principal, and `from SPINIFEX_EBS_PROVIDER` on the rest.
+
+Under the instance principal, `ocibsd` uses the instance's own certificate. An API-key run runs the credential hook before formation instead, and seeds each node's `spinifex.toml` with `provider = "ocibsd"` and the hook's `/etc/spinifex/oci/config`, which init keeps as an existing config, so its `EBS provider:` line reads `from the existing config`.
+
+Every volume and backup `ocibsd` creates is tagged `spinifex-cluster=<cluster_id>`, and none of them is in any Terraform state, so the run records the ID in `.validate-<topology>/ocibsd-clusters` before any volume exists.
+
+The `ebs provider` gate then checks the units, the viperblock mask, the daemon's device drop-in, and that the compartment and availability domain init detected are the ones the state reports on every node, imports an AMI onto OCI and boots from it, and drives stop and start, attach and detach, snapshot and restore with a guest-written marker read back over the serial console. On `vm-multi-ocibsd` it also starts the stopped instance on another node. It ends by terminating everything and polling OCI until nothing tagged with the instance's volumes or snapshot is left.
+
+`e2e-oci.sh` deletes everything tagged with each recorded `cluster_id` once that topology's nodes are destroyed, including from `--sweep-only`, and fails the run if anything tagged remains. A standalone `validate-topology.sh` run does not sweep, so follow it with `./e2e-oci.sh --sweep-only`.
+
 The workbook runs **on the node** against `127.0.0.1`, because the node certificate carries no SAN for its public address — a workbook driven from outside the VCN is still blocked.
 
 ### Validate infrastructure Terraform already created

scripts/terraform/oci-spx/compute.tf

@@ -45,6 +45,11 @@ locals {
   # after the boot disk (mulga-poc has oraclevda1 pointing at sdb1).
   data_device = "/dev/oracleoci/oraclevdb"
 
+  # The Block Volume Management plugin logs iSCSI sessions in and out by itself,
+  # and on an ocibsd node those sessions are ocibsd's to manage. With the plugin
+  # off, the node's data volume attaches paravirtualized, which needs no login.
+  ocibsd_nodes = var.ebs_provider == "ocibsd"
+
   # adopt and create differ only in who creates the dynamic group and policy; a node
   # authenticates identically either way, because the matching rule covers the whole
   # compartment and so names no instance.
@@ -106,11 +111,12 @@ resource "oci_core_instance" "mulgadc" {
     user_data           = base64encode(local.user_data)
   }
 
-  # Required for automatic UHP iSCSI multipath setup and login.
+  # Required for automatic UHP iSCSI multipath setup and login, and disabled on
+  # an ocibsd node, where ocibsd owns every iSCSI session (see ocibsd_nodes).
   agent_config {
     plugins_config {
       name          = "Block Volume Management"
-      desired_state = "ENABLED"
+      desired_state = local.ocibsd_nodes ? "DISABLED" : "ENABLED"
     }
   }
 
@@ -165,10 +171,10 @@ resource "oci_core_volume" "mulgadc_data" {
 
 resource "oci_core_volume_attachment" "mulgadc_data" {
   count           = var.node_count
-  attachment_type = "iscsi"
+  attachment_type = local.ocibsd_nodes ? "paravirtualized" : "iscsi"
   # Required by OCI for Ultra High Performance iSCSI volume attachments.
   device                            = local.data_device
-  is_agent_auto_iscsi_login_enabled = true
+  is_agent_auto_iscsi_login_enabled = local.ocibsd_nodes ? null : true
   instance_id                       = oci_core_instance.mulgadc[count.index].id
   volume_id                         = oci_core_volume.mulgadc_data[count.index].id
 }

scripts/terraform/oci-spx/e2e-oci.sh

@@ -8,8 +8,10 @@
 #
 #   SPX_REF           Ref to prove. Default: the spinifex checkout's current branch.
 #   OCI_TOPOLOGIES    Space-separated. Default: "vm-single vm-multi bm
-#                     vm-single-principal". Set one name to test one thing: a
-#                     single VM topology is roughly a quarter of the full run.
+#                     vm-single-principal vm-single-ocibsd vm-multi-ocibsd". Set
+#                     one name to test one thing. The two ocibsd topologies back
+#                     EBS with OCI Block Volumes, need OCI_SOURCE=tree and an
+#                     external pool, and are recorded SKIPPED without them.
 #   OCI_SOURCE        tree (build SPX_REF) or release (the published installer).
 #   OCI_VERSION       With OCI_SOURCE=release, install this exact release tag and
 #                     ignore OCI_CHANNEL. Preferred for anything repeatable: a tag
@@ -82,7 +84,7 @@ SPINIFEX_ROOT="$(cd "$HERE/../../.." && pwd)"
 # under the workspace. Deriving one from the other is only right in the first case.
 MULGA_ROOT="${MULGA_ROOT:-$SPINIFEX_ROOT/..}"
 
-TOPOLOGIES="${OCI_TOPOLOGIES:-vm-single vm-multi bm vm-single-principal}"
+TOPOLOGIES="${OCI_TOPOLOGIES:-vm-single vm-multi bm vm-single-principal vm-single-ocibsd vm-multi-ocibsd}"
 SOURCE="${OCI_SOURCE:-tree}"
 CHANNEL="${OCI_CHANNEL:-latest}"
 INSTALL_VERSION="${OCI_VERSION:-}"
@@ -289,6 +291,35 @@ publish_summary() {
     fi
 }
 
+# ocibsd's volumes and backups live in no Terraform state, so a destroy leaves any
+# a failed run made. Each cluster_id a topology recorded is swept by its tag once
+# the nodes are gone, and anything still tagged afterwards fails the run.
+sweep_ocibsd() {
+    local dir="$1" record="$1/ocibsd-clusters" cluster compartment region left
+    [ -s "$record" ] || return 0
+    left="$(mktemp)"
+    while IFS=$'\t' read -r cluster compartment region; do
+        [ -n "$cluster" ] || continue
+        log "sweep: OCI volumes and backups of ocibsd cluster $cluster"
+        if OCIBSD_COMPARTMENT="$compartment" python3 "$HERE/scripts/oci_env.py" \
+            ${OCI_PROFILE:+--profile "$OCI_PROFILE"} --region "$region" -- \
+            python3 "$HERE/scripts/oci_volumes.py" sweep --cluster "$cluster" \
+            < /dev/null >> "$ARTIFACT_DIR/sweep.log" 2>&1; then
+            log "sweep: nothing tagged with ocibsd cluster $cluster remains"
+        else
+            log "sweep: OCI still holds volumes or backups of ocibsd cluster $cluster. See $ARTIFACT_DIR/sweep.log"
+            echo "sweep ${dir##*/.validate-} ocibsd cluster $cluster: FAIL" >> "$VERDICT"
+            printf '%s\t%s\t%s\n' "$cluster" "$compartment" "$region" >> "$left"
+        fi
+    done < "$record"
+    # Kept only for what is left, so the next sweep retries exactly that.
+    if [ -s "$left" ]; then
+        mv "$left" "$record"
+    else
+        rm -f "$left" "$record"
+    fi
+}
+
 # A topology destroys itself from its own EXIT trap, so this only catches a run
 # the trap never reached: a killed process, a runner reclaimed mid-job, a reboot.
 # It works from the state files rather than by name filter, so it can only ever
@@ -297,12 +328,16 @@ sweep() {
     local found=0 dir topology
     local sweep_env=(env)
     for dir in "$STATE_ROOT"/.validate-*; do
-        [ -s "$dir/terraform.tfstate" ] || continue
+        [ -d "$dir" ] || continue
+        if [ ! -s "$dir/terraform.tfstate" ]; then
+            sweep_ocibsd "$dir"
+            continue
+        fi
         python3 -c '
 import json, sys
 state = json.load(open(sys.argv[1]))
 sys.exit(0 if any(r["instances"] for r in state.get("resources", [])) else 1)
-' "$dir/terraform.tfstate" 2>/dev/null || continue
+' "$dir/terraform.tfstate" 2>/dev/null || { sweep_ocibsd "$dir"; continue; }
         topology="${dir##*/.validate-}"
         # The child reads OCI_STATE_ROOT from the environment, so it looks in the
         # same place this loop found the state rather than beside its own script.
@@ -319,6 +354,7 @@ sys.exit(0 if any(r["instances"] for r in state.get("resources", [])) else 1)
         if "${sweep_env[@]}" "$HERE/validate-topology.sh" --topology "$topology" --destroy-only \
             >> "$ARTIFACT_DIR/sweep.log" 2>&1; then
             log "sweep: $topology destroyed"
+            sweep_ocibsd "$dir"
         else
             log "sweep: $topology WOULD NOT DESTROY -- it is still billing. See $ARTIFACT_DIR/sweep.log"
             echo "sweep $topology: FAIL" >> "$VERDICT"
@@ -342,6 +378,10 @@ sweep
 
 if [ "$SWEEP_ONLY" = 1 ]; then
     log "--sweep-only: done"
+    if grep -q FAIL "$VERDICT"; then
+        cat "$VERDICT"
+        exit 1
+    fi
     exit 0
 fi
 
@@ -394,6 +434,22 @@ for topology in $TOPOLOGIES; do
         build_principal_env
         run_env=(env "${PRINCIPAL_ENV[@]}")
     fi
+    # ocibsd's files ship in the distro this checkout builds, and its volumes
+    # need the credential the pool uses, so neither can be stood in for.
+    case "$topology" in
+        *-ocibsd)
+            if [ "$SOURCE" != tree ]; then
+                log "$topology: SKIPPED -- the ocibsd files ship only with a tree build"
+                echo "$topology: SKIPPED (OCI_SOURCE=$SOURCE)" >> "$VERDICT"
+                continue
+            fi
+            if [ "${OCI_NO_EXTERNAL_POOL:-0}" = 1 ]; then
+                log "$topology: SKIPPED -- ocibsd needs the OCI credential the external pool installs"
+                echo "$topology: SKIPPED (OCI_NO_EXTERNAL_POOL=1)" >> "$VERDICT"
+                continue
+            fi
+            ;;
+    esac
     args=(--topology "$topology" --ssh-public-key "$SSH_PUBLIC_KEY" --ssh-private-key "$SSH_PRIVATE_KEY")
     [ "$DRY_RUN" = 1 ] && args+=(--dry-run)
     # Two separate choices. --no-external-pool is the one that forms without any
@@ -460,5 +516,8 @@ if [ "$RUN_RC" = 0 ] && ! grep -q FAIL "$VERDICT"; then
 else
     log "FAIL"
     printf 'FAIL\te2e-oci\t%d.000s\n' "$SECONDS"
+    # A failed sweep is a FAIL in the verdict with every topology passing, and is
+    # still resources left billing, so it fails the run as well.
+    RUN_RC=1
 fi
 exit "$RUN_RC"

scripts/terraform/oci-spx/instance-principal.tf

@@ -29,26 +29,44 @@ resource "oci_identity_dynamic_group" "nodes" {
 }
 
 # Narrower than it looks. The allocator creates, moves and deletes secondary
-# private IPs on a node's own VNIC and attaches public IPs to them, which is what
-# these four verbs cover; nothing here grants compute, storage or identity.
+# private IPs on a node's own VNIC and attaches public IPs to them, and ocibsd
+# creates, attaches and backs up EBS volumes. Nothing here grants identity.
+locals {
+  network_grants = [
+    "use vnics",
+    "manage private-ips",
+    "manage public-ips",
+
+    # CreatePrivateIp needs SUBNET_ATTACH and DeletePrivateIp needs SUBNET_DETACH,
+    # both of which only the subnets `use` verb carries. Without it every allocation
+    # is refused as a 404, which reads as a missing resource rather than a missing grant.
+    "use subnets",
+  ]
+
+  # ocibsd: volumes, backups and attachments, and attaching them to an instance.
+  # Without these every ocibsd call is refused and its service fails at start.
+  # A viperblockd deployment touches no OCI volume, so it is not granted them.
+  ocibsd_grants = [
+    "manage volume-family",
+    "use instances",
+  ]
+
+  node_grants = concat(local.network_grants, var.ebs_provider == "ocibsd" ? local.ocibsd_grants : [])
+}
+
 resource "oci_identity_policy" "nodes" {
   provider       = oci.home
   count          = var.instance_principal == "create" ? 1 : 0
   compartment_id = var.compartment_ocid
   name           = "${var.deployment_name}-nodes-network"
-  description    = "Allows Spinifex nodes to manage the private and public IPs backing external addresses."
+  description    = "Allows Spinifex nodes to manage the IPs backing external addresses and the OCI volumes backing EBS."
 
   statements = [
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use vnics in compartment id ${var.compartment_ocid}",
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage private-ips in compartment id ${var.compartment_ocid}",
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage public-ips in compartment id ${var.compartment_ocid}",
-
-    # CreatePrivateIp needs SUBNET_ATTACH and DeletePrivateIp needs SUBNET_DETACH,
-    # both of which only the subnets `use` verb carries. Without it every allocation
-    # is refused as a 404, which reads as a missing resource rather than a missing grant.
-    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use subnets in compartment id ${var.compartment_ocid}",
+    for grant in local.node_grants :
+    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to ${grant} in compartment id ${var.compartment_ocid}"
   ]
-    lifecycle {
+
+  lifecycle {
     precondition {
       condition     = local.tenancy_home_region != ""
       error_message = "OCI did not report a tenancy home region; cannot create instance-principal IAM resources."

scripts/terraform/oci-spx/outputs.tf

@@ -31,3 +31,20 @@ output "nodes" {
 output "hosts_file" {
   value = join("\n", [for instance in oci_core_instance.mulgadc : instance.public_ip])
 }
+
+# The api_key route's equivalent of the instance-principal policy, for a tenancy
+# admin to apply to the API user's group. Sensitive only to keep the compartment
+# OCID out of apply.log; read it with terraform output api_key_policy_statements.
+output "api_key_policy_statements" {
+  sensitive = true
+  value = [
+    for grant in local.node_grants :
+    "Allow group ${var.api_key_group_name} to ${grant} in compartment id ${var.compartment_ocid}"
+  ]
+}
+
+# Where spx admin init should detect the nodes on ocibsd, so the run can check
+# what init read from the instance metadata and knows where to sweep afterwards.
+output "availability_domain" { value = local.compute_availability_domain }
+output "region" { value = var.region }
+output "ebs_provider" { value = var.ebs_provider }

scripts/terraform/oci-spx/scripts/oci_volumes.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""Find and sweep the OCI volumes and backups ocibsd tags, which no Terraform state holds.
+
+Run under oci_env.py for the credential, with the compartment in OCIBSD_COMPARTMENT.
+Standard library and the openssl binary only; the key reaches openssl through a pipe.
+
+  oci_volumes.py volumes --tag spinifex-volume-id=vol-0abc
+  oci_volumes.py backups --tag spinifex-snapshot-id=snap-0abc
+  oci_volumes.py sweep --cluster <cluster_id> [--timeout 900]
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import email.utils
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+API = "/20160918"
+# Gone for good; OCI keeps the record for a while and it cannot be deleted again.
+GONE = {"TERMINATED"}
+# OCI's own answers to a delete it will accept later: an attachment or a backup
+# still in progress (409), or a precondition such as a busy lifecycle (412).
+RETRY_LATER = {409, 412, 429, 500, 502, 503, 504}
+
+
+class OCIError(Exception):
+    def __init__(self, status: int, body: str):
+        super().__init__(f"OCI answered {status}: {body[:300]}")
+        self.status = status
+
+
+class Client:
+    def __init__(self, env: dict[str, str]):
+        missing = [k for k in ("TF_VAR_tenancy_ocid", "TF_VAR_user_ocid", "TF_VAR_fingerprint", "TF_VAR_region") if not env.get(k)]
+        if missing:
+            raise ValueError(f"run this under oci_env.py: {', '.join(missing)} unset")
+        self.key_id = f"{env['TF_VAR_tenancy_ocid']}/{env['TF_VAR_user_ocid']}/{env['TF_VAR_fingerprint']}"
+        self.host = f"iaas.{env['TF_VAR_region']}.oraclecloud.com"
+        if env.get("TF_VAR_private_key"):
+            self.key = env["TF_VAR_private_key"].encode()
+        elif env.get("TF_VAR_private_key_path"):
+            self.key = Path(env["TF_VAR_private_key_path"]).read_bytes()
+        else:
+            raise ValueError("run this under oci_env.py: no private key in TF_VAR_private_key or TF_VAR_private_key_path")
+
+    def _sign(self, data: bytes) -> str:
+        read_end, write_end = os.pipe()
+        try:
+            os.write(write_end, self.key)
+        finally:
+            os.close(write_end)
+        try:
+            result = subprocess.run(
+                ["openssl", "dgst", "-sha256", "-sign", f"/dev/fd/{read_end}"],
+                input=data,
+                capture_output=True,
+                pass_fds=(read_end,),
+                check=False,
+            )
+        finally:
+            os.close(read_end)
+        if result.returncode != 0:
+            raise ValueError(f"openssl could not sign with the OCI key: {result.stderr.decode(errors='replace').strip()}")
+        return base64.b64encode(result.stdout).decode()
+
+    def request(self, method: str, path: str, query: dict[str, str] | None = None) -> tuple[dict[str, str], object]:
+        target = API + path
+        if query:
+            target += "?" + urllib.parse.urlencode(query)
+        date = email.utils.formatdate(usegmt=True)
+        headers = {"date": date, "host": self.host}
+        signed = ["(request-target)", "date", "host"]
+        lines = [f"(request-target): {method.lower()} {target}", f"date: {date}", f"host: {self.host}"]
+        # A PUT or POST also signs its body, which this signer does not do.
+        if method not in ("GET", "DELETE"):
+            raise ValueError(f"unsupported method {method}")
+        signature = self._sign("\n".join(lines).encode())
+        headers["authorization"] = (
+            f'Signature version="1",keyId="{self.key_id}",algorithm="rsa-sha256",'
+            f'headers="{" ".join(signed)}",signature="{signature}"'
+        )
+        req = urllib.request.Request(f"https://{self.host}{target}", method=method, headers=headers)
+        try:
+            with urllib.request.urlopen(req, timeout=60) as resp:
+                body = resp.read()
+                return dict(resp.headers), json.loads(body) if body else None
+        except urllib.error.HTTPError as error:
+            raise OCIError(error.code, error.read().decode(errors="replace")) from None
+
+    def list_all(self, kind: str, compartment: str) -> list[dict]:
+        items: list[dict] = []
+        query = {"compartmentId": compartment, "limit": "100"}
+        while True:
+            headers, page = self.request("GET", f"/{kind}", query)
+            items.extend(page or [])
+            next_page = {k.lower(): v for k, v in headers.items()}.get("opc-next-page")
+            if not next_page:
+                return items
+            query["page"] = next_page
+
+
+def matching(items: list[dict], tags: dict[str, str]) -> list[dict]:
+    """The live items carrying every tag, with no OCID beyond the item's own."""
+    found = []
+    for item in items:
+        freeform = item.get("freeformTags") or {}
+        if item.get("lifecycleState") in GONE:
+            continue
+        if all(freeform.get(k) == v for k, v in tags.items()):
+            found.append(
+                {
+                    "id": item["id"],
+                    "displayName": item.get("displayName"),
+                    "lifecycleState": item.get("lifecycleState"),
+                    "freeformTags": freeform,
+                }
+            )
+    return found
+
+
+def parse_tags(pairs: list[str]) -> dict[str, str]:
+    tags = {}
+    for pair in pairs:
+        key, sep, value = pair.partition("=")
+        if not sep or not key:
+            raise ValueError(f"--tag wants key=value, got {pair!r}")
+        tags[key] = value
+    return tags
+
+
+def sweep(client: Client, compartment: str, cluster: str, timeout: int, interval: int) -> int:
+    """Delete every live volume and backup tagged with this cluster until none is left.
+
+    OCI refuses both deletes while a backup is CREATING, so a refusal is retried
+    until the deadline, which is what makes a run killed mid-snapshot sweepable.
+    """
+    tags = {"spinifex-cluster": cluster}
+    deadline = time.monotonic() + timeout
+    while True:
+        left = []
+        for kind in ("volumeBackups", "volumes"):
+            for item in matching(client.list_all(kind, compartment), tags):
+                left.append((kind, item))
+                if item["lifecycleState"] == "TERMINATING":
+                    continue
+                try:
+                    client.request("DELETE", f"/{kind}/{item['id']}")
+                    print(f"sweep: deleting {kind[:-1]} {item['displayName']} ({item['lifecycleState']})")
+                except OCIError as error:
+                    if error.status == 404:
+                        continue
+                    if error.status not in RETRY_LATER:
+                        raise
+                    print(f"sweep: {kind[:-1]} {item['displayName']} not deletable yet: {error.status}")
+        if not left:
+            print(f"sweep: nothing tagged spinifex-cluster={cluster} is left")
+            return 0
+        if time.monotonic() >= deadline:
+            for kind, item in left:
+                print(f"sweep: STILL PRESENT {kind[:-1]} {item['displayName']} {item['lifecycleState']}")
+            print(f"sweep: {len(left)} tagged resource(s) left after {timeout}s")
+            return 1
+        time.sleep(interval)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = parser.add_subparsers(dest="command", required=True)
+    for name in ("volumes", "backups"):
+        listing = sub.add_parser(name, help=f"print the live {name} carrying every --tag, as JSON")
+        listing.add_argument("--tag", action="append", default=[], required=True)
+    sweeping = sub.add_parser("sweep", help="delete every live volume and backup tagged with --cluster")
+    sweeping.add_argument("--cluster", required=True)
+    sweeping.add_argument("--timeout", type=int, default=900)
+    sweeping.add_argument("--interval", type=int, default=15)
+    args = parser.parse_args()
+
+    compartment = os.environ.get("OCIBSD_COMPARTMENT", "")
+    if not compartment.startswith("ocid1."):
+        raise ValueError("OCIBSD_COMPARTMENT is not set to a compartment OCID")
+    client = Client(dict(os.environ))
+
+    if args.command == "sweep":
+        if not args.cluster.strip():
+            raise ValueError("--cluster is empty, which would match nothing and report clean")
+        return sweep(client, compartment, args.cluster, args.timeout, args.interval)
+    kind = "volumes" if args.command == "volumes" else "volumeBackups"
+    print(json.dumps(matching(client.list_all(kind, compartment), parse_tags(args.tag))))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (OCIError, ValueError, OSError) as error:
+        print(f"oci_volumes.py: {error}", file=sys.stderr)
+        raise SystemExit(1)

scripts/terraform/oci-spx/setup-identity.sh

@@ -21,6 +21,7 @@ STATE_DIR="${OCI_STATE_ROOT:-$HERE}/.identity"
 SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}"
 DESTROY=0
 DRY_RUN=0
+EBS_PROVIDER=viperblockd
 
 log() { printf '[setup-identity] %s\n' "$*"; }
 die() {
@@ -30,14 +31,17 @@ die() {
 
 usage() {
     cat >&2 <<EOF
-usage: ${0##*/} [--dry-run] [--destroy]
+usage: ${0##*/} [--dry-run] [--destroy] [--ebs-provider viperblockd|ocibsd]
 
 Creates the OCI dynamic group and policy for instance-principal authentication.
 Needs a tenancy-admin OCI credential; the compartment-scoped user the rest of this
 configuration runs as cannot create either resource.
 
-  --dry-run   Show the plan, change nothing.
-  --destroy   Remove them again. Every node using "adopt" loses its credential.
+  --dry-run       Show the plan, change nothing.
+  --destroy       Remove them again. Every node using "adopt" loses its credential.
+  --ebs-provider  ocibsd also grants manage volume-family and use instances. The
+                  policy is shared by every deployment in the compartment, so a
+                  later run without it removes those grants. Default viperblockd.
 EOF
     exit 2
 }
@@ -46,11 +50,16 @@ while [ $# -gt 0 ]; do
     case "$1" in
         --dry-run) DRY_RUN=1; shift ;;
         --destroy) DESTROY=1; shift ;;
+        --ebs-provider) EBS_PROVIDER="${2:?}"; shift 2 ;;
         -h | --help) usage ;;
         *) die "unknown option $1" ;;
     esac
 done
 
+case "$EBS_PROVIDER" in
+    viperblockd | ocibsd) ;;
+    *) die "--ebs-provider must be viperblockd or ocibsd, got '$EBS_PROVIDER'" ;;
+esac
 [ -r "$SSH_PUBLIC_KEY" ] || die "no readable SSH public key at $SSH_PUBLIC_KEY (set OCI_SSH_PUBLIC_KEY)"
 mkdir -p "$STATE_DIR"
 
@@ -75,7 +84,7 @@ if [ "$DESTROY" = 1 ]; then
     exit 0
 fi
 
-args=(-var 'instance_principal=create' -state "$STATE_DIR/terraform.tfstate" "${targets[@]}")
+args=(-var 'instance_principal=create' -var "ebs_provider=$EBS_PROVIDER" -state "$STATE_DIR/terraform.tfstate" "${targets[@]}")
 
 if [ "$DRY_RUN" = 1 ]; then
     log "dry run: planning the dynamic group and policy only"

scripts/terraform/oci-spx/validate-topology.sh

@@ -49,7 +49,7 @@ WORKBOOKS=""
 # set -u an unset array is an error rather than an empty one.
 HOSTS=()
 
-TOPOLOGIES="bm vm-single vm-multi vm-single-principal"
+TOPOLOGIES="bm vm-single vm-multi vm-single-principal vm-single-ocibsd vm-multi-ocibsd"
 
 # What a topology is worth when nothing says otherwise, so a CI run of a named
 # topology is the same every time. Supplied as TF_VAR_, which terraform.auto.tfvars
@@ -61,8 +61,14 @@ TOPOLOGIES="bm vm-single vm-multi vm-single-principal"
 #
 # A case rather than an associative array: macOS ships bash 3.2, where declare -A
 # is an indexed assignment and the key is read as an arithmetic variable.
+#
+# TOPO_EBS_PROVIDER is what backs EBS. The -ocibsd topologies are their viperblock
+# twins on OCI Block Volumes, so a difference between a pair is the provider's.
+# They set no provider and rely on init's detection on an OCI VM; every other
+# topology names viperblockd in SPINIFEX_EBS_PROVIDER to keep its baseline.
 topo_defaults() {
     TOPO_FORCE_PRINCIPAL=0
+    TOPO_EBS_PROVIDER=viperblockd
     case "$1" in
         bm)        TOPO_DEFAULT_SHAPE="BM.Standard.E2.64";  TOPO_DEFAULT_NODES=1 ;;
         vm-single) TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=1 ;;
@@ -74,6 +80,10 @@ topo_defaults() {
             TOPO_DEFAULT_SHAPE="${OCI_PRINCIPAL_SHAPE:-VM.Standard.E6.Flex}"
             TOPO_DEFAULT_NODES=1
             TOPO_FORCE_PRINCIPAL=1 ;;
+        vm-single-ocibsd)
+            TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=1; TOPO_EBS_PROVIDER=ocibsd ;;
+        vm-multi-ocibsd)
+            TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=3; TOPO_EBS_PROVIDER=ocibsd ;;
         *) return 1 ;;
     esac
 }
@@ -86,7 +96,7 @@ die() {
 
 usage() {
     cat >&2 <<EOF
-usage: ${0##*/} --topology <bm|vm-single|vm-multi|vm-single-principal> [options]
+usage: ${0##*/} --topology <bm|vm-single|vm-multi|vm-single-principal|vm-single-ocibsd|vm-multi-ocibsd> [options]
 
 Builds a topology, or installs on existing OCI hosts, then installs Spinifex,
 forms the cluster, and verifies the OCI address allocator.
@@ -95,6 +105,13 @@ forms the cluster, and verifies the OCI address allocator.
                           topology is the easiest expensive mistake here.
                           vm-single-principal is vm-single authenticating as the
                           instance, and implies --instance-principal.
+                          vm-single-ocibsd and vm-multi-ocibsd set no EBS
+                          provider, so init detects ocibsd on the OCI VM, and
+                          add the ebs provider gate; the others set
+                          SPINIFEX_EBS_PROVIDER=viperblockd. The ocibsd pair
+                          need --distro. Their OCI volumes and backups are
+                          in no Terraform state: the destroy here leaves any a
+                          failed run made, and e2e-oci.sh sweeps them by tag.
   --ssh-public-key PATH   Public half installed on each node. Default $SSH_PUBLIC_KEY
   --ssh-private-key PATH  Private half used to reach them. Default $SSH_PRIVATE_KEY
   --instance-principal    Configure the pool with oci_auth="instance_principal"
@@ -106,7 +123,9 @@ forms the cluster, and verifies the OCI address allocator.
                           Create the dynamic group and policy before deployment.
                           Needs a tenancy-admin profile. Opt-in because these are
                           tenancy-level resources. Use --identity-oci-profile to
-                          use a different profile for this one-time setup.
+                          use a different profile for this one-time setup. The
+                          policy carries the volume grants only on an ocibsd
+                          topology, and it is shared, so run it from one of those.
   --identity-oci-profile NAME
                           Tenancy-admin profile for --setup-instance-principal.
                           Defaults to --oci-profile / OCI_PROFILE.
@@ -209,6 +228,19 @@ fi
 
 [ -z "$EXISTING_HOSTS_FILE" ] || [ -z "$EXISTING_STATE_FILE" ] \
     || die "--existing-hosts-file and --existing-state-file are mutually exclusive"
+
+# ocibsd needs a node credential, which a run with no pool never installs, and the
+# compartment and region from the state, which a hosts file does not carry, to
+# sweep and check its volumes. setup.sh stages its files from the tarball, and a
+# published release predates them.
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ] && [ "$DESTROY_ONLY" = 0 ]; then
+    [ "$SKIP_POOL" != 1 ] \
+        || die "--no-external-pool installs no OCI credential, so ocibsd on $TOPOLOGY could not reach the OCI API"
+    [ -z "$EXISTING_HOSTS_FILE" ] \
+        || die "$TOPOLOGY reads its compartment and availability domain from Terraform state; use --existing-state-file"
+    [ -n "$DISTRO" ] || [ "$DRY_RUN" = 1 ] \
+        || die "$TOPOLOGY needs --distro: a published release predates the ocibsd files setup.sh stages and init installs"
+fi
 if [ -n "$EXISTING_HOSTS_FILE" ] || [ -n "$EXISTING_STATE_FILE" ]; then
     EXISTING_INFRA=1
     [ "$DESTROY_ONLY" = 0 ] \
@@ -281,7 +313,7 @@ tf() {
     # deployment sizes itself in that file and the topology only supplies the
     # default a CI run wants.
     TF_VAR_compute_shape="$TOPO_DEFAULT_SHAPE" TF_VAR_node_count="$TOPO_DEFAULT_NODES" \
-        python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" \
+        TF_VAR_ebs_provider="$TOPO_EBS_PROVIDER" python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" \
         ${OCI_PROFILE:+--profile "$OCI_PROFILE"} -- \
         terraform -chdir="$HERE" "$@"
 }
@@ -383,18 +415,18 @@ identity_resources() {
                 "$HERE/setup-identity.sh" --destroy
         elif [ "$DRY_RUN" = 1 ]; then
             OCI_CLI_PROFILE="$identity_profile" OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \
-                "$HERE/setup-identity.sh" --dry-run
+                "$HERE/setup-identity.sh" --dry-run --ebs-provider "$TOPO_EBS_PROVIDER"
         else
             OCI_CLI_PROFILE="$identity_profile" OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \
-                "$HERE/setup-identity.sh"
+                "$HERE/setup-identity.sh" --ebs-provider "$TOPO_EBS_PROVIDER"
         fi
     else
         if [ "$action" = destroy ]; then
             OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --destroy
         elif [ "$DRY_RUN" = 1 ]; then
-            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --dry-run
+            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --dry-run --ebs-provider "$TOPO_EBS_PROVIDER"
         else
-            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh"
+            OCI_SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" "$HERE/setup-identity.sh" --ebs-provider "$TOPO_EBS_PROVIDER"
         fi
     fi
 }
@@ -443,8 +475,10 @@ mark_stage() { : > "$STATE_DIR/stage-$1.done"; }
 formation_complete() {
     local host
     stage_done formation || return 1
+    # A line only init writes, not a non-empty file: an ocibsd API-key run seeds the
+    # file with the credential before init, so a fresh node already has one.
     for host in "${HOSTS[@]}"; do
-        ssh_node "$host" 'test -s /etc/spinifex/spinifex.toml' || return 1
+        ssh_node "$host" "grep -q '^data_dir = ' /etc/spinifex/spinifex.toml" || return 1
     done
 }
 
@@ -634,6 +668,7 @@ if [ "$DRY_RUN" = 1 ]; then
         | sed -E 's/ +=/ =/' | sort -u | sed 's/^/  /'
     log "full plan in $STATE_DIR/plan.txt"
     plan="install Spinifex on $NODES node(s), form the cluster"
+    [ "$TOPO_EBS_PROVIDER" = ocibsd ] && plan="$plan on ocibsd, run the ebs provider gate"
     [ "$SKIP_POOL" = 1 ] && plan="$plan, configure no external pool"
     if [ "$SKIP_WORKLOAD" = 1 ]; then
         plan="$plan, launch no guests"
@@ -669,6 +704,29 @@ else
         || die "read instance_principal='$PRINCIPAL_MODE' from the tfvars and the apply built '$built_principal'; set it in one place"
 fi
 
+# Where ocibsd should keep the cluster's volumes, read back from what was built
+# rather than from this run's inputs, to check what init detected and to sweep.
+# Never logged: the compartment is an account's OCID.
+topo_output() {
+    if [ "$EXISTING_INFRA" = 1 ]; then
+        state_output "$1"
+    else
+        tf output -raw "${tf_state[@]}" "$1"
+    fi
+}
+OCIBSD_COMPARTMENT=""
+OCIBSD_AD=""
+OCIBSD_REGION=""
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+    built_provider="$(topo_output ebs_provider 2>/dev/null)" \
+        || die "the state has no ebs_provider output; it predates ocibsd, so rebuild it"
+    [ "$built_provider" = ocibsd ] \
+        || die "$TOPOLOGY needs nodes built for ocibsd and the state says ebs_provider='$built_provider'; a tfvars override wins over the topology"
+    OCIBSD_COMPARTMENT="$(topo_output compartment_ocid)" || die "could not read compartment_ocid from the state"
+    OCIBSD_AD="$(topo_output availability_domain)" || die "could not read availability_domain from the state"
+    OCIBSD_REGION="$(topo_output region)" || die "could not read region from the state"
+fi
+
 # Written for every topology, not just the multi-node one that passes it to
 # install-node.sh: it is also how an operator gets the addresses afterwards
 # without a terraform invocation and the right -state path.
@@ -768,6 +826,42 @@ REMOTE
 done
 record install PASS "$INSTALLED_VERSION"
 
+
+# spx admin init's EBS choice, written to every node once (ADR-0018:S3). The ocibsd
+# topologies set nothing, so init detects ocibsd on the OCI VM; the rest name
+# viperblockd, which keeps the viperblock baseline on OCI.
+INIT_EBS_PROVIDER=""
+[ "$TOPO_EBS_PROVIDER" = ocibsd ] || INIT_EBS_PROVIDER=viperblockd
+
+# ocibsd defaults to the instance principal. An API-key run gives init the key
+# instead: the hook installs it first, and each node's config is seeded with
+# provider and credential, which init keeps as an existing config's values.
+API_KEY_EBS=0
+[ "$TOPO_EBS_PROVIDER" = ocibsd ] && [ "$PRINCIPAL_MODE" = off ] && API_KEY_EBS=1
+PRESEEDED_EBS=0
+if [ "$API_KEY_EBS" = 1 ] && ! formation_complete; then
+    log "running the credential hook before formation, so init can check the ocibsd credential"
+    "$CREDENTIAL_HOOK" "$SSH_PRIVATE_KEY" "${HOSTS[@]}" > "$STATE_DIR/credential-hook.log" 2>&1 \
+        || die "the credential hook failed; see $STATE_DIR/credential-hook.log"
+    for i in "${!HOSTS[@]}"; do
+        # Named as init and install-node.sh name them: the host on one node,
+        # node1..nodeN on several.
+        if [ "$NODES" = 1 ]; then name='$(hostname -s)'; else name="node$((i + 1))"; fi
+        ssh_node "${HOSTS[$i]}" "sed \"s/@NODE@/$name/\" | sudo tee /etc/spinifex/spinifex.toml >/dev/null" <<'TOML' \
+            || die "could not seed the ocibsd credential on ${HOSTS[$i]}"
+[nodes.@NODE@.ebs]
+provider = "ocibsd"
+
+[nodes.@NODE@.ebs.oci]
+auth = "config_file"
+config_file = "/etc/spinifex/oci/config"
+config_profile = "spinifex"
+TOML
+    done
+    PRESEEDED_EBS=1
+    record "ocibsd credential" PASS "API key installed and seeded before init"
+fi
+
 # A single node is its own documented path: install-node.sh refuses fewer than two
 # hosts, because there is nothing to join. Both branches use the flags the guide
 # names, so what is validated is what the guide tells a customer to type.
@@ -776,19 +870,34 @@ if formation_complete; then
     record formation SKIP "checkpoint verified"
 elif [ "$NODES" = 1 ]; then
     log "initializing the single node"
-    ssh_node "${HOSTS[0]}" '
-        set -e
-        sudo spx admin init --node "$(hostname -s)" --nodes 1 \
-            --region ap-southeast-2 --az ap-southeast-2a \
-            --external-mode=nat --ipsec=false
-        sudo systemctl start spinifex.target
-    ' > "$STATE_DIR/form.log" 2>&1 \
+    ssh_node "${HOSTS[0]}" "INIT_EBS_PROVIDER=$INIT_EBS_PROVIDER PRESEEDED_EBS=$PRESEEDED_EBS bash -s" \
+        > "$STATE_DIR/form.log" 2>&1 <<'REMOTE' \
         || die "single-node init failed; see $STATE_DIR/form.log"
+set -e
+ebs_env=()
+[ -z "$INIT_EBS_PROVIDER" ] || ebs_env=("SPINIFEX_EBS_PROVIDER=$INIT_EBS_PROVIDER")
+# The seeded config is the only one there, so replacing it needs --force.
+force=()
+[ "$PRESEEDED_EBS" = 0 ] || force=(--force)
+sudo env -u SPINIFEX_EBS_PROVIDER ${ebs_env[@]+"${ebs_env[@]}"} spx admin init ${force[@]+"${force[@]}"} \
+    --node "$(hostname -s)" --nodes 1 \
+    --region ap-southeast-2 --az ap-southeast-2a \
+    --external-mode=nat --ipsec=false
+sudo systemctl start spinifex.target
+REMOTE
     mark_stage formation
     record formation PASS
 else
     log "forming the cluster"
-    "$REPO_ROOT/scripts/install-node.sh" \
+    # OCI reuses public addresses, and install-node.sh checks host keys against this
+    # runner's known_hosts, so a key an earlier run saw at an address would refuse it.
+    for host in "${HOSTS[@]}"; do
+        ssh-keygen -R "$host" >/dev/null 2>&1 || true
+    done
+    # env -u first, so a SPINIFEX_EBS_PROVIDER in this shell cannot stand in for
+    # detection on the ocibsd topologies.
+    env -u SPINIFEX_EBS_PROVIDER ${INIT_EBS_PROVIDER:+SPINIFEX_EBS_PROVIDER=$INIT_EBS_PROVIDER} \
+        "$REPO_ROOT/scripts/install-node.sh" \
         --hosts-file "$STATE_DIR/hosts" \
         --user ubuntu \
         --identity "$SSH_PRIVATE_KEY" \
@@ -800,6 +909,53 @@ else
     record formation PASS
 fi
 
+# What init printed about its EBS choice, from the log formation left: init's own
+# output on one node, the init log install-node.sh keeps on the first host on several.
+init_ebs_line() {
+    if [ "$NODES" = 1 ]; then
+        grep -m1 '^EBS provider: ' "$STATE_DIR/form.log"
+    else
+        ssh_node "${HOSTS[0]}" "sudo grep -m1 '^EBS provider: ' /var/log/spx-init.log"
+    fi
+}
+# The provider has to be the topology's for the reason the topology names: the
+# ocibsd pair detected on the VM, the rest told viperblockd by the environment,
+# so a baseline flipped by detection or an ocibsd run that was configured fails.
+case "$TOPO_EBS_PROVIDER:$API_KEY_EBS" in
+    ocibsd:0) want_ebs='EBS provider: ocibsd (detected: OCI virtual machine ' ;;
+    ocibsd:1) want_ebs='EBS provider: ocibsd (from the existing config)' ;;
+    *)        want_ebs='EBS provider: viperblockd (from SPINIFEX_EBS_PROVIDER)' ;;
+esac
+ebs_line="$(init_ebs_line)" || die "init printed no EBS provider line; see $STATE_DIR/form.log"
+case "$ebs_line" in
+    "$want_ebs"*) record "ebs provider choice" PASS "${ebs_line#EBS provider: }" ;;
+    *) die "init chose '$ebs_line', and $TOPOLOGY needs '$want_ebs'" ;;
+esac
+
+# ocibsd tags every volume and backup with the cluster_id spx admin init generated,
+# and no Terraform state holds them, so e2e-oci.sh sweeps by that tag after the
+# destroy. Recorded before any volume exists, so a killed run is still sweepable.
+OCIBSD_CLUSTERS="$STATE_DIR/ocibsd-clusters"
+OCIBSD_CLUSTER_ID=""
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+    OCIBSD_CLUSTER_ID="$(ssh_node "${HOSTS[0]}" 'sudo python3 -' <<'PY'
+import sys, tomllib
+nodes = tomllib.load(open("/etc/spinifex/spinifex.toml", "rb")).get("nodes", {})
+# A leader's config also lists the other nodes, without their EBS tables.
+ids = {n["ebs"]["oci"].get("cluster_id", "") for n in nodes.values() if "oci" in n.get("ebs", {})}
+if len(ids) != 1 or "" in ids:
+    sys.exit("expected one cluster_id across [nodes.*.ebs.oci], found %r" % sorted(ids))
+print(ids.pop())
+PY
+)" || die "could not read the ocibsd cluster_id from ${HOSTS[0]}"
+    [[ "$OCIBSD_CLUSTER_ID" =~ ^[0-9a-f]{32}$ ]] \
+        || die "${HOSTS[0]} reports cluster_id '$OCIBSD_CLUSTER_ID', which is not the 128-bit hex value init writes"
+    # Appended: a record a failed sweep left behind must survive this run's.
+    line="$(printf '%s\t%s\t%s' "$OCIBSD_CLUSTER_ID" "$OCIBSD_COMPARTMENT" "$OCIBSD_REGION")"
+    grep -qxF "$line" "$OCIBSD_CLUSTERS" 2>/dev/null || printf '%s\n' "$line" >> "$OCIBSD_CLUSTERS"
+    log "ocibsd cluster_id $OCIBSD_CLUSTER_ID, recorded for the sweep"
+fi
+
 # Runs after formation and before the pool is configured, which is the only window
 # where a node has /etc/spinifex but has not yet started the allocator. Nothing in
 # this repository knows what it does: an API-key deployment needs a credential on
@@ -809,7 +965,7 @@ if pool_complete; then
     log "OCI pool configuration already completed (resuming; pool setup skipped)"
     record pool SKIP "checkpoint verified"
 else
-    if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then
+    if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ] && [ "$PRESEEDED_EBS" = 0 ]; then
         log "running the credential hook"
         # Arguments, not a file: the hook is told where the nodes are and how to reach
         # them, and decides for itself what to put there.
@@ -936,9 +1092,9 @@ fi
 
 # Parse the STATUS column, never grep for Ready: NotReady contains it, and that
 # read passed a single-node run whose only node was NotReady on 0.0.0.0. The table
-# is colourised, so the escapes come off first.
+# is colourised and drawn with box characters, so both come off first.
 count_ready() {
-    sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/nodes.txt" \
+    sed -e 's/\x1b\[[0-9;]*m//g' -e 's/│/|/g' "$STATE_DIR/nodes.txt" \
         | awk -F'|' 'NR > 1 { gsub(/^[ \t]+|[ \t]+$/, "", $2); if ($2 == "Ready") n++ } END { print n + 0 }'
 }
 
@@ -962,10 +1118,429 @@ fi
 
 if [ "$SKIP_WORKLOAD" = 1 ]; then
     log "--skip-workload: stopping before the workbook"
+    if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+        record "ebs provider" SKIPPED "--skip-workload"
+    fi
     record workbooks SKIPPED "--skip-workload"
     exit 0
 fi
 
+# The ebs provider gate, ocibsd only. The workbooks reach EBS only through RDS,
+# ECS and EKS, so this boots from an imported AMI, stops, attaches, snapshots,
+# restores and terminates, checking each against OCI's own view of the volumes.
+
+GATE_IMAGE=""
+GATE_LOG="$STATE_DIR/ebs-gate.log"
+
+# The AWS CLI on node 1, as the workbook driver runs it. Each argument is quoted
+# for the remote shell, so a JMESPath expression arrives as one word.
+node_aws() {
+    ssh_node "${HOSTS[0]}" "AWS_PROFILE=spinifex aws --output text $(printf '%q ' "$@")"
+}
+
+# OCI's live volumes or backups carrying every key=value given, as JSON, read from
+# this runner with the credential the apply used; nothing on a node is trusted.
+oci_tagged() {
+    local kind="$1" tag tags=()
+    shift
+    for tag in "$@"; do tags+=(--tag "$tag"); done
+    OCIBSD_COMPARTMENT="$OCIBSD_COMPARTMENT" python3 "$HERE/scripts/oci_env.py" \
+        ${OCI_PROFILE:+--profile "$OCI_PROFILE"} --region "$OCIBSD_REGION" -- \
+        python3 "$HERE/scripts/oci_volumes.py" "$kind" "${tags[@]}" 2>>"$GATE_LOG"
+}
+
+oci_count() {
+    oci_tagged "$@" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)))'
+}
+
+# get-console-output is base64 on the wire and the CLI may or may not decode it.
+console_text() {
+    node_aws ec2 get-console-output --instance-id "$1" --query Output 2>/dev/null \
+        | python3 -c '
+import base64, binascii, sys
+raw = sys.stdin.read().strip()
+try:
+    sys.stdout.write(base64.b64decode(raw, validate=True).decode("utf-8", "replace"))
+except (binascii.Error, ValueError):
+    sys.stdout.write(raw)
+' | tr -d '\r'
+}
+
+# wait_console <instance> <ERE> <seconds> prints the last console line matching.
+# On a timeout the guest's last gate reports go to the gate log, so a failure
+# shows what the guest saw instead.
+wait_console() {
+    local id="$1" pattern="$2" deadline=$((SECONDS + $3)) line
+    while [ "$SECONDS" -lt "$deadline" ]; do
+        line="$(console_text "$id" | grep -E "$pattern" | tail -1 || true)"
+        if [ -n "$line" ]; then
+            printf '%s\n' "$line"
+            return 0
+        fi
+        sleep 10
+    done
+    {
+        printf '\n=== %s console, last gate reports, waiting for: %s ===\n' "$id" "$pattern"
+        console_text "$id" | grep 'SPXGATE' | tail -20 || true
+    } >> "$GATE_LOG" 2>&1
+    return 1
+}
+
+# wait_new_boot <instance> <token> <boots seen> <seconds> prints the boot ID of
+# the first boot not yet seen that found the root marker, proving a restart kept it.
+wait_new_boot() {
+    local id="$1" tok="$2" seen="$3" deadline=$((SECONDS + $4)) boot
+    while [ "$SECONDS" -lt "$deadline" ]; do
+        for boot in $(console_text "$id" \
+            | sed -n "s/.*SPXGATE boot=\([0-9a-f-]*\) root-present serial=[^ ]* $tok\$/\1/p"); do
+            case " $seen " in
+                *" $boot "*) ;;
+                *) printf '%s\n' "$boot"; return 0 ;;
+            esac
+        done
+        sleep 10
+    done
+    return 1
+}
+
+# The node whose QEMU runs the instance, by process, as the multinode suite finds it.
+hosting_node() {
+    local host found=""
+    for host in "${HOSTS[@]}"; do
+        if ssh_node "$host" "ps -eo args | grep -F -- '$1' | grep -q '[q]emu-system'"; then
+            [ -z "$found" ] || die "$1 runs on both $found and $host"
+            found="$host"
+        fi
+    done
+    [ -n "$found" ] || die "no node runs a QEMU for $1"
+    printf '%s\n' "$found"
+}
+
+wait_gone_from_oci() {
+    local deadline=$((SECONDS + 600)) left vol
+    while :; do
+        left=0
+        for vol in "$@"; do
+            case "$vol" in
+                snap-*) left=$((left + $(oci_count backups "spinifex-snapshot-id=$vol"))) ;;
+                *) left=$((left + $(oci_count volumes "spinifex-volume-id=$vol"))) ;;
+            esac
+        done
+        [ "$left" = 0 ] && return 0
+        [ "$SECONDS" -lt "$deadline" ] || return 1
+        sleep 15
+    done
+}
+
+# Reports on the serial console, which get-console-output reads: the root marker
+# at each boot, then each Spinifex disk (serial vol...) as it appears, writing a
+# marker to a blank one and reading one back from a restored one.
+write_gate_user_data() {
+    cat > "$1" <<'USERDATA'
+#cloud-config
+write_files:
+  # Ubuntu 26.04's dd (uutils) fails O_DIRECT reads with EINVAL, so the gate's
+  # direct I/O uses a page-aligned buffer of its own.
+  - path: /usr/local/sbin/spx-gate-io
+    permissions: "0755"
+    content: |
+      #!/usr/bin/python3
+      import mmap, os, sys
+      mode, dev = sys.argv[1], sys.argv[2]
+      buf = mmap.mmap(-1, 4096)
+      if mode == "read":
+          fd = os.open(dev, os.O_RDONLY | os.O_DIRECT)
+          n = os.readv(fd, [buf])
+          if n != 4096:
+              sys.exit(f"short read: {n} bytes")
+          sys.stdout.buffer.write(buf[:])
+      else:
+          buf.write((sys.argv[3] + "\n").encode())
+          fd = os.open(dev, os.O_WRONLY | os.O_DIRECT | os.O_SYNC)
+          if os.writev(fd, [buf]) != 4096:
+              sys.exit("short write")
+          os.fsync(fd)
+  - path: /usr/local/sbin/spx-ebs-gate
+    permissions: "0755"
+    content: |
+      #!/bin/sh
+      BOOT=$(cat /proc/sys/kernel/random/boot_id)
+      say() { echo "SPXGATE boot=$BOOT $*" > /dev/ttyS0; }
+      serial() { cat "/sys/block/$1/serial" 2>/dev/null || cat "/sys/block/$1/device/serial" 2>/dev/null || true; }
+      src=$(findmnt -no SOURCE /)
+      rootdisk=$(lsblk -no PKNAME "$src" | head -1)
+      [ -n "$rootdisk" ] || rootdisk=${src##*/}
+      mark=/var/lib/spx-ebs-gate/root
+      if [ -s "$mark" ]; then
+          say "root-present serial=$(serial "$rootdisk") $(cat "$mark")"
+      else
+          mkdir -p /var/lib/spx-ebs-gate
+          cat /proc/sys/kernel/random/uuid > "$mark.tmp" && sync && mv "$mark.tmp" "$mark" && sync
+          say "root-written serial=$(serial "$rootdisk") $(cat "$mark")"
+      fi
+      seen=" "
+      while :; do
+          next=" "
+          for sys in /sys/block/*; do
+              d=${sys##*/}
+              [ "$d" != "$rootdisk" ] || continue
+              s=$(serial "$d")
+              case "$s" in vol*) ;; *) continue ;; esac
+              case "$seen" in *" $d "*) next="$next$d "; continue ;; esac
+              # A failed or short read is not a blank disk: overwriting it would destroy the marker.
+              if ! err=$(spx-gate-io read "/dev/$d" 2>&1 >/run/spx-gate-head); then
+                  say "data-unreadable serial=$s $(printf '%s' "$err" | tr '\n' ' ' | tail -c 200)"
+                  continue
+              fi
+              head=$(head -c 64 /run/spx-gate-head | tr -d '\000')
+              case "$head" in
+                  "SPXGATE-DATA "*)
+                      say "data-present serial=$s ${head#SPXGATE-DATA }"; next="$next$d " ;;
+                  "")
+                      if blkid -p "/dev/$d" >/dev/null 2>&1; then
+                          say "data-foreign serial=$s"; next="$next$d "
+                      else
+                          tok=$(cat /proc/sys/kernel/random/uuid)
+                          if spx-gate-io write "/dev/$d" "SPXGATE-DATA $tok" 2>/dev/null; then
+                              say "data-written serial=$s $tok"; next="$next$d "
+                          fi
+                      fi ;;
+                  *) say "data-foreign serial=$s"; next="$next$d " ;;
+              esac
+          done
+          seen=$next
+          sleep 3
+      done
+  - path: /etc/systemd/system/spx-ebs-gate.service
+    content: |
+      [Unit]
+      Description=Spinifex EBS provider gate reporter
+      After=local-fs.target
+      [Service]
+      ExecStart=/usr/local/sbin/spx-ebs-gate
+      Restart=always
+      [Install]
+      WantedBy=multi-user.target
+runcmd:
+  - [systemctl, daemon-reload]
+  - [systemctl, enable, --now, spx-ebs-gate.service]
+USERDATA
+}
+
+# What the gate leaves for a failure it cannot explain from its own output: each
+# node's iSCSI sessions and records, block devices and their holders, and the
+# provider's and kernel's recent log. Best effort, so a failed read never masks
+# the gate's own error.
+gate_diagnose() {
+    local host
+    for host in "${HOSTS[@]}"; do
+        {
+            printf '\n=== diagnostics from %s ===\n' "$host"
+            ssh_node "$host" 'sudo bash -s' <<'DIAG' 2>&1
+for c in "iscsiadm -m session -P 3" "iscsiadm -m node" "lsblk -o NAME,SIZE,TYPE,MOUNTPOINTS,HCTL" "ls -l /dev/disk/by-path"; do
+    printf -- '--- %s\n' "$c"; $c 2>&1
+done
+printf -- '--- holders of iSCSI devices\n'
+for d in /dev/disk/by-path/*iqn.2015-12.com.oracleiaas:*; do
+    [ -e "$d" ] || continue
+    dev=$(readlink -f "$d"); printf '%s -> %s\n' "$d" "$dev"
+    ls /sys/class/block/"${dev##*/}"/holders 2>&1; fuser -v "$dev" 2>&1
+done
+printf -- '--- journalctl -u spinifex-ocibsd\n'; journalctl -u spinifex-ocibsd -n 300 --no-pager 2>&1
+printf -- '--- journalctl -k\n'; journalctl -k -n 150 --no-pager 2>&1
+DIAG
+        } >> "$GATE_LOG" || true
+    done
+}
+gate_die() {
+    gate_diagnose
+    die "$@"
+}
+
+ebs_provider_gate() {
+    local host out ami snap_ami itype subnet iid az root_vol dot tok boot boots line
+    local data_vol data_serial data_tok snap restored from to
+    : > "$GATE_LOG"
+
+    log "ebs provider gate: units, masks, drop-ins and the detected placement on every node"
+    for host in "${HOSTS[@]}"; do
+        # The placement init read from the instance metadata must be where this run
+        # built the nodes; compared on the node, so the compartment is never printed.
+        ssh_node "$host" "WANT_COMPARTMENT='$OCIBSD_COMPARTMENT' WANT_AD='$OCIBSD_AD' bash -s" >> "$GATE_LOG" 2>&1 <<'REMOTE' \
+            || gate_die "$host is not an ocibsd node as installed; see $GATE_LOG"
+fail() { echo "FAIL $(hostname -s): $*"; exit 1; }
+[ "$(systemctl is-enabled spinifex-viperblock.service 2>&1)" = masked ] \
+    || fail "spinifex-viperblock.service is not masked"
+if systemctl is-active --quiet spinifex-viperblock.service; then fail "spinifex-viperblock.service is running"; fi
+systemctl is-active --quiet spinifex-ocibsd.service || fail "spinifex-ocibsd.service is not active"
+[ "$(systemctl is-enabled spinifex-ocibsd.service)" = enabled ] || fail "spinifex-ocibsd.service is not enabled"
+systemctl show -p DropInPaths --value spinifex-daemon.service \
+    | grep -q /etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf \
+    || fail "the daemon's ocibsd device drop-in is not loaded"
+systemctl show -p DeviceAllow --value spinifex-daemon.service | grep -q 'block-sd rw' \
+    || fail "the daemon's device cgroup does not allow block-sd"
+test -f /etc/udev/rules.d/99-spinifex-ocibsd.rules || fail "no ocibsd udev rule"
+sudo WANT_COMPARTMENT="$WANT_COMPARTMENT" WANT_AD="$WANT_AD" python3 - <<"PY" \
+    || fail "a node in spinifex.toml is not on ocibsd where the nodes were built"
+import os, sys, tomllib
+nodes = tomllib.load(open("/etc/spinifex/spinifex.toml", "rb")).get("nodes", {})
+# Only tables with an ebs section: a leader also lists the other nodes, bare.
+# Each node's own table always has one, and this runs on every node.
+nodes = {n: v for n, v in nodes.items() if "ebs" in v}
+other = {n: v["ebs"].get("provider") for n, v in nodes.items() if v["ebs"].get("provider") != "ocibsd"}
+if not nodes or other:
+    sys.exit("providers other than ocibsd: %r" % other)
+for n, v in sorted(nodes.items()):
+    oci = v["ebs"].get("oci", {})
+    if oci.get("compartment_id") != os.environ["WANT_COMPARTMENT"]:
+        sys.exit("%s: compartment_id is not the compartment the nodes were built in" % n)
+    if oci.get("availability_domain") != os.environ["WANT_AD"]:
+        sys.exit("%s: availability_domain %r, and the nodes were built in %r" % (n, oci.get("availability_domain"), os.environ["WANT_AD"]))
+print("ebs providers:", ", ".join("%s=ocibsd" % n for n in sorted(nodes)))
+PY
+echo "ok $(hostname -s)"
+REMOTE
+    done
+    record "ebs provider units" PASS "ocibsd active, viperblock masked, device drop-in, placement from metadata on $NODES node(s)"
+
+    GATE_IMAGE=ubuntu-26.04-x86_64
+    log "ebs provider gate: importing $GATE_IMAGE onto an OCI volume"
+    out="$(ssh_node "${HOSTS[0]}" "sudo spx admin images import --name $GATE_IMAGE --config /etc/spinifex/spinifex.toml" 2>&1)" \
+        || { printf '%s\n' "$out" >> "$GATE_LOG"; gate_die "importing $GATE_IMAGE failed; see $GATE_LOG"; }
+    printf '%s\n' "$out" >> "$GATE_LOG"
+    ami="$(printf '%s\n' "$out" | sed -n 's/.*Image-ID (AMI): \(ami-[0-9a-f]*\).*/\1/p' | tail -1)"
+    [ -n "$ami" ] || gate_die "the import printed no AMI ID; see $GATE_LOG"
+    snap_ami="$(node_aws ec2 describe-images --image-ids "$ami" --query 'Images[0].BlockDeviceMappings[0].Ebs.SnapshotId')"
+    [ "$(oci_count backups "spinifex-snapshot-id=$snap_ami" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$ami's snapshot $snap_ami is not one OCI backup tagged with this cluster"
+    record "ami import" PASS "$ami, snapshot $snap_ami is an OCI backup"
+
+    itype="$(node_aws ec2 describe-instance-types \
+        --query 'sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus>=`2` && MemoryInfo.SizeInMiB>=`2048`], &MemoryInfo.SizeInMiB)[0].InstanceType')"
+    { [ -n "$itype" ] && [ "$itype" != None ]; } || gate_die "no instance type with 2 vCPU and 2 GiB on this cluster"
+    subnet="$(node_aws ec2 describe-subnets --filters Name=default-for-az,Values=true \
+        --query 'Subnets[0].SubnetId' 2>>"$GATE_LOG")" || subnet=""
+    [ "$subnet" != None ] || subnet=""
+    write_gate_user_data "$STATE_DIR/ebs-gate-user-data.yaml"
+    scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o LogLevel=ERROR -q "$STATE_DIR/ebs-gate-user-data.yaml" "ubuntu@${HOSTS[0]}:/tmp/ebs-gate-user-data.yaml" \
+        || gate_die "could not copy the gate's user-data"
+
+    log "ebs provider gate: booting $itype from $ami"
+    iid="$(node_aws ec2 run-instances --image-id "$ami" --instance-type "$itype" --count 1 \
+        ${subnet:+--subnet-id "$subnet"} --user-data file:///tmp/ebs-gate-user-data.yaml \
+        --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=ebs-provider-gate}]' \
+        --query 'Instances[0].InstanceId')" || gate_die "run-instances from $ami failed"
+    node_aws ec2 wait instance-running --instance-ids "$iid" || gate_die "$iid never reached running"
+    read -r az root_vol dot <<EOF
+$(node_aws ec2 describe-instances --instance-ids "$iid" \
+    --query 'Reservations[0].Instances[0].[Placement.AvailabilityZone,BlockDeviceMappings[0].Ebs.VolumeId,BlockDeviceMappings[0].Ebs.DeleteOnTermination]')
+EOF
+    [ "$dot" = True ] || gate_die "$iid's root volume $root_vol is not delete-on-termination"
+    line="$(wait_console "$iid" 'SPXGATE boot=[0-9a-f-]+ root-written ' 900)" \
+        || gate_die "$iid never reported its root marker on the console"
+    boot="$(printf '%s\n' "$line" | sed -n 's/.*boot=\([0-9a-f-]*\) .*/\1/p')"
+    tok="${line##* }"
+    boots="$boot"
+    [ "$(oci_count volumes "spinifex-volume-id=$root_vol" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$iid's root volume $root_vol is not one OCI volume tagged with this cluster"
+    record "boot from ocibsd" PASS "$iid on $root_vol, an OCI volume tagged with this cluster"
+
+    log "ebs provider gate: stop and start $iid"
+    node_aws ec2 stop-instances --instance-ids "$iid" >/dev/null || gate_die "stop-instances $iid failed"
+    node_aws ec2 wait instance-stopped --instance-ids "$iid" || gate_die "$iid never stopped"
+    node_aws ec2 start-instances --instance-ids "$iid" >/dev/null || gate_die "start-instances $iid failed"
+    node_aws ec2 wait instance-running --instance-ids "$iid" || gate_die "$iid never ran again"
+    boot="$(wait_new_boot "$iid" "$tok" "$boots" 900)" \
+        || gate_die "$iid did not find its root marker after a stop and start"
+    boots="$boots $boot"
+    record "stop and start" PASS "the root marker survived"
+
+    log "ebs provider gate: attach and detach a data volume"
+    data_vol="$(node_aws ec2 create-volume --availability-zone "$az" --size 1 --volume-type gp3 \
+        --query VolumeId)" || gate_die "create-volume failed"
+    node_aws ec2 wait volume-available --volume-ids "$data_vol" || gate_die "$data_vol never became available"
+    node_aws ec2 attach-volume --volume-id "$data_vol" --instance-id "$iid" --device /dev/sdf >/dev/null \
+        || gate_die "attach-volume $data_vol failed"
+    node_aws ec2 wait volume-in-use --volume-ids "$data_vol" || gate_die "$data_vol never attached"
+    data_serial="${data_vol//-/}"
+    line="$(wait_console "$iid" "SPXGATE boot=[0-9a-f-]+ data-written serial=$data_serial " 600)" \
+        || gate_die "the guest never wrote to $data_vol"
+    data_tok="${line##* }"
+    [ "$(oci_count volumes "spinifex-volume-id=$data_vol" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$data_vol is not one OCI volume tagged with this cluster"
+    node_aws ec2 detach-volume --volume-id "$data_vol" >/dev/null || gate_die "detach-volume $data_vol failed"
+    node_aws ec2 wait volume-available --volume-ids "$data_vol" || gate_die "$data_vol never detached"
+    record "attach and detach" PASS "$data_vol written in the guest"
+
+    # Read back before the snapshot, so a lost marker is told apart from a lost restore.
+    node_aws ec2 attach-volume --volume-id "$data_vol" --instance-id "$iid" --device /dev/sdf >/dev/null \
+        || gate_die "reattach-volume $data_vol failed"
+    node_aws ec2 wait volume-in-use --volume-ids "$data_vol" || gate_die "$data_vol never reattached"
+    wait_console "$iid" "SPXGATE boot=[0-9a-f-]+ data-present serial=$data_serial $data_tok\$" 600 >/dev/null \
+        || gate_die "the guest did not read its marker back from $data_vol after a detach"
+    node_aws ec2 detach-volume --volume-id "$data_vol" >/dev/null || gate_die "detach-volume $data_vol failed"
+    node_aws ec2 wait volume-available --volume-ids "$data_vol" || gate_die "$data_vol never detached again"
+    record "reattach" PASS "$data_vol kept its marker across a detach"
+
+    log "ebs provider gate: snapshot, restore and read back"
+    snap="$(node_aws ec2 create-snapshot --volume-id "$data_vol" --query SnapshotId)" \
+        || gate_die "create-snapshot $data_vol failed"
+    node_aws ec2 wait snapshot-completed --snapshot-ids "$snap" || gate_die "$snap never completed"
+    [ "$(oci_count backups "spinifex-snapshot-id=$snap" "spinifex-cluster=$OCIBSD_CLUSTER_ID")" = 1 ] \
+        || gate_die "$snap is not one OCI backup tagged with this cluster"
+    restored="$(node_aws ec2 create-volume --availability-zone "$az" --snapshot-id "$snap" \
+        --volume-type gp3 --query VolumeId)" || gate_die "create-volume from $snap failed"
+    node_aws ec2 wait volume-available --volume-ids "$restored" || gate_die "$restored never became available"
+    node_aws ec2 attach-volume --volume-id "$restored" --instance-id "$iid" --device /dev/sdg >/dev/null \
+        || gate_die "attach-volume $restored failed"
+    node_aws ec2 wait volume-in-use --volume-ids "$restored" || gate_die "$restored never attached"
+    wait_console "$iid" "SPXGATE boot=[0-9a-f-]+ data-present serial=${restored//-/} $data_tok\$" 600 >/dev/null \
+        || gate_die "the guest did not read $data_vol's marker back from $restored"
+    node_aws ec2 detach-volume --volume-id "$restored" >/dev/null || gate_die "detach-volume $restored failed"
+    node_aws ec2 wait volume-available --volume-ids "$restored" || gate_die "$restored never detached"
+    record "snapshot and restore" PASS "$snap restored to $restored with the marker intact"
+
+    # A stopped instance restarts on its last node unless that node's daemon does
+    # not answer, so the daemon is stopped there for the start. The guest is down
+    # first, so no write is in flight; this is not a fence test.
+    if [ "$NODES" -gt 1 ]; then
+        from="$(hosting_node "$iid")"
+        log "ebs provider gate: moving $iid off $from"
+        node_aws ec2 stop-instances --instance-ids "$iid" >/dev/null || gate_die "stop-instances $iid failed"
+        node_aws ec2 wait instance-stopped --instance-ids "$iid" || gate_die "$iid never stopped"
+        ssh_node "$from" 'sudo systemctl stop spinifex-daemon.service' || gate_die "could not stop the daemon on $from"
+        node_aws ec2 start-instances --instance-ids "$iid" >/dev/null || gate_die "start-instances $iid failed"
+        node_aws ec2 wait instance-running --instance-ids "$iid" || gate_die "$iid never ran on another node"
+        ssh_node "$from" 'sudo systemctl start spinifex-daemon.service' || gate_die "could not restart the daemon on $from"
+        to="$(hosting_node "$iid")"
+        [ "$to" != "$from" ] || gate_die "$iid started on $from again, so nothing moved"
+        boot="$(wait_new_boot "$iid" "$tok" "$boots" 900)" \
+            || gate_die "$iid did not find its root marker on $to"
+        record "cross-node start" PASS "$iid moved from $from to $to with its root marker"
+    fi
+
+    log "ebs provider gate: terminate and delete"
+    node_aws ec2 terminate-instances --instance-ids "$iid" >/dev/null || gate_die "terminate-instances $iid failed"
+    node_aws ec2 wait instance-terminated --instance-ids "$iid" || gate_die "$iid never terminated"
+    # Spinifex refuses DeleteVolume on a volume with a snapshot and DeleteSnapshot on
+    # one a live volume came from, so the restore goes first and the source last.
+    node_aws ec2 delete-volume --volume-id "$restored" || gate_die "delete-volume $restored failed"
+    node_aws ec2 delete-snapshot --snapshot-id "$snap" || gate_die "delete-snapshot $snap failed"
+    local deadline=$((SECONDS + 300))
+    until node_aws ec2 delete-volume --volume-id "$data_vol" 2>>"$GATE_LOG"; do
+        [ "$SECONDS" -lt "$deadline" ] || gate_die "delete-volume $data_vol still refused 300s after $snap was deleted"
+        sleep 10
+    done
+    wait_gone_from_oci "$root_vol" "$data_vol" "$restored" "$snap" \
+        || gate_die "OCI still holds a live volume or backup for $root_vol, $data_vol, $restored or $snap after 600s"
+    record "terminate" PASS "no OCI volume or backup left for the instance, its volumes or the snapshot"
+}
+
+if [ "$TOPO_EBS_PROVIDER" = ocibsd ]; then
+    ebs_provider_gate
+fi
+
 # The same driver the GitHub nightly runs (cell 17), on the same published
 # workbooks, so a workbook that passes on a hypervisor and fails on OCI is a
 # difference in OCI and not in the test. It owns its own per-workbook assertions
@@ -1002,12 +1577,18 @@ scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev
 # to absorb parallel uploads. Three are appliances rather than distros: RDS, ECS
 # and EKS each boot their own, found by a spinifex:managed-by tag and never by name.
 log "importing the images the workbooks need"
-ssh_node "${HOSTS[0]}" '
-    set -e
-    for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do
-        sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml >/dev/null
-    done
-' > "$STATE_DIR/images.log" 2>&1 || die "image import failed; see $STATE_DIR/images.log"
+# The ebs provider gate has already imported its own, and is not asked twice.
+WORKBOOK_IMAGES=""
+for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do
+    [ "$img" = "$GATE_IMAGE" ] || WORKBOOK_IMAGES="$WORKBOOK_IMAGES $img"
+done
+ssh_node "${HOSTS[0]}" "WORKBOOK_IMAGES='$WORKBOOK_IMAGES' bash -s" > "$STATE_DIR/images.log" 2>&1 <<'REMOTE' \
+    || die "image import failed; see $STATE_DIR/images.log"
+set -e
+for img in $WORKBOOK_IMAGES; do
+    sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml >/dev/null
+done
+REMOTE
 
 # Every workbook that is a workbook. The shared driver's own default is five, which
 # leaves ECS and all three EKS variants untested on every platform -- their

scripts/terraform/oci-spx/variables.tf

@@ -68,6 +68,28 @@ variable "instance_principal" {
   }
 }
 
+# What backs EBS on these nodes. ocibsd attaches an OCI block volume per EBS
+# volume at runtime, so the Oracle Cloud Agent's Block Volume Management plugin
+# is disabled and the node's own data volume attaches paravirtualized instead.
+variable "ebs_provider" {
+  description = "EBS provider the nodes are installed with: viperblockd, or ocibsd for OCI Block Volumes."
+  type        = string
+  default     = "viperblockd"
+
+  validation {
+    condition     = contains(["viperblockd", "ocibsd"], var.ebs_provider)
+    error_message = "ebs_provider must be viperblockd or ocibsd."
+  }
+}
+
+# Named only in the api_key_policy_statements output, which a tenancy admin
+# applies to the group holding the API user that spx-oci-config.sh installs.
+variable "api_key_group_name" {
+  description = "IAM group of the API-key user the nodes authenticate as when instance_principal is off."
+  type        = string
+  default     = "spinifex-nodes"
+}
+
 variable "vcn_cidr" {
   description = "VCN address space. A /22 leaves room for the per-node secondary private IPs Spinifex allocates for external addresses."
   type        = string

scripts/uninstall-spx.sh

@@ -349,7 +349,8 @@ rm_path /etc/systemd/system/spinifex-*.service \
         /etc/systemd/system/spinifex.slice \
         /etc/systemd/system/spinifex-system.slice \
         /etc/systemd/system/spinifex-guests.slice \
-        /etc/systemd/system/system.slice.d/spinifex-reserve.conf
+        /etc/systemd/system/system.slice.d/spinifex-reserve.conf \
+        /etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf
 
 # Drop-ins we wrote into *other people's* units. These are the dangerous ones:
 # they are not in a spinifex-named path, so a namespace glob misses them, and

spec/EBSProvider/EBSProvider.tla

@@ -42,6 +42,7 @@ ASSUME Capabilities \in [online_expansion          : BOOLEAN,
                          read_only_publish         : BOOLEAN,
                          volume_enumeration        : BOOLEAN,
                          snapshot_enumeration      : BOOLEAN,
+                         snapshot_copy             : BOOLEAN,
                          owner_routing             : BOOLEAN,
                          exclusion                 : [scope             : {"none", "node", "cluster"},
                                                       claim_ttl_seconds : Nat,

spec/EBSProvider/README.md

@@ -17,6 +17,7 @@ The contract that every implementation of the `EBSProvider` interface must meet.
   A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
 - Is the schema version part of the contract?
   Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
+  A block export arrives at contract version 2: a PublishVolume call below it, for a volume that only a host block device can export, is refused with `unsupported_version`.
   A snapshot's `state` in a ListSnapshots ref, and the `snapshot_id` filter, arrive at contract version 4: a ref carries no state below it, and a filtered call below it is refused with `unsupported_version` rather than answered with every snapshot.
 - Are `internal` and `unavailable` the only errors that a failed provider may cause?
 - Must a present but malformed ID be refused with `invalid_argument`?

spec/EBSProvider/Snapshots.tla

@@ -70,13 +70,14 @@ CopySnapshot(req, val, err) ==
       volID     == req.volume_id
 
       Errors ==
-        LET checks == [invalid_argument |-> \/ None \in {srcSnapID, dstSnapID, volID}
-                                            \/ srcSnapID = dstSnapID
-                                            \* Only compared, not looked up, since a snapshot outlives its volume.
-                                            \/ /\ KeyExists(snapshots, srcSnapID)
-                                               /\ snapshots[srcSnapID].source_volume_id /= volID,
-                       already_exists   |-> KeyExists(snapshots, dstSnapID),
-                       not_found        |-> srcSnapID \notin Optional(DOMAIN snapshots)]
+        LET checks == [invalid_argument       |-> \/ None \in {srcSnapID, dstSnapID, volID}
+                                                  \/ srcSnapID = dstSnapID
+                                                  \* Only compared, not looked up, since a snapshot outlives its volume.
+                                                  \/ /\ KeyExists(snapshots, srcSnapID)
+                                                     /\ snapshots[srcSnapID].source_volume_id /= volID,
+                       already_exists         |-> KeyExists(snapshots, dstSnapID),
+                       not_found              |-> srcSnapID \notin Optional(DOMAIN snapshots),
+                       unsupported_capability |-> ~Capabilities.snapshot_copy]
         IN {e \in DOMAIN checks : checks[e]}
 
      \* The request was rejected, or the provider failed to answer.

spinifex/bootstrap/config/config.go

@@ -322,14 +322,27 @@ type ViperblockConfig struct {
 }
 
 // EBS provider selectors. Viperblockd routes EBS calls to the viperblockd
-// daemon and qemunbd to the qemunbdd daemon, both over the versioned
-// ebs.provider.v1.* NATS contract. Embedded named the removed in-process
+// daemon, qemunbd to the qemunbdd daemon and ocibsd to the ocibsd daemon, all
+// over the versioned ebs.provider.v1.* NATS contract. Embedded named the removed in-process
 // engine and survives solely so a config that still selects it can be
 // rejected by name.
 const (
 	EBSProviderEmbedded    = "embedded"
 	EBSProviderViperblockd = "viperblockd"
 	EBSProviderQEMUNBD     = "qemunbd"
+	EBSProviderOCIBSD      = "ocibsd"
+)
+
+// OCI credential selections, as in oci_auth on an external pool and auth in
+// [nodes.<name>.ebs.oci]. An unset auth reads a config file.
+const (
+	// OCIAuthConfigFile authenticates from an API-key config file. The default,
+	// because it is what an operator already has from `oci setup config`.
+	OCIAuthConfigFile = "config_file"
+	// OCIAuthInstancePrincipal authenticates as the instance, from the
+	// certificate its metadata service serves. No key material on any node and
+	// nothing to rotate, at the cost of a dynamic group and an IAM policy.
+	OCIAuthInstancePrincipal = "instance_principal"
 )
 
 // Per-volume export tunables. Both are applied per nbdkit process, and
@@ -353,10 +366,13 @@ const (
 // boundary, not one provider's settings, so a second provider never needs a
 // rename.
 type EBSConfig struct {
-	// Provider is "viperblockd" or "qemunbd" and may be left unset. Volumes
-	// are persisted in ebsmetadata.
+	// Provider is "viperblockd", "qemunbd" or "ocibsd" and may be left unset.
+	// Volumes are persisted in ebsmetadata.
 	Provider string `json:"Provider" mapstructure:"provider"`
 
+	// OCI is the [nodes.<name>.ebs.oci] table, read only by ocibsd.
+	OCI EBSOCIConfig `json:"OCI" mapstructure:"oci"`
+
 	// DefaultThreads is nbdkit's -t: worker threads per NBD connection, which
 	// bounds how many requests are in flight inside viperblock for one volume.
 	// QEMU opens a single connection per volume, so this is the whole
@@ -370,9 +386,27 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// EBSOCIConfig is where ocibsd keeps the cluster's volumes and how it
+// authenticates. Auth, ConfigFile and ConfigProfile mean what oci_auth,
+// oci_config_file and oci_config_profile mean on an oci external pool.
+type EBSOCIConfig struct {
+	// CompartmentID and AvailabilityDomain are the same on every node: a
+	// volume is created there whichever node answers, and attached anywhere.
+	CompartmentID      string `json:"CompartmentID" mapstructure:"compartment_id"`
+	AvailabilityDomain string `json:"AvailabilityDomain" mapstructure:"availability_domain"`
+	// ClusterID is generated once by spx admin init and tags every OCI
+	// resource ocibsd creates, so one compartment can hold several clusters.
+	ClusterID     string `json:"ClusterID" mapstructure:"cluster_id"`
+	Auth          string `json:"Auth" mapstructure:"auth"`
+	ConfigFile    string `json:"ConfigFile" mapstructure:"config_file"`
+	ConfigProfile string `json:"ConfigProfile" mapstructure:"config_profile"`
+}
+
+func (c EBSOCIConfig) isZero() bool { return c == EBSOCIConfig{} }
+
 // SupportedEBSProvider reports whether name is a provider a node may run.
 func SupportedEBSProvider(name string) bool {
-	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD || name == EBSProviderOCIBSD
 }
 
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
@@ -889,6 +923,63 @@ func validateSingleEBSProvider(cc *ClusterConfig) error {
 	return nil
 }
 
+// validateOCIAuth checks an OCI credential selection, where prefix is what
+// the keys are named with ("oci_" on an external pool, "" in [ebs.oci]).
+func validateOCIAuth(prefix, auth, configFile, configProfile string) error {
+	// Named explicitly rather than inferred, so a node that was meant to
+	// use its own identity and has no dynamic group fails at config load
+	// instead of falling back to a key file nobody installed.
+	switch auth {
+	case "", OCIAuthConfigFile, OCIAuthInstancePrincipal:
+	default:
+		return fmt.Errorf("%sauth must be %q or %q, got %q", prefix, OCIAuthConfigFile, OCIAuthInstancePrincipal, auth)
+	}
+	// Credentials come from the metadata service under instance principal,
+	// so a config file named beside it is one of two sources of truth and
+	// the one that would be silently ignored.
+	if auth == OCIAuthInstancePrincipal && (configFile != "" || configProfile != "") {
+		return fmt.Errorf("%[1]sconfig_file/%[1]sconfig_profile are not valid with %[1]sauth=\"instance_principal\" (the instance's own certificate is the credential)", prefix)
+	}
+	return nil
+}
+
+// validateEBSOCI checks every node's [ebs.oci] table: present and complete
+// exactly when the node runs ocibsd, and naming one compartment and one AD
+// across the cluster, since any node may create a volume another attaches.
+func validateEBSOCI(cc *ClusterConfig) error {
+	var first string
+	var want EBSOCIConfig
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		ebs := cc.Nodes[name].EBS
+		if ebs.ResolvedProvider() != EBSProviderOCIBSD {
+			if !ebs.OCI.isZero() {
+				return fmt.Errorf("config: [nodes.%s.ebs.oci] is only valid with provider = %q", name, EBSProviderOCIBSD)
+			}
+			continue
+		}
+		oci := ebs.OCI
+		if oci.CompartmentID == "" || oci.AvailabilityDomain == "" {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] compartment_id and availability_domain are required with provider = %q", name, EBSProviderOCIBSD)
+		}
+		if err := validateOCIAuth("", oci.Auth, oci.ConfigFile, oci.ConfigProfile); err != nil {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] %w", name, err)
+		}
+		if first == "" {
+			first, want = name, oci
+			continue
+		}
+		if oci.ClusterID != want.ClusterID {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] cluster_id %q differs from [nodes.%s.ebs.oci] cluster_id %q; every node must carry the cluster's one ID",
+				name, oci.ClusterID, first, want.ClusterID)
+		}
+		if oci.CompartmentID != want.CompartmentID || oci.AvailabilityDomain != want.AvailabilityDomain {
+			return fmt.Errorf("config: [nodes.%s.ebs.oci] names compartment %q in %q but [nodes.%s.ebs.oci] names %q in %q; every node must name the same compartment and availability domain",
+				first, want.CompartmentID, want.AvailabilityDomain, name, oci.CompartmentID, oci.AvailabilityDomain)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -905,7 +996,7 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			// ebsmetadata one-way without the operator ever asking for it.
 			return fmt.Errorf("config: [nodes.%s.ebs] provider=%q has been removed; set provider = %q or remove the key (this is a one-way switch: volumes move to ebsmetadata)", nodeName, EBSProviderEmbedded, EBSProviderViperblockd)
 		default:
-			return fmt.Errorf("config: [nodes.%s.ebs] provider=%q unsupported; use %q or %q, or remove the key", nodeName, nodeCfg.EBS.Provider, EBSProviderViperblockd, EBSProviderQEMUNBD)
+			return fmt.Errorf("config: [nodes.%s.ebs] provider=%q unsupported; use %q, %q or %q, or remove the key", nodeName, nodeCfg.EBS.Provider, EBSProviderViperblockd, EBSProviderQEMUNBD, EBSProviderOCIBSD)
 		}
 		// Range checks only. Both settings are deliberately unbounded above by
 		// anything host-aware: they are operator tunables, and the host cost is
@@ -923,6 +1014,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 	if err := validateSingleEBSProvider(cc); err != nil {
 		return err
 	}
+	if err := validateEBSOCI(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err
@@ -993,19 +1087,8 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			if (p.OCIVNICID == "") == (p.OCIVNICIface == "") {
 				return fmt.Errorf("config: [[network.external_pools]] %q: source=\"oci\" requires exactly one of oci_vnic_id or oci_vnic_iface", p.Name)
 			}
-			// Named explicitly rather than inferred, so a node that was meant to
-			// use its own identity and has no dynamic group fails at config load
-			// instead of falling back to a key file nobody installed.
-			switch p.OCIAuth {
-			case "", "config_file", "instance_principal":
-			default:
-				return fmt.Errorf("config: [[network.external_pools]] %q: oci_auth must be \"config_file\" or \"instance_principal\", got %q", p.Name, p.OCIAuth)
-			}
-			// Credentials come from the metadata service under instance principal,
-			// so a config file named beside it is one of two sources of truth and
-			// the one that would be silently ignored.
-			if p.OCIAuth == "instance_principal" && (p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
-				return fmt.Errorf("config: [[network.external_pools]] %q: oci_config_file/oci_config_profile are not valid with oci_auth=\"instance_principal\" (the instance's own certificate is the credential)", p.Name)
+			if err := validateOCIAuth("oci_", p.OCIAuth, p.OCIConfigFile, p.OCIConfigProfile); err != nil {
+				return fmt.Errorf("config: [[network.external_pools]] %q: %w", p.Name, err)
 			}
 			if p.BindBridge != "" || p.DHCPMAC != "" {
 				return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)

spinifex/bootstrap/config/config_test.go

@@ -1053,6 +1053,7 @@ region = "us-east-1"
 		{"unset defaults to viperblockd", "", EBSProviderViperblockd},
 		{"viperblockd explicit", "[nodes.n1.ebs]\nprovider = \"viperblockd\"\n", EBSProviderViperblockd},
 		{"qemunbd explicit", "[nodes.n1.ebs]\nprovider = \"qemunbd\"\n", EBSProviderQEMUNBD},
+		{"ocibsd explicit", "[nodes.n1.ebs]\nprovider = \"ocibsd\"\n\n[nodes.n1.ebs.oci]\ncompartment_id = \"c\"\navailability_domain = \"ad\"\n", EBSProviderOCIBSD},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -1148,6 +1149,59 @@ func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
 	}
 }
 
+func TestLoadConfig_EBSOCI(t *testing.T) {
+	oci := func(node, compartment, ad, extra string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = \"ocibsd\"\n\n[nodes.%s.ebs.oci]\ncompartment_id = %q\navailability_domain = %q\n%s", node, node, compartment, ad, extra)
+	}
+	const comp, ad = "ocid1.compartment.oc1..a", "Uocm:US-ASHBURN-AD-1"
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"one node", []string{oci("n1", comp, ad, "")}, ""},
+		{"every node agrees, with unset peers", []string{oci("n1", comp, ad, ""), "", oci("n3", comp, ad, "auth = \"instance_principal\"\n")}, ""},
+		{"config file credentials", []string{oci("n1", comp, ad, "auth = \"config_file\"\nconfig_file = \"/etc/spinifex/oci/config\"\nconfig_profile = \"SPX\"\n")}, ""},
+		{"no compartment", []string{oci("n1", "", ad, "")}, "[nodes.n1.ebs.oci] compartment_id and availability_domain are required"},
+		{"no availability domain", []string{oci("n1", comp, "", "")}, "[nodes.n1.ebs.oci] compartment_id and availability_domain are required"},
+		{"no table at all", []string{"[nodes.n1.ebs]\nprovider = \"ocibsd\"\n"}, "compartment_id and availability_domain are required"},
+		{"unknown auth", []string{oci("n1", comp, ad, "auth = \"api_key\"\n")}, `[nodes.n1.ebs.oci] auth must be "config_file" or "instance_principal", got "api_key"`},
+		{"instance principal beside a config file", []string{oci("n1", comp, ad, "auth = \"instance_principal\"\nconfig_file = \"/x\"\n")},
+			`config_file/config_profile are not valid with auth="instance_principal"`},
+		{"nodes disagree on the AD", []string{oci("n1", comp, ad, ""), oci("n2", comp, "Uocm:US-ASHBURN-AD-2", "")},
+			"every node must name the same compartment and availability domain"},
+		{"nodes agree on the cluster ID", []string{oci("n1", comp, ad, "cluster_id = \"c1\"\n"), oci("n2", comp, ad, "cluster_id = \"c1\"\n")}, ""},
+		{"nodes disagree on the cluster ID", []string{oci("n1", comp, ad, "cluster_id = \"c1\"\n"), oci("n2", comp, ad, "cluster_id = \"c2\"\n")},
+			`[nodes.n2.ebs.oci] cluster_id "c2" differs from [nodes.n1.ebs.oci] cluster_id "c1"`},
+		{"nodes disagree on the compartment", []string{oci("n1", comp, ad, ""), oci("n2", "ocid1.compartment.oc1..b", ad, "")},
+			"every node must name the same compartment and availability domain"},
+		{"the table on another provider", []string{"[nodes.n1.ebs]\nprovider = \"viperblockd\"\n\n[nodes.n1.ebs.oci]\ncompartment_id = \"x\"\n"},
+			`[nodes.n1.ebs.oci] is only valid with provider = "ocibsd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			cfg, err := LoadConfig(path)
+			if tc.wantErr != "" {
+				require.ErrorContains(t, err, tc.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, EBSProviderOCIBSD, cfg.Nodes["n1"].EBS.ResolvedProvider())
+			got := cfg.Nodes["n1"].EBS.OCI
+			assert.Equal(t, comp, got.CompartmentID)
+			assert.Equal(t, ad, got.AvailabilityDomain)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/domains/ec2/instance/provider_root_volume_test.go

@@ -76,13 +76,25 @@ func rootVolumeAMILoader() *fakeAMILoader {
 	}
 }
 
+// createCapturingProvider records the last CreateVolume request it served.
+type createCapturingProvider struct {
+	*ebsprovider.MemoryProvider
+
+	last ebsprovider.CreateVolumeRequest
+}
+
+func (p *createCapturingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
+	p.last = req
+	return p.MemoryProvider.CreateVolume(ctx, req)
+}
+
 // TestPrepareRootVolume_Provider_CreatesFromAMISnapshot locks that an injected
 // provider owns root-volume creation: the volume is restored from the AMI's
 // snapshot rather than copied block-by-block by the control plane.
 func TestPrepareRootVolume_Provider_CreatesFromAMISnapshot(t *testing.T) {
 	defer goleak.VerifyNone(t)
 
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true})
+	provider := &createCapturingProvider{MemoryProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true})}
 	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
 	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
 
@@ -97,6 +109,9 @@ func TestPrepareRootVolume_Provider_CreatesFromAMISnapshot(t *testing.T) {
 	})
 	require.NoError(t, err, "the root volume must exist in the provider")
 	assert.Equal(t, int64(testRootVolumeBytes), created.CapacityBytes)
+	params, err := ebsprovider.ParseVolumeParameters(provider.last.Parameters)
+	require.NoError(t, err)
+	assert.Equal(t, ebspolicy.VolumeTypeGP3, params.VolumeType, "the provider must be told the root volume's type")
 
 	require.Len(t, instance.EBSRequests.Requests, 1)
 	assert.Equal(t, "vol-root", instance.EBSRequests.Requests[0].Name)

spinifex/domains/ec2/instance/service_impl.go

@@ -1861,6 +1861,12 @@ func (s *InstanceServiceImpl) createRootVolumeViaProvider(ctx context.Context, s
 	slog.InfoContext(ctx, "Creating root volume from AMI snapshot via EBS provider",
 		"imageId", spec.amiID, "volumeId", spec.volumeID, "snapshotId", amiConfig.SnapshotID, "sourceVolumeId", sourceVolumeID)
 
+	params, err := ebsprovider.NewVolumeParameters(ebspolicy.VolumeTypeGP3)
+	if err != nil {
+		slog.ErrorContext(ctx, "Encode root volume provider parameters failed", "volumeId", spec.volumeID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
 	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
 		Versioned:              ebsprovider.NewVersioned(),
 		VolumeID:               spec.volumeID,
@@ -1868,6 +1874,7 @@ func (s *InstanceServiceImpl) createRootVolumeViaProvider(ctx context.Context, s
 		AvailabilityZone:       s.config.AZ,
 		SourceSnapshotID:       amiConfig.SnapshotID,
 		SourceSnapshotVolumeID: sourceVolumeID,
+		Parameters:             params,
 	})
 	if err != nil {
 		slog.ErrorContext(ctx, "Provider root volume creation failed",

spinifex/domains/ec2/snapshot/service_impl.go

@@ -1111,6 +1111,12 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS
 		DestinationSnapshotID: newSnapshotID,
 		VolumeID:              sourceCfg.VolumeID,
 	})
+	// A provider without SnapshotCopy refuses as unsupported, which no retry
+	// changes, so the caller is told the operation is unsupported here.
+	if errors.Is(err, ebsprovider.ErrUnsupportedCapability) {
+		slog.WarnContext(ctx, "CopySnapshot is not supported by the EBS provider", "sourceSnapshotId", sourceSnapshotID)
+		return nil, errors.New(awserrors.ErrorUnsupportedOperation)
+	}
 	if err != nil || copied == nil {
 		slog.ErrorContext(ctx, "CopySnapshot: provider copy failed", "sourceSnapshotId", sourceSnapshotID, "newSnapshotId", newSnapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1293,7 +1293,7 @@ func providerSnapshotService(t *testing.T, volumeID string) (*SnapshotServiceImp
 	store := objectstore.NewMemoryObjectStore()
 	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
 	svc := NewSnapshotServiceImplWithStore(cfg, store, nil)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{SnapshotCopy: true})
 	svc.SetEBSProvider(provider)
 	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
 		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4, State: "available",
@@ -1413,6 +1413,34 @@ func TestCopySnapshot_Provider_ForeignSnapshotRefused(t *testing.T) {
 	require.True(t, ok, "the source must survive a refused copy")
 }
 
+// A provider that cannot copy in place answers UnsupportedOperation, and the
+// refusal leaves no copy behind in the control plane.
+func TestCopySnapshot_Provider_WithoutSnapshotCopyIsUnsupported(t *testing.T) {
+	ctx := context.Background()
+	svc, provider := providerSnapshotService(t, "vol-copyunsup")
+
+	src, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-copyunsup")}, testAccountID)
+	require.NoError(t, err)
+
+	svc.SetEBSProvider(&noCopyProvider{EBSProvider: provider})
+	_, err = svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{SourceSnapshotId: src.SnapshotId}, testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorUnsupportedOperation, err.Error())
+
+	listed, err := svc.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Len(t, listed.Snapshots, 1, "only the source snapshot should exist")
+}
+
+// noCopyProvider refuses CopySnapshot as a provider without SnapshotCopy does.
+type noCopyProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *noCopyProvider) CopySnapshot(context.Context, ebsprovider.CopySnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, ebsprovider.ErrUnsupportedCapability
+}
+
 // copyFailingProvider fails only CopySnapshot, leaving every other operation
 // intact so the rollback path is what the test exercises.
 type copyFailingProvider struct {

spinifex/domains/ec2/volume/provider_snapshot_source_test.go

@@ -96,6 +96,25 @@ func TestCreateVolume_Provider_WithoutSnapshotSendsNoSource(t *testing.T) {
 	assert.Empty(t, provider.lastRequest.SourceSnapshotVolumeID)
 }
 
+// TestCreateVolume_Provider_SendsVolumeType locks that the provider is told the
+// volume type, which is how a provider with performance tiers picks one.
+func TestCreateVolume_Provider_SendsVolumeType(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	cfg := &config.Config{AZ: providerTestAZ, Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
+	svc := NewVolumeServiceImplWithStore(cfg, store, nil)
+	provider := newSnapshotSourceProvider()
+	svc.SetEBSProvider(provider)
+
+	_, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{
+		Size: aws.Int64(8), AvailabilityZone: aws.String(providerTestAZ), VolumeType: aws.String("GP3"),
+	}, "000000000001")
+	require.NoError(t, err)
+
+	params, err := ebsprovider.ParseVolumeParameters(provider.lastRequest.Parameters)
+	require.NoError(t, err)
+	assert.Equal(t, ebspolicy.VolumeTypeGP3, params.VolumeType)
+}
+
 // fixedAMILoader resolves one AMI to a snapshot on a known source volume.
 type fixedAMILoader struct {
 	meta           ebsmetadata.AMI

spinifex/domains/ec2/volume/service_impl.go

@@ -294,6 +294,12 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 		return nil, err
 	}
 
+	params, err := ebsprovider.NewVolumeParameters(volumeType)
+	if err != nil {
+		slog.ErrorContext(ctx, "CreateVolume: encode provider parameters failed", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	// SourceVolumeName must travel with the snapshot ID: a clone resolves its
 	// base blocks against the source volume's prefix, and the provider rejects
 	// a snapshot source without it.
@@ -304,6 +310,7 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 		AvailabilityZone:       *input.AvailabilityZone,
 		SourceSnapshotID:       snapshotID,
 		SourceSnapshotVolumeID: sourceVolumeName,
+		Parameters:             params,
 	})
 	if err != nil {
 		slog.ErrorContext(ctx, "CreateVolume: provider allocation failed", "volumeId", volumeID, "err", err)

spinifex/domains/lambda/awsapi/get.go

@@ -56,27 +56,6 @@ func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (functi
 	return rec, revision, err
 }
 
-// render opens the record's environment and renders its configuration; an
-// environment that cannot be opened fails rather than being left out.
-func (d Deps) render(ctx context.Context, rec function.Record, revision uint64) (*lambda.FunctionConfiguration, error) {
-	var env map[string]string
-	if rec.EnvCipher != "" {
-		if d.AEAD == nil {
-			return nil, serviceError("environment decryption is unavailable")
-		}
-		var err error
-		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
-			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return nil, serviceError("could not read the function environment")
-		}
-	}
-	cfg, err := configuration(rec, revision, env)
-	if err != nil {
-		return nil, serviceError("could not build the function configuration")
-	}
-	return cfg, nil
-}
-
 // loadConfiguration resolves the requested $LATEST function and renders its
 // configuration; it is a metadata read and never touches the code store.
 func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (function.Record, *lambda.FunctionConfiguration, error) {
@@ -85,9 +64,19 @@ func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (fun
 		return function.Record{}, nil, err
 	}
 
-	cfg, err := d.render(ctx, rec, revision)
+	var env map[string]string
+	if rec.EnvCipher != "" {
+		if d.AEAD == nil {
+			return function.Record{}, nil, serviceError("environment decryption is unavailable")
+		}
+		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
+			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
+			return function.Record{}, nil, serviceError("could not read the function environment")
+		}
+	}
+	cfg, err := configuration(rec, revision, env)
 	if err != nil {
-		return function.Record{}, nil, err
+		return function.Record{}, nil, serviceError("could not build the function configuration")
 	}
 	return rec, cfg, nil
 }

spinifex/domains/lambda/awsapi/handler_test.go

@@ -60,14 +60,6 @@ type localClient struct {
 	configErr error
 	// configCalls counts UpdateConfiguration requests sent to the owner.
 	configCalls int
-	// deleteCalls counts Delete requests sent to the owner.
-	deleteCalls int
-	// deleteErr fails Delete before the owner sees it; loseDeleteReply
-	// reports a transport error after the owner committed it.
-	deleteErr       error
-	loseDeleteReply bool
-	// listErr fails List before the owner sees it.
-	listErr error
 }
 
 var _ function.Client = (*localClient)(nil)
@@ -177,42 +169,6 @@ func (c *localClient) UpdateConfiguration(ctx context.Context, account, region s
 	return r.Record, r.Revision, nil
 }
 
-func (c *localClient) List(ctx context.Context, account, region, marker string, maxItems int) ([]function.ListedFunction, string, error) {
-	c.mu.Lock()
-	fail := c.listErr
-	c.mu.Unlock()
-	if fail != nil {
-		return nil, "", fail
-	}
-	r, err := c.svc.List(ctx, &function.ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, account)
-	if err != nil {
-		return nil, "", err
-	}
-	return r.Functions, r.NextMarker, nil
-}
-
-func (c *localClient) Delete(ctx context.Context, account, region string, rec function.Record, rev uint64) error {
-	c.mu.Lock()
-	hook, fail, lose := c.beforeCommit, c.deleteErr, c.loseDeleteReply
-	c.beforeCommit = nil
-	c.deleteCalls++
-	c.mu.Unlock()
-	if fail != nil {
-		return fail
-	}
-	if hook != nil {
-		hook()
-	}
-	if _, err := c.svc.Delete(ctx, &function.DeleteRequest{Region: region, FunctionName: rec.FunctionName,
-		FunctionID: rec.FunctionID, Revision: rev}, account); err != nil {
-		return err
-	}
-	if lose {
-		return errors.New("nats: timeout")
-	}
-	return nil
-}
-
 // memStore is an in-memory code store; objects is keyed by Ref.
 type memStore struct {
 	mu      sync.Mutex
@@ -908,7 +864,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "DeleteFunction", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListFunctions", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
 	assert.Equal(t, "UpdateFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "DeleteEventSourceMapping", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/event-source-mappings/x", nil)))
@@ -917,10 +873,7 @@ func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
 	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "UpdateFunctionCode", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/code", nil)))
-	assert.Equal(t, "DeleteFunction", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
-	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions", nil)))
-	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/", nil)))
-	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x/configuration", nil)))
+	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
 }
 
 func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {

spinifex/domains/lambda/awsapi/invoke.go

@@ -57,13 +57,13 @@ func (r *RefusalError) Error() string { return "invocation refused before dispat
 
 func (r *RefusalError) Unwrap() error { return r.Err }
 
-// functionTarget is the function and qualifier a request names, once
+// invokeTarget is the function and qualifier an Invoke request names, once
 // every deterministic check on the request has passed.
-type functionTarget struct{ Name, Qualifier string }
+type invokeTarget struct{ Name, Qualifier string }
 
 // validateInvoke runs every check that needs no stored state: options, size,
 // payload JSON, function name and qualifier. Nothing here reads metadata.
-func validateInvoke(c *call) (*functionTarget, error) {
+func validateInvoke(c *call) (*invokeTarget, error) {
 	if err := checkInvokeOptions(c.Header); err != nil {
 		return nil, err
 	}
@@ -76,12 +76,6 @@ func validateInvoke(c *call) (*functionTarget, error) {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidRequestContentException,
 			"Could not parse request body into json")
 	}
-	return qualifiedTarget(c)
-}
-
-// qualifiedTarget resolves the function name and the qualifier from either
-// its suffix or the Qualifier query, which c.Query must hold strictly parsed.
-func qualifiedTarget(c *call) (*functionTarget, error) {
 	if len(c.Params[0]) > maxFunctionNameLen {
 		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must have length less than or equal to 256")
 	}
@@ -107,7 +101,7 @@ func qualifiedTarget(c *call) (*functionTarget, error) {
 		}
 		q = query[0]
 	}
-	return &functionTarget{Name: name, Qualifier: q}, nil
+	return &invokeTarget{Name: name, Qualifier: q}, nil
 }
 
 // invoke implements Invoke for RequestResponse on $LATEST. serve has already

spinifex/domains/lambda/awsapi/listdelete.go

@@ -1,107 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"log/slog"
-	"net/http"
-	"strconv"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/lambda"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// singleQuery returns the one value of a query member; present but repeated
-// or empty is refused.
-func singleQuery(c *call, member, field string) (string, bool, error) {
-	v, ok := c.Query[member]
-	if !ok {
-		return "", false, nil
-	}
-	if len(v) != 1 || v[0] == "" {
-		return "", true, invalid("1 validation error detected: Value at '%s' failed to satisfy constraint: Member must have length greater than or equal to 1", field)
-	}
-	return v[0], true, nil
-}
-
-// listFunctions implements ListFunctions over $LATEST. Pages hold at most
-// function.MaxListPage functions, and may be short or empty while a
-// NextMarker remains, since each owner request also bounds its scan.
-func (d Deps) listFunctions(ctx context.Context, c *call) (reply, error) {
-	if c.Query.Has("MasterRegion") {
-		return reply{}, unsupported("MasterRegion")
-	}
-	if v, ok, err := singleQuery(c, "FunctionVersion", "functionVersion"); err != nil {
-		return reply{}, err
-	} else if ok && v != lambda.FunctionVersionAll {
-		return reply{}, invalid("1 validation error detected: Value '%s' at 'functionVersion' failed to satisfy constraint: Member must satisfy enum value set: [ALL]", v)
-	}
-	maxItems := function.MaxListPage
-	if v, ok, err := singleQuery(c, "MaxItems", "maxItems"); err != nil {
-		return reply{}, err
-	} else if ok {
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 1 || n > maxListItems {
-			return reply{}, invalid("MaxItems must be between 1 and %d", maxListItems)
-		}
-		maxItems = min(n, function.MaxListPage)
-	}
-	marker, _, err := singleQuery(c, "Marker", "marker")
-	if err != nil {
-		return reply{}, err
-	}
-
-	listed, next, err := d.Meta.List(ctx, c.AccountID, c.Region, marker, maxItems)
-	if err != nil {
-		if _, coded := awserrors.ResolveErrorCode(err); coded {
-			return reply{}, err
-		}
-		slog.ErrorContext(ctx, "Lambda: list owner request failed", "err", err)
-		return reply{}, serviceError("the functions could not be listed; retry the request")
-	}
-	out := &lambda.ListFunctionsOutput{Functions: make([]*lambda.FunctionConfiguration, 0, len(listed))}
-	for _, l := range listed {
-		cfg, err := d.render(ctx, l.Record, l.Revision)
-		if err != nil {
-			return reply{}, err
-		}
-		// ListFunctions returns a subset of the configuration: AWS leaves the
-		// state and last-update members to GetFunction.
-		cfg.State, cfg.StateReason, cfg.StateReasonCode = nil, nil, nil
-		cfg.LastUpdateStatus, cfg.LastUpdateStatusReason, cfg.LastUpdateStatusReasonCode = nil, nil, nil
-		out.Functions = append(out.Functions, cfg)
-	}
-	if next != "" {
-		out.NextMarker = aws.String(next)
-	}
-	return reply{Status: http.StatusOK, Body: out}, nil
-}
-
-// deleteFunction implements DeleteFunction for a whole function. Any explicit
-// qualifier is refused before anything is read, so a request scoped to a
-// version can never delete the function.
-func (d Deps) deleteFunction(ctx context.Context, c *call) (reply, error) {
-	if c.Delete == nil {
-		return reply{}, serviceError("delete request was not validated")
-	}
-	if q := c.Delete.Qualifier; q != "" {
-		if q == "$LATEST" {
-			return reply{}, invalid("$LATEST version cannot be deleted without deleting the function.")
-		}
-		return reply{}, notFound(c.Region, c.AccountID, c.Delete.Name+":"+q)
-	}
-	rec, revision, err := d.lookupTarget(ctx, c, c.Delete.Name, "")
-	if err != nil {
-		return reply{}, err
-	}
-	if err := d.Meta.Delete(ctx, c.AccountID, c.Region, rec, revision); err != nil {
-		// A lost reply may follow the unbinding; the caller reads the function to learn which.
-		if _, coded := awserrors.ResolveErrorCode(err); coded {
-			return reply{}, err
-		}
-		slog.ErrorContext(ctx, "Lambda: delete owner request failed", "function_id", rec.FunctionID, "err", err)
-		return reply{}, serviceError("the function could not be deleted; retry the request")
-	}
-	return reply{Status: http.StatusNoContent}, nil
-}

spinifex/domains/lambda/awsapi/listdelete_test.go

@@ -1,385 +0,0 @@
-package awsapi_test
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/service/lambda"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	listPath  = "/2015-03-31/functions"
-	helloPath = "/2015-03-31/functions/hello"
-	helloARN  = "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
-)
-
-// listAll walks ListFunctions from path, following NextMarker, and returns the
-// pages it read.
-func listAll(t *testing.T, e *env, path string) []lambda.ListFunctionsOutput {
-	t.Helper()
-	var pages []lambda.ListFunctionsOutput
-	marker := ""
-	for range 100 {
-		p := path
-		if marker != "" {
-			p += "&Marker=" + url.QueryEscape(marker)
-		}
-		rec, err := e.do(http.MethodGet, p, nil)
-		require.NoError(t, err)
-		require.Equal(t, http.StatusOK, rec.Code)
-		var out lambda.ListFunctionsOutput
-		require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
-		pages = append(pages, out)
-		if out.NextMarker == nil {
-			return pages
-		}
-		marker = *out.NextMarker
-	}
-	t.Fatal("the walk does not end")
-	return nil
-}
-
-func listedNames(pages []lambda.ListFunctionsOutput) []string {
-	var names []string
-	for _, p := range pages {
-		for _, f := range p.Functions {
-			names = append(names, *f.FunctionName)
-		}
-	}
-	return names
-}
-
-func createActive(t *testing.T, e *env, names ...string) {
-	t.Helper()
-	for _, name := range names {
-		_, err := e.do(http.MethodPost, listPath, createBody(name, nil))
-		require.NoError(t, err)
-	}
-	e.activate(t)
-}
-
-func TestListFunctions_WalksEveryFunctionAcrossPagesOnBothPaths(t *testing.T) {
-	e := newUpdateEnv(t)
-	createActive(t, e, "second", "third")
-
-	pages := listAll(t, e, listPath+"?MaxItems=1")
-	assert.Len(t, pages, 3, "a page that fills on the last function has no NextMarker")
-	assert.ElementsMatch(t, []string{"hello", "second", "third"}, listedNames(pages))
-	slash := listAll(t, e, listPath+"/?MaxItems=1")
-	assert.Equal(t, listedNames(pages), listedNames(slash))
-
-	rec, err := e.do(http.MethodGet, listPath, nil)
-	require.NoError(t, err)
-	assert.NotContains(t, rec.Body.String(), "NextMarker")
-}
-
-// ListFunctions renders each function as GetFunctionConfiguration does, with
-// its environment opened, less the state and last-update members.
-func TestListFunctions_RendersTheConfigurationSubset(t *testing.T) {
-	e := newUpdateEnv(t)
-	rec, err := e.do(http.MethodGet, helloPath+"/configuration", nil)
-	require.NoError(t, err)
-	want := decodeConfig(t, rec.Body.Bytes())
-	want.State, want.StateReason, want.StateReasonCode = nil, nil, nil
-	want.LastUpdateStatus, want.LastUpdateStatusReason, want.LastUpdateStatusReasonCode = nil, nil, nil
-
-	rec, err = e.do(http.MethodGet, listPath, nil)
-	require.NoError(t, err)
-	for _, member := range []string{`"State"`, `"StateReason`, `"LastUpdateStatus`} {
-		assert.NotContains(t, rec.Body.String(), member)
-	}
-	var out lambda.ListFunctionsOutput
-	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
-	require.Len(t, out.Functions, 1)
-	assert.Equal(t, want, *out.Functions[0])
-	assert.Equal(t, "prod", *out.Functions[0].Environment.Variables["MODE"])
-}
-
-func TestListFunctions_EmptyAccountReturnsAnEmptyList(t *testing.T) {
-	e := newEnv(t)
-	rec, err := e.do(http.MethodGet, listPath+"?FunctionVersion=ALL", nil)
-	require.NoError(t, err)
-	assert.JSONEq(t, `{"Functions":[]}`, rec.Body.String())
-}
-
-func TestListFunctions_PageHoldsAtMostFifty(t *testing.T) {
-	e := newEnv(t)
-	for i := range function.MaxListPage + 1 {
-		_, err := e.do(http.MethodPost, listPath, createBody(fmt.Sprintf("fn-%02d", i), nil))
-		require.NoError(t, err)
-	}
-	pages := listAll(t, e, listPath+"?MaxItems=10000")
-	require.Len(t, pages, 2)
-	assert.Len(t, pages[0].Functions, function.MaxListPage)
-	assert.Len(t, listedNames(pages), function.MaxListPage+1)
-}
-
-// Two accounts holding the same function names see only their own functions,
-// with their own environments, on first and continuation pages, and neither
-// can continue the other's walk.
-func TestListFunctions_AccountsAreIsolated(t *testing.T) {
-	e := newEnv(t)
-	const other = "210987654321"
-	e.deps.Roles = roleSet{roleARN: true, "arn:aws:iam::" + other + ":role/lambda-exec": true}
-	names := map[string][]string{
-		gwtest.AccountID: {"shared-1", "only-a", "shared-2", "shared-3"},
-		other:            {"shared-2", "shared-1", "only-b", "shared-3"},
-	}
-	for _, acct := range []string{gwtest.AccountID, other} {
-		e.h.AccountID = acct
-		for _, name := range names[acct] {
-			rec, err := e.do(http.MethodPost, listPath, createBody(name, func(b map[string]any) {
-				b["Role"] = "arn:aws:iam::" + acct + ":role/lambda-exec"
-				b["Environment"] = map[string]any{"Variables": map[string]string{"OWNER": acct}}
-			}))
-			require.NoError(t, err)
-			require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String())
-		}
-	}
-	e.activate(t)
-
-	markers := map[string]string{}
-	for _, acct := range []string{gwtest.AccountID, other} {
-		e.h.AccountID = acct
-		pages := listAll(t, e, listPath+"?MaxItems=2")
-		require.Len(t, pages, 2, "the walk has a continuation page")
-		markers[acct] = *pages[0].NextMarker
-		assert.Equal(t, names[acct], listedNames(pages))
-		for _, p := range pages {
-			for _, f := range p.Functions {
-				assert.Equal(t, "arn:aws:lambda:"+gwtest.Region+":"+acct+":function:"+*f.FunctionName, *f.FunctionArn)
-				assert.Equal(t, "arn:aws:iam::"+acct+":role/lambda-exec", *f.Role)
-				assert.Equal(t, acct, *f.Environment.Variables["OWNER"])
-			}
-		}
-	}
-	for acct, foreign := range map[string]string{gwtest.AccountID: markers[other], other: markers[gwtest.AccountID]} {
-		e.h.AccountID = acct
-		_, err := e.do(http.MethodGet, listPath+"?Marker="+url.QueryEscape(foreign), nil)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}
-
-func TestListFunctions_ParameterValidation(t *testing.T) {
-	e := newUpdateEnv(t)
-	for name, query := range map[string]string{
-		"MaxItems zero":           "MaxItems=0",
-		"MaxItems over 10000":     "MaxItems=10001",
-		"MaxItems not a number":   "MaxItems=x",
-		"MaxItems repeated":       "MaxItems=1&MaxItems=2",
-		"empty Marker":            "Marker=",
-		"malformed Marker":        "Marker=not-a-marker",
-		"MasterRegion":            "MasterRegion=us-east-1",
-		"unknown FunctionVersion": "FunctionVersion=LATEST",
-		"malformed query":         "MaxItems=%zz",
-	} {
-		t.Run(name, func(t *testing.T) {
-			_, err := e.do(http.MethodGet, listPath+"?"+query, nil)
-			testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-		})
-	}
-}
-
-func TestListFunctions_AuthorizesOnEveryResource(t *testing.T) {
-	e := newUpdateEnv(t)
-	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", helloARN))
-	_, err := e.do(http.MethodGet, listPath, nil)
-	gwtest.AssertDenied(t, err)
-	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", "*"))
-	_, err = e.do(http.MethodGet, listPath, nil)
-	require.NoError(t, err)
-}
-
-func TestListFunctions_OwnerFailuresAreRetryable(t *testing.T) {
-	e := newUpdateEnv(t)
-	e.meta.listErr = fmt.Errorf("NATS request to %s: %w", function.SubjectList, nats.ErrNoResponders)
-	_, err := e.do(http.MethodGet, listPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.False(t, awserrors.IsTerminal(err))
-
-	e.meta.listErr = nil
-	e.deps.AEAD = nil
-	_, err = e.do(http.MethodGet, listPath, nil)
-	// An environment that cannot be opened fails the page.
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-}
-
-// deletion reads the retained obligation for functionID.
-func (e *env) deletion(t *testing.T, functionID string) (function.Deletion, bool) {
-	t.Helper()
-	entry, err := e.kv(t).Get(t.Context(), function.KVDeletedKey(gwtest.Region, functionID))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return function.Deletion{}, false
-	}
-	require.NoError(t, err)
-	var d function.Deletion
-	require.NoError(t, json.Unmarshal(entry.Value(), &d))
-	return d, true
-}
-
-func TestDeleteFunction_RemovesTheFunctionAndRetainsItsRecordsAndCode(t *testing.T) {
-	e := newUpdateEnv(t)
-	before, _ := e.record(t, "hello")
-	copies := e.store.count()
-	// Deletion reads neither the code store nor the environment.
-	codes, aead := e.deps.Codes, e.deps.AEAD
-	e.deps.Codes, e.deps.AEAD = nil, nil
-
-	rec, err := e.do(http.MethodDelete, helloPath, nil)
-	require.NoError(t, err)
-	assert.Equal(t, http.StatusNoContent, rec.Code)
-	assert.Empty(t, rec.Body.Bytes())
-
-	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	assert.Empty(t, listedNames(listAll(t, e, listPath+"?MaxItems=1")))
-	_, err = e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-
-	d, ok := e.deletion(t, before.FunctionID)
-	require.True(t, ok)
-	assert.Equal(t, "hello", d.FunctionName)
-	assert.Contains(t, e.uploads(t), before.FunctionID)
-	assert.Equal(t, copies, e.store.count(), "no code copy is removed")
-
-	e.deps.Codes, e.deps.AEAD = codes, aead
-	_, err = e.do(http.MethodPost, listPath, createBody("hello", nil))
-	require.NoError(t, err)
-	again, _ := e.record(t, "hello")
-	assert.NotEqual(t, before.FunctionID, again.FunctionID)
-	_, ok = e.deletion(t, before.FunctionID)
-	assert.True(t, ok, "the old obligation survives the name's reuse")
-}
-
-// Every explicit qualifier, in the query or the name, and every query that
-// cannot be read exactly, is refused before the owner is asked to delete.
-func TestDeleteFunction_RefusesEveryQualifierWithoutMutation(t *testing.T) {
-	cases := map[string]struct {
-		path string
-		code string
-	}{
-		"query $LATEST":         {helloPath + "?Qualifier=%24LATEST", awserrors.ErrorInvalidParameterValueException},
-		"query version":         {helloPath + "?Qualifier=1", awserrors.ErrorResourceNotFoundException},
-		"query alias":           {helloPath + "?Qualifier=live", awserrors.ErrorResourceNotFoundException},
-		"empty query":           {helloPath + "?Qualifier=", awserrors.ErrorInvalidParameterValueException},
-		"bare query key":        {helloPath + "?Qualifier", awserrors.ErrorInvalidParameterValueException},
-		"repeated query":        {helloPath + "?Qualifier=1&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
-		"malformed escape":      {helloPath + "?Qualifier=%zz", awserrors.ErrorInvalidParameterValueException},
-		"malformed other pair":  {helloPath + "?x=%zz&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
-		"semicolon":             {helloPath + "?Qualifier=1;x=2", awserrors.ErrorInvalidParameterValueException},
-		"invalid pattern":       {helloPath + "?Qualifier=a.b", awserrors.ErrorInvalidParameterValueException},
-		"name suffix $LATEST":   {helloPath + ":%24LATEST", awserrors.ErrorInvalidParameterValueException},
-		"name suffix version":   {helloPath + ":1", awserrors.ErrorResourceNotFoundException},
-		"ARN suffix version":    {"/2015-03-31/functions/" + url.PathEscape(helloARN+":1"), awserrors.ErrorResourceNotFoundException},
-		"empty name suffix":     {helloPath + ":", awserrors.ErrorInvalidParameterValueException},
-		"mismatched qualifiers": {helloPath + ":1?Qualifier=2", awserrors.ErrorInvalidParameterValueException},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			e := newUpdateEnv(t)
-			before, beforeRev := e.record(t, "hello")
-			_, err := e.do(http.MethodDelete, tc.path, nil)
-			testutil.RequireAWSCode(t, err, tc.code)
-			assert.Zero(t, e.meta.deleteCalls)
-			requireConfigUnchanged(t, e, before, beforeRev)
-			_, ok := e.deletion(t, before.FunctionID)
-			assert.False(t, ok)
-		})
-	}
-}
-
-func TestDeleteFunction_Authorization(t *testing.T) {
-	allowAll := gwtest.Statement("Allow", "*", "*")
-	t.Run("denied on the function ARN", func(t *testing.T) {
-		e := newUpdateEnv(t)
-		e.h = gwtest.Scoped(allowAll, gwtest.Statement("Deny", "lambda:DeleteFunction", helloARN))
-		before, beforeRev := e.record(t, "hello")
-		_, err := e.do(http.MethodDelete, helloPath, nil)
-		gwtest.AssertDenied(t, err)
-		requireConfigUnchanged(t, e, before, beforeRev)
-	})
-	t.Run("another function's grant does not reach it", func(t *testing.T) {
-		e := newUpdateEnv(t)
-		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN+"-other"))
-		_, err := e.do(http.MethodDelete, helloPath, nil)
-		gwtest.AssertDenied(t, err)
-	})
-	t.Run("allowed on the function ARN", func(t *testing.T) {
-		e := newUpdateEnv(t)
-		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN))
-		rec, err := e.do(http.MethodDelete, helloPath, nil)
-		require.NoError(t, err)
-		assert.Equal(t, http.StatusNoContent, rec.Code)
-	})
-}
-
-func TestDeleteFunction_LostReplyIsAnErrorButTheDeleteStands(t *testing.T) {
-	e := newUpdateEnv(t)
-	e.meta.loseDeleteReply = true
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.False(t, awserrors.IsTerminal(err))
-	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-}
-
-func TestDeleteFunction_OwnerWithoutTheSubjectIsRetryableAndChangesNothing(t *testing.T) {
-	e := newUpdateEnv(t)
-	before, beforeRev := e.record(t, "hello")
-	e.meta.deleteErr = fmt.Errorf("NATS request to %s: %w", function.SubjectDelete, nats.ErrNoResponders)
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.False(t, awserrors.IsTerminal(err))
-	requireConfigUnchanged(t, e, before, beforeRev)
-}
-
-func TestDeleteFunction_UpdateBetweenReadAndDeleteWins(t *testing.T) {
-	e := newConfigEnv(t)
-	e.meta.beforeCommit = func() {
-		_, err := e.do(http.MethodPut, configPath, map[string]any{"Handler": "handler.winner"})
-		assert.NoError(t, err)
-	}
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-	after, _ := e.record(t, "hello")
-	assert.Equal(t, "handler.winner", after.Handler)
-}
-
-func TestDeleteFunction_LeavesMappingsInPlace(t *testing.T) {
-	e, store, _ := newMappingEnv(t)
-	rec, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
-	require.NoError(t, err)
-	uuid, _ := decodeMapping(t, rec.Body.Bytes())["UUID"].(string)
-	before, beforeRev, err := store.Get(t.Context(), uuid)
-	require.NoError(t, err)
-
-	_, err = e.do(http.MethodDelete, helloPath, nil)
-	require.NoError(t, err)
-	after, afterRev, err := store.Get(t.Context(), uuid)
-	require.NoError(t, err)
-	assert.Equal(t, beforeRev, afterRev)
-	assert.Equal(t, before, after)
-	assert.Equal(t, eventsource.StateEnabled, after.State)
-}
-
-func TestDeleteFunction_LaterInvokeIsNotFound(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	_, err := e.do(http.MethodDelete, helloPath, nil)
-	require.NoError(t, err)
-	_, err = e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	assert.Empty(t, ex.calls)
-}

spinifex/domains/lambda/awsapi/registration.go

@@ -149,20 +149,6 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		}
 	case "GetEventSourceMapping", "DeleteEventSourceMapping":
 		resource = mappingARN(inv.Region, inv.AccountID, params[0])
-	case "ListFunctions":
-		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
-			return invalid("The request query string could not be parsed.")
-		}
-	case "DeleteFunction":
-		// A dropped Qualifier would widen the request to the whole function,
-		// so the query is parsed strictly and the target resolved before anything else.
-		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
-			return invalid("The request query string could not be parsed.")
-		}
-		if c.Delete, err = qualifiedTarget(c); err != nil {
-			return err
-		}
-		resource = qualifiedFunctionARN(inv.Region, inv.AccountID, c.Delete.Name, c.Delete.Qualifier)
 	case "Invoke":
 		// Invoke validates deterministically before authorizing, and its
 		// resource comes from that same validated target. URL.Query drops
@@ -248,11 +234,6 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }
 
 func writeJSON(w http.ResponseWriter, out reply) {
-	if out.Status == http.StatusNoContent {
-		maps.Copy(w.Header(), out.Header)
-		w.WriteHeader(out.Status)
-		return
-	}
 	var body []byte
 	var err error
 	if raw, ok := out.Body.(rawJSON); ok {

spinifex/domains/lambda/awsapi/routes.go

@@ -22,10 +22,7 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 	// Invoke is Invoke's validated target, resolved ahead of the
 	// authorization gate whose resource it names.
-	Invoke *functionTarget
-	// Delete is DeleteFunction's validated target, resolved from the strictly
-	// parsed query ahead of the authorization gate.
-	Delete *functionTarget
+	Invoke *invokeTarget
 	// UpdateConfig is the decoded UpdateFunctionConfiguration body, read
 	// ahead of the PassRole check on a new role.
 	UpdateConfig *lambda.UpdateFunctionConfigurationInput
@@ -52,18 +49,6 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.createFunction(ctx, c)
 		}},
-	{Method: "GET", Pattern: "/2015-03-31/functions", Action: "ListFunctions",
-		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
-			return d.listFunctions(ctx, c)
-		}},
-	{Method: "GET", Pattern: "/2015-03-31/functions/", Action: "ListFunctions",
-		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
-			return d.listFunctions(ctx, c)
-		}},
-	{Method: "DELETE", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "DeleteFunction",
-		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
-			return d.deleteFunction(ctx, c)
-		}},
 	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "GetFunction",
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)

spinifex/domains/lambda/function/client.go

@@ -26,10 +26,6 @@ type Client interface {
 	CodeUpdated(ctx context.Context, accountID, region string, rec Record, revision uint64, codeRevisionID, sha256Hex string, size int64) (Record, uint64, error)
 	// UpdateConfiguration commits cfg for the function at revision and returns the updated record and revision.
 	UpdateConfiguration(ctx context.Context, accountID, region string, rec Record, revision uint64, cfg Configuration) (Record, uint64, error)
-	// List returns up to maxItems of the Region's functions after marker and the next marker, empty at the end.
-	List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error)
-	// Delete unbinds the function read at revision, retaining its cleanup obligation.
-	Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error
 }
 
 // NATSClient is the Client over NATS request/reply to the daemon.
@@ -102,19 +98,3 @@ func (c *NATSClient) UpdateConfiguration(ctx context.Context, accountID, region
 	}
 	return resp.Record, resp.Revision, nil
 }
-
-func (c *NATSClient) List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error) {
-	resp, err := natsmsg.NATSRequest[ListResponse](ctx, c.conn, SubjectList,
-		&ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, metaRequestTimeout, accountID)
-	if err != nil {
-		return nil, "", err
-	}
-	return resp.Functions, resp.NextMarker, nil
-}
-
-func (c *NATSClient) Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error {
-	_, err := natsmsg.NATSRequest[DeleteResponse](ctx, c.conn, SubjectDelete,
-		&DeleteRequest{Region: region, FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, Revision: revision},
-		metaRequestTimeout, accountID)
-	return err
-}

spinifex/domains/lambda/function/delete.go

@@ -1,87 +0,0 @@
-package function
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Delete records the function's cleanup obligation, then unbinds its name by
-// compare-and-swap at the revision the caller read. Code copies and upload
-// records are retained; nothing here removes them.
-func (s *MetaService) Delete(ctx context.Context, req *DeleteRequest, accountID string) (*DeleteResponse, error) {
-	if !regionRE.MatchString(req.Region) {
-		return nil, badRegion()
-	}
-	if !ValidName(req.FunctionName) || !idRE.MatchString(req.FunctionID) || accountID == "" {
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "delete request is incomplete")
-	}
-	kv, err := s.bucket(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-	key := KVFunctionKey(req.Region, req.FunctionName)
-	entry, err := kv.Get(ctx, key)
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return nil, awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s", req.FunctionName)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("lambda: get function record: %w", err)
-	}
-	rec, err := decode(entry.Value())
-	if err != nil {
-		return nil, err
-	}
-	if entry.Revision() != req.Revision || rec.FunctionID != req.FunctionID {
-		return nil, conflict("Function was modified by another request: %s", req.FunctionName)
-	}
-	if err := s.recordDeletion(ctx, kv, accountID, rec); err != nil {
-		return nil, err
-	}
-	if err := kv.Delete(ctx, key, jetstream.LastRevision(req.Revision)); err != nil {
-		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, conflict("Function was modified by another request: %s", req.FunctionName)
-		}
-		return nil, fmt.Errorf("lambda: delete function record: %w", err)
-	}
-	return &DeleteResponse{}, nil
-}
-
-// recordDeletion writes the obligation for rec. One already present, from an
-// earlier attempt whose outcome was uncertain, is accepted only for the same
-// function; anything else refuses the delete.
-func (s *MetaService) recordDeletion(ctx context.Context, kv jetstream.KeyValue, accountID string, rec Record) error {
-	want := Deletion{
-		FormatVersion: FormatVersion, AccountID: accountID, Region: rec.Region,
-		FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, RecordedAt: s.now().UTC(),
-	}
-	data, err := json.Marshal(want)
-	if err != nil {
-		return err
-	}
-	key := KVDeletedKey(rec.Region, rec.FunctionID)
-	_, err = kv.Create(ctx, key, data)
-	if err == nil {
-		return nil
-	}
-	if !errors.Is(err, jetstream.ErrKeyExists) {
-		return fmt.Errorf("lambda: record function deletion: %w", err)
-	}
-	entry, err := kv.Get(ctx, key)
-	if err != nil {
-		return fmt.Errorf("lambda: read function deletion: %w", err)
-	}
-	var have Deletion
-	if err := json.Unmarshal(entry.Value(), &have); err != nil {
-		return fmt.Errorf("lambda: function deletion record is unreadable: %w", err)
-	}
-	if have.FormatVersion != FormatVersion || have.AccountID != accountID || have.Region != rec.Region ||
-		have.FunctionName != rec.FunctionName || have.FunctionID != rec.FunctionID {
-		return fmt.Errorf("lambda: function deletion record %s names another function", key)
-	}
-	return nil
-}

spinifex/domains/lambda/function/delete_test.go

@@ -1,296 +0,0 @@
-package function
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// serveAs creates name with stored code under functionID and activates it.
-func serveAs(t *testing.T, svc *MetaService, name, functionID string) *GetResponse {
-	t.Helper()
-	rev := mustCreate(t, svc, name, functionID)
-	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: name, Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-	require.NoError(t, err)
-	svc.SetReadiness(func(context.Context, Record) error { return nil })
-	require.NoError(t, svc.ActivatePending(t.Context()))
-	got := mustGet(t, svc, name)
-	require.Equal(t, StateActive, got.Record.State)
-	return got
-}
-
-func deleteReq(got *GetResponse) *DeleteRequest {
-	return &DeleteRequest{Region: testRegion, FunctionName: got.Record.FunctionName, FunctionID: got.Record.FunctionID, Revision: got.Revision}
-}
-
-// deletion reads the retained obligation for functionID, or reports it absent.
-func deletion(t *testing.T, svc *MetaService, functionID string) (Deletion, bool) {
-	t.Helper()
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	entry, err := kv.Get(t.Context(), KVDeletedKey(testRegion, functionID))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return Deletion{}, false
-	}
-	require.NoError(t, err)
-	var d Deletion
-	require.NoError(t, json.Unmarshal(entry.Value(), &d))
-	return d, true
-}
-
-func requireRetained(t *testing.T, svc *MetaService, name, functionID string) {
-	t.Helper()
-	d, ok := deletion(t, svc, functionID)
-	require.True(t, ok, "deletion obligation for %s is missing", functionID)
-	assert.Equal(t, Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion,
-		FunctionName: name, FunctionID: functionID, RecordedAt: d.RecordedAt}, d)
-	_, ok = upload(t, svc, functionID)
-	assert.True(t, ok, "upload record for %s was removed", functionID)
-}
-
-func TestDelete_UnbindsTheNameAndRetainsTheObligationAndUploads(t *testing.T) {
-	svc, c := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
-	require.NoError(t, err)
-
-	assert.False(t, mustGet(t, svc, "hello").Found)
-	requireRetained(t, svc, "hello", "f1")
-	d, _ := deletion(t, svc, "f1")
-	assert.Equal(t, c.now(), d.RecordedAt)
-	up, _ := upload(t, svc, "f1")
-	assert.Equal(t, UploadAttached, up.State, "the upload record is left exactly as it was")
-}
-
-func TestDelete_MissingFunctionIsNotFound(t *testing.T) {
-	svc, _ := newTestService(t)
-	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: 1}, testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	_, ok := deletion(t, svc, "f1")
-	assert.False(t, ok)
-}
-
-func TestDelete_StaleRevisionOrAnotherFunctionIsRefusedWithoutAnObligation(t *testing.T) {
-	svc, _ := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	for name, req := range map[string]*DeleteRequest{
-		"stale revision": {Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: got.Revision - 1},
-		"other function": {Region: testRegion, FunctionName: "hello", FunctionID: "f0", Revision: got.Revision},
-	} {
-		_, err := svc.Delete(t.Context(), req, testAccount)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-		assert.Equal(t, got.Revision, mustGet(t, svc, "hello").Revision, name)
-		_, ok := deletion(t, svc, req.FunctionID)
-		assert.False(t, ok, name)
-	}
-}
-
-func TestDelete_RefusesMalformedRequests(t *testing.T) {
-	svc, _ := newTestService(t)
-	for name, req := range map[string]*DeleteRequest{
-		"bad region": {Region: "Bad.Region", FunctionName: "hello", FunctionID: "f1", Revision: 1},
-		"bad name":   {Region: testRegion, FunctionName: "a.b", FunctionID: "f1", Revision: 1},
-		"bad id":     {Region: testRegion, FunctionName: "hello", FunctionID: "f.1", Revision: 1},
-	} {
-		_, err := svc.Delete(t.Context(), req, testAccount)
-		require.Error(t, err, name)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}
-
-// The delete passes its read, then pauses before writing while an update
-// commits; the conditional delete loses and the function stays bound. The
-// obligation it left proves nothing was deleted.
-func TestDelete_InterleavedUpdateLosesTheConditionalDelete(t *testing.T) {
-	svc, c := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	paused, release := make(chan struct{}), make(chan struct{})
-	var calls atomic.Int32
-	svc.now = func() time.Time {
-		if calls.Add(1) == 1 {
-			close(paused)
-			<-release
-		}
-		return c.now()
-	}
-	deleteErr := make(chan error, 1)
-	go func() {
-		_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
-		deleteErr <- err
-	}()
-	<-paused
-	cfg := Configuration{Role: got.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
-	won, err := svc.UpdateConfiguration(t.Context(), configReq(got.Revision, cfg), testAccount)
-	require.NoError(t, err)
-	close(release)
-	testutil.RequireAWSCode(t, <-deleteErr, awserrors.ErrorResourceConflictException)
-
-	after := mustGet(t, svc, "hello")
-	require.True(t, after.Found)
-	assert.Equal(t, won.Revision, after.Revision)
-	assert.Equal(t, "f1", after.Record.FunctionID)
-	_, ok := deletion(t, svc, "f1")
-	assert.True(t, ok, "the obligation from the losing attempt is retained")
-}
-
-func TestDelete_ObligationNamingAnotherFunctionRefusesTheDelete(t *testing.T) {
-	svc, _ := newTestService(t)
-	got := serveAs(t, svc, "hello", "f1")
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	data, err := json.Marshal(Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion, FunctionName: "other", FunctionID: "f1"})
-	require.NoError(t, err)
-	_, err = kv.Create(t.Context(), KVDeletedKey(testRegion, "f1"), data)
-	require.NoError(t, err)
-
-	_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
-	require.Error(t, err)
-	_, coded := awserrors.ResolveErrorCode(err)
-	assert.False(t, coded, "an inconsistent obligation is an internal fault, retried by the caller")
-	assert.True(t, mustGet(t, svc, "hello").Found)
-}
-
-// A delete of a create still storing its code wins; the late CodeStored and
-// RollbackCreate of that create change nothing, and its upload stays owed.
-func TestDelete_PendingCreateRacingDeletionChangesNothingLater(t *testing.T) {
-	svc, _ := newTestService(t)
-	rev := mustCreate(t, svc, "hello", "f1")
-	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: rev}, testAccount)
-	require.NoError(t, err)
-	recreated := serveAs(t, svc, "hello", "f2")
-
-	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-	require.Error(t, err)
-	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
-	require.Error(t, err)
-
-	got := mustGet(t, svc, "hello")
-	assert.Equal(t, recreated.Revision, got.Revision)
-	assert.Equal(t, "f2", got.Record.FunctionID)
-	requireOwed(t, svc, "f1")
-	requireRetained(t, svc, "hello", "f1")
-}
-
-func TestDelete_SameNameRecreationKeepsEachObligationApart(t *testing.T) {
-	svc, _ := newTestService(t)
-	first := serveAs(t, svc, "hello", "f1")
-	_, err := svc.Delete(t.Context(), deleteReq(first), testAccount)
-	require.NoError(t, err)
-	second := serveAs(t, svc, "hello", "f2")
-	_, ok := deletion(t, svc, "f2")
-	assert.False(t, ok, "a recreated function has no obligation of its own")
-
-	_, err = svc.Delete(t.Context(), deleteReq(second), testAccount)
-	require.NoError(t, err)
-	requireRetained(t, svc, "hello", "f1")
-	requireRetained(t, svc, "hello", "f2")
-}
-
-// faultKV fails one write to key: before it commits, or after, losing the
-// acknowledgement.
-type faultKV struct {
-	jetstream.KeyValue
-
-	key   string
-	after bool
-	fired atomic.Bool
-}
-
-var errInjected = errors.New("nats: timeout")
-
-func (f *faultKV) trip(key string) bool { return key == f.key && f.fired.CompareAndSwap(false, true) }
-
-func (f *faultKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
-	if !f.trip(key) {
-		return f.KeyValue.Create(ctx, key, value, opts...)
-	}
-	if f.after {
-		if _, err := f.KeyValue.Create(ctx, key, value, opts...); err != nil {
-			return 0, err
-		}
-	}
-	return 0, errInjected
-}
-
-func (f *faultKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
-	if !f.trip(key) {
-		return f.KeyValue.Delete(ctx, key, opts...)
-	}
-	if f.after {
-		if err := f.KeyValue.Delete(ctx, key, opts...); err != nil {
-			return err
-		}
-	}
-	return errInjected
-}
-
-// Each write of a delete is interrupted before it commits, or commits with its
-// acknowledgement lost. A fresh owner then finds the binding and obligation the
-// stream actually holds, a retry converges, and a recreated name stays apart.
-func TestDelete_InterruptionAndLostAcknowledgementAtEachWrite(t *testing.T) {
-	cases := []struct {
-		name       string
-		key        string
-		after      bool
-		bound      bool
-		obligation bool
-	}{
-		{"obligation write interrupted", KVDeletedKey(testRegion, "f1"), false, true, false},
-		{"obligation acknowledgement lost", KVDeletedKey(testRegion, "f1"), true, true, true},
-		{"name delete interrupted", KVFunctionKey(testRegion, "hello"), false, true, true},
-		{"name delete acknowledgement lost", KVFunctionKey(testRegion, "hello"), true, false, true},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			_, nc, _ := testutil.StartTestJetStream(t)
-			js := testutil.NewJetStream(t, nc)
-			svc := NewMetaService(js)
-			got := serveAs(t, svc, "hello", "f1")
-			kv, err := svc.bucket(t.Context(), testAccount)
-			require.NoError(t, err)
-			svc.mu.Lock()
-			svc.buckets[testAccount] = &faultKV{KeyValue: kv, key: tc.key, after: tc.after}
-			svc.mu.Unlock()
-
-			_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
-			require.ErrorIs(t, err, errInjected)
-			_, coded := awserrors.ResolveErrorCode(err)
-			assert.False(t, coded, "an uncertain write is a retryable fault")
-
-			reopened := NewMetaService(js)
-			now := mustGet(t, reopened, "hello")
-			assert.Equal(t, tc.bound, now.Found, "binding")
-			if tc.bound {
-				assert.Equal(t, got.Revision, now.Revision)
-			}
-			_, ok := deletion(t, reopened, "f1")
-			assert.Equal(t, tc.obligation, ok, "obligation")
-			_, ok = upload(t, reopened, "f1")
-			assert.True(t, ok, "upload record retained")
-
-			_, err = reopened.Delete(t.Context(), deleteReq(got), testAccount)
-			if tc.bound {
-				require.NoError(t, err, "the retry completes the delete")
-			} else {
-				testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-			}
-			assert.False(t, mustGet(t, reopened, "hello").Found)
-			requireRetained(t, reopened, "hello", "f1")
-
-			recreated := serveAs(t, reopened, "hello", "f2")
-			assert.Equal(t, "f2", recreated.Record.FunctionID)
-			requireRetained(t, reopened, "hello", "f1")
-			_, ok = deletion(t, reopened, "f2")
-			assert.False(t, ok)
-		})
-	}
-}

spinifex/domains/lambda/function/list.go

@@ -1,141 +0,0 @@
-package function
-
-import (
-	"context"
-	"encoding/base64"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"math"
-	"strings"
-
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// List bounds: a page returns at most MaxListPage functions and examines at
-// most maxListScan stream sequences, whatever each one holds.
-const (
-	MaxListPage = 50
-	maxListScan = 200
-)
-
-// listCursor is the decoded marker: the last stream sequence a page examined,
-// bound to the account and Region it was issued for.
-type listCursor struct {
-	Version   int    `json:"v"`
-	AccountID string `json:"a"`
-	Region    string `json:"r"`
-	After     uint64 `json:"s"`
-}
-
-func encodeCursor(c listCursor) (string, error) {
-	data, err := json.Marshal(c)
-	if err != nil {
-		return "", err
-	}
-	return base64.RawURLEncoding.EncodeToString(data), nil
-}
-
-func decodeCursor(marker, accountID, region string) (uint64, error) {
-	bad := awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "Invalid marker: %s", marker)
-	data, err := base64.RawURLEncoding.DecodeString(marker)
-	if err != nil {
-		return 0, bad
-	}
-	var c listCursor
-	if err := json.Unmarshal(data, &c); err != nil {
-		return 0, bad
-	}
-	if c.Version != 1 || c.AccountID != accountID || c.Region != region || c.After == 0 || c.After == math.MaxUint64 {
-		return 0, bad
-	}
-	return c.After, nil
-}
-
-// List returns the Region's functions in stream order after the marker. Each
-// step is one leader get of an exact sequence, bounded by a fast stream-state
-// read, so a page costs at most its scan bound whatever the bucket holds.
-// NextMarker is set unless the page reached the stream end it read.
-func (s *MetaService) List(ctx context.Context, req *ListRequest, accountID string) (*ListResponse, error) {
-	if !regionRE.MatchString(req.Region) {
-		return nil, badRegion()
-	}
-	if accountID == "" || req.MaxItems < 1 || req.MaxItems > MaxListPage {
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "list request is incomplete")
-	}
-	var after uint64
-	if req.Marker != "" {
-		var err error
-		if after, err = decodeCursor(req.Marker, accountID, req.Region); err != nil {
-			return nil, err
-		}
-	}
-	kv, err := s.bucket(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-	stream := "KV_" + kv.Bucket()
-	// An unfiltered stream info without deleted details reads counters only.
-	info, err := s.js.Stream(ctx, stream)
-	if err != nil {
-		return nil, fmt.Errorf("lambda: list function records: %w", err)
-	}
-	state := info.CachedInfo().State
-	if state.FirstSeq > 0 {
-		after = max(after, state.FirstSeq-1)
-	}
-	keyPrefix := "$KV." + kv.Bucket() + "." + kvFunctionKeyPrefix + req.Region + "."
-	out := &ListResponse{Functions: []ListedFunction{}}
-	for gets := 0; gets < maxListScan && after < state.LastSeq; gets++ {
-		seq := after + 1
-		m, err := kvstore.LeaderMsgGet(ctx, s.js, stream, kvstore.MsgGetRequest{Seq: seq})
-		if errors.Is(err, jetstream.ErrMsgNotFound) {
-			after = seq
-			continue
-		}
-		if err != nil {
-			return nil, fmt.Errorf("lambda: list function records: %w", err)
-		}
-		if m.Sequence != seq {
-			return nil, fmt.Errorf("lambda: list function records: %w", jetstream.ErrInvalidJetStreamResponse)
-		}
-		if !strings.HasPrefix(m.Subject, keyPrefix) || removed(m.Header) {
-			after = seq
-			continue
-		}
-		if len(out.Functions) == req.MaxItems {
-			// A full page reached a function it cannot return; the marker
-			// stays before it.
-			break
-		}
-		rec, err := decode(m.Data)
-		if err != nil {
-			return nil, err
-		}
-		out.Functions = append(out.Functions, ListedFunction{Record: rec, Revision: seq})
-		after = seq
-	}
-	if after >= state.LastSeq {
-		return out, nil
-	}
-	if out.NextMarker, err = encodeCursor(listCursor{Version: 1, AccountID: accountID, Region: req.Region, After: after}); err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// removed reports a key's delete or purge marker, as the KV Get of nats.go
-// reads one.
-func removed(h nats.Header) bool {
-	if op := h.Get("KV-Operation"); op != "" {
-		return op == "DEL" || op == "PURGE"
-	}
-	switch h.Get(jetstream.MarkerReasonHeader) {
-	case "MaxAge", "Purge", "Remove":
-		return true
-	}
-	return false
-}

spinifex/domains/lambda/function/list_test.go

@@ -1,325 +0,0 @@
-package function
-
-import (
-	"fmt"
-	"math"
-	"testing"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func listPage(t *testing.T, svc *MetaService, region, marker string, maxItems int) *ListResponse {
-	t.Helper()
-	resp, err := svc.List(t.Context(), &ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, testAccount)
-	require.NoError(t, err)
-	return resp
-}
-
-// walk follows NextMarker to the end and returns the names in order and the
-// number of pages.
-func walk(t *testing.T, svc *MetaService, region string, maxItems int) ([]string, int) {
-	t.Helper()
-	var names []string
-	marker, pages := "", 0
-	for {
-		resp := listPage(t, svc, region, marker, maxItems)
-		pages++
-		require.LessOrEqual(t, len(resp.Functions), maxItems)
-		for _, f := range resp.Functions {
-			names = append(names, f.Record.FunctionName)
-		}
-		if resp.NextMarker == "" {
-			return names, pages
-		}
-		marker = resp.NextMarker
-		require.Less(t, pages, 1000, "the walk does not end")
-	}
-}
-
-func TestList_EmptyAccountHasNoFunctionsAndNoMarker(t *testing.T) {
-	svc, _ := newTestService(t)
-	resp := listPage(t, svc, testRegion, "", MaxListPage)
-	assert.Empty(t, resp.Functions)
-	assert.NotNil(t, resp.Functions)
-	assert.Empty(t, resp.NextMarker)
-}
-
-func TestList_PagesReturnEveryFunctionOnceWithItsRevision(t *testing.T) {
-	svc, _ := newTestService(t)
-	want := make([]string, 0, 7)
-	for i := range 7 {
-		name := fmt.Sprintf("fn-%d", i)
-		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
-		want = append(want, name)
-	}
-	names, pages := walk(t, svc, testRegion, 2)
-	assert.Equal(t, want, names, "stream order, each once")
-	assert.Equal(t, 4, pages)
-
-	page := listPage(t, svc, testRegion, "", 1)
-	require.Len(t, page.Functions, 1)
-	got := mustGet(t, svc, "fn-0")
-	assert.Equal(t, got.Revision, page.Functions[0].Revision)
-	assert.Equal(t, got.Record, page.Functions[0].Record)
-}
-
-// A page that fills on the last function looks ahead, finds the walk over,
-// and returns no marker; one that fills before it keeps the next one.
-func TestList_FullPageAtTheEndHasNoMarker(t *testing.T) {
-	svc, _ := newTestService(t)
-	for i := range 4 {
-		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
-	}
-	_, pages := walk(t, svc, testRegion, 2)
-	assert.Equal(t, 2, pages)
-	assert.Empty(t, listPage(t, svc, testRegion, "", 4).NextMarker)
-
-	first := listPage(t, svc, testRegion, "", 3)
-	require.NotEmpty(t, first.NextMarker)
-	rest := listPage(t, svc, testRegion, first.NextMarker, 3)
-	require.Len(t, rest.Functions, 1, "the record the full page looked at is not skipped")
-	assert.Equal(t, "fn-3", rest.Functions[0].Record.FunctionName)
-	assert.Empty(t, rest.NextMarker)
-}
-
-func TestList_SkipsDeletedFunctionsUploadsObligationsAndOtherRegions(t *testing.T) {
-	svc, _ := newTestService(t)
-	gone := serveAs(t, svc, "gone", "f0")
-	mustCreate(t, svc, "kept", "f1")
-	_, err := svc.Delete(t.Context(), deleteReq(gone), testAccount)
-	require.NoError(t, err)
-	other := createReq("elsewhere", "f2")
-	other.Region = "us-east-1"
-	_, err = svc.Create(t.Context(), other, testAccount)
-	require.NoError(t, err)
-
-	names, _ := walk(t, svc, testRegion, MaxListPage)
-	assert.Equal(t, []string{"kept"}, names)
-	names, _ = walk(t, svc, "us-east-1", MaxListPage)
-	assert.Equal(t, []string{"elsewhere"}, names)
-}
-
-// cursorAfter is the last sequence a marker says its page examined.
-func cursorAfter(t *testing.T, marker string) uint64 {
-	t.Helper()
-	after, err := decodeCursor(marker, testAccount, testRegion)
-	require.NoError(t, err)
-	return after
-}
-
-// walkFrom follows NextMarker from marker to the end, checking that no page
-// examines more than maxListScan sequences, and returns the names it listed.
-func walkFrom(t *testing.T, svc *MetaService, marker string, maxItems int) ([]string, int) {
-	t.Helper()
-	var names []string
-	var after uint64
-	if marker != "" {
-		after = cursorAfter(t, marker)
-	}
-	for pages := 1; ; pages++ {
-		resp := listPage(t, svc, testRegion, marker, maxItems)
-		require.LessOrEqual(t, len(resp.Functions), maxItems)
-		for _, f := range resp.Functions {
-			names = append(names, f.Record.FunctionName)
-		}
-		if resp.NextMarker == "" {
-			return names, pages
-		}
-		next := cursorAfter(t, resp.NextMarker)
-		require.LessOrEqual(t, next-after, uint64(maxListScan), "a page examined more than its scan bound")
-		require.True(t, next > after || len(resp.Functions) > 0, "a page made no progress")
-		marker, after = resp.NextMarker, next
-		require.Less(t, pages, 1000, "the walk does not end")
-	}
-}
-
-// tombstones appends n deleted functions to the stream.
-func tombstones(t *testing.T, svc *MetaService, n int) {
-	t.Helper()
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	for i := range n {
-		name := fmt.Sprintf("tomb-%d", i)
-		mustCreate(t, svc, name, fmt.Sprintf("t%d", i))
-		require.NoError(t, kv.Delete(t.Context(), KVFunctionKey(testRegion, name)))
-	}
-}
-
-// More delete markers than one request may examine give empty pages that
-// still continue, and the walk reaches the live function after them.
-func TestList_ScanBoundGivesShortPagesThatContinue(t *testing.T) {
-	svc, _ := newTestService(t)
-	tombstones(t, svc, maxListScan+5)
-	mustCreate(t, svc, "live", "f1")
-
-	first := listPage(t, svc, testRegion, "", MaxListPage)
-	assert.Empty(t, first.Functions)
-	require.NotEmpty(t, first.NextMarker)
-	assert.Equal(t, uint64(maxListScan), cursorAfter(t, first.NextMarker))
-	names, _ := walkFrom(t, svc, first.NextMarker, MaxListPage)
-	assert.Equal(t, []string{"live"}, names)
-}
-
-// Messages that are not this Region's function records count toward the scan
-// bound like any other sequence: uploads, obligations, other keys and other
-// Regions' functions.
-func TestList_ExcludedRecordsCountTowardTheScanBound(t *testing.T) {
-	svc, _ := newTestService(t)
-	mustCreate(t, svc, "first", "f1")
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	for i := range maxListScan + 30 {
-		var key string
-		switch i % 4 {
-		case 0:
-			key = KVFunctionKey("us-east-1", fmt.Sprintf("x-%d", i))
-		case 1:
-			key = fmt.Sprintf("upload.%s.f9.%d", testRegion, i)
-		case 2:
-			key = KVDeletedKey(testRegion, fmt.Sprintf("d%d", i))
-		default:
-			key = fmt.Sprintf("other.%d", i)
-		}
-		_, err := kv.Put(t.Context(), key, []byte("{}"))
-		require.NoError(t, err)
-	}
-	mustCreate(t, svc, "last", "f2")
-
-	names, pages := walkFrom(t, svc, "", MaxListPage)
-	assert.Equal(t, []string{"first", "last"}, names)
-	assert.GreaterOrEqual(t, pages, 2)
-}
-
-// A key rewritten many times leaves gaps behind its latest revision; missing
-// sequences count toward the bound and the walk crosses them.
-func TestList_SparseSequencesCountTowardTheScanBound(t *testing.T) {
-	svc, _ := newTestService(t)
-	mustCreate(t, svc, "first", "f1")
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	for range 2*maxListScan + 7 {
-		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
-		require.NoError(t, err)
-	}
-	mustCreate(t, svc, "middle", "f2")
-	for range maxListScan {
-		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
-		require.NoError(t, err)
-	}
-	mustCreate(t, svc, "last", "f3")
-
-	page := listPage(t, svc, testRegion, "", MaxListPage)
-	require.Len(t, page.Functions, 1)
-	assert.Equal(t, uint64(maxListScan), cursorAfter(t, page.NextMarker))
-	for _, size := range []int{1, MaxListPage} {
-		names, _ := walkFrom(t, svc, "", size)
-		assert.Equal(t, []string{"first", "middle", "last"}, names, "page size %d", size)
-	}
-}
-
-// A large live set is returned once each, in stream order, in pages that
-// never exceed the requested size.
-func TestList_LargeLiveSetPagesEveryFunctionOnce(t *testing.T) {
-	svc, _ := newTestService(t)
-	want := make([]string, 0, 3*MaxListPage+7)
-	for i := range cap(want) {
-		name := fmt.Sprintf("fn-%03d", i)
-		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
-		want = append(want, name)
-	}
-	for _, size := range []int{1, 7, MaxListPage} {
-		names, _ := walkFrom(t, svc, "", size)
-		assert.Equal(t, want, names, "page size %d", size)
-	}
-}
-
-// A full page looking ahead through delete markers to the end of the stream
-// returns no marker; when the scan bound runs out first, it continues and a
-// later page finds the function beyond them.
-func TestList_LookAheadThroughDeleteMarkers(t *testing.T) {
-	t.Run("reaches the end", func(t *testing.T) {
-		svc, _ := newTestService(t)
-		mustCreate(t, svc, "live", "f1")
-		tombstones(t, svc, 3)
-		page := listPage(t, svc, testRegion, "", 1)
-		require.Len(t, page.Functions, 1)
-		assert.Empty(t, page.NextMarker)
-	})
-	t.Run("scan bound exhausted", func(t *testing.T) {
-		svc, _ := newTestService(t)
-		mustCreate(t, svc, "live", "f1")
-		tombstones(t, svc, maxListScan)
-		mustCreate(t, svc, "beyond", "f2")
-		page := listPage(t, svc, testRegion, "", 1)
-		require.Len(t, page.Functions, 1)
-		assert.Equal(t, "live", page.Functions[0].Record.FunctionName)
-		require.NotEmpty(t, page.NextMarker, "the bound ran out before the walk was known to end")
-		names, _ := walkFrom(t, svc, page.NextMarker, 1)
-		assert.Equal(t, []string{"beyond"}, names)
-	})
-}
-
-// An update moves a function to a later sequence, so this walk, begun before
-// the update, meets it again after the function it had not yet reached.
-func TestList_UpdateDuringWalkMovesFunctionLater(t *testing.T) {
-	svc, _ := newTestService(t)
-	first := serveAs(t, svc, "a", "f1")
-	serveAs(t, svc, "b", "f2")
-	page := listPage(t, svc, testRegion, "", 1)
-	require.Equal(t, "a", page.Functions[0].Record.FunctionName)
-	cfg := Configuration{Role: first.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
-	_, err := svc.UpdateConfiguration(t.Context(), &ConfigurationUpdatedRequest{Region: testRegion, FunctionName: "a",
-		FunctionID: "f1", Revision: first.Revision, Configuration: cfg}, testAccount)
-	require.NoError(t, err)
-
-	var rest []string
-	for marker := page.NextMarker; marker != ""; {
-		p := listPage(t, svc, testRegion, marker, 1)
-		for _, f := range p.Functions {
-			rest = append(rest, f.Record.FunctionName)
-		}
-		marker = p.NextMarker
-	}
-	assert.Equal(t, []string{"b", "a"}, rest)
-}
-
-func TestList_RefusesForeignMalformedAndExhaustedMarkers(t *testing.T) {
-	svc, _ := newTestService(t)
-	for i := range 3 {
-		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
-	}
-	marker := listPage(t, svc, testRegion, "", 1).NextMarker
-	require.NotEmpty(t, marker)
-
-	foreign, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: marker, MaxItems: 1}, "000000000002")
-	assert.Nil(t, foreign)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	_, err = svc.List(t.Context(), &ListRequest{Region: "us-east-1", Marker: marker, MaxItems: 1}, testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-
-	exhausted, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion, After: math.MaxUint64})
-	require.NoError(t, err)
-	zero, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion})
-	require.NoError(t, err)
-	future, err := encodeCursor(listCursor{Version: 2, AccountID: testAccount, Region: testRegion, After: 1})
-	require.NoError(t, err)
-	for _, bad := range []string{"not base64!", "e30", exhausted, zero, future} {
-		_, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: bad, MaxItems: 1}, testAccount)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}
-
-func TestList_RefusesMalformedRequests(t *testing.T) {
-	svc, _ := newTestService(t)
-	for _, req := range []*ListRequest{
-		{Region: "Bad.Region", MaxItems: 1},
-		{Region: testRegion, MaxItems: 0},
-		{Region: testRegion, MaxItems: MaxListPage + 1},
-	} {
-		_, err := svc.List(t.Context(), req, testAccount)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-}

spinifex/domains/lambda/function/record.go

@@ -63,7 +63,6 @@ const (
 	KVBucketAccountHistory = 1
 	kvFunctionKeyPrefix    = "fn."
 	kvUploadKeyPrefix      = "upload."
-	kvDeletedKeyPrefix     = "deleted."
 )
 
 // KVUploadKey returns the key of the record that tracks one code upload. The
@@ -112,10 +111,6 @@ const (
 	SubjectCodeUpdated     = "lambda.function.code-updated"
 	// SubjectConfigurationUpdated commits new settings for an Active function.
 	SubjectConfigurationUpdated = "lambda.function.configuration-updated"
-	// SubjectList pages the function records of one Region.
-	SubjectList = "lambda.function.list"
-	// SubjectDelete unbinds a function name and retains its cleanup obligation.
-	SubjectDelete = "lambda.function.delete"
 )
 
 // CreateRequest carries the record the gateway built; the daemon owns the
@@ -223,53 +218,3 @@ type ConfigurationUpdatedResponse struct {
 	Record   Record `json:"record"`
 	Revision uint64 `json:"revision"`
 }
-
-// KVDeletedKey returns the key of the obligation a deleted function leaves
-// behind. A recreated function receives a new random ID, and an obligation
-// whose identity does not match the deleted function is refused.
-func KVDeletedKey(region, functionID string) string {
-	return kvDeletedKeyPrefix + region + "." + functionID
-}
-
-// Deletion is the retained cleanup obligation of a deleted function: its code
-// copies and upload records are kept until a later owner proves them unused.
-// It is written before the name is unbound, so it alone proves no deletion.
-type Deletion struct {
-	FormatVersion int       `json:"formatVersion"`
-	AccountID     string    `json:"accountId"`
-	Region        string    `json:"region"`
-	FunctionName  string    `json:"functionName"`
-	FunctionID    string    `json:"functionId"`
-	RecordedAt    time.Time `json:"recordedAt"`
-}
-
-// ListRequest asks for one page of a Region's functions after Marker, which
-// is empty for the first page or a NextMarker the owner returned.
-type ListRequest struct {
-	Region   string `json:"region"`
-	Marker   string `json:"marker,omitempty"`
-	MaxItems int    `json:"maxItems"`
-}
-
-// ListedFunction is one function record and its revision.
-type ListedFunction struct {
-	Record   Record `json:"record"`
-	Revision uint64 `json:"revision"`
-}
-
-// ListResponse is one page; NextMarker is empty once the walk has ended.
-type ListResponse struct {
-	Functions  []ListedFunction `json:"functions"`
-	NextMarker string           `json:"nextMarker,omitempty"`
-}
-
-// DeleteRequest unbinds the function the caller read at Revision.
-type DeleteRequest struct {
-	Region       string `json:"region"`
-	FunctionName string `json:"functionName"`
-	FunctionID   string `json:"functionId"`
-	Revision     uint64 `json:"revision"`
-}
-
-// DeleteResponse is empty: success means the name was unbound.
-type DeleteResponse struct{}

spinifex/domains/network/external/ocinet/wiring_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/external/ocinet"
 	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
 )
@@ -25,21 +25,21 @@ func TestNewClientSelectsTheCredentialThePoolNames(t *testing.T) {
 	}{
 		{
 			name:          "instance principal",
-			auth:          external.OCIAuthInstancePrincipal,
+			auth:          config.OCIAuthInstancePrincipal,
 			wantPrincipal: true,
 		},
 		{
 			// A pool configured for an instance principal still must not fall
 			// back to a key file when one happens to be configured as well.
 			name:          "instance principal with a config file also set",
-			auth:          external.OCIAuthInstancePrincipal,
+			auth:          config.OCIAuthInstancePrincipal,
 			configFile:    "/etc/spinifex/oci/config",
 			profile:       "spinifex",
 			wantPrincipal: true,
 		},
 		{
 			name:       "config file",
-			auth:       external.OCIAuthConfigFile,
+			auth:       config.OCIAuthConfigFile,
 			configFile: "/etc/spinifex/oci/config",
 			profile:    "spinifex",
 		},
@@ -94,6 +94,6 @@ func TestNewClientReturnsTheConstructorError(t *testing.T) {
 	)
 	defer restore()
 
-	_, err := ocinet.NewClient(ocinet.PoolWithAuth(external.OCIAuthConfigFile, "/missing", "spinifex"))
+	_, err := ocinet.NewClient(ocinet.PoolWithAuth(config.OCIAuthConfigFile, "/missing", "spinifex"))
 	require.ErrorIs(t, err, wantErr)
 }

spinifex/domains/network/external/types.go

@@ -1,5 +1,7 @@
 package external
 
+import "github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+
 // ExternalPoolConfig describes one external IP pool wired to a VPC.
 // Mirrors [external_pools] in spinifex.toml.
 type ExternalPoolConfig struct {
@@ -32,8 +34,8 @@ type ExternalPoolConfig struct {
 	OCIVNICIface     string
 	OCISubnetID      string
 	OCIPublicIPPool  string
-	// OCIAuth selects how the provider authenticates: OCIAuthInstancePrincipal
-	// for the instance's own certificate, or OCIAuthConfigFile (the default) for
+	// OCIAuth selects how the provider authenticates: config.OCIAuthInstancePrincipal
+	// for the instance's own certificate, or config.OCIAuthConfigFile (the default) for
 	// an API key, which OCIConfigFile and OCIConfigProfile then locate.
 	OCIAuth          string
 	OCIConfigFile    string
@@ -77,7 +79,7 @@ func (p *ExternalPoolConfig) IsOCI() bool {
 // UsesInstancePrincipal reports whether the OCI provider authenticates as the
 // instance rather than from an API-key config file.
 func (p *ExternalPoolConfig) UsesInstancePrincipal() bool {
-	return p != nil && p.OCIAuth == OCIAuthInstancePrincipal
+	return p != nil && p.OCIAuth == config.OCIAuthInstancePrincipal
 }
 
 // IsExoscale reports whether the pool sources IPs from Exoscale Elastic IPs.
@@ -123,14 +125,6 @@ const (
 	// between servers. A routed Elastic IP arrives addressed to the host's MAC.
 	SourceLatitude = "latitude"
 
-	// OCIAuthConfigFile authenticates from an API-key config file. The default,
-	// because it is what an operator already has from `oci setup config`.
-	OCIAuthConfigFile = "config_file"
-	// OCIAuthInstancePrincipal authenticates as the instance, from the
-	// certificate its metadata service serves. No key material on any node and
-	// nothing to rotate, at the cost of a dynamic group and an IAM policy.
-	OCIAuthInstancePrincipal = "instance_principal"
-
 	// DHCPMACDerived leases with deterministic per-client-id 02:xx MACs.
 	DHCPMACDerived = "derived"
 	// DHCPMACInterface leases with the bind interface's own MAC; the

spinifex/domains/sqs/storevolume/prepare.go

@@ -48,9 +48,10 @@ type Preparer struct {
 	Provider   Publisher
 	NodeID     string
 	ScratchDir string
-	// WriteImage copies a raw image onto an NBD export. The composition root
-	// supplies it, since a domain does not import the runtime copy helper.
-	WriteImage func(ctx context.Context, sourcePath, nbdURI string, progress io.Writer) error
+	// WriteImage copies a raw image onto the published export, refusing a kind
+	// it cannot write. The composition root supplies it, since a domain does
+	// not import the runtime copy helper.
+	WriteImage func(ctx context.Context, sourcePath string, export ebsprovider.VolumeExport, progress io.Writer) error
 
 	lookPath  func(string) (string, error)
 	runMkfs   func(ctx context.Context, mkfs, path string) error
@@ -163,8 +164,9 @@ func (e *prepError) Error() string { return e.err.Error() }
 // fill publishes the volume, copies the image and unpublishes it. Its error
 // says whether the export is known to be closed.
 func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
+	// A block export has no version 1 form, so publish as the VM path does.
 	published, err := p.Provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
+		Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
 	})
 	if err != nil {
 		// These refusals may mean another export exists, which an unpublish
@@ -181,10 +183,10 @@ func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
 	}
 
 	var writeErr error
-	if published == nil || published.Export.Kind != ebsprovider.ExportKindNBD || published.Export.NBD == nil {
-		writeErr = errors.New("volume was not published as an nbd export")
+	if published == nil {
+		writeErr = errors.New("publish returned no export")
 	} else {
-		writeErr = p.WriteImage(ctx, image, published.Export.NBD.NBDURI, nil)
+		writeErr = p.WriteImage(ctx, image, published.Export, nil)
 	}
 
 	if err := p.unpublish(ctx, volumeID); err != nil {

spinifex/domains/sqs/storevolume/prepare_test.go

@@ -1,6 +1,6 @@
 package storevolume
 
-//test:in-package — the host tools, mkfs and NBD copy are swapped for fakes
+//test:in-package — the host tools, mkfs and image copy are swapped for fakes
 // through unexported fields so no test formats or writes a real device.
 
 import (
@@ -32,6 +32,7 @@ const (
 	testVolume  = "vol-0123456789abcdef0"
 	testNode    = "node-a"
 	testURI     = "nbd://127.0.0.1:10809"
+	testDevice  = "/dev/sdx"
 )
 
 // harness records every call, in order, across the fakes.
@@ -99,8 +100,14 @@ func (h *harness) PublishVolume(_ context.Context, req ebsprovider.PublishVolume
 		kind = ebsprovider.ExportKindNBD
 	}
 	export := ebsprovider.VolumeExport{Kind: kind}
-	if kind == ebsprovider.ExportKindNBD {
+	switch kind {
+	case ebsprovider.ExportKindNBD:
 		export.NBD = &ebsprovider.NBDExport{NBDURI: testURI}
+	case ebsprovider.ExportKindBlock:
+		if req.SchemaVersion < ebsprovider.DescriptorSchemaVersion {
+			return nil, fmt.Errorf("%w: a block export has no version %d form", ebsprovider.ErrUnsupportedVersion, req.SchemaVersion)
+		}
+		export.Block = &ebsprovider.BlockExport{DevicePath: testDevice, SizeBytes: 1 << 30}
 	}
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }
@@ -133,8 +140,15 @@ func (h *harness) preparer(t *testing.T) *Preparer {
 			h.record("mkfs " + filepath.Base(path))
 			return h.mkfsErr
 		},
-		WriteImage: func(_ context.Context, src, uri string, _ io.Writer) error {
-			h.record("write " + uri)
+		WriteImage: func(_ context.Context, src string, export ebsprovider.VolumeExport, _ io.Writer) error {
+			switch {
+			case export.Kind == ebsprovider.ExportKindNBD && export.NBD != nil:
+				h.record("write " + export.NBD.NBDURI)
+			case export.Kind == ebsprovider.ExportKindBlock && export.Block != nil:
+				h.record("write " + export.Block.DevicePath)
+			default:
+				return fmt.Errorf("no writer for a %q export", export.Kind)
+			}
 			h.writtenImage = src
 			return h.writeErr
 		},
@@ -265,12 +279,22 @@ func TestPrepare_FailedCopyUnpublishesThenDiscards(t *testing.T) {
 	}, h.calls)
 }
 
-func TestPrepare_NonNBDExportIsNotWrittenAndIsDiscarded(t *testing.T) {
-	h := &harness{exportKind: "iscsi"}
-	_, err := prepare(t, h)
-	require.ErrorContains(t, err, "not published as an nbd export")
-	assert.NotContains(t, h.calls, "write "+testURI)
-	assert.Equal(t, "discard "+testVolume, h.calls[len(h.calls)-1])
+// A block export has no version 1 form, so the block path works only because
+// preparation publishes at the latest version.
+func TestPrepare_BlockExportIsWrittenAtItsDevicePathAndReleased(t *testing.T) {
+	h := &harness{exportKind: ebsprovider.ExportKindBlock}
+	id, err := prepare(t, h)
+	require.NoError(t, err)
+	assert.Equal(t, testVolume, id)
+	assert.Equal(t, []string{
+		"mkfs ext4.raw",
+		"create 000000000000 1GiB",
+		"publish " + testVolume + " on " + testNode,
+		"write " + testDevice,
+		"unpublish " + testVolume + " on " + testNode,
+		"complete " + testVolume,
+	}, h.calls)
+	assert.Equal(t, "available", h.state)
 }
 
 // An in-use or already-exists refusal may mean another export of the volume
@@ -359,7 +383,7 @@ func TestPrepare_StuckUnpublishIsBounded(t *testing.T) {
 	p := h.preparer(t)
 	p.cleanupTimeout = 50 * time.Millisecond
 	ctx, cancel := context.WithCancel(context.Background())
-	p.WriteImage = func(context.Context, string, string, io.Writer) error {
+	p.WriteImage = func(context.Context, string, ebsprovider.VolumeExport, io.Writer) error {
 		cancel()
 		return context.Canceled
 	}
@@ -391,7 +415,7 @@ func TestPrepare_CancelledCopyStillUnpublishes(t *testing.T) {
 	h := &harness{}
 	ctx, cancel := context.WithCancel(context.Background())
 	p := h.preparer(t)
-	p.WriteImage = func(context.Context, string, string, io.Writer) error {
+	p.WriteImage = func(context.Context, string, ebsprovider.VolumeExport, io.Writer) error {
 		cancel()
 		return context.Canceled
 	}

spinifex/ingress/aws/pipeline/lambda_invoke_wire_test.go

@@ -59,14 +59,6 @@ func (activeMeta) UpdateConfiguration(context.Context, string, string, function.
 	return function.Record{}, 0, errors.New("unused")
 }
 
-func (activeMeta) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
-	return nil, "", errors.New("unused")
-}
-
-func (activeMeta) Delete(context.Context, string, string, function.Record, uint64) error {
-	return errors.New("unused")
-}
-
 // countingExec records how many invocations reached it.
 type countingExec struct{ calls atomic.Int32 }
 

spinifex/operator/admin/admin.go

@@ -122,6 +122,18 @@ type ConfigSettings struct {
 	BootstrapCidr       string
 	BootstrapSubnetCidr string
 
+	// EBSProvider is rendered as [nodes.X.ebs] provider; empty renders the
+	// default. The OCI fields fill [nodes.X.ebs.oci] for ocibsd.
+	EBSProvider           string
+	OCICompartmentID      string
+	OCIAvailabilityDomain string
+	OCIClusterID          string
+	// OCIAuth, OCIConfigFile and OCIConfigProfile are this node's credential,
+	// rendered only when set.
+	OCIAuth          string
+	OCIConfigFile    string
+	OCIConfigProfile string
+
 	// GPUPassthrough enables VFIO GPU passthrough in the daemon config.
 	// Sets gpu_passthrough = true under [nodes.<node>.daemon].
 	GPUPassthrough bool

spinifex/operator/admin/image_import.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/imagewrite"
 )
 
 // ImportOpts describes one raw disk image to import into a new volume.
@@ -16,7 +16,7 @@ type ImportOpts struct {
 	// VolumeID names the volume to create. For an AMI import this is the
 	// image ID, so the snapshot lands at SnapPrefix(imageID).
 	VolumeID string
-	// NodeID is the node whose provider serves the NBD export written to. It
+	// NodeID is the node whose provider serves the export written to. It
 	// must be the node this process runs on: the export is local to it.
 	NodeID string
 	// SizeBytes is the volume's capacity, which the caller rounds to a whole
@@ -41,8 +41,8 @@ type ImportOpts struct {
 // shortens it so the wait does not sleep.
 var snapshotPollInterval = 5 * time.Second
 
-// ImportImage writes a local raw disk image into a new provider volume over
-// NBD, then optionally snapshots it. Everything here goes through the provider
+// ImportImage writes a local raw disk image into a new provider volume through
+// the export the provider publishes, then optionally snapshots it. Everything here goes through the provider
 // contract, so an import works against any conforming provider rather than
 // only the one whose storage library this process happens to link.
 //
@@ -66,8 +66,9 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 		return fmt.Errorf("import: create volume %s: %w", opts.VolumeID, err)
 	}
 
+	// A block export has no version 1 form, so publish as the VM path does.
 	published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned: ebsprovider.NewLatestVersioned(),
 		VolumeID:  opts.VolumeID,
 		NodeID:    opts.NodeID,
 	})
@@ -75,12 +76,7 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 		return fmt.Errorf("import: publish volume %s: %w", opts.VolumeID, err)
 	}
 
-	var writeErr error
-	if published.Export.Kind == ebsprovider.ExportKindNBD && published.Export.NBD != nil {
-		writeErr = writeImage(ctx, opts.SourcePath, published.Export.NBD.NBDURI, opts.Progress)
-	} else {
-		writeErr = fmt.Errorf("volume is published as a %q export; image import writes only through nbd", published.Export.Kind)
-	}
+	writeErr := writeImage(ctx, opts.SourcePath, published.Export, opts.Progress)
 
 	// Unpublish regardless: a volume left published cannot be snapshotted or
 	// attached, and on the failure path it would strand the export too.
@@ -157,5 +153,5 @@ func waitForSnapshot(ctx context.Context, provider ebsprovider.EBSProvider, snap
 }
 
 // writeImage is the bulk copy step, indirected so a test can drive the verb
-// sequence around it without a live NBD export to write to.
-var writeImage = nbd.WriteRawImage
+// sequence around it without a live export to write to.
+var writeImage = imagewrite.Writer{}.Write

spinifex/operator/admin/image_import_test.go

@@ -3,11 +3,15 @@ package admin
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/imagewrite"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -55,6 +59,9 @@ func (p *recordingProvider) CreateVolume(ctx context.Context, req ebsprovider.Cr
 func (p *recordingProvider) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
 	p.calls = append(p.calls, "publish")
 	p.publishes = append(p.publishes, req)
+	if p.publishExport.Kind == ebsprovider.ExportKindBlock && req.SchemaVersion < ebsprovider.DescriptorSchemaVersion {
+		return nil, fmt.Errorf("%w: a block export has no version %d form", ebsprovider.ErrUnsupportedVersion, req.SchemaVersion)
+	}
 	if _, err := p.MemoryProvider.PublishVolume(ctx, req); err != nil {
 		return nil, err
 	}
@@ -81,9 +88,9 @@ func (p *recordingProvider) CreateSnapshot(ctx context.Context, req ebsprovider.
 func stubWriteImage(t *testing.T, err error, seen *string) {
 	t.Helper()
 	original := writeImage
-	writeImage = func(_ context.Context, _, nbdURI string, _ io.Writer) error {
-		if seen != nil {
-			*seen = nbdURI
+	writeImage = func(_ context.Context, _ string, export ebsprovider.VolumeExport, _ io.Writer) error {
+		if seen != nil && export.NBD != nil {
+			*seen = export.NBD.NBDURI
 		}
 		return err
 	}
@@ -140,30 +147,54 @@ func TestImportImage_WriteFailureStillUnpublishes(t *testing.T) {
 		"a failed copy must not be snapshotted, and must still be unpublished")
 }
 
-// TestImportImage_BlockExportIsRefused covers a provider that publishes a host
-// block device: the import writes only through NBD, so it must fail without
-// copying and still release the export.
-func TestImportImage_BlockExportIsRefused(t *testing.T) {
-	var wroteTo string
-	stubWriteImage(t, nil, &wroteTo)
+// blockImport sets up an import against a provider publishing a host block
+// device, with the shared writer running a fake qemu-img.
+func blockImport(t *testing.T, imageBytes int64, deviceBytes uint64) (*recordingProvider, ImportOpts, *[][]string) {
+	t.Helper()
 	provider := newRecordingProvider()
 	provider.publishExport = ebsprovider.VolumeExport{
 		Kind:  ebsprovider.ExportKindBlock,
-		Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: 1 << 30},
+		Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: deviceBytes},
 	}
+	image := filepath.Join(t.TempDir(), "image.raw")
+	require.NoError(t, os.WriteFile(image, nil, 0o600))
+	require.NoError(t, os.Truncate(image, imageBytes))
 
-	err := ImportImage(context.Background(), provider, ImportOpts{
-		VolumeID:   "ami-import005",
-		NodeID:     "node-a",
-		SizeBytes:  1 << 30,
-		SourcePath: "/dev/null",
-		Snapshot:   true,
-	})
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), `"block" export`)
-	assert.Empty(t, wroteTo, "a block export must not reach the NBD writer")
+	var ran [][]string
+	original := writeImage
+	writeImage = imagewrite.Writer{QemuImg: func(_ context.Context, _ io.Writer, args ...string) error {
+		ran = append(ran, args)
+		return nil
+	}}.Write
+	t.Cleanup(func() { writeImage = original })
+	return provider, ImportOpts{VolumeID: "ami-import005", NodeID: "node-a", SizeBytes: 1 << 30, SourcePath: image, Snapshot: true}, &ran
+}
+
+// A block export has no version 1 form, so only a publish at the latest
+// version reaches the device at all.
+func TestImportImage_BlockExportIsWrittenAtItsDevicePath(t *testing.T) {
+	provider, opts, ran := blockImport(t, 1<<20, 1<<30)
+
+	require.NoError(t, ImportImage(context.Background(), provider, opts))
+
+	require.Len(t, provider.publishes, 1)
+	assert.Equal(t, ebsprovider.SchemaVersion, provider.publishes[0].SchemaVersion)
+	require.Len(t, *ran, 1)
+	assert.Equal(t, "/dev/sdx", (*ran)[0][len((*ran)[0])-1], "the copy must target the device the provider published")
+	assert.Equal(t, []string{"create", "publish", "unpublish", "snapshot"}, provider.calls)
+	vol, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: opts.VolumeID})
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.VolumeStateAvailable, vol.State)
+}
+
+func TestImportImage_ImageLargerThanTheBlockExportIsRefusedAndUnpublished(t *testing.T) {
+	provider, opts, ran := blockImport(t, 1<<20+1, 1<<20)
+
+	err := ImportImage(context.Background(), provider, opts)
+	require.ErrorContains(t, err, "larger than")
+	assert.Empty(t, *ran, "nothing may be written to a device too small for the image")
 	assert.Equal(t, []string{"create", "publish", "unpublish"}, provider.calls,
-		"a refused export must not be snapshotted, and must still be unpublished")
+		"a refused write must not be snapshotted, and must still be unpublished")
 }
 
 // TestImportImage_NoSnapshotWhenNotAsked covers the weights-style import that

spinifex/operator/cli/admin.go

@@ -1660,6 +1660,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(0)
 	}
 
+	// Resolved once and written to every node (ADR-0018:S3), before anything
+	// is written, so a refused OCI credential leaves the node untouched.
+	ebsChoice, err := resolveInitEBS(context.Background(), spinifexTomlPath, node, productionEBSResolver())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+
 	if spxRoot == "" {
 		spxRoot = DefaultDataDir()
 	}
@@ -1848,6 +1856,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		// external networking is disabled.
 		networkConfig := &formation.NetworkConfig{
 			IPSecEnabled: ipsecEnabled,
+			EBS:          formationEBS(ebsChoice.EBSConfig),
 		}
 		if externalMode != "" {
 			bootstrapVpcId := awsidentifiers.GenerateResourceID("vpc")
@@ -1867,7 +1876,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey,
 			masterKey, viperblockKey, natsToken, clusterName,
 			configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email,
-			port, nodes, formationTimeoutStr, tokenTTLStr, services, nameserver, networkConfig, northstarCreds)
+			port, nodes, formationTimeoutStr, tokenTTLStr, services, nameserver, networkConfig, northstarCreds, ebsChoice)
 		return
 	}
 
@@ -2029,6 +2038,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}
 
+	ebsChoice.apply(&configSettings)
+
 	// Generate config files
 	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, predastoreNodesStr != ""); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
@@ -2036,6 +2047,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 
 	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)
+	exitOnActivationError(activateEBSProvider(context.Background(), ebsChoice, productionOCIBSDActivation()))
 
 	skipHostDNS, _ := cmd.Flags().GetBool("skip-host-dns")
 	configureHostDNS(configSettings, skipHostDNS)
@@ -2077,7 +2089,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	masterKey, viperblockKey []byte, natsToken, clusterName,
 	configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email string,
 	port, expectedNodes int, formationTimeoutStr, tokenTTLStr string, services []string, nameserver bool, networkConfig *formation.NetworkConfig,
-	northstarCreds admin.NorthstarCredentials) {
+	northstarCreds admin.NorthstarCredentials, ebsChoice nodeEBS) {
 	formationTimeout, err := time.ParseDuration(formationTimeoutStr)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err)
@@ -2304,6 +2316,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	if networkConfig != nil {
 		applyNetworkConfig(&configSettings, networkConfig)
 	}
+	ebsChoice.apply(&configSettings)
 
 	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
@@ -2311,6 +2324,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	}
 
 	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)
+	exitOnActivationError(activateEBSProvider(context.Background(), ebsChoice, productionOCIBSDActivation()))
 
 	skipHostDNS, _ := cmd.Flags().GetBool("skip-host-dns")
 	configureHostDNS(configSettings, skipHostDNS)
@@ -2885,6 +2899,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	if statusResp.NetworkConfig != nil {
 		applyNetworkConfig(&configSettings, statusResp.NetworkConfig)
 	}
+	// The provider is the cluster's, from formation; the credential is this
+	// node's, and is checked before its config is written.
+	ebsChoice, err := resolveJoinEBS(context.Background(), spinifexTomlPath, node, statusResp.NetworkConfig, productionEBSResolver())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+	ebsChoice.apply(&configSettings)
 
 	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
@@ -2892,6 +2914,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 
 	finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP)
+	exitOnActivationError(activateEBSProvider(context.Background(), ebsChoice, productionOCIBSDActivation()))
 
 	skipHostDNS, _ := cmd.Flags().GetBool("skip-host-dns")
 	configureHostDNS(configSettings, skipHostDNS)

spinifex/operator/cli/admin_ebs.go

@@ -0,0 +1,405 @@
+package cli
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	toml "github.com/pelletier/go-toml/v2"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/operator/admin"
+	"github.com/mulgadc/spinifex/spinifex/operator/host/systemd"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	ebsocibsd "github.com/mulgadc/spinifex/spinifex/providers/ebs/ocibsd"
+	"github.com/mulgadc/spinifex/spinifex/runtime/formation"
+	ocibsdrole "github.com/mulgadc/spinifex/spinifex/runtime/roles/ocibsd"
+)
+
+// ebsProviderEnv names the provider for an install with no existing config,
+// in place of detection.
+const ebsProviderEnv = "SPINIFEX_EBS_PROVIDER"
+
+const (
+	// imdsDetectTimeout bounds the metadata read detection makes, so an
+	// install off OCI pays at most this for it.
+	imdsDetectTimeout = time.Second
+	// credentialCheckTimeout bounds the OCI read the credential check makes.
+	credentialCheckTimeout = 30 * time.Second
+)
+
+// Where a node's provider came from, printed by init and join so a run can
+// tell a detected provider from a configured one.
+const (
+	ebsFromConfig    = "from the existing config"
+	ebsFromEnv       = "from " + ebsProviderEnv
+	ebsFromDetection = "detected"
+	ebsFromFormation = "from formation"
+)
+
+// nodeEBS is one node's EBS choice: the cluster-wide part formation carries
+// to joining nodes, and this node's own OCI credential.
+type nodeEBS struct {
+	formation.EBSConfig
+
+	Auth          string
+	ConfigFile    string
+	ConfigProfile string
+	// Source is one of the ebsFrom* values; Detail says what detection saw.
+	Source string
+	Detail string
+}
+
+// describe is the line init and join print for the choice.
+func (e nodeEBS) describe() string {
+	provider := (config.EBSConfig{Provider: e.Provider}).ResolvedProvider()
+	if e.Detail != "" {
+		return fmt.Sprintf("EBS provider: %s (%s: %s)", provider, e.Source, e.Detail)
+	}
+	return fmt.Sprintf("EBS provider: %s (%s)", provider, e.Source)
+}
+
+// apply copies the choice into the settings spinifex.toml is rendered from.
+func (e nodeEBS) apply(s *admin.ConfigSettings) {
+	s.EBSProvider = e.Provider
+	s.OCICompartmentID = e.OCICompartmentID
+	s.OCIAvailabilityDomain = e.OCIAvailabilityDomain
+	s.OCIClusterID = e.ClusterID
+	s.OCIAuth = e.Auth
+	s.OCIConfigFile = e.ConfigFile
+	s.OCIConfigProfile = e.ConfigProfile
+}
+
+// ebsResolver holds the host and cloud seams provider resolution reads.
+type ebsResolver struct {
+	getenv   func(string) string
+	instance func(ctx context.Context) (oci.InstanceMetadata, error)
+	// newClient builds the OCI client the credential check reads through.
+	newClient func(auth, configFile, configProfile string) (oci.BlockClient, error)
+}
+
+func productionEBSResolver() ebsResolver {
+	md := &oci.MetadataClient{BaseURL: oci.IMDSBaseURL, HTTP: &http.Client{Timeout: imdsDetectTimeout}}
+	return ebsResolver{
+		getenv:   os.Getenv,
+		instance: md.Instance,
+		newClient: func(auth, configFile, configProfile string) (oci.BlockClient, error) {
+			return ocibsdrole.BlockClientFor(&ocibsdrole.Config{Auth: auth, ConfigFile: configFile, ConfigProfile: configProfile})
+		},
+	}
+}
+
+// existingEBS is a node's [nodes.<node>.ebs] in a spinifex.toml already on
+// disk, kept as written: an empty Provider is the default, viperblockd.
+type existingEBS struct {
+	Provider string `toml:"provider"`
+	OCI      struct {
+		CompartmentID      string `toml:"compartment_id"`
+		AvailabilityDomain string `toml:"availability_domain"`
+		ClusterID          string `toml:"cluster_id"`
+		Auth               string `toml:"auth"`
+		ConfigFile         string `toml:"config_file"`
+		ConfigProfile      string `toml:"config_profile"`
+	} `toml:"oci"`
+}
+
+// readExistingEBS returns node's EBS table from tomlPath, nil when there is
+// no file or it has no [nodes.<node>]. A file that cannot be read is an
+// error: the provider it names must never be guessed.
+func readExistingEBS(tomlPath, node string) (*existingEBS, error) {
+	raw, err := os.ReadFile(tomlPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read the existing config %s: %w", tomlPath, err)
+	}
+	var cfg struct {
+		Nodes map[string]struct {
+			EBS existingEBS `toml:"ebs"`
+		} `toml:"nodes"`
+	}
+	if err := toml.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("parse the existing config %s, whose EBS provider this node keeps: %w", tomlPath, err)
+	}
+	n, ok := cfg.Nodes[node]
+	if !ok {
+		return nil, nil
+	}
+	return &n.EBS, nil
+}
+
+// isOCIVM reports whether md is an OCI virtual machine; bare-metal shapes
+// stay on viperblockd until a bare-metal iSCSI run is proven.
+func isOCIVM(md oci.InstanceMetadata) bool {
+	return md.Shape != "" && !strings.HasPrefix(md.Shape, "BM.")
+}
+
+// detect reads the OCI instance metadata within imdsDetectTimeout; ok is
+// false when nothing answers, which is every host off OCI.
+func (r ebsResolver) detect(ctx context.Context) (oci.InstanceMetadata, bool) {
+	ctx, cancel := context.WithTimeout(ctx, imdsDetectTimeout)
+	defer cancel()
+	md, err := r.instance(ctx)
+	if err != nil {
+		return oci.InstanceMetadata{}, false
+	}
+	return md, true
+}
+
+// resolveProvider picks init's provider: an existing node config's, then
+// ebsProviderEnv, then detection. md is set when detection read it.
+func (r ebsResolver) resolveProvider(ctx context.Context, existing *existingEBS) (nodeEBS, *oci.InstanceMetadata) {
+	switch {
+	case existing != nil:
+		return nodeEBS{
+			EBSConfig: formation.EBSConfig{Provider: (config.EBSConfig{Provider: existing.Provider}).ResolvedProvider()},
+			Source:    ebsFromConfig,
+		}, nil
+	case r.getenv(ebsProviderEnv) != "":
+		return nodeEBS{EBSConfig: formation.EBSConfig{Provider: r.getenv(ebsProviderEnv)}, Source: ebsFromEnv}, nil
+	}
+	md, ok := r.detect(ctx)
+	switch {
+	case !ok:
+		return nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderViperblockd}, Source: ebsFromDetection, Detail: "no OCI instance metadata"}, nil
+	case !isOCIVM(md):
+		return nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderViperblockd}, Source: ebsFromDetection, Detail: "OCI bare metal " + md.Shape}, nil
+	}
+	return nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderOCIBSD}, Source: ebsFromDetection, Detail: "OCI virtual machine " + md.Shape}, &md
+}
+
+// fillOCIBSD completes an ocibsd choice: the existing config's values win,
+// then the instance metadata, and auth defaults to instance_principal.
+// md is read here when the caller has none and a value is missing.
+func (r ebsResolver) fillOCIBSD(ctx context.Context, e *nodeEBS, existing *existingEBS, md *oci.InstanceMetadata) error {
+	var prior existingEBS
+	if existing != nil {
+		prior = *existing
+	}
+	if e.OCICompartmentID == "" {
+		e.OCICompartmentID = prior.OCI.CompartmentID
+	}
+	if e.OCIAvailabilityDomain == "" {
+		e.OCIAvailabilityDomain = prior.OCI.AvailabilityDomain
+	}
+	if e.OCICompartmentID == "" || e.OCIAvailabilityDomain == "" {
+		if md == nil {
+			got, ok := r.detect(ctx)
+			if !ok {
+				return fmt.Errorf("EBS provider %s (%s) needs this node's OCI instance metadata for its compartment and availability domain, and none answered", config.EBSProviderOCIBSD, e.Source)
+			}
+			md = &got
+		}
+		if e.OCICompartmentID == "" {
+			e.OCICompartmentID = md.CompartmentID
+		}
+		if e.OCIAvailabilityDomain == "" {
+			e.OCIAvailabilityDomain = md.AvailabilityDomain
+		}
+	}
+	e.Auth, e.ConfigFile, e.ConfigProfile = prior.OCI.Auth, prior.OCI.ConfigFile, prior.OCI.ConfigProfile
+	if e.Auth == "" && e.ConfigFile == "" && e.ConfigProfile == "" {
+		e.Auth = config.OCIAuthInstancePrincipal
+	}
+	if e.ClusterID == "" {
+		e.ClusterID = prior.OCI.ClusterID
+	}
+	if e.ClusterID == "" {
+		id, err := generateClusterID()
+		if err != nil {
+			return err
+		}
+		e.ClusterID = id
+	}
+	return nil
+}
+
+// checkCredential makes the read ocibsd makes at start with this node's
+// credential, and never falls back to another provider when it is refused.
+func (r ebsResolver) checkCredential(ctx context.Context, e nodeEBS) error {
+	client, err := r.newClient(e.Auth, e.ConfigFile, e.ConfigProfile)
+	if err != nil {
+		return fmt.Errorf("ocibsd: build the OCI client for auth %q: %w", e.Auth, err)
+	}
+	ctx, cancel := context.WithTimeout(ctx, credentialCheckTimeout)
+	defer cancel()
+	return ebsocibsd.CheckCredential(ctx, client, e.OCICompartmentID, e.OCIAvailabilityDomain, e.ClusterID)
+}
+
+// resolveInitEBS resolves init's provider once, for every node (ADR-0018:S3),
+// and for ocibsd completes its table and checks the credential.
+func resolveInitEBS(ctx context.Context, tomlPath, node string, r ebsResolver) (nodeEBS, error) {
+	existing, err := readExistingEBS(tomlPath, node)
+	if err != nil {
+		return nodeEBS{}, err
+	}
+	e, md := r.resolveProvider(ctx, existing)
+	if !config.SupportedEBSProvider(e.Provider) {
+		return nodeEBS{}, fmt.Errorf("EBS provider %q (%s) is unsupported; use %q, %q or %q", e.Provider, e.Source, config.EBSProviderViperblockd, config.EBSProviderQEMUNBD, config.EBSProviderOCIBSD)
+	}
+	fmt.Println(e.describe())
+	if e.Provider != config.EBSProviderOCIBSD {
+		return e, nil
+	}
+	if err := r.fillOCIBSD(ctx, &e, existing, md); err != nil {
+		return nodeEBS{}, err
+	}
+	if err := r.checkCredential(ctx, e); err != nil {
+		return nodeEBS{}, err
+	}
+	return e, nil
+}
+
+// resolveJoinEBS takes the provider from formation; for ocibsd this node's
+// own credential comes from its existing config, else instance_principal,
+// and is checked before its config is written.
+func resolveJoinEBS(ctx context.Context, tomlPath, node string, nc *formation.NetworkConfig, r ebsResolver) (nodeEBS, error) {
+	e := nodeEBS{Source: ebsFromFormation}
+	if nc != nil && nc.EBS != nil {
+		e.EBSConfig = *nc.EBS
+	}
+	fmt.Println(e.describe())
+	if e.Provider != config.EBSProviderOCIBSD {
+		return e, nil
+	}
+	if e.OCICompartmentID == "" || e.OCIAvailabilityDomain == "" || e.ClusterID == "" {
+		return nodeEBS{}, errors.New("formation named ocibsd without its compartment, availability domain and cluster ID")
+	}
+	existing, err := readExistingEBS(tomlPath, node)
+	if err != nil {
+		return nodeEBS{}, err
+	}
+	if err := r.fillOCIBSD(ctx, &e, existing, nil); err != nil {
+		return nodeEBS{}, err
+	}
+	if err := r.checkCredential(ctx, e); err != nil {
+		return nodeEBS{}, err
+	}
+	return e, nil
+}
+
+// formationEBS is the EBS object sent to joining nodes: nil for the default
+// provider, so a viperblock cluster's formation payload has no ebs object.
+func formationEBS(c formation.EBSConfig) *formation.EBSConfig {
+	if (config.EBSConfig{Provider: c.Provider}).ResolvedProvider() == config.EBSProviderViperblockd {
+		return nil
+	}
+	return &c
+}
+
+// generateClusterID returns a random 128-bit value in hex.
+func generateClusterID() (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("generate the cluster ID: %w", err)
+	}
+	return hex.EncodeToString(b[:]), nil
+}
+
+// ocibsdStageDir is where setup.sh stages the tarball's ocibsd/ files on an
+// OCI virtual machine.
+const ocibsdStageDir = "/usr/local/share/spinifex/ocibsd"
+
+// ocibsdFiles maps each staged file to where activation installs it.
+var ocibsdFiles = []struct{ staged, installed string }{
+	{"spinifex-ocibsd.service", "etc/systemd/system/spinifex-ocibsd.service"},
+	{"99-spinifex-ocibsd.rules", "etc/udev/rules.d/99-spinifex-ocibsd.rules"},
+	{"spinifex-daemon.service.d/ocibsd.conf", "etc/systemd/system/spinifex-daemon.service.d/ocibsd.conf"},
+}
+
+// viperblockUnit is masked on an ocibsd node, so exactly one service answers
+// ebs.provider.v1.* there.
+const viperblockUnit = "etc/systemd/system/spinifex-viperblock.service"
+
+// ocibsdActivation installs the staged ocibsd files under Root and runs the
+// systemd and udev steps through Runner.
+type ocibsdActivation struct {
+	Root   string
+	Stage  string
+	Runner ebsocibsd.CommandRunner
+}
+
+func productionOCIBSDActivation() ocibsdActivation {
+	return ocibsdActivation{Root: "/", Stage: ocibsdStageDir, Runner: ebsocibsd.ExecRunner{}}
+}
+
+// activateEBSProvider readies an ocibsd node before spinifex.target starts;
+// any other provider's node is left as setup.sh installed it.
+func activateEBSProvider(ctx context.Context, e nodeEBS, a ocibsdActivation) error {
+	if e.Provider != config.EBSProviderOCIBSD {
+		return nil
+	}
+	for _, f := range ocibsdFiles {
+		if err := installStaged(filepath.Join(a.Stage, f.staged), filepath.Join(a.Root, f.installed)); err != nil {
+			return err
+		}
+	}
+	if err := maskUnit(filepath.Join(a.Root, viperblockUnit)); err != nil {
+		return err
+	}
+	for _, cmd := range [][]string{
+		{"systemctl", "enable", "spinifex-ocibsd.service"},
+		{"systemctl", "daemon-reload"},
+		{"udevadm", "control", "--reload-rules"},
+	} {
+		if _, err := a.Runner.Run(ctx, cmd[0], cmd[1:]...); err != nil {
+			return fmt.Errorf("activate ocibsd: %w", err)
+		}
+	}
+	fmt.Println("✅ ocibsd installed: spinifex-ocibsd.service enabled, spinifex-viperblock.service masked")
+	return nil
+}
+
+// installStaged copies src to dst with mode 0644, through a temporary file
+// so a unit is never read half-written.
+func installStaged(src, dst string) error {
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return fmt.Errorf("activate ocibsd: read the staged %s (setup.sh stages it on an OCI virtual machine): %w", src, err)
+	}
+	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { //nolint:gosec // systemd and udev directories are world-readable
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	tmp := dst + ".spx-tmp"
+	if err := os.WriteFile(tmp, data, 0o644); err != nil { //nolint:gosec // unit files and udev rules are world-readable, as setup.sh installs them
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	if err := os.Rename(tmp, dst); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	return nil
+}
+
+// maskUnit links path to /dev/null. systemctl mask refuses a real unit file
+// in /etc, which is where setup.sh installs the viperblock unit.
+func maskUnit(path string) error {
+	if systemd.IsMasked(path) {
+		return nil
+	}
+	if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("activate ocibsd: mask %s: %w", path, err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // the systemd unit directory is world-readable
+		return fmt.Errorf("activate ocibsd: %w", err)
+	}
+	if err := os.Symlink(os.DevNull, path); err != nil {
+		return fmt.Errorf("activate ocibsd: mask %s: %w", path, err)
+	}
+	return nil
+}
+
+func exitOnActivationError(err error) {
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+}

spinifex/operator/cli/admin_ebs_test.go

@@ -0,0 +1,530 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/operator/admin"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/runtime/formation"
+)
+
+const (
+	testOCICompartment = "ocid1.compartment.oc1..test"
+	testOCIAD          = "Uocm:US-ASHBURN-AD-1"
+	testClusterID      = "5f0c3a9e1b7d42c68e2a4f1d9b3c7e05"
+)
+
+var errRefused = errors.New("oci ListVolumes: Authorization failed or requested resource not found (NotAuthorizedOrNotFound, http 404)")
+
+// fakeResolver is an ebsResolver over a staged environment, instance
+// metadata and OCI client, counting the metadata reads it answers.
+type fakeResolver struct {
+	env       map[string]string
+	md        *oci.InstanceMetadata
+	client    *oci.BlockFake
+	imdsReads int
+	clientFor []string
+}
+
+func newFakeResolver() *fakeResolver {
+	return &fakeResolver{env: map[string]string{}, client: oci.NewBlockFake()}
+}
+
+func (f *fakeResolver) onOCI(shape string) *fakeResolver {
+	f.md = &oci.InstanceMetadata{ID: "ocid1.instance.oc1..node", CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, Shape: shape}
+	return f
+}
+
+func (f *fakeResolver) resolver() ebsResolver {
+	return ebsResolver{
+		getenv: func(k string) string { return f.env[k] },
+		instance: func(ctx context.Context) (oci.InstanceMetadata, error) {
+			f.imdsReads++
+			if _, ok := ctx.Deadline(); !ok {
+				return oci.InstanceMetadata{}, errors.New("the metadata read has no timeout")
+			}
+			if f.md == nil {
+				return oci.InstanceMetadata{}, errors.New("dial tcp 169.254.169.254:80: i/o timeout")
+			}
+			return *f.md, nil
+		},
+		newClient: func(auth, file, profile string) (oci.BlockClient, error) {
+			f.clientFor = append(f.clientFor, strings.Join([]string{auth, file, profile}, "|"))
+			return f.client, nil
+		},
+	}
+}
+
+// writeExisting writes a spinifex.toml holding only node1's EBS tables.
+func writeExisting(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, os.WriteFile(path, []byte(body), 0o600))
+	return path
+}
+
+func absentConfig(t *testing.T) string {
+	return filepath.Join(t.TempDir(), "spinifex.toml")
+}
+
+func TestResolveInitEBS_Precedence(t *testing.T) {
+	cases := []struct {
+		name     string
+		existing string
+		env      string
+		shape    string
+		want     string
+		source   string
+	}{
+		{name: "an existing config that omits the provider keeps viperblockd over the environment and detection", existing: "[nodes.node1.daemon]\nhost = \"x\"\n", env: config.EBSProviderOCIBSD, shape: "VM.Standard.E6.Flex", want: config.EBSProviderViperblockd, source: ebsFromConfig},
+		{name: "an existing config's provider wins over the environment", existing: "[nodes.node1.ebs]\nprovider = \"qemunbd\"\n", env: config.EBSProviderViperblockd, shape: "VM.Standard.E6.Flex", want: config.EBSProviderQEMUNBD, source: ebsFromConfig},
+		{name: "the environment wins over detection", env: config.EBSProviderViperblockd, shape: "VM.Standard.E6.Flex", want: config.EBSProviderViperblockd, source: ebsFromEnv},
+		{name: "detection on an OCI virtual machine", shape: "VM.Standard.E6.Flex", want: config.EBSProviderOCIBSD, source: ebsFromDetection},
+		{name: "detection on an OCI bare-metal shape", shape: "BM.Standard.E2.64", want: config.EBSProviderViperblockd, source: ebsFromDetection},
+		{name: "detection off OCI", want: config.EBSProviderViperblockd, source: ebsFromDetection},
+		{name: "another node's table is not this node's config", existing: "[nodes.node2.ebs]\nprovider = \"qemunbd\"\n", want: config.EBSProviderViperblockd, source: ebsFromDetection},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newFakeResolver()
+			if tc.shape != "" {
+				f.onOCI(tc.shape)
+			}
+			if tc.env != "" {
+				f.env[ebsProviderEnv] = tc.env
+			}
+			path := absentConfig(t)
+			if tc.existing != "" {
+				path = writeExisting(t, tc.existing)
+			}
+			got, err := resolveInitEBS(context.Background(), path, "node1", f.resolver())
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got.Provider)
+			assert.Equal(t, tc.source, got.Source)
+			if tc.source != ebsFromDetection {
+				assert.Zero(t, f.imdsReads, "a configured provider never reads the metadata service")
+			}
+			if tc.want != config.EBSProviderOCIBSD {
+				assert.Equal(t, formation.EBSConfig{Provider: tc.want}, got.EBSConfig, "no OCI table or cluster ID off ocibsd")
+				assert.Empty(t, f.clientFor, "no OCI call off ocibsd")
+			}
+		})
+	}
+}
+
+func TestResolveInitEBS_DetectionFillsTheOCITable(t *testing.T) {
+	f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+	got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err)
+
+	assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+	assert.Equal(t, testOCICompartment, got.OCICompartmentID)
+	assert.Equal(t, testOCIAD, got.OCIAvailabilityDomain)
+	assert.Equal(t, "instance_principal", got.Auth)
+	assert.Empty(t, got.ConfigFile)
+	assert.Regexp(t, `^[0-9a-f]{32}$`, got.ClusterID, "128 random bits in hex")
+	assert.Equal(t, 1, f.imdsReads, "detection's read also fills the table")
+	assert.Equal(t, []string{"instance_principal||"}, f.clientFor, "the credential is checked as instance principal")
+	assert.Equal(t, "EBS provider: ocibsd (detected: OCI virtual machine VM.Standard.E6.Flex)", got.describe())
+
+	again, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err)
+	assert.NotEqual(t, got.ClusterID, again.ClusterID, "each new cluster gets its own ID")
+}
+
+// An existing config's [nodes.<node>.ebs.oci] values win over the metadata
+// and the instance_principal default, and its cluster ID is kept.
+func TestResolveInitEBS_ExistingOCIValuesWin(t *testing.T) {
+	f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+	path := writeExisting(t, `[nodes.node1.ebs]
+provider = "ocibsd"
+
+[nodes.node1.ebs.oci]
+compartment_id = "ocid1.compartment.oc1..mine"
+cluster_id = "`+testClusterID+`"
+auth = "config_file"
+config_file = "/etc/spinifex/oci/config"
+config_profile = "spinifex"
+`)
+	got, err := resolveInitEBS(context.Background(), path, "node1", f.resolver())
+	require.NoError(t, err)
+
+	assert.Equal(t, ebsFromConfig, got.Source)
+	assert.Equal(t, "ocid1.compartment.oc1..mine", got.OCICompartmentID)
+	assert.Equal(t, testOCIAD, got.OCIAvailabilityDomain, "a value the config omits comes from the metadata")
+	assert.Equal(t, testClusterID, got.ClusterID, "a re-init keeps the ID its volumes are tagged with")
+	assert.Equal(t, []string{"config_file|/etc/spinifex/oci/config|spinifex"}, f.clientFor)
+}
+
+func TestResolveInitEBS_OCIBSDFromTheEnvironmentNeedsTheMetadata(t *testing.T) {
+	f := newFakeResolver()
+	f.env[ebsProviderEnv] = config.EBSProviderOCIBSD
+	_, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.ErrorContains(t, err, "needs this node's OCI instance metadata")
+
+	f.onOCI("BM.Standard.E2.64")
+	got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err, "a named provider is the operator's choice, whatever the shape")
+	assert.Equal(t, ebsFromEnv, got.Source)
+	assert.Equal(t, testOCIAD, got.OCIAvailabilityDomain)
+}
+
+func TestResolveInitEBS_Refusals(t *testing.T) {
+	t.Run("an unknown provider in the environment", func(t *testing.T) {
+		f := newFakeResolver()
+		f.env[ebsProviderEnv] = "ceph"
+		_, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.ErrorContains(t, err, `EBS provider "ceph" (from SPINIFEX_EBS_PROVIDER) is unsupported`)
+	})
+	t.Run("an existing config that cannot be parsed", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		_, err := resolveInitEBS(context.Background(), writeExisting(t, "[nodes.node1.ebs\n"), "node1", f.resolver())
+		require.ErrorContains(t, err, "parse the existing config")
+		assert.Zero(t, f.imdsReads, "an unreadable config is not replaced by detection")
+	})
+}
+
+// The check is the read ocibsd makes at start: it stops init naming the
+// policy when refused, and an inspect-only credential passes it.
+func TestResolveInitEBS_CredentialCheck(t *testing.T) {
+	t.Run("refused", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		f.client.FailOp("ListVolumes", errRefused)
+		_, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.Error(t, err)
+		assert.ErrorIs(t, err, errRefused)
+		assert.ErrorContains(t, err, "manage volume-family in compartment id "+testOCICompartment)
+		assert.ErrorContains(t, err, "use instances in compartment id "+testOCICompartment)
+	})
+	t.Run("inspect only", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		for _, op := range []string{"CreateVolume", "AttachVolume", "CreateVolumeBackup"} {
+			f.client.FailOp(op, errRefused)
+		}
+		got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+		_, err = f.client.CreateVolume(context.Background(), oci.CreateVolumeInput{CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, SizeGB: oci.MinVolumeSizeGB})
+		assert.ErrorIs(t, err, errRefused, "the check made no create")
+		assert.Empty(t, f.client.Volumes())
+	})
+	t.Run("ok", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		got, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+	})
+}
+
+func TestResolveJoinEBS(t *testing.T) {
+	ocibsdFormation := &formation.NetworkConfig{EBS: &formation.EBSConfig{
+		Provider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD, ClusterID: testClusterID,
+	}}
+
+	t.Run("a viperblock cluster checks nothing", func(t *testing.T) {
+		f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+		got, err := resolveJoinEBS(context.Background(), absentConfig(t), "node2", &formation.NetworkConfig{}, f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, nodeEBS{Source: ebsFromFormation}, got)
+		assert.Zero(t, f.imdsReads)
+		assert.Empty(t, f.clientFor)
+	})
+	t.Run("ocibsd defaults this node to its instance principal", func(t *testing.T) {
+		f := newFakeResolver()
+		got, err := resolveJoinEBS(context.Background(), absentConfig(t), "node2", ocibsdFormation, f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, *ocibsdFormation.EBS, got.EBSConfig, "the cluster's values come from formation unchanged")
+		assert.Equal(t, "instance_principal", got.Auth)
+		assert.Zero(t, f.imdsReads, "formation carries the compartment and AD")
+		assert.Equal(t, []string{"instance_principal||"}, f.clientFor)
+	})
+	t.Run("this node's existing credential wins", func(t *testing.T) {
+		f := newFakeResolver()
+		path := writeExisting(t, "[nodes.node2.ebs]\nprovider = \"ocibsd\"\n\n[nodes.node2.ebs.oci]\nauth = \"config_file\"\nconfig_file = \"/etc/spinifex/oci/config\"\ncluster_id = \"0000\"\n")
+		got, err := resolveJoinEBS(context.Background(), path, "node2", ocibsdFormation, f.resolver())
+		require.NoError(t, err)
+		assert.Equal(t, "config_file", got.Auth)
+		assert.Equal(t, "/etc/spinifex/oci/config", got.ConfigFile)
+		assert.Equal(t, testClusterID, got.ClusterID, "the cluster ID is the cluster's, not a stale local one")
+	})
+	t.Run("a refused credential stops the join", func(t *testing.T) {
+		f := newFakeResolver()
+		f.client.FailOp("ListVolumes", errRefused)
+		_, err := resolveJoinEBS(context.Background(), absentConfig(t), "node2", ocibsdFormation, f.resolver())
+		require.ErrorContains(t, err, "manage volume-family")
+	})
+}
+
+// fakeRunner records each command it is given.
+type fakeRunner struct{ calls []string }
+
+func (r *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) {
+	r.calls = append(r.calls, strings.Join(append([]string{name}, args...), " "))
+	return nil, nil
+}
+
+// stageOCIBSD stages the repository's build/ocibsd files as setup.sh does.
+func stageOCIBSD(t *testing.T) string {
+	t.Helper()
+	stage := filepath.Join(t.TempDir(), "ocibsd")
+	for _, f := range ocibsdFiles {
+		data, err := os.ReadFile(filepath.Join("..", "..", "..", "build", "ocibsd", f.staged))
+		require.NoError(t, err)
+		require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(stage, f.staged)), 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(stage, f.staged), data, 0o644))
+	}
+	return stage
+}
+
+func TestActivateEBSProvider_OCIBSD(t *testing.T) {
+	root := t.TempDir()
+	stage := stageOCIBSD(t)
+	unitDir := filepath.Join(root, "etc", "systemd", "system")
+	require.NoError(t, os.MkdirAll(unitDir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(unitDir, "spinifex-viperblock.service"), []byte("[Unit]\n"), 0o644))
+	runner := &fakeRunner{}
+
+	e := nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderOCIBSD}}
+	require.NoError(t, activateEBSProvider(context.Background(), e, ocibsdActivation{Root: root, Stage: stage, Runner: runner}))
+
+	for _, f := range ocibsdFiles {
+		want, err := os.ReadFile(filepath.Join(stage, f.staged))
+		require.NoError(t, err)
+		got, err := os.ReadFile(filepath.Join(root, f.installed))
+		require.NoError(t, err, f.installed)
+		assert.Equal(t, string(want), string(got), f.installed)
+	}
+	target, err := os.Readlink(filepath.Join(unitDir, "spinifex-viperblock.service"))
+	require.NoError(t, err, "the viperblock unit file is replaced by a mask")
+	assert.Equal(t, "/dev/null", target)
+	assert.Equal(t, []string{
+		"systemctl enable spinifex-ocibsd.service",
+		"systemctl daemon-reload",
+		"udevadm control --reload-rules",
+	}, runner.calls)
+
+	// A repeat, as an init --force makes, refreshes the files and keeps the mask.
+	runner.calls = nil
+	require.NoError(t, activateEBSProvider(context.Background(), e, ocibsdActivation{Root: root, Stage: stage, Runner: runner}))
+	target, err = os.Readlink(filepath.Join(unitDir, "spinifex-viperblock.service"))
+	require.NoError(t, err)
+	assert.Equal(t, "/dev/null", target)
+	assert.Len(t, runner.calls, 3)
+}
+
+func TestActivateEBSProvider_OtherProvidersAreUntouched(t *testing.T) {
+	for _, provider := range []string{"", config.EBSProviderViperblockd, config.EBSProviderQEMUNBD} {
+		root := t.TempDir()
+		runner := &fakeRunner{}
+		e := nodeEBS{EBSConfig: formation.EBSConfig{Provider: provider}}
+		require.NoError(t, activateEBSProvider(context.Background(), e, ocibsdActivation{Root: root, Stage: filepath.Join(root, "absent"), Runner: runner}))
+		assert.Empty(t, runner.calls, provider)
+		entries, err := os.ReadDir(root)
+		require.NoError(t, err)
+		assert.Empty(t, entries, provider)
+	}
+}
+
+func TestActivateEBSProvider_MissingStage(t *testing.T) {
+	runner := &fakeRunner{}
+	e := nodeEBS{EBSConfig: formation.EBSConfig{Provider: config.EBSProviderOCIBSD}}
+	err := activateEBSProvider(context.Background(), e, ocibsdActivation{Root: t.TempDir(), Stage: filepath.Join(t.TempDir(), "absent"), Runner: runner})
+	require.ErrorContains(t, err, "setup.sh stages it on an OCI virtual machine")
+	assert.Empty(t, runner.calls, "nothing is enabled without its files")
+}
+
+// A viperblock cluster's formation payload is the JSON it was before the
+// provider choice existed: no ebs object at all.
+func TestFormationEBS_ViperblockPayloadIsUnchanged(t *testing.T) {
+	cases := []struct {
+		name, provider string
+	}{
+		{name: "viperblockd", provider: config.EBSProviderViperblockd},
+		{name: "an empty provider", provider: ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ebs := formationEBS(formation.EBSConfig{Provider: tc.provider})
+			assert.Nil(t, ebs)
+
+			minimal, err := json.Marshal(formation.NetworkConfig{IPSecEnabled: true, EBS: ebs})
+			require.NoError(t, err)
+			assert.Equal(t, `{"external_mode":"","ipsec_enabled":true}`, string(minimal))
+
+			external, err := json.Marshal(formation.NetworkConfig{
+				ExternalMode: "nat", BootstrapVpcId: "vpc-1", BootstrapCidr: "172.31.0.0/16", EBS: ebs,
+			})
+			require.NoError(t, err)
+			assert.Equal(t, `{"external_mode":"nat","ipsec_enabled":false,"bootstrap_vpc_id":"vpc-1","bootstrap_cidr":"172.31.0.0/16"}`, string(external))
+		})
+	}
+}
+
+// A non-default provider is sent with no cluster ID; only ocibsd has one.
+func TestFormationEBS_OtherProvidersAreSent(t *testing.T) {
+	f := newFakeResolver()
+	f.env[ebsProviderEnv] = config.EBSProviderQEMUNBD
+	qemu, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", f.resolver())
+	require.NoError(t, err)
+	wire, err := json.Marshal(formation.NetworkConfig{EBS: formationEBS(qemu.EBSConfig)})
+	require.NoError(t, err)
+	assert.Equal(t, `{"external_mode":"","ipsec_enabled":false,"ebs":{"provider":"qemunbd"}}`, string(wire))
+}
+
+// A joining node of a viperblock cluster receives no ebs object and writes
+// the default provider with no OCI table.
+func TestApplyNetworkConfig_ViperblockJoinerWritesTheDefault(t *testing.T) {
+	var received formation.NetworkConfig
+	require.NoError(t, json.Unmarshal([]byte(`{"external_mode":"","ipsec_enabled":false}`), &received))
+	assert.Nil(t, received.EBS)
+
+	settings := admin.ConfigSettings{}
+	applyNetworkConfig(&settings, &received)
+	choice, err := resolveJoinEBS(context.Background(), absentConfig(t), "node1", &received, newFakeResolver().resolver())
+	require.NoError(t, err)
+	choice.apply(&settings)
+	assert.Empty(t, settings.EBSProvider)
+	assert.Empty(t, settings.OCIClusterID)
+
+	got := renderEBS(t, settings)
+	assert.Equal(t, config.EBSProviderViperblockd, got.Provider)
+	assert.Equal(t, config.EBSOCIConfig{}, got.OCI)
+}
+
+// A joining node writes the provider and cluster ID the init node chose,
+// and its own instance-principal credential.
+func TestApplyNetworkConfig_PropagatesTheEBSProvider(t *testing.T) {
+	init, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", newFakeResolver().onOCI("VM.Standard.E6.Flex").resolver())
+	require.NoError(t, err)
+	wire, err := json.Marshal(formation.NetworkConfig{EBS: formationEBS(init.EBSConfig)})
+	require.NoError(t, err)
+	assert.Contains(t, string(wire), `"cluster_id":"`+init.ClusterID+`"`)
+	assert.NotContains(t, string(wire), "instance_principal", "a node's credential is its own")
+	var received formation.NetworkConfig
+	require.NoError(t, json.Unmarshal(wire, &received))
+	require.NotNil(t, received.EBS)
+
+	settings := admin.ConfigSettings{}
+	applyNetworkConfig(&settings, &received)
+	choice, err := resolveJoinEBS(context.Background(), absentConfig(t), "node1", &received, newFakeResolver().resolver())
+	require.NoError(t, err)
+	choice.apply(&settings)
+
+	got := renderEBS(t, settings)
+	assert.Equal(t, config.EBSProviderOCIBSD, got.Provider)
+	assert.Equal(t, config.EBSOCIConfig{
+		CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, ClusterID: init.ClusterID, Auth: "instance_principal",
+	}, got.OCI, "the joining node's config carries the init node's ID")
+}
+
+// renderSpinifexToml renders the spinifex.toml template for one node.
+func renderSpinifexToml(t *testing.T, settings admin.ConfigSettings) string {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings.Node, settings.Az, settings.Region, settings.Port = "node1", "ap-southeast-2a", "ap-southeast-2", "4432"
+	settings.BindIP, settings.ConfigDir = "10.11.12.1", "/etc/spinifex"
+	settings.OVNNBAddr, settings.OVNSBAddr = "tcp:127.0.0.1:6641", "tcp:127.0.0.1:6642"
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+	return path
+}
+
+// renderEBS renders the template and loads node1's [ebs] back.
+func renderEBS(t *testing.T, settings admin.ConfigSettings) config.EBSConfig {
+	t.Helper()
+	path := renderSpinifexToml(t, settings)
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+	cfg, err := config.LoadConfig(path)
+	require.NoError(t, err)
+	return cfg.Nodes["node1"].EBS
+}
+
+func TestSpinifexTomlTemplate_EBSProvider(t *testing.T) {
+	cases := []struct {
+		name     string
+		settings admin.ConfigSettings
+		want     config.EBSConfig
+	}{
+		{name: "unset renders viperblockd", want: config.EBSConfig{Provider: config.EBSProviderViperblockd}},
+		{name: "qemunbd", settings: admin.ConfigSettings{EBSProvider: config.EBSProviderQEMUNBD}, want: config.EBSConfig{Provider: config.EBSProviderQEMUNBD}},
+		{
+			name: "ocibsd as the instance principal",
+			settings: admin.ConfigSettings{EBSProvider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD,
+				OCIClusterID: testClusterID, OCIAuth: "instance_principal"},
+			want: config.EBSConfig{Provider: config.EBSProviderOCIBSD, OCI: config.EBSOCIConfig{
+				CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, ClusterID: testClusterID, Auth: "instance_principal",
+			}},
+		},
+		{
+			name: "ocibsd with a config file",
+			settings: admin.ConfigSettings{EBSProvider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD,
+				OCIClusterID: testClusterID, OCIAuth: "config_file", OCIConfigFile: "/etc/spinifex/oci/config", OCIConfigProfile: "spinifex"},
+			want: config.EBSConfig{Provider: config.EBSProviderOCIBSD, OCI: config.EBSOCIConfig{
+				CompartmentID: testOCICompartment, AvailabilityDomain: testOCIAD, ClusterID: testClusterID, Auth: "config_file", ConfigFile: "/etc/spinifex/oci/config", ConfigProfile: "spinifex",
+			}},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := renderEBS(t, tc.settings)
+			assert.Equal(t, tc.want.Provider, got.Provider)
+			assert.Equal(t, tc.want.OCI, got.OCI)
+		})
+	}
+}
+
+// A viperblock init renders the same spinifex.toml as one that never chose a
+// provider, so resolution and the cluster ID leave its config untouched.
+func TestSpinifexTomlTemplate_ViperblockInitIsUnchanged(t *testing.T) {
+	ebs, err := resolveInitEBS(context.Background(), absentConfig(t), "node1", newFakeResolver().resolver())
+	require.NoError(t, err)
+	settings := admin.ConfigSettings{}
+	ebs.apply(&settings)
+
+	before, err := os.ReadFile(renderSpinifexToml(t, admin.ConfigSettings{}))
+	require.NoError(t, err)
+	after, err := os.ReadFile(renderSpinifexToml(t, settings))
+	require.NoError(t, err)
+	assert.Equal(t, string(before), string(after))
+	assert.NotContains(t, string(after), "cluster_id")
+	assert.NotContains(t, string(after), ".ebs.oci]")
+}
+
+// What init renders for ocibsd is what a re-init reads back as the existing
+// config, so a --force keeps the provider, the table and the cluster ID.
+func TestReadExistingEBS_RoundTrip(t *testing.T) {
+	path := renderSpinifexToml(t, admin.ConfigSettings{
+		EBSProvider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD, OCIClusterID: testClusterID, OCIAuth: "instance_principal",
+	})
+	f := newFakeResolver().onOCI("VM.Standard.E6.Flex")
+	f.env[ebsProviderEnv] = config.EBSProviderViperblockd
+	got, err := resolveInitEBS(context.Background(), path, "node1", f.resolver())
+	require.NoError(t, err)
+	assert.Equal(t, ebsFromConfig, got.Source)
+	assert.Equal(t, formation.EBSConfig{
+		Provider: config.EBSProviderOCIBSD, OCICompartmentID: testOCICompartment, OCIAvailabilityDomain: testOCIAD, ClusterID: testClusterID,
+	}, got.EBSConfig)
+	assert.Equal(t, "instance_principal", got.Auth)
+	assert.Zero(t, f.imdsReads, "a complete existing table needs no metadata")
+
+	viperblock, err := readExistingEBS(renderSpinifexToml(t, admin.ConfigSettings{}), "node1")
+	require.NoError(t, err)
+	require.NotNil(t, viperblock)
+	assert.Equal(t, config.EBSProviderViperblockd, viperblock.Provider)
+
+	absent, err := readExistingEBS(absentConfig(t), "node1")
+	require.NoError(t, err)
+	assert.Nil(t, absent)
+}

spinifex/operator/cli/admin_upgrade.go

@@ -203,6 +203,8 @@ func reportUnitStatus(result systemd.Result) {
 			fmt.Printf("  %-38s %d → %d (stale, will replace)\n", s.Name+":", s.InstalledVersion, s.EmbeddedVersion)
 		case systemd.ActionConflict:
 			fmt.Printf("  %-38s %d (operator-modified, not touched — see `systemctl edit %s`)\n", s.Name+":", s.InstalledVersion, s.Name)
+		case systemd.ActionMasked:
+			fmt.Printf("  %-38s masked, not touched\n", s.Name+":")
 		}
 	}
 	if result.HasConflicts() {

spinifex/operator/cli/admin_upgrade_test.go

@@ -76,6 +76,7 @@ func TestReportUnitStatus_PrintsEachActionAndConflictFooter(t *testing.T) {
 		{Name: "install.service", Action: systemd.ActionInstall, EmbeddedVersion: 2},
 		{Name: "replace.service", Action: systemd.ActionReplace, InstalledVersion: 1, EmbeddedVersion: 2},
 		{Name: "conflict.service", Action: systemd.ActionConflict, InstalledVersion: 2},
+		{Name: "masked.service", Action: systemd.ActionMasked},
 	}}
 
 	out := captureStdout(t, func() { reportUnitStatus(result) })
@@ -85,6 +86,7 @@ func TestReportUnitStatus_PrintsEachActionAndConflictFooter(t *testing.T) {
 		"install.service:", "missing → will install (v2)",
 		"replace.service:", "1 → 2 (stale, will replace)",
 		"conflict.service:", "2 (operator-modified, not touched",
+		"masked.service:", "masked, not touched",
 		"Operator-modified units above are left as-is",
 	} {
 		assert.Contains(t, out, want)

spinifex/operator/cli/service.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/nats"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/northstar"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/ocibsd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/predastore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qemunbdd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qmpcollector"
@@ -249,6 +250,11 @@ var viperblockStartCmd = &cobra.Command{
 		if err != nil {
 			return fmt.Errorf("load config file: %w", err)
 		}
+		// A reinstalled unit can outlive its mask on a node with another
+		// provider, which would answer the same subjects.
+		if err := requireEBSProvider(clusterConfig, config.EBSProviderViperblockd); err != nil {
+			return err
+		}
 		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		natsHost := viper.GetString("nats-host")
@@ -379,6 +385,15 @@ var viperblockStartCmd = &cobra.Command{
 	},
 }
 
+// requireEBSProvider refuses to start a provider service on a node whose
+// configured provider is another one; an unset provider is viperblockd.
+func requireEBSProvider(clusterConfig *config.ClusterConfig, want string) error {
+	if got := clusterConfig.Nodes[clusterConfig.Node].EBS.ResolvedProvider(); got != want {
+		return fmt.Errorf("[nodes.%s.ebs] provider is %q, not %q", clusterConfig.Node, got, want)
+	}
+	return nil
+}
+
 var qemunbdCmd = &cobra.Command{
 	Use:   "qemunbd",
 	Short: "Manage the qemunbd service",
@@ -453,6 +468,68 @@ var qemunbdStartCmd = &cobra.Command{
 	},
 }
 
+var ocibsdCmd = &cobra.Command{
+	Use:   "ocibsd",
+	Short: "Manage the ocibsd service",
+}
+
+// ocibsd answers the same ebs.provider.v1.* subjects as viperblock and
+// qemunbd, so only the cluster's one provider may run against its NATS.
+var ocibsdStartCmd = &cobra.Command{
+	Use:           "start",
+	Short:         "Start the ocibsd service",
+	SilenceErrors: true,
+	SilenceUsage:  true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cfgFile := viper.GetString("config")
+		if cfgFile == "" {
+			return fmt.Errorf("config file is not set")
+		}
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		if err != nil {
+			return fmt.Errorf("load config file: %w", err)
+		}
+		if err := requireEBSProvider(clusterConfig, config.EBSProviderOCIBSD); err != nil {
+			return err
+		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
+		if natsHost := viper.GetString("nats-host"); natsHost != "" {
+			nodeConfig.NATS.Host = natsHost
+		}
+		baseDir := nodeConfig.Predastore.BaseDir
+		if override := viper.GetString("base-dir"); override != "" {
+			baseDir = override
+		}
+
+		debug := viper.GetBool("ocibsd-debug")
+		defer initTelemetry("ocibsd", debug)()
+
+		oci := nodeConfig.EBS.OCI
+		service, err := service.New("ocibsd", &ocibsd.Config{
+			BaseDir:            baseDir,
+			NatsHost:           nodeConfig.NATS.Host,
+			NatsToken:          nodeConfig.NATS.ACL.Token,
+			NatsCACert:         nodeConfig.NATS.CACert,
+			NodeName:           clusterConfig.Node,
+			AvailabilityZone:   nodeConfig.AZ,
+			ClusterID:          oci.ClusterID,
+			CompartmentID:      oci.CompartmentID,
+			AvailabilityDomain: oci.AvailabilityDomain,
+			Auth:               oci.Auth,
+			ConfigFile:         oci.ConfigFile,
+			ConfigProfile:      oci.ConfigProfile,
+			Debug:              debug,
+		})
+		if err != nil {
+			return fmt.Errorf("create ocibsd service: %w", err)
+		}
+		if _, err := service.Start(); err != nil {
+			return fmt.Errorf("start ocibsd service: %w", err)
+		}
+		return nil
+	},
+}
+
 // Repeat for nats.
 var natsStartCmd = &cobra.Command{
 	Use:           "start",
@@ -1147,6 +1224,14 @@ func init() {
 
 	qemunbdCmd.AddCommand(qemunbdStartCmd)
 
+	serviceCmd.AddCommand(ocibsdCmd)
+
+	ocibsdCmd.PersistentFlags().Bool("debug", false, "ocibsd (EBS) debug logging")
+	mustBindEnv("ocibsd-debug", "SPINIFEX_OCIBSD_DEBUG")
+	mustBindPFlag("ocibsd-debug", ocibsdCmd.PersistentFlags(), "debug")
+
+	ocibsdCmd.AddCommand(ocibsdStartCmd)
+
 	// Nats
 	serviceCmd.AddCommand(natsCmd)
 

spinifex/operator/cli/service_exit_test.go

@@ -188,6 +188,48 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
 			},
 			wantErr: "start viperblock service",
 		},
+		{
+			name: "viperblock start (explicit viperblockd provider passes the guard)",
+			cmd:  viperblockStartCmd,
+			setup: func(t *testing.T) {
+				resetGlobalViper(t)
+				toml := minimalClusterToml + "\n[nodes.node1.ebs]\nprovider = \"viperblockd\"\n"
+				viper.Set("config", writeSpinifexToml(t, toml))
+				viper.Set("plugin-path", t.TempDir())
+				viper.Set("base-dir", blockedDir(t))
+			},
+			wantErr: "start viperblock service",
+		},
+		{
+			name: "viperblock start (refused on a qemunbd node)",
+			cmd:  viperblockStartCmd,
+			setup: func(t *testing.T) {
+				resetGlobalViper(t)
+				toml := minimalClusterToml + "\n[nodes.node1.ebs]\nprovider = \"qemunbd\"\n"
+				viper.Set("config", writeSpinifexToml(t, toml))
+				viper.Set("plugin-path", t.TempDir())
+			},
+			wantErr: `[nodes.node1.ebs] provider is "qemunbd", not "viperblockd"`,
+		},
+		{
+			name: "viperblock start (refused on an ocibsd node)",
+			cmd:  viperblockStartCmd,
+			setup: func(t *testing.T) {
+				resetGlobalViper(t)
+				toml := minimalClusterToml + `
+[nodes.node1.ebs]
+provider = "ocibsd"
+
+[nodes.node1.ebs.oci]
+compartment_id = "ocid1.compartment.oc1..test"
+availability_domain = "Uocm:US-ASHBURN-AD-1"
+cluster_id = "5f0c3a9e1b7d42c68e2a4f1d9b3c7e05"
+`
+				viper.Set("config", writeSpinifexToml(t, toml))
+				viper.Set("plugin-path", t.TempDir())
+			},
+			wantErr: `[nodes.node1.ebs] provider is "ocibsd", not "viperblockd"`,
+		},
 		{name: "qemunbd start (missing config)", cmd: qemunbdStartCmd},
 		{
 			name: "qemunbd start (config file load failure)",

spinifex/operator/cli/templates/spinifex.toml

@@ -139,7 +139,7 @@ encryption_key_file = "{{.EncryptionKeyFile}}"
 # provider: every node in a cluster must name the same one, and the daemon
 # refuses to start on a provider other than the one the cluster recorded.
 # Changing it needs the cluster wiped and reformed.
-provider = "viperblockd"
+provider = "{{or .EBSProvider "viperblockd"}}"
 # Per-volume export tunables. Both are applied per nbdkit process and there is
 # one of those per volume, so the host cost of each is the value times the
 # volume count. Uncomment to override; a service restart plus a volume
@@ -152,6 +152,27 @@ provider = "viperblockd"
 # cache_size_mb: per-volume plaintext read cache. 0 disables it. Auxiliary -efi
 # volumes are uncached regardless.
 #cache_size_mb = 128
+{{- if eq .EBSProvider "ocibsd"}}
+
+[nodes.{{.Node}}.ebs.oci]
+# Every node names the same compartment and availability domain.
+compartment_id = "{{.OCICompartmentID}}"
+availability_domain = "{{.OCIAvailabilityDomain}}"
+# Generated once at init; tags every OCI volume and backup of this cluster.
+cluster_id = "{{.OCIClusterID}}"
+# auth: "config_file" (the default when unset) reads config_file and
+# config_profile, which default to ~/.oci/config and DEFAULT;
+# "instance_principal" uses the instance's own certificate and takes neither.
+{{- if .OCIAuth}}
+auth = "{{.OCIAuth}}"
+{{- end}}
+{{- if .OCIConfigFile}}
+config_file = "{{.OCIConfigFile}}"
+{{- end}}
+{{- if .OCIConfigProfile}}
+config_profile = "{{.OCIConfigProfile}}"
+{{- end}}
+{{- end}}
 
 [nodes.{{.Node}}.vpcd]
 ovn_nb_addr = "{{.OVNNBAddr}}"

spinifex/operator/host/systemd/reconcile.go

@@ -30,6 +30,7 @@ const (
 	ActionInstall  Action = "install"  // unit missing on disk
 	ActionReplace  Action = "replace"  // installed marker version is older than embedded
 	ActionConflict Action = "conflict" // marker version matches but content differs — operator-modified
+	ActionMasked   Action = "masked"   // unit is linked to /dev/null; left masked
 )
 
 // UnitStatus reports the reconcile decision for a single unit.
@@ -103,6 +104,12 @@ func sortedUnitNames() []string {
 	return slices.Sorted(maps.Keys(Units))
 }
 
+// IsMasked reports whether the unit file at path is masked, a link to /dev/null.
+func IsMasked(path string) bool {
+	target, err := os.Readlink(path)
+	return err == nil && target == os.DevNull
+}
+
 // Reconcile compares each embedded unit (Units, generated from
 // build/systemd) against the copy installed under root. Missing or
 // version-stale units are replaced after a timestamped backup; units whose
@@ -117,6 +124,13 @@ func Reconcile(root string, opts Options) (Result, error) {
 		embeddedVersion := unitVersion(embedded)
 		path := filepath.Join(root, name)
 
+		if IsMasked(path) {
+			result.Statuses = append(result.Statuses, UnitStatus{
+				Name: name, Action: ActionMasked, EmbeddedVersion: embeddedVersion,
+			})
+			continue
+		}
+
 		installed, err := os.ReadFile(path)
 		if os.IsNotExist(err) {
 			result.Statuses = append(result.Statuses, UnitStatus{

spinifex/operator/host/systemd/reconcile_test.go

@@ -521,3 +521,33 @@ func TestBackupUnit_UnreadableSourceReturnsError(t *testing.T) {
 		t.Fatal("backupUnit: want an error for a nonexistent source path, got nil")
 	}
 }
+
+func TestReconcile_MaskedUnitStaysMasked(t *testing.T) {
+	root := t.TempDir()
+	reloadCalls := stubDaemonReload(t)
+	const name = "spinifex-viperblock.service"
+	path := filepath.Join(root, name)
+	if err := os.Symlink(os.DevNull, path); err != nil {
+		t.Fatalf("mask: %v", err)
+	}
+
+	result, err := Reconcile(root, Options{})
+	if err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+
+	st := findStatus(t, result, name)
+	if st.Action != ActionMasked || st.Applied || st.Backup != "" {
+		t.Errorf("status = %+v, want masked, not applied, no backup", st)
+	}
+	if target, err := os.Readlink(path); err != nil || target != os.DevNull {
+		t.Errorf("unit no longer masked: target %q, err %v", target, err)
+	}
+	backups, _ := filepath.Glob(path + ".pre-reconcile-*")
+	if len(backups) != 0 {
+		t.Errorf("backups written for a masked unit: %v", backups)
+	}
+	if reloadCalls() != 1 {
+		t.Errorf("daemon-reload calls = %d, want 1 for the other units", reloadCalls())
+	}
+}

spinifex/providers/cloud/oci/apiclient_test.go

@@ -29,17 +29,27 @@ type capture struct {
 	method string
 	path   string
 	query  url.Values
+	header http.Header
 	body   map[string]any
 }
 
-// testClient wires the real SDK-backed client to a handler. The signing key is
-// generated per test — the SDK refuses to build a client without one, and a
-// throwaway key is enough because nothing verifies the signature here.
+// testClient wires the real SDK-backed client to a handler.
 func testClient(t *testing.T, h http.HandlerFunc) (oci.Client, *[]capture) {
+	t.Helper()
+	provider, endpoint, hc, seen := testEndpoint(t, h)
+	c, err := oci.NewTestClient(provider, endpoint, hc)
+	require.NoError(t, err)
+	return c, seen
+}
+
+// testEndpoint serves h over TLS and returns a configuration provider for it.
+// The signing key is generated per test — the SDK refuses to build a client
+// without one, and nothing verifies the signature here.
+func testEndpoint(t *testing.T, h http.HandlerFunc) (common.ConfigurationProvider, string, *http.Client, *[]capture) {
 	t.Helper()
 	var seen []capture
 	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		c := capture{method: r.Method, path: r.URL.Path, query: r.URL.Query()}
+		c := capture{method: r.Method, path: r.URL.Path, query: r.URL.Query(), header: r.Header.Clone()}
 		if b, _ := io.ReadAll(r.Body); len(b) > 0 {
 			_ = json.Unmarshal(b, &c.body)
 		}
@@ -57,10 +67,7 @@ func testClient(t *testing.T, h http.HandlerFunc) (oci.Client, *[]capture) {
 	provider := common.NewRawConfigurationProvider(
 		"ocid1.tenancy.oc1..t", "ocid1.user.oc1..u", "ap-sydney-1",
 		"aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", string(pemKey), nil)
-
-	c, err := oci.NewTestClient(provider, strings.TrimPrefix(srv.URL, "https://"), srv.Client())
-	require.NoError(t, err)
-	return c, &seen
+	return provider, strings.TrimPrefix(srv.URL, "https://"), srv.Client(), &seen
 }
 
 // OCI pages list responses and a VNIC can hold 64 addresses. A client that

spinifex/providers/cloud/oci/block.go

@@ -0,0 +1,513 @@
+package oci
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"time"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/common/auth"
+	"github.com/oracle/oci-go-sdk/v65/core"
+)
+
+// BlockClient is the narrow block-storage surface an EBS provider needs: OCI
+// block volumes, their iSCSI attachments and their backups. It is separate
+// from Client so the address allocator and the storage provider stay apart.
+type BlockClient interface {
+	CreateVolume(ctx context.Context, in CreateVolumeInput) (BlockVolume, error)
+	GetVolume(ctx context.Context, volumeID string) (BlockVolume, error)
+	// ListVolumes lists the compartment, then keeps only volumes carrying
+	// every tag in the filter: OCI's list call has no tag filter of its own.
+	// Each volume is listed once, though OCI may list one twice.
+	ListVolumes(ctx context.Context, filter VolumeFilter) ([]BlockVolume, error)
+	UpdateVolume(ctx context.Context, volumeID string, in UpdateVolumeInput) (BlockVolume, error)
+	DeleteVolume(ctx context.Context, volumeID string) error
+
+	AttachVolume(ctx context.Context, in AttachVolumeInput) (VolumeAttachment, error)
+	GetVolumeAttachment(ctx context.Context, attachmentID string) (VolumeAttachment, error)
+	ListVolumeAttachments(ctx context.Context, filter AttachmentFilter) ([]VolumeAttachment, error)
+	DetachVolume(ctx context.Context, attachmentID string) error
+
+	CreateVolumeBackup(ctx context.Context, in CreateVolumeBackupInput) (VolumeBackup, error)
+	GetVolumeBackup(ctx context.Context, backupID string) (VolumeBackup, error)
+	// ListVolumeBackups filters and lists each backup once, as ListVolumes does.
+	ListVolumeBackups(ctx context.Context, filter BackupFilter) ([]VolumeBackup, error)
+	DeleteVolumeBackup(ctx context.Context, backupID string) error
+}
+
+// VolumeState is an OCI block volume's lifecycle state.
+type VolumeState string
+
+const (
+	VolumeProvisioning VolumeState = "PROVISIONING"
+	VolumeRestoring    VolumeState = "RESTORING"
+	VolumeAvailable    VolumeState = "AVAILABLE"
+	VolumeTerminating  VolumeState = "TERMINATING"
+	VolumeTerminated   VolumeState = "TERMINATED"
+	VolumeFaulty       VolumeState = "FAULTY"
+)
+
+// Gone reports whether the volume is deleted or being deleted. OCI keeps a
+// terminated volume readable and listed for a while after the delete.
+func (s VolumeState) Gone() bool { return s == VolumeTerminating || s == VolumeTerminated }
+
+// BackupState is an OCI volume backup's lifecycle state.
+type BackupState string
+
+const (
+	BackupRequestReceived BackupState = "REQUEST_RECEIVED"
+	BackupCreating        BackupState = "CREATING"
+	BackupAvailable       BackupState = "AVAILABLE"
+	BackupTerminating     BackupState = "TERMINATING"
+	BackupTerminated      BackupState = "TERMINATED"
+	BackupFaulty          BackupState = "FAULTY"
+)
+
+// Gone reports whether the backup is deleted or being deleted.
+func (s BackupState) Gone() bool { return s == BackupTerminating || s == BackupTerminated }
+
+// InProgress reports whether OCI is still taking the backup. OCI refuses to
+// delete the backup, or its source volume, while this is true.
+func (s BackupState) InProgress() bool { return s == BackupRequestReceived || s == BackupCreating }
+
+// AttachmentState is an OCI volume attachment's lifecycle state.
+type AttachmentState string
+
+const (
+	AttachmentAttaching AttachmentState = "ATTACHING"
+	AttachmentAttached  AttachmentState = "ATTACHED"
+	AttachmentDetaching AttachmentState = "DETACHING"
+	AttachmentDetached  AttachmentState = "DETACHED"
+)
+
+// MinVolumeSizeGB is the smallest block volume OCI creates.
+const MinVolumeSizeGB = 50
+
+// IQNPrefix starts the IQN of every OCI block volume attachment.
+const IQNPrefix = "iqn.2015-12.com.oracleiaas:"
+
+// BlockVolume is an OCI block volume. SourceBackupID is set on a volume
+// restored from a backup.
+type BlockVolume struct {
+	ID                 string
+	CompartmentID      string
+	AvailabilityDomain string
+	DisplayName        string
+	SizeGB             int64
+	VPUsPerGB          int64
+	State              VolumeState
+	SourceBackupID     string
+	FreeformTags       map[string]string
+	TimeCreated        time.Time
+	// ETag is the revision GetVolume or UpdateVolume answered with; a listed
+	// volume has none.
+	ETag string
+}
+
+// Tags is the volume's freeform tags.
+func (v BlockVolume) Tags() map[string]string { return v.FreeformTags }
+
+// IsGone reports whether the volume is deleted or being deleted.
+func (v BlockVolume) IsGone() bool { return v.State.Gone() }
+
+// CreateVolumeInput creates a volume, empty or restored from SourceBackupID.
+// A zero VPUsPerGB takes OCI's default tier. RetryToken is sent as
+// opc-retry-token, so OCI answers a repeat with the first call's volume.
+type CreateVolumeInput struct {
+	CompartmentID      string
+	AvailabilityDomain string
+	DisplayName        string
+	SizeGB             int64
+	VPUsPerGB          int64
+	SourceBackupID     string
+	FreeformTags       map[string]string
+	RetryToken         string
+}
+
+// VolumeFilter narrows ListVolumes. CompartmentID is required; DisplayName
+// and AvailabilityDomain are matched by OCI and Tags by the client.
+type VolumeFilter struct {
+	CompartmentID      string
+	AvailabilityDomain string
+	DisplayName        string
+	Tags               map[string]string
+}
+
+// UpdateVolumeInput changes a volume. A zero SizeGB leaves the size alone and
+// a nil FreeformTags leaves the tags alone; a non-nil map replaces them all.
+type UpdateVolumeInput struct {
+	SizeGB       int64
+	FreeformTags map[string]string
+	// IfMatch, when set, refuses the update with HTTP 412 unless the volume
+	// is still at this ETag.
+	IfMatch string
+}
+
+// VolumeAttachment is a volume attached to an instance. IQN, IPv4 and Port
+// are the iSCSI target and are empty on an attachment of another type.
+type VolumeAttachment struct {
+	ID                 string
+	InstanceID         string
+	VolumeID           string
+	AvailabilityDomain string
+	State              AttachmentState
+	ReadOnly           bool
+	IQN                string
+	IPv4               string
+	Port               int
+}
+
+// AttachVolumeInput attaches VolumeID to InstanceID over iSCSI, never
+// shareable, so OCI refuses a second attachment of the volume anywhere.
+type AttachVolumeInput struct {
+	InstanceID  string
+	VolumeID    string
+	DisplayName string
+	ReadOnly    bool
+}
+
+// AttachmentFilter narrows ListVolumeAttachments. CompartmentID is required.
+type AttachmentFilter struct {
+	CompartmentID string
+	VolumeID      string
+	InstanceID    string
+}
+
+// VolumeBackup is an OCI volume backup. SizeGB is the source volume's size.
+type VolumeBackup struct {
+	ID            string
+	CompartmentID string
+	VolumeID      string
+	DisplayName   string
+	SizeGB        int64
+	UniqueSizeGB  int64
+	State         BackupState
+	FreeformTags  map[string]string
+	TimeCreated   time.Time
+}
+
+// Tags is the backup's freeform tags.
+func (b VolumeBackup) Tags() map[string]string { return b.FreeformTags }
+
+// IsGone reports whether the backup is deleted or being deleted.
+func (b VolumeBackup) IsGone() bool { return b.State.Gone() }
+
+// CreateVolumeBackupInput takes an incremental backup of VolumeID, in the
+// volume's compartment. RetryToken is sent as opc-retry-token.
+type CreateVolumeBackupInput struct {
+	VolumeID     string
+	DisplayName  string
+	FreeformTags map[string]string
+	RetryToken   string
+}
+
+// BackupFilter narrows ListVolumeBackups the way VolumeFilter does.
+type BackupFilter struct {
+	CompartmentID string
+	DisplayName   string
+	Tags          map[string]string
+}
+
+// blockAPIClient is the BlockClient over the OCI Go SDK.
+type blockAPIClient struct {
+	storage core.BlockstorageClient
+	compute core.ComputeClient
+}
+
+var _ BlockClient = (*blockAPIClient)(nil)
+
+// NewBlockClientWithProvider builds a BlockClient from an explicit
+// configuration provider.
+func NewBlockClientWithProvider(p common.ConfigurationProvider) (BlockClient, error) {
+	storage, err := core.NewBlockstorageClientWithConfigurationProvider(p)
+	if err != nil {
+		return nil, fmt.Errorf("oci block storage client: %w", err)
+	}
+	compute, err := core.NewComputeClientWithConfigurationProvider(p)
+	if err != nil {
+		return nil, fmt.Errorf("oci compute client: %w", err)
+	}
+	return &blockAPIClient{storage: storage, compute: compute}, nil
+}
+
+// NewInstancePrincipalBlockClient authenticates as the instance itself, as
+// NewInstancePrincipalClient does.
+func NewInstancePrincipalBlockClient() (BlockClient, error) {
+	provider, err := auth.InstancePrincipalConfigurationProvider()
+	if err != nil {
+		return nil, fmt.Errorf("oci instance principal: %w", err)
+	}
+	return NewBlockClientWithProvider(provider)
+}
+
+// NewConfigFileBlockClient authenticates from an API-key config file, with the
+// same defaults as NewConfigFileClient.
+func NewConfigFileBlockClient(path, profile string) (BlockClient, error) {
+	provider, err := configFileProvider(path, profile)
+	if err != nil {
+		return nil, err
+	}
+	return NewBlockClientWithProvider(provider)
+}
+
+func (c *blockAPIClient) CreateVolume(ctx context.Context, in CreateVolumeInput) (BlockVolume, error) {
+	details := core.CreateVolumeDetails{
+		CompartmentId:      &in.CompartmentID,
+		AvailabilityDomain: &in.AvailabilityDomain,
+		DisplayName:        optional(in.DisplayName),
+		SizeInGBs:          &in.SizeGB,
+		FreeformTags:       in.FreeformTags,
+	}
+	if in.VPUsPerGB > 0 {
+		details.VpusPerGB = &in.VPUsPerGB
+	}
+	if in.SourceBackupID != "" {
+		details.SourceDetails = core.VolumeSourceFromVolumeBackupDetails{Id: &in.SourceBackupID}
+	}
+	resp, err := c.storage.CreateVolume(ctx, core.CreateVolumeRequest{
+		CreateVolumeDetails: details,
+		OpcRetryToken:       optional(in.RetryToken),
+	})
+	if err != nil {
+		return BlockVolume{}, wrap("CreateVolume", err)
+	}
+	return toBlockVolume(resp.Volume), nil
+}
+
+func (c *blockAPIClient) GetVolume(ctx context.Context, volumeID string) (BlockVolume, error) {
+	resp, err := c.storage.GetVolume(ctx, core.GetVolumeRequest{VolumeId: &volumeID})
+	if err != nil {
+		return BlockVolume{}, wrap("GetVolume", err)
+	}
+	v := toBlockVolume(resp.Volume)
+	v.ETag = deref(resp.Etag)
+	return v, nil
+}
+
+func (c *blockAPIClient) ListVolumes(ctx context.Context, filter VolumeFilter) ([]BlockVolume, error) {
+	seen := map[string]bool{}
+	return listAll(func(page *string) ([]core.Volume, *string, error) {
+		resp, err := c.storage.ListVolumes(ctx, core.ListVolumesRequest{
+			CompartmentId:      &filter.CompartmentID,
+			AvailabilityDomain: optional(filter.AvailabilityDomain),
+			DisplayName:        optional(filter.DisplayName),
+			Page:               page,
+		})
+		return resp.Items, resp.OpcNextPage, wrap("ListVolumes", err)
+	}, func(item core.Volume) bool {
+		return hasTags(item.FreeformTags, filter.Tags) && firstSight(seen, item.Id)
+	}, infallible(toBlockVolume))
+}
+
+func (c *blockAPIClient) UpdateVolume(ctx context.Context, volumeID string, in UpdateVolumeInput) (BlockVolume, error) {
+	details := core.UpdateVolumeDetails{FreeformTags: in.FreeformTags}
+	if in.SizeGB > 0 {
+		details.SizeInGBs = &in.SizeGB
+	}
+	resp, err := c.storage.UpdateVolume(ctx, core.UpdateVolumeRequest{
+		VolumeId: &volumeID, UpdateVolumeDetails: details, IfMatch: optional(in.IfMatch),
+	})
+	if err != nil {
+		return BlockVolume{}, wrap("UpdateVolume", err)
+	}
+	v := toBlockVolume(resp.Volume)
+	v.ETag = deref(resp.Etag)
+	return v, nil
+}
+
+func (c *blockAPIClient) DeleteVolume(ctx context.Context, volumeID string) error {
+	_, err := c.storage.DeleteVolume(ctx, core.DeleteVolumeRequest{VolumeId: &volumeID})
+	return wrap("DeleteVolume", err)
+}
+
+func (c *blockAPIClient) AttachVolume(ctx context.Context, in AttachVolumeInput) (VolumeAttachment, error) {
+	notShareable := false
+	resp, err := c.compute.AttachVolume(ctx, core.AttachVolumeRequest{
+		AttachVolumeDetails: core.AttachIScsiVolumeDetails{
+			InstanceId:  &in.InstanceID,
+			VolumeId:    &in.VolumeID,
+			DisplayName: optional(in.DisplayName),
+			IsReadOnly:  &in.ReadOnly,
+			IsShareable: &notShareable,
+		},
+	})
+	if err != nil {
+		return VolumeAttachment{}, wrap("AttachVolume", err)
+	}
+	return toVolumeAttachment(resp.VolumeAttachment), nil
+}
+
+func (c *blockAPIClient) GetVolumeAttachment(ctx context.Context, attachmentID string) (VolumeAttachment, error) {
+	resp, err := c.compute.GetVolumeAttachment(ctx, core.GetVolumeAttachmentRequest{VolumeAttachmentId: &attachmentID})
+	if err != nil {
+		return VolumeAttachment{}, wrap("GetVolumeAttachment", err)
+	}
+	return toVolumeAttachment(resp.VolumeAttachment), nil
+}
+
+func (c *blockAPIClient) ListVolumeAttachments(ctx context.Context, filter AttachmentFilter) ([]VolumeAttachment, error) {
+	return listAll(func(page *string) ([]core.VolumeAttachment, *string, error) {
+		resp, err := c.compute.ListVolumeAttachments(ctx, core.ListVolumeAttachmentsRequest{
+			CompartmentId: &filter.CompartmentID,
+			VolumeId:      optional(filter.VolumeID),
+			InstanceId:    optional(filter.InstanceID),
+			Page:          page,
+		})
+		return resp.Items, resp.OpcNextPage, wrap("ListVolumeAttachments", err)
+	}, nil, infallible(toVolumeAttachment))
+}
+
+func (c *blockAPIClient) DetachVolume(ctx context.Context, attachmentID string) error {
+	_, err := c.compute.DetachVolume(ctx, core.DetachVolumeRequest{VolumeAttachmentId: &attachmentID})
+	return wrap("DetachVolume", err)
+}
+
+func (c *blockAPIClient) CreateVolumeBackup(ctx context.Context, in CreateVolumeBackupInput) (VolumeBackup, error) {
+	resp, err := c.storage.CreateVolumeBackup(ctx, core.CreateVolumeBackupRequest{
+		CreateVolumeBackupDetails: core.CreateVolumeBackupDetails{
+			VolumeId:     &in.VolumeID,
+			DisplayName:  optional(in.DisplayName),
+			FreeformTags: in.FreeformTags,
+			Type:         core.CreateVolumeBackupDetailsTypeIncremental,
+		},
+		OpcRetryToken: optional(in.RetryToken),
+	})
+	if err != nil {
+		return VolumeBackup{}, wrap("CreateVolumeBackup", err)
+	}
+	return toVolumeBackup(resp.VolumeBackup), nil
+}
+
+func (c *blockAPIClient) GetVolumeBackup(ctx context.Context, backupID string) (VolumeBackup, error) {
+	resp, err := c.storage.GetVolumeBackup(ctx, core.GetVolumeBackupRequest{VolumeBackupId: &backupID})
+	if err != nil {
+		return VolumeBackup{}, wrap("GetVolumeBackup", err)
+	}
+	return toVolumeBackup(resp.VolumeBackup), nil
+}
+
+func (c *blockAPIClient) ListVolumeBackups(ctx context.Context, filter BackupFilter) ([]VolumeBackup, error) {
+	seen := map[string]bool{}
+	return listAll(func(page *string) ([]core.VolumeBackup, *string, error) {
+		resp, err := c.storage.ListVolumeBackups(ctx, core.ListVolumeBackupsRequest{
+			CompartmentId: &filter.CompartmentID,
+			DisplayName:   optional(filter.DisplayName),
+			Page:          page,
+		})
+		return resp.Items, resp.OpcNextPage, wrap("ListVolumeBackups", err)
+	}, func(item core.VolumeBackup) bool {
+		return hasTags(item.FreeformTags, filter.Tags) && firstSight(seen, item.Id)
+	}, infallible(toVolumeBackup))
+}
+
+func (c *blockAPIClient) DeleteVolumeBackup(ctx context.Context, backupID string) error {
+	_, err := c.storage.DeleteVolumeBackup(ctx, core.DeleteVolumeBackupRequest{VolumeBackupId: &backupID})
+	return wrap("DeleteVolumeBackup", err)
+}
+
+func toBlockVolume(v core.Volume) BlockVolume {
+	out := BlockVolume{
+		ID:                 deref(v.Id),
+		CompartmentID:      deref(v.CompartmentId),
+		AvailabilityDomain: deref(v.AvailabilityDomain),
+		DisplayName:        deref(v.DisplayName),
+		SizeGB:             derefInt64(v.SizeInGBs),
+		VPUsPerGB:          derefInt64(v.VpusPerGB),
+		State:              VolumeState(v.LifecycleState),
+		FreeformTags:       maps.Clone(v.FreeformTags),
+	}
+	switch src := v.SourceDetails.(type) {
+	case core.VolumeSourceFromVolumeBackupDetails:
+		out.SourceBackupID = deref(src.Id)
+	case *core.VolumeSourceFromVolumeBackupDetails:
+		out.SourceBackupID = deref(src.Id)
+	}
+	if v.TimeCreated != nil {
+		out.TimeCreated = v.TimeCreated.Time
+	}
+	return out
+}
+
+func toVolumeAttachment(a core.VolumeAttachment) VolumeAttachment {
+	if a == nil {
+		return VolumeAttachment{}
+	}
+	out := VolumeAttachment{
+		ID:                 deref(a.GetId()),
+		InstanceID:         deref(a.GetInstanceId()),
+		VolumeID:           deref(a.GetVolumeId()),
+		AvailabilityDomain: deref(a.GetAvailabilityDomain()),
+		State:              AttachmentState(a.GetLifecycleState()),
+		ReadOnly:           a.GetIsReadOnly() != nil && *a.GetIsReadOnly(),
+	}
+	var iscsi *core.IScsiVolumeAttachment
+	switch v := a.(type) {
+	case core.IScsiVolumeAttachment:
+		iscsi = &v
+	case *core.IScsiVolumeAttachment:
+		iscsi = v
+	}
+	if iscsi != nil {
+		out.IQN = deref(iscsi.Iqn)
+		out.IPv4 = deref(iscsi.Ipv4)
+		if iscsi.Port != nil {
+			out.Port = *iscsi.Port
+		}
+	}
+	return out
+}
+
+func toVolumeBackup(b core.VolumeBackup) VolumeBackup {
+	out := VolumeBackup{
+		ID:            deref(b.Id),
+		CompartmentID: deref(b.CompartmentId),
+		VolumeID:      deref(b.VolumeId),
+		DisplayName:   deref(b.DisplayName),
+		SizeGB:        derefInt64(b.SizeInGBs),
+		UniqueSizeGB:  derefInt64(b.UniqueSizeInGBs),
+		State:         BackupState(b.LifecycleState),
+		FreeformTags:  maps.Clone(b.FreeformTags),
+	}
+	if b.TimeCreated != nil {
+		out.TimeCreated = b.TimeCreated.Time
+	}
+	return out
+}
+
+// hasTags reports whether tags holds every key and value in want.
+func hasTags(tags, want map[string]string) bool {
+	for k, v := range want {
+		if got, ok := tags[k]; !ok || got != v {
+			return false
+		}
+	}
+	return true
+}
+
+// firstSight reports whether id is new to seen, and records it. OCI's list
+// calls can return a resource twice while it changes state, as a new backup
+// leaving REQUEST_RECEIVED does.
+func firstSight(seen map[string]bool, id *string) bool {
+	key := deref(id)
+	if seen[key] {
+		return false
+	}
+	seen[key] = true
+	return true
+}
+
+// optional is nil for the empty string, which the SDK omits from a request.
+func optional(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return &s
+}
+
+func derefInt64(n *int64) int64 {
+	if n == nil {
+		return 0
+	}
+	return *n
+}

spinifex/providers/cloud/oci/block_apiclient_test.go

@@ -0,0 +1,293 @@
+package oci_test
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+)
+
+func testBlockClient(t *testing.T, h http.HandlerFunc) (oci.BlockClient, *[]capture) {
+	t.Helper()
+	provider, endpoint, hc, seen := testEndpoint(t, h)
+	c, err := oci.NewTestBlockClient(provider, endpoint, hc)
+	require.NoError(t, err)
+	return c, seen
+}
+
+const volumeJSON = `{"id":"ocid1.volume.oc1..v1","compartmentId":"c1","availabilityDomain":"AD-1",
+"displayName":"vol-1","sizeInGBs":50,"sizeInMBs":51200,"vpusPerGB":20,"lifecycleState":"PROVISIONING",
+"timeCreated":"2026-11-10T03:00:00Z","freeformTags":{"spinifex-volume-id":"vol-1"},
+"sourceDetails":{"type":"volumeBackup","id":"ocid1.volumebackup.oc1..b1"}}`
+
+// The retry token is what makes a create after a lost reply return the first
+// volume instead of a second one, so it has to reach OCI as the header, with
+// the tags and the restore source in the body.
+func TestBlockCreateVolumeSendsRetryTokenTagsAndSource(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(volumeJSON))
+	})
+
+	got, err := c.CreateVolume(context.Background(), oci.CreateVolumeInput{
+		CompartmentID: "c1", AvailabilityDomain: "AD-1", DisplayName: "vol-1",
+		SizeGB: 50, VPUsPerGB: 20, SourceBackupID: "ocid1.volumebackup.oc1..b1",
+		FreeformTags: map[string]string{"spinifex-volume-id": "vol-1"},
+		RetryToken:   "token-1",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, oci.BlockVolume{
+		ID: "ocid1.volume.oc1..v1", CompartmentID: "c1", AvailabilityDomain: "AD-1", DisplayName: "vol-1",
+		SizeGB: 50, VPUsPerGB: 20, State: oci.VolumeProvisioning, SourceBackupID: "ocid1.volumebackup.oc1..b1",
+		FreeformTags: map[string]string{"spinifex-volume-id": "vol-1"}, TimeCreated: got.TimeCreated,
+	}, got)
+	assert.False(t, got.TimeCreated.IsZero())
+
+	require.Len(t, *seen, 1)
+	req := (*seen)[0]
+	assert.Equal(t, http.MethodPost, req.method)
+	assert.Equal(t, "/20160918/volumes", req.path)
+	assert.Equal(t, "token-1", req.header.Get("Opc-Retry-Token"))
+	assert.Equal(t, "c1", req.body["compartmentId"])
+	assert.Equal(t, "AD-1", req.body["availabilityDomain"])
+	assert.Equal(t, "vol-1", req.body["displayName"])
+	assert.EqualValues(t, 50, req.body["sizeInGBs"])
+	assert.EqualValues(t, 20, req.body["vpusPerGB"])
+	assert.Equal(t, map[string]any{"spinifex-volume-id": "vol-1"}, req.body["freeformTags"])
+	assert.Equal(t, map[string]any{"type": "volumeBackup", "id": "ocid1.volumebackup.oc1..b1"}, req.body["sourceDetails"])
+}
+
+// A zero tier is OCI's Lower Cost tier, so the client must leave the field out
+// rather than send a zero nobody asked for.
+func TestBlockCreateVolumeOmitsUnsetTierAndSource(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"id":"v1","lifecycleState":"AVAILABLE"}`))
+	})
+
+	_, err := c.CreateVolume(context.Background(), oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50})
+	require.NoError(t, err)
+	body := (*seen)[0].body
+	assert.NotContains(t, body, "vpusPerGB")
+	assert.NotContains(t, body, "sourceDetails")
+	assert.NotContains(t, body, "displayName")
+}
+
+// OCI's list call has no tag filter, so the client filters every page; a
+// volume carrying a different cluster's tag must not be returned.
+func TestBlockListVolumesFiltersTagsAcrossPages(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("page") == "" {
+			w.Header().Set("Opc-Next-Page", "page2")
+			_, _ = w.Write([]byte(`[{"id":"v1","freeformTags":{"spinifex-cluster":"a"}},{"id":"v2","freeformTags":{"spinifex-cluster":"b"}}]`))
+			return
+		}
+		_, _ = w.Write([]byte(`[{"id":"v3","freeformTags":{"spinifex-cluster":"a","other":"x"}},{"id":"v4"}]`))
+	})
+
+	got, err := c.ListVolumes(context.Background(), oci.VolumeFilter{
+		CompartmentID: "c1", DisplayName: "vol-1", Tags: map[string]string{"spinifex-cluster": "a"},
+	})
+	require.NoError(t, err)
+	require.Len(t, got, 2)
+	assert.Equal(t, "v1", got[0].ID)
+	assert.Equal(t, "v3", got[1].ID)
+	require.Len(t, *seen, 2)
+	assert.Equal(t, "c1", (*seen)[0].query.Get("compartmentId"))
+	assert.Equal(t, "vol-1", (*seen)[0].query.Get("displayName"))
+	assert.False(t, (*seen)[0].query.Has("availabilityDomain"), "an unset filter must not be sent")
+}
+
+func TestBlockUpdateVolumeSendsOnlyWhatChanges(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"id":"v1","sizeInGBs":60,"lifecycleState":"AVAILABLE"}`))
+	})
+
+	got, err := c.UpdateVolume(context.Background(), "v1", oci.UpdateVolumeInput{FreeformTags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	assert.Equal(t, int64(60), got.SizeGB)
+	_, err = c.UpdateVolume(context.Background(), "v1", oci.UpdateVolumeInput{SizeGB: 60})
+	require.NoError(t, err)
+
+	require.Len(t, *seen, 2)
+	assert.Equal(t, http.MethodPut, (*seen)[0].method)
+	assert.Equal(t, "/20160918/volumes/v1", (*seen)[0].path)
+	assert.NotContains(t, (*seen)[0].body, "sizeInGBs", "a tag update must not resize the volume")
+	assert.Equal(t, map[string]any{"k": "v"}, (*seen)[0].body["freeformTags"])
+	assert.EqualValues(t, 60, (*seen)[1].body["sizeInGBs"])
+	assert.NotContains(t, (*seen)[1].body, "freeformTags", "a resize must not clear the tags")
+}
+
+// The attachment is never shareable: that is what makes OCI refuse a second
+// writer anywhere, which the provider's cluster-wide exclusion rests on.
+func TestBlockAttachVolumeIsISCSIAndNotShareable(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"attachmentType":"iscsi","id":"a1","instanceId":"i1","volumeId":"v1",
+"availabilityDomain":"AD-1","lifecycleState":"ATTACHING","isReadOnly":true,
+"iqn":"iqn.2015-12.com.oracleiaas:x","ipv4":"169.254.2.2","port":3260,"compartmentId":"c1","timeCreated":"2026-11-10T03:00:00Z"}`))
+	})
+
+	got, err := c.AttachVolume(context.Background(), oci.AttachVolumeInput{InstanceID: "i1", VolumeID: "v1", DisplayName: "vol-1", ReadOnly: true})
+	require.NoError(t, err)
+	assert.Equal(t, oci.VolumeAttachment{
+		ID: "a1", InstanceID: "i1", VolumeID: "v1", AvailabilityDomain: "AD-1", State: oci.AttachmentAttaching,
+		ReadOnly: true, IQN: "iqn.2015-12.com.oracleiaas:x", IPv4: "169.254.2.2", Port: 3260,
+	}, got)
+
+	body := (*seen)[0].body
+	assert.Equal(t, "/20160918/volumeAttachments", (*seen)[0].path)
+	assert.Equal(t, "iscsi", body["type"])
+	assert.Equal(t, true, body["isReadOnly"])
+	assert.Equal(t, false, body["isShareable"])
+	assert.Equal(t, "i1", body["instanceId"])
+	assert.Equal(t, "v1", body["volumeId"])
+}
+
+// A paravirtualized attachment has no iSCSI target, and must read back with
+// none rather than fail the whole list.
+func TestBlockListVolumeAttachmentsReadsEveryType(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`[
+{"attachmentType":"iscsi","id":"a1","instanceId":"i1","volumeId":"v1","availabilityDomain":"AD-1","lifecycleState":"ATTACHED","iqn":"iqn.x","ipv4":"169.254.2.2","port":3260,"compartmentId":"c1","timeCreated":"2026-11-10T03:00:00Z"},
+{"attachmentType":"paravirtualized","id":"a2","instanceId":"i2","volumeId":"v1","availabilityDomain":"AD-1","lifecycleState":"DETACHED","compartmentId":"c1","timeCreated":"2026-11-10T03:00:00Z"}]`))
+	})
+
+	got, err := c.ListVolumeAttachments(context.Background(), oci.AttachmentFilter{CompartmentID: "c1", VolumeID: "v1"})
+	require.NoError(t, err)
+	require.Len(t, got, 2)
+	assert.Equal(t, "iqn.x", got[0].IQN)
+	assert.Equal(t, 3260, got[0].Port)
+	assert.Equal(t, oci.AttachmentDetached, got[1].State)
+	assert.Empty(t, got[1].IQN)
+	assert.Equal(t, "v1", (*seen)[0].query.Get("volumeId"))
+	assert.False(t, (*seen)[0].query.Has("instanceId"))
+}
+
+func TestBlockCreateVolumeBackupIsIncrementalWithRetryToken(t *testing.T) {
+	c, seen := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"id":"b1","compartmentId":"c1","volumeId":"v1","displayName":"snap-1","sizeInGBs":50,
+"uniqueSizeInGBs":2,"lifecycleState":"CREATING","type":"INCREMENTAL","timeCreated":"2026-11-10T03:00:00Z","freeformTags":{"spinifex-snapshot-id":"snap-1"}}`))
+	})
+
+	got, err := c.CreateVolumeBackup(context.Background(), oci.CreateVolumeBackupInput{
+		VolumeID: "v1", DisplayName: "snap-1", FreeformTags: map[string]string{"spinifex-snapshot-id": "snap-1"}, RetryToken: "token-2",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, oci.BackupCreating, got.State)
+	assert.Equal(t, int64(2), got.UniqueSizeGB)
+	assert.Equal(t, "v1", got.VolumeID)
+
+	req := (*seen)[0]
+	assert.Equal(t, "/20160918/volumeBackups", req.path)
+	assert.Equal(t, "token-2", req.header.Get("Opc-Retry-Token"))
+	assert.Equal(t, "INCREMENTAL", req.body["type"])
+	assert.Equal(t, map[string]any{"spinifex-snapshot-id": "snap-1"}, req.body["freeformTags"])
+}
+
+func TestBlockListVolumeBackupsFiltersTags(t *testing.T) {
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`[{"id":"b1","freeformTags":{"spinifex-cluster":"a"}},{"id":"b2","freeformTags":{"spinifex-cluster":"b"}}]`))
+	})
+
+	got, err := c.ListVolumeBackups(context.Background(), oci.BackupFilter{CompartmentID: "c1", Tags: map[string]string{"spinifex-cluster": "a"}})
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "b1", got[0].ID)
+}
+
+// OCI listed one new backup twice in a live run, as it left REQUEST_RECEIVED,
+// and a provider counting the entries saw two backups for one snapshot.
+func TestBlockListsAnswerEachResourceOnce(t *testing.T) {
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("page") == "" {
+			w.Header().Set("Opc-Next-Page", "page2")
+			_, _ = w.Write([]byte(`[{"id":"x1","lifecycleState":"REQUEST_RECEIVED"},{"id":"x1","lifecycleState":"CREATING"},{"id":"x2"}]`))
+			return
+		}
+		_, _ = w.Write([]byte(`[{"id":"x2"},{"id":"x3"}]`))
+	})
+
+	backups, err := c.ListVolumeBackups(context.Background(), oci.BackupFilter{CompartmentID: "c1"})
+	require.NoError(t, err)
+	ids := make([]string, 0, len(backups))
+	for _, b := range backups {
+		ids = append(ids, b.ID)
+	}
+	assert.Equal(t, []string{"x1", "x2", "x3"}, ids)
+
+	volumes, err := c.ListVolumes(context.Background(), oci.VolumeFilter{CompartmentID: "c1"})
+	require.NoError(t, err)
+	ids = ids[:0]
+	for _, v := range volumes {
+		ids = append(ids, v.ID)
+	}
+	assert.Equal(t, []string{"x1", "x2", "x3"}, ids)
+}
+
+// The two 409s OCI answers while a backup is in progress are what the provider
+// maps to volume_in_use and unavailable, so they must stay classified.
+func TestBlockConflictsAreClassified(t *testing.T) {
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusConflict)
+		_, _ = w.Write([]byte(`{"code":"Conflict","message":"There is currently a backup operation in progress"}`))
+	})
+
+	for name, call := range map[string]func() error{
+		"DeleteVolume":       func() error { return c.DeleteVolume(context.Background(), "v1") },
+		"DeleteVolumeBackup": func() error { return c.DeleteVolumeBackup(context.Background(), "b1") },
+		"DetachVolume":       func() error { return c.DetachVolume(context.Background(), "a1") },
+	} {
+		t.Run(name, func(t *testing.T) {
+			err := call()
+			require.ErrorIs(t, err, oci.ErrConflict)
+			var api *oci.APIError
+			require.ErrorAs(t, err, &api)
+			assert.Equal(t, name, api.Op)
+		})
+	}
+}
+
+func TestBlockNotFoundAndLimitAreClassified(t *testing.T) {
+	var status int
+	var code string
+	c, _ := testBlockClient(t, func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(`{"code":"` + code + `","message":"m"}`))
+	})
+
+	ctx := context.Background()
+	for name, tc := range map[string]struct {
+		status int
+		code   string
+		want   error
+		call   func() error
+	}{
+		"GetVolume": {http.StatusNotFound, "NotAuthorizedOrNotFound", oci.ErrNotFound, func() error {
+			_, err := c.GetVolume(ctx, "v1")
+			return err
+		}},
+		"GetVolumeBackup": {http.StatusNotFound, "NotAuthorizedOrNotFound", oci.ErrNotFound, func() error {
+			_, err := c.GetVolumeBackup(ctx, "b1")
+			return err
+		}},
+		"GetVolumeAttachment": {http.StatusNotFound, "NotAuthorizedOrNotFound", oci.ErrNotFound, func() error {
+			_, err := c.GetVolumeAttachment(ctx, "a1")
+			return err
+		}},
+		"AttachVolume": {http.StatusBadRequest, "LimitExceeded", oci.ErrLimitExceeded, func() error {
+			_, err := c.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: "v1"})
+			return err
+		}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			status, code = tc.status, tc.code
+			err := tc.call()
+			require.ErrorIs(t, err, tc.want)
+			var api *oci.APIError
+			require.ErrorAs(t, err, &api)
+			assert.Equal(t, name, api.Op)
+		})
+	}
+}

spinifex/providers/cloud/oci/block_fake.go

@@ -0,0 +1,501 @@
+package oci
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// MaxISCSIAttachmentsPerInstance is OCI's cap on iSCSI attachments to one
+// instance, the same on every VM and bare-metal shape.
+const MaxISCSIAttachmentsPerInstance = 32
+
+// ErrReplyLost is what BlockFake returns for a call LoseReply staged: the
+// call took effect in OCI but its answer never arrived.
+var ErrReplyLost = errors.New("oci fake: reply lost after the call took effect")
+
+// BlockFake is an in-memory BlockClient. It models the OCI behaviours an EBS
+// provider has to handle: the 50 GB floor, retry tokens, backups that stay
+// CREATING until a test settles them, and attachments that move by state.
+type BlockFake struct {
+	stagedFailures
+
+	mu sync.Mutex
+
+	// AttachAfter and DetachAfter are how many GetVolumeAttachment calls an
+	// attachment spends ATTACHING or DETACHING before it settles. Zero means
+	// it settles in the call that started it.
+	AttachAfter int
+	DetachAfter int
+
+	// AttachmentCap caps iSCSI attachments per instance; zero means
+	// MaxISCSIAttachmentsPerInstance.
+	AttachmentCap int
+
+	volumes     map[string]*BlockVolume
+	backups     map[string]*VolumeBackup
+	attachments map[string]*fakeAttachment
+	tokens      map[string]string // op and retry token -> resource OCID
+	tokenOf     map[string]string // resource OCID -> the retry token it was created with
+	loseReply   map[string]bool
+	seq         int
+}
+
+type fakeAttachment struct {
+	VolumeAttachment
+
+	countdown int
+}
+
+var _ BlockClient = (*BlockFake)(nil)
+
+// NewBlockFake returns an empty BlockFake whose attachments spend one read in
+// each transitional state, so a caller that does not wait is caught.
+func NewBlockFake() *BlockFake {
+	return &BlockFake{
+		AttachAfter: 1,
+		DetachAfter: 1,
+		volumes:     map[string]*BlockVolume{},
+		backups:     map[string]*VolumeBackup{},
+		attachments: map[string]*fakeAttachment{},
+		tokens:      map[string]string{},
+		tokenOf:     map[string]string{},
+		loseReply:   map[string]bool{},
+
+		stagedFailures: stagedFailures{FailWith: map[string]error{}},
+	}
+}
+
+// LoseReply makes the next call to op take effect and return ErrReplyLost.
+func (f *BlockFake) LoseReply(op string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.loseReply[op] = true
+}
+
+// SettleBackup moves a backup still in progress to state, as OCI does once
+// the backup finishes or fails.
+func (f *BlockFake) SettleBackup(backupID string, state BackupState) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	b, ok := f.backups[backupID]
+	if !ok {
+		return fmt.Errorf("oci fake: no backup %s", backupID)
+	}
+	if !b.State.InProgress() {
+		return fmt.Errorf("oci fake: backup %s is %s, not in progress", backupID, b.State)
+	}
+	b.State = state
+	return nil
+}
+
+// SeedAttachment inserts an attachment directly, for state OCI holds that no
+// call in the test made, such as another instance's attachment.
+func (f *BlockFake) SeedAttachment(a VolumeAttachment) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.attachments[a.ID] = &fakeAttachment{VolumeAttachment: a}
+}
+
+// Volumes returns every volume the fake holds, terminated ones included.
+func (f *BlockFake) Volumes() []BlockVolume {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]BlockVolume, 0, len(f.volumes))
+	for _, v := range f.volumes {
+		out = append(out, cloneVolume(*v))
+	}
+	return out
+}
+
+// Backups returns every backup the fake holds, terminated ones included.
+func (f *BlockFake) Backups() []VolumeBackup {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]VolumeBackup, 0, len(f.backups))
+	for _, b := range f.backups {
+		out = append(out, cloneBackup(*b))
+	}
+	return out
+}
+
+// RetryToken is the opc-retry-token the resource was created with.
+func (f *BlockFake) RetryToken(resourceID string) string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.tokenOf[resourceID]
+}
+
+func (f *BlockFake) CreateVolume(_ context.Context, in CreateVolumeInput) (BlockVolume, error) {
+	const op = "CreateVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return BlockVolume{}, err
+	}
+	if id, ok := f.replayed(op, in.RetryToken); ok {
+		return answerOf(f, op, cloneVolume(*f.volumes[id]))
+	}
+	if in.CompartmentID == "" || in.AvailabilityDomain == "" {
+		return BlockVolume{}, fakeError(op, 400, "InvalidParameter", "compartmentId and availabilityDomain are required")
+	}
+	if in.SizeGB < MinVolumeSizeGB {
+		return BlockVolume{}, fakeError(op, 400, "InvalidParameter", fmt.Sprintf("sizeInGBs must be at least %d", MinVolumeSizeGB))
+	}
+	if in.SourceBackupID != "" {
+		b, ok := f.backups[in.SourceBackupID]
+		if !ok || b.State.Gone() {
+			return BlockVolume{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume backup not found")
+		}
+		if b.State != BackupAvailable {
+			return BlockVolume{}, fakeError(op, 409, "Conflict", "volume backup is not available")
+		}
+		if in.SizeGB < b.SizeGB {
+			return BlockVolume{}, fakeError(op, 400, "InvalidParameter", "sizeInGBs is smaller than the backup")
+		}
+	}
+	vpus := in.VPUsPerGB
+	if vpus == 0 {
+		vpus = 10
+	}
+	id := f.newID("volume")
+	f.volumes[id] = &BlockVolume{
+		ID: id, CompartmentID: in.CompartmentID, AvailabilityDomain: in.AvailabilityDomain,
+		DisplayName: in.DisplayName, SizeGB: in.SizeGB, VPUsPerGB: vpus, State: VolumeAvailable,
+		SourceBackupID: in.SourceBackupID, FreeformTags: maps.Clone(in.FreeformTags), TimeCreated: time.Now().UTC(),
+		ETag: "1",
+	}
+	f.remember(op, in.RetryToken, id)
+	return answerOf(f, op, cloneVolume(*f.volumes[id]))
+}
+
+func (f *BlockFake) GetVolume(_ context.Context, volumeID string) (BlockVolume, error) {
+	const op = "GetVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return BlockVolume{}, err
+	}
+	v, ok := f.volumes[volumeID]
+	if !ok {
+		return BlockVolume{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	return cloneVolume(*v), nil
+}
+
+func (f *BlockFake) ListVolumes(_ context.Context, filter VolumeFilter) ([]BlockVolume, error) {
+	const op = "ListVolumes"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return nil, err
+	}
+	var out []BlockVolume
+	for _, v := range f.volumes {
+		if v.CompartmentID != filter.CompartmentID ||
+			(filter.AvailabilityDomain != "" && v.AvailabilityDomain != filter.AvailabilityDomain) ||
+			(filter.DisplayName != "" && v.DisplayName != filter.DisplayName) ||
+			!hasTags(v.FreeformTags, filter.Tags) {
+			continue
+		}
+		listed := cloneVolume(*v)
+		listed.ETag = ""
+		out = append(out, listed)
+	}
+	return out, nil
+}
+
+func (f *BlockFake) UpdateVolume(_ context.Context, volumeID string, in UpdateVolumeInput) (BlockVolume, error) {
+	const op = "UpdateVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return BlockVolume{}, err
+	}
+	v, ok := f.volumes[volumeID]
+	if !ok || v.State.Gone() {
+		return BlockVolume{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	if in.IfMatch != "" && in.IfMatch != v.ETag {
+		return BlockVolume{}, fakeError(op, 412, "NoEtagMatch", "the volume changed since the given etag")
+	}
+	if in.SizeGB > 0 && in.SizeGB < v.SizeGB {
+		return BlockVolume{}, fakeError(op, 400, "InvalidParameter", "a volume cannot shrink")
+	}
+	if in.SizeGB > 0 {
+		v.SizeGB = in.SizeGB
+	}
+	if in.FreeformTags != nil {
+		v.FreeformTags = maps.Clone(in.FreeformTags)
+	}
+	rev, _ := strconv.Atoi(v.ETag)
+	v.ETag = strconv.Itoa(rev + 1)
+	return answerOf(f, op, cloneVolume(*v))
+}
+
+func (f *BlockFake) DeleteVolume(_ context.Context, volumeID string) error {
+	const op = "DeleteVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return err
+	}
+	v, ok := f.volumes[volumeID]
+	if !ok || v.State.Gone() {
+		return fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	for _, b := range f.backups {
+		if b.VolumeID == volumeID && b.State.InProgress() {
+			return fakeError(op, 409, "Conflict", "There is currently a backup operation in progress for Volume "+volumeID+".")
+		}
+	}
+	for _, a := range f.attachments {
+		if a.VolumeID == volumeID && a.State != AttachmentDetached {
+			return fakeError(op, 409, "Conflict", "volume is attached")
+		}
+	}
+	v.State = VolumeTerminated
+	_, err := answerOf(f, op, struct{}{})
+	return err
+}
+
+func (f *BlockFake) AttachVolume(_ context.Context, in AttachVolumeInput) (VolumeAttachment, error) {
+	const op = "AttachVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeAttachment{}, err
+	}
+	v, ok := f.volumes[in.VolumeID]
+	if !ok || v.State.Gone() {
+		return VolumeAttachment{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	if v.State != VolumeAvailable {
+		return VolumeAttachment{}, fakeError(op, 409, "Conflict", "volume is not available")
+	}
+	limit := f.AttachmentCap
+	if limit == 0 {
+		limit = MaxISCSIAttachmentsPerInstance
+	}
+	onInstance := 0
+	for _, a := range f.attachments {
+		if a.State == AttachmentDetached {
+			continue
+		}
+		if a.VolumeID == in.VolumeID {
+			return VolumeAttachment{}, fakeError(op, 409, "Conflict", "volume is already attached and is not shareable")
+		}
+		if a.InstanceID == in.InstanceID {
+			onInstance++
+		}
+	}
+	if onInstance >= limit {
+		return VolumeAttachment{}, fakeError(op, 400, "LimitExceeded", "the instance has reached its volume attachment limit")
+	}
+	id := f.newID("volumeattachment")
+	a := &fakeAttachment{
+		VolumeAttachment: VolumeAttachment{
+			ID: id, InstanceID: in.InstanceID, VolumeID: in.VolumeID, AvailabilityDomain: v.AvailabilityDomain,
+			State: AttachmentAttaching, ReadOnly: in.ReadOnly,
+			IQN: IQNPrefix + id, IPv4: fmt.Sprintf("169.254.2.%d", 2+f.seq%250), Port: 3260,
+		},
+		countdown: f.AttachAfter,
+	}
+	if a.countdown == 0 {
+		a.State = AttachmentAttached
+	}
+	f.attachments[id] = a
+	return answerOf(f, op, a.VolumeAttachment)
+}
+
+func (f *BlockFake) GetVolumeAttachment(_ context.Context, attachmentID string) (VolumeAttachment, error) {
+	const op = "GetVolumeAttachment"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeAttachment{}, err
+	}
+	a, ok := f.attachments[attachmentID]
+	if !ok {
+		return VolumeAttachment{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume attachment not found")
+	}
+	if a.State == AttachmentAttaching || a.State == AttachmentDetaching {
+		if a.countdown > 0 {
+			a.countdown--
+		}
+		if a.countdown == 0 && a.State == AttachmentAttaching {
+			a.State = AttachmentAttached
+		} else if a.countdown == 0 {
+			a.State = AttachmentDetached
+		}
+	}
+	return a.VolumeAttachment, nil
+}
+
+func (f *BlockFake) ListVolumeAttachments(_ context.Context, filter AttachmentFilter) ([]VolumeAttachment, error) {
+	const op = "ListVolumeAttachments"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return nil, err
+	}
+	var out []VolumeAttachment
+	for _, a := range f.attachments {
+		if (filter.VolumeID != "" && a.VolumeID != filter.VolumeID) ||
+			(filter.InstanceID != "" && a.InstanceID != filter.InstanceID) {
+			continue
+		}
+		out = append(out, a.VolumeAttachment)
+	}
+	slices.SortFunc(out, func(a, b VolumeAttachment) int { return strings.Compare(a.ID, b.ID) })
+	return out, nil
+}
+
+func (f *BlockFake) DetachVolume(_ context.Context, attachmentID string) error {
+	const op = "DetachVolume"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return err
+	}
+	a, ok := f.attachments[attachmentID]
+	if !ok {
+		return fakeError(op, 404, "NotAuthorizedOrNotFound", "volume attachment not found")
+	}
+	if a.State == AttachmentDetaching || a.State == AttachmentDetached {
+		return fakeError(op, 409, "Conflict", "volume attachment is "+string(a.State))
+	}
+	a.State, a.countdown = AttachmentDetaching, f.DetachAfter
+	if a.countdown == 0 {
+		a.State = AttachmentDetached
+	}
+	_, err := answerOf(f, op, struct{}{})
+	return err
+}
+
+func (f *BlockFake) CreateVolumeBackup(_ context.Context, in CreateVolumeBackupInput) (VolumeBackup, error) {
+	const op = "CreateVolumeBackup"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeBackup{}, err
+	}
+	if id, ok := f.replayed(op, in.RetryToken); ok {
+		return answerOf(f, op, cloneBackup(*f.backups[id]))
+	}
+	v, ok := f.volumes[in.VolumeID]
+	if !ok || v.State.Gone() {
+		return VolumeBackup{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume not found")
+	}
+	id := f.newID("volumebackup")
+	f.backups[id] = &VolumeBackup{
+		ID: id, CompartmentID: v.CompartmentID, VolumeID: in.VolumeID, DisplayName: in.DisplayName,
+		SizeGB: v.SizeGB, State: BackupCreating, FreeformTags: maps.Clone(in.FreeformTags), TimeCreated: time.Now().UTC(),
+	}
+	f.remember(op, in.RetryToken, id)
+	return answerOf(f, op, cloneBackup(*f.backups[id]))
+}
+
+func (f *BlockFake) GetVolumeBackup(_ context.Context, backupID string) (VolumeBackup, error) {
+	const op = "GetVolumeBackup"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return VolumeBackup{}, err
+	}
+	b, ok := f.backups[backupID]
+	if !ok {
+		return VolumeBackup{}, fakeError(op, 404, "NotAuthorizedOrNotFound", "volume backup not found")
+	}
+	return cloneBackup(*b), nil
+}
+
+func (f *BlockFake) ListVolumeBackups(_ context.Context, filter BackupFilter) ([]VolumeBackup, error) {
+	const op = "ListVolumeBackups"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return nil, err
+	}
+	var out []VolumeBackup
+	for _, b := range f.backups {
+		if b.CompartmentID != filter.CompartmentID ||
+			(filter.DisplayName != "" && b.DisplayName != filter.DisplayName) ||
+			!hasTags(b.FreeformTags, filter.Tags) {
+			continue
+		}
+		out = append(out, cloneBackup(*b))
+	}
+	return out, nil
+}
+
+func (f *BlockFake) DeleteVolumeBackup(_ context.Context, backupID string) error {
+	const op = "DeleteVolumeBackup"
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.take(op); err != nil {
+		return err
+	}
+	b, ok := f.backups[backupID]
+	if !ok || b.State.Gone() {
+		return fakeError(op, 404, "NotAuthorizedOrNotFound", "volume backup not found")
+	}
+	if b.State.InProgress() {
+		return fakeError(op, 409, "Conflict", "Cannot delete Volume Backup "+backupID+" while in state Creating.")
+	}
+	b.State = BackupTerminated
+	_, err := answerOf(f, op, struct{}{})
+	return err
+}
+
+// answerOf returns out, or ErrReplyLost once if LoseReply staged op.
+func answerOf[T any](f *BlockFake, op string, out T) (T, error) {
+	if f.loseReply[op] {
+		delete(f.loseReply, op)
+		var zero T
+		return zero, ErrReplyLost
+	}
+	return out, nil
+}
+
+// replayed finds the resource an earlier call with the same retry token made.
+func (f *BlockFake) replayed(op, token string) (string, bool) {
+	if token == "" {
+		return "", false
+	}
+	id, ok := f.tokens[op+"/"+token]
+	return id, ok
+}
+
+func (f *BlockFake) remember(op, token, id string) {
+	if token == "" {
+		return
+	}
+	f.tokens[op+"/"+token] = id
+	f.tokenOf[id] = token
+}
+
+func (f *BlockFake) newID(kind string) string {
+	f.seq++
+	return fmt.Sprintf("ocid1.%s.oc1..fake%06d", kind, f.seq)
+}
+
+func fakeError(op string, status int, code, message string) error {
+	return newAPIError(op, status, code, message, "fake")
+}
+
+func cloneVolume(v BlockVolume) BlockVolume {
+	v.FreeformTags = maps.Clone(v.FreeformTags)
+	return v
+}
+
+func cloneBackup(b VolumeBackup) VolumeBackup {
+	b.FreeformTags = maps.Clone(b.FreeformTags)
+	return b
+}

spinifex/providers/cloud/oci/block_fake_test.go

@@ -0,0 +1,193 @@
+package oci_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+)
+
+func fakeVolume(t *testing.T, f *oci.BlockFake, name string) oci.BlockVolume {
+	t.Helper()
+	v, err := f.CreateVolume(context.Background(), oci.CreateVolumeInput{
+		CompartmentID: "c1", AvailabilityDomain: "AD-1", DisplayName: name, SizeGB: oci.MinVolumeSizeGB,
+		FreeformTags: map[string]string{"spinifex-volume-id": name},
+	})
+	require.NoError(t, err)
+	return v
+}
+
+func TestBlockFakeVolumeFloorAndRetryToken(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+
+	_, err := f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: oci.MinVolumeSizeGB - 1})
+	require.Error(t, err, "OCI refuses a volume below its 50 GB floor")
+
+	in := oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 60, RetryToken: "t1"}
+	first, err := f.CreateVolume(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, int64(10), first.VPUsPerGB, "an unset tier is OCI's Balanced default")
+	again, err := f.CreateVolume(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, first.ID, again.ID, "a repeated retry token answers with the first volume")
+	assert.Equal(t, "t1", f.RetryToken(first.ID))
+	assert.Len(t, f.Volumes(), 1)
+}
+
+func TestBlockFakeLostReplyStillTakesEffect(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	f.LoseReply("CreateVolume")
+
+	_, err := f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50})
+	require.ErrorIs(t, err, oci.ErrReplyLost)
+	assert.Len(t, f.Volumes(), 1, "the volume exists although its creator never heard back")
+
+	f.FailOp("GetVolume", fmt.Errorf("boom"))
+	_, err = f.GetVolume(ctx, f.Volumes()[0].ID)
+	require.EqualError(t, err, "boom")
+	_, err = f.GetVolume(ctx, f.Volumes()[0].ID)
+	require.NoError(t, err, "a staged failure is used once")
+}
+
+// The experiment's two refusals: OCI will not delete a volume or a backup
+// while a backup of it is still being taken.
+func TestBlockFakeBackupInProgressRefusesDeletes(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+
+	b, err := f.CreateVolumeBackup(ctx, oci.CreateVolumeBackupInput{VolumeID: v.ID, DisplayName: "snap-1", RetryToken: "t2"})
+	require.NoError(t, err)
+	assert.Equal(t, oci.BackupCreating, b.State)
+	assert.Equal(t, v.SizeGB, b.SizeGB)
+	assert.Equal(t, "c1", b.CompartmentID)
+
+	require.ErrorIs(t, f.DeleteVolume(ctx, v.ID), oci.ErrConflict)
+	require.ErrorIs(t, f.DeleteVolumeBackup(ctx, b.ID), oci.ErrConflict)
+	_, err = f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50, SourceBackupID: b.ID})
+	require.ErrorIs(t, err, oci.ErrConflict, "a backup still being taken cannot be restored")
+
+	require.NoError(t, f.SettleBackup(b.ID, oci.BackupAvailable))
+	require.Error(t, f.SettleBackup(b.ID, oci.BackupFaulty), "a settled backup does not settle again")
+	require.Error(t, f.SettleBackup("absent", oci.BackupAvailable))
+
+	restored, err := f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50, SourceBackupID: b.ID})
+	require.NoError(t, err)
+	assert.Equal(t, b.ID, restored.SourceBackupID)
+
+	require.NoError(t, f.DeleteVolume(ctx, v.ID))
+	got, err := f.GetVolume(ctx, v.ID)
+	require.NoError(t, err, "OCI keeps a terminated volume readable")
+	assert.Equal(t, oci.VolumeTerminated, got.State)
+	require.ErrorIs(t, f.DeleteVolume(ctx, v.ID), oci.ErrNotFound)
+
+	require.NoError(t, f.DeleteVolumeBackup(ctx, b.ID))
+	require.ErrorIs(t, f.DeleteVolumeBackup(ctx, b.ID), oci.ErrNotFound)
+	_, err = f.CreateVolume(ctx, oci.CreateVolumeInput{CompartmentID: "c1", AvailabilityDomain: "AD-1", SizeGB: 50, SourceBackupID: b.ID})
+	require.ErrorIs(t, err, oci.ErrNotFound)
+}
+
+func TestBlockFakeAttachmentLifecycle(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+
+	a, err := f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: v.ID})
+	require.NoError(t, err)
+	assert.Equal(t, oci.AttachmentAttaching, a.State)
+	assert.NotEmpty(t, a.IQN)
+	assert.Equal(t, 3260, a.Port)
+
+	_, err = f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i2", VolumeID: v.ID})
+	require.ErrorIs(t, err, oci.ErrConflict, "a volume that is not shareable attaches once")
+	require.ErrorIs(t, f.DeleteVolume(ctx, v.ID), oci.ErrConflict, "an attached volume cannot be deleted")
+
+	got, err := f.GetVolumeAttachment(ctx, a.ID)
+	require.NoError(t, err)
+	assert.Equal(t, oci.AttachmentAttached, got.State)
+
+	require.NoError(t, f.DetachVolume(ctx, a.ID))
+	listed, err := f.ListVolumeAttachments(ctx, oci.AttachmentFilter{CompartmentID: "c1", VolumeID: v.ID})
+	require.NoError(t, err)
+	require.Len(t, listed, 1)
+	assert.Equal(t, oci.AttachmentDetaching, listed[0].State)
+	require.ErrorIs(t, f.DetachVolume(ctx, a.ID), oci.ErrConflict)
+
+	got, err = f.GetVolumeAttachment(ctx, a.ID)
+	require.NoError(t, err)
+	assert.Equal(t, oci.AttachmentDetached, got.State)
+	none, err := f.ListVolumeAttachments(ctx, oci.AttachmentFilter{CompartmentID: "c1", InstanceID: "i2"})
+	require.NoError(t, err)
+	assert.Empty(t, none)
+
+	_, err = f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i2", VolumeID: v.ID})
+	require.NoError(t, err, "a detached volume attaches again")
+}
+
+func TestBlockFakeAttachmentCap(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	f.AttachAfter = 0
+	for i := range oci.MaxISCSIAttachmentsPerInstance {
+		v := fakeVolume(t, f, fmt.Sprintf("vol-%d", i))
+		_, err := f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: v.ID})
+		require.NoError(t, err)
+	}
+	extra := fakeVolume(t, f, "vol-extra")
+	_, err := f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i1", VolumeID: extra.ID})
+	require.ErrorIs(t, err, oci.ErrLimitExceeded)
+	_, err = f.AttachVolume(ctx, oci.AttachVolumeInput{InstanceID: "i2", VolumeID: extra.ID})
+	require.NoError(t, err, "the cap is per instance")
+}
+
+func TestBlockFakeListsFilterByCompartmentNameAndTag(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+	fakeVolume(t, f, "vol-2")
+	_, err := f.CreateVolumeBackup(ctx, oci.CreateVolumeBackupInput{VolumeID: v.ID, DisplayName: "snap-1", FreeformTags: map[string]string{"k": "a"}})
+	require.NoError(t, err)
+
+	vols, err := f.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: "c1", Tags: map[string]string{"spinifex-volume-id": "vol-1"}})
+	require.NoError(t, err)
+	require.Len(t, vols, 1)
+	assert.Equal(t, v.ID, vols[0].ID)
+	vols, err = f.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: "c1", DisplayName: "vol-2", AvailabilityDomain: "AD-1"})
+	require.NoError(t, err)
+	assert.Len(t, vols, 1)
+	vols, err = f.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: "c2"})
+	require.NoError(t, err)
+	assert.Empty(t, vols)
+
+	backups, err := f.ListVolumeBackups(ctx, oci.BackupFilter{CompartmentID: "c1", Tags: map[string]string{"k": "a"}})
+	require.NoError(t, err)
+	assert.Len(t, backups, 1)
+	backups, err = f.ListVolumeBackups(ctx, oci.BackupFilter{CompartmentID: "c1", DisplayName: "snap-2"})
+	require.NoError(t, err)
+	assert.Empty(t, backups)
+}
+
+func TestBlockFakeUpdateVolume(t *testing.T) {
+	ctx := context.Background()
+	f := oci.NewBlockFake()
+	v := fakeVolume(t, f, "vol-1")
+
+	got, err := f.UpdateVolume(ctx, v.ID, oci.UpdateVolumeInput{FreeformTags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	assert.Equal(t, v.SizeGB, got.SizeGB)
+	assert.Equal(t, map[string]string{"k": "v"}, got.FreeformTags)
+	got, err = f.UpdateVolume(ctx, v.ID, oci.UpdateVolumeInput{SizeGB: 80})
+	require.NoError(t, err)
+	assert.Equal(t, int64(80), got.SizeGB)
+	assert.Equal(t, map[string]string{"k": "v"}, got.FreeformTags, "a resize keeps the tags")
+	_, err = f.UpdateVolume(ctx, v.ID, oci.UpdateVolumeInput{SizeGB: 60})
+	require.Error(t, err, "a volume cannot shrink")
+	_, err = f.UpdateVolume(ctx, "absent", oci.UpdateVolumeInput{SizeGB: 60})
+	require.ErrorIs(t, err, oci.ErrNotFound)
+}

spinifex/providers/cloud/oci/client.go

@@ -94,6 +94,16 @@ const DefaultConfigProfile = "DEFAULT"
 // The default, because it needs nothing from a tenancy admin. It costs a private
 // key on every node, which NewInstancePrincipalClient avoids.
 func NewConfigFileClient(path, profile string) (Client, error) {
+	provider, err := configFileProvider(path, profile)
+	if err != nil {
+		return nil, err
+	}
+	return NewClientWithProvider(provider)
+}
+
+// configFileProvider reads API-key credentials from path and profile, each
+// defaulting as the OCI CLI does.
+func configFileProvider(path, profile string) (common.ConfigurationProvider, error) {
 	if path == "" {
 		path = DefaultConfigFile
 	}
@@ -108,7 +118,7 @@ func NewConfigFileClient(path, profile string) (Client, error) {
 	if err != nil {
 		return nil, fmt.Errorf("oci config file %s (profile %s): %w", expanded, profile, err)
 	}
-	return NewClientWithProvider(provider)
+	return provider, nil
 }
 
 // expandHome resolves a leading ~ so operators can configure the conventional
@@ -176,54 +186,56 @@ func (c *apiClient) GetPrivateIP(ctx context.Context, privateIPID string) (Priva
 }
 
 func (c *apiClient) ListPrivateIPs(ctx context.Context, vnicID string) ([]PrivateIP, error) {
-	var out []PrivateIP
-	var page *string
-	for {
+	return listAll(func(page *string) ([]core.PrivateIp, *string, error) {
 		resp, err := c.net.ListPrivateIps(ctx, core.ListPrivateIpsRequest{VnicId: &vnicID, Page: page})
-		if err != nil {
-			return nil, wrap("ListPrivateIPs", err)
-		}
-		for _, item := range resp.Items {
-			ip, err := toPrivateIP(item)
-			if err != nil {
-				return nil, err
-			}
-			out = append(out, ip)
-		}
-		if resp.OpcNextPage == nil {
-			return out, nil
-		}
-		page = resp.OpcNextPage
-	}
+		return resp.Items, resp.OpcNextPage, wrap("ListPrivateIPs", err)
+	}, nil, toPrivateIP)
 }
 
 func (c *apiClient) ListReservedPublicIPs(ctx context.Context, compartmentID string) ([]PublicIP, error) {
-	var out []PublicIP
-	var page *string
-	for {
+	return listAll(func(page *string) ([]core.PublicIp, *string, error) {
 		resp, err := c.net.ListPublicIps(ctx, core.ListPublicIpsRequest{
 			CompartmentId: &compartmentID,
 			Scope:         core.ListPublicIpsScopeRegion,
 			Lifetime:      core.ListPublicIpsLifetimeReserved,
 			Page:          page,
 		})
+		return resp.Items, resp.OpcNextPage, wrap("ListReservedPublicIPs", err)
+	}, nil, toPublicIP)
+}
+
+// listAll reads every page fetch returns, from the first, and converts each
+// item keep accepts; a nil keep accepts every item.
+func listAll[I, T any](fetch func(page *string) ([]I, *string, error), keep func(I) bool, convert func(I) (T, error)) ([]T, error) {
+	var out []T
+	var page *string
+	for {
+		items, next, err := fetch(page)
 		if err != nil {
-			return nil, wrap("ListReservedPublicIPs", err)
+			return nil, err
 		}
-		for _, item := range resp.Items {
-			ip, err := toPublicIP(item)
+		for _, item := range items {
+			if keep != nil && !keep(item) {
+				continue
+			}
+			v, err := convert(item)
 			if err != nil {
 				return nil, err
 			}
-			out = append(out, ip)
+			out = append(out, v)
 		}
-		if resp.OpcNextPage == nil {
+		if next == nil {
 			return out, nil
 		}
-		page = resp.OpcNextPage
+		page = next
 	}
 }
 
+// infallible adapts a conversion that cannot fail to listAll.
+func infallible[I, T any](convert func(I) T) func(I) (T, error) {
+	return func(item I) (T, error) { return convert(item), nil }
+}
+
 func (c *apiClient) GetSubnet(ctx context.Context, subnetID string) (Subnet, error) {
 	resp, err := c.net.GetSubnet(ctx, core.GetSubnetRequest{SubnetId: &subnetID})
 	if err != nil {
@@ -361,21 +373,21 @@ func wrap(op string, err error) error {
 	if !ok {
 		return fmt.Errorf("oci %s: %w", op, err)
 	}
-	api := &APIError{
-		Op:         op,
-		Code:       svc.GetCode(),
-		Message:    svc.GetMessage(),
-		StatusCode: svc.GetHTTPStatusCode(),
-		RequestID:  svc.GetOpcRequestID(),
-	}
+	return newAPIError(op, svc.GetHTTPStatusCode(), svc.GetCode(), svc.GetMessage(), svc.GetOpcRequestID())
+}
+
+// newAPIError builds an APIError classified by its HTTP status, as wrap does
+// for the SDK's errors and the fakes do for the failures they model.
+func newAPIError(op string, status int, code, message, requestID string) *APIError {
+	api := &APIError{Op: op, Code: code, Message: message, StatusCode: status, RequestID: requestID}
 	switch {
-	case svc.GetHTTPStatusCode() == 404:
+	case status == 404:
 		api.kind = ErrNotFound
-	case svc.GetHTTPStatusCode() == 409, svc.GetHTTPStatusCode() == 412:
+	case status == 409, status == 412:
 		api.kind = ErrConflict
-	case svc.GetHTTPStatusCode() == 400 && svc.GetCode() == "LimitExceeded":
+	case status == 400 && code == "LimitExceeded":
 		api.kind = ErrLimitExceeded
-	case svc.GetHTTPStatusCode() == 429:
+	case status == 429:
 		api.kind = ErrLimitExceeded
 	}
 	return api

spinifex/providers/cloud/oci/export_test.go

@@ -1,6 +1,7 @@
 package oci
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
@@ -38,3 +39,19 @@ func NewTestClient(provider common.ConfigurationProvider, endpoint string, hc *h
 	c.HTTPClient = hc
 	return &apiClient{net: c}, nil
 }
+
+// NewTestBlockClient points the real SDK-backed BlockClient at a local
+// endpoint, for both the block storage and the compute half.
+func NewTestBlockClient(provider common.ConfigurationProvider, endpoint string, hc *http.Client) (BlockClient, error) {
+	client, err := NewBlockClientWithProvider(provider)
+	if err != nil {
+		return nil, err
+	}
+	c, ok := client.(*blockAPIClient)
+	if !ok {
+		return nil, fmt.Errorf("NewBlockClientWithProvider built a %T", client)
+	}
+	c.storage.Host, c.storage.HTTPClient = endpoint, hc
+	c.compute.Host, c.compute.HTTPClient = endpoint, hc
+	return c, nil
+}

spinifex/providers/cloud/oci/fake.go

@@ -15,6 +15,8 @@ import (
 // models the two behaviours that actually bite: OCI picks the address, and
 // assignment is asynchronous so a create is not immediately ASSIGNED.
 type Fake struct {
+	stagedFailures
+
 	mu sync.Mutex
 
 	// NextAddr supplies successive private addresses when the caller does not
@@ -27,10 +29,6 @@ type Fake struct {
 	// reporting ASSIGNED. Zero means assigned immediately.
 	AssignAfter int
 
-	// FailWith, when non-nil, is returned by the next call to the named
-	// operation and then cleared. Keyed by the Op strings used in APIError.
-	FailWith map[string]error
-
 	// PrivateLimit caps private IPs per VNIC; 0 means unlimited. OCI's real
 	// cap is 64.
 	PrivateLimit int
@@ -48,10 +46,11 @@ func NewFake() *Fake {
 	return &Fake{
 		NextAddr:   netip.MustParseAddr("10.200.0.100"),
 		NextPublic: netip.MustParseAddr("203.0.113.10"),
-		FailWith:   map[string]error{},
 		privateIPs: map[string]PrivateIP{},
 		publicIPs:  map[string]PublicIP{},
 		pending:    map[string]int{},
+
+		stagedFailures: stagedFailures{FailWith: map[string]error{}},
 	}
 }
 
@@ -102,18 +101,29 @@ func (f *Fake) ListReservedPublicIPs(_ context.Context, _ string) ([]PublicIP, e
 	return out, nil
 }
 
-// FailOp stages err for the next call to op. Locked, so a test may stage one
-// against a Fake that a loop under test is already calling.
-func (f *Fake) FailOp(op string, err error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.FailWith[op] = err
+// stagedFailures is the failure a fake returns from an operation's next call.
+type stagedFailures struct {
+	lock sync.Mutex
+
+	// FailWith, when non-nil, is returned by the next call to the named
+	// operation and then cleared. Keyed by the Op strings used in APIError.
+	FailWith map[string]error
+}
+
+// FailOp stages err for the next call to op, which then changes nothing.
+// Locked, so a test may stage one against a fake a loop is already calling.
+func (s *stagedFailures) FailOp(op string, err error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	s.FailWith[op] = err
 }
 
 // take returns and clears a staged failure for op.
-func (f *Fake) take(op string) error {
-	if err, ok := f.FailWith[op]; ok {
-		delete(f.FailWith, op)
+func (s *stagedFailures) take(op string) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if err, ok := s.FailWith[op]; ok {
+		delete(s.FailWith, op)
 		return err
 	}
 	return nil

spinifex/providers/cloud/oci/types.go

@@ -1,5 +1,6 @@
 // Package oci wraps the subset of the Oracle Cloud API that Spinifex needs to
-// hand out publicly reachable addresses on an OCI instance.
+// hand out publicly reachable addresses on an OCI instance, and, in block.go,
+// to keep EBS volumes on OCI block volumes.
 //
 // OCI enforces the source IP per VNIC: a packet leaving the instance with a
 // source address that is not a registered private IP object on that VNIC is
@@ -21,6 +22,7 @@ package oci
 import (
 	"errors"
 	"fmt"
+	"net/http"
 	"net/netip"
 	"time"
 )
@@ -119,3 +121,10 @@ func (e *APIError) Error() string {
 
 // Unwrap returns the sentinel so errors.Is(err, ErrLimitExceeded) matches.
 func (e *APIError) Unwrap() error { return e.kind }
+
+// PreconditionFailed reports an HTTP 412: the resource changed since the ETag
+// an If-Match named.
+func PreconditionFailed(err error) bool {
+	var api *APIError
+	return errors.As(err, &api) && api.StatusCode == http.StatusPreconditionFailed
+}

spinifex/providers/ebs/conformance/bench.go

@@ -131,7 +131,7 @@ func RunBenchSuite(b *testing.B, provider ebsprovider.EBSProvider, cfg BenchConf
 		b.ResetTimer()
 		for b.Loop() {
 			if _, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{
-				Versioned: ebsprovider.NewVersioned(),
+				Versioned: ebsprovider.NewLatestVersioned(),
 				VolumeID:  volumeID,
 				NodeID:    cfg.node(),
 			}); err != nil {

spinifex/providers/ebs/conformance/conformance.go

@@ -30,6 +30,7 @@ var ReferenceCapabilities = ebsprovider.Capabilities{
 	ReadOnlyPublish:         true,
 	VolumeEnumeration:       true,
 	SnapshotEnumeration:     true,
+	SnapshotCopy:            true,
 	// MemoryProvider keeps publication in one process's map, so node is the
 	// honest answer. A reference set that claimed cluster would make the
 	// suite's concurrency subtests pass against something that cannot hold.
@@ -50,6 +51,18 @@ func capabilitiesOf(t *testing.T, provider ebsprovider.EBSProvider) *ebsprovider
 // fails the test instead of hanging it.
 const maxListPages = 100
 
+// unreadableAt reports whether a caller at version cannot read the export a
+// provider publishes at the latest version: a block export has no v1 form.
+func unreadableAt(latest *ebsprovider.PublishedVolume, version uint16) bool {
+	return version < ebsprovider.DescriptorSchemaVersion && latest.Export.Kind == ebsprovider.ExportKindBlock
+}
+
+func requireUnreadableRefused(t *testing.T, err error, version uint16) {
+	t.Helper()
+	require.ErrorIsf(t, err, ebsprovider.ErrUnsupportedVersion,
+		"a version %d caller cannot read a block export, so its publish must be refused", version)
+}
+
 // unpublish is UnpublishVolume for a case that asserts only on the error.
 func unpublish(ctx context.Context, provider ebsprovider.EBSProvider, req ebsprovider.UnpublishVolumeRequest) error {
 	_, err := provider.UnpublishVolume(ctx, req)
@@ -153,7 +166,14 @@ func readSnapshot(t *testing.T, provider ebsprovider.EBSProvider, snapshotID str
 
 // settleTimeout bounds the wait for a pending snapshot to complete, so a
 // provider that never settles one fails the test instead of hanging it.
-const settleTimeout = 30 * time.Second
+const settleTimeout = 5 * time.Minute
+
+// The settle poll starts short, so a fast provider settles at once, and
+// doubles to a cap, so a slow backend is read every few seconds.
+const (
+	settlePollMin = 10 * time.Millisecond
+	settlePollMax = 2 * time.Second
+)
 
 // createSnapshot takes snapshotID of volumeID and waits, through the filtered
 // ListSnapshots, until it is completed: CreateSnapshot may answer pending, so
@@ -168,6 +188,7 @@ func createSnapshot(t *testing.T, provider ebsprovider.EBSProvider, snapshotID,
 	require.Equal(t, snapshotID, snap.ID)
 
 	deadline := time.Now().Add(settleTimeout)
+	poll := settlePollMin
 	for {
 		ref := readSnapshot(t, provider, snapshotID)
 		switch ref.State {
@@ -179,7 +200,8 @@ func createSnapshot(t *testing.T, provider ebsprovider.EBSProvider, snapshotID,
 			require.Failf(t, "snapshot did not complete", "snapshot %s settled to %q", snapshotID, ref.State)
 		}
 		require.Truef(t, time.Now().Before(deadline), "snapshot %s still pending after %s", snapshotID, settleTimeout)
-		time.Sleep(10 * time.Millisecond)
+		time.Sleep(poll)
+		poll = min(2*poll, settlePollMax)
 	}
 }
 
@@ -611,7 +633,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-inuse"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-inuse"), NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.id("vol-expand-inuse"), NodeID: cfg.node()})
 			require.NoError(t, err)
 
 			expanded, err := provider.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-inuse"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}})
@@ -648,7 +670,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-inuse"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-inuse"), NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.id("vol-delete-inuse"), NodeID: cfg.node()})
 			require.NoError(t, err)
 			err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-inuse")})
 			require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
@@ -752,7 +774,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			volumeID := cfg.id("vol-snap-held")
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 
 			for _, version := range contractVersions {
@@ -787,6 +809,38 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 	})
 
 	t.Run("CopySnapshot", func(t *testing.T) {
+		t.Run("unsupported_version", func(t *testing.T) {
+			provider := newProvider(t)
+			_, err := provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{
+				SourceSnapshotID: cfg.id("snap-a"), DestinationSnapshotID: cfg.id("snap-b"), VolumeID: cfg.id("vol-a"),
+			})
+			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+		})
+
+		// A provider that cannot copy must say so, or a caller reads the
+		// refusal as a failed copy worth retrying.
+		t.Run("unsupported_capability when copy is not advertised", func(t *testing.T) {
+			if capabilities.SnapshotCopy {
+				t.Skip("provider advertises snapshot copy")
+			}
+			provider := newProvider(t)
+			ctx := context.Background()
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-unsup"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			createSnapshot(t, provider, cfg.id("snap-copysnap-unsup"), cfg.id("vol-copysnap-unsup"))
+			_, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{
+				Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-unsup"), DestinationSnapshotID: cfg.id("snap-copysnap-unsup-dst"), VolumeID: cfg.id("vol-copysnap-unsup"),
+			})
+			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability)
+			if capabilities.SnapshotEnumeration {
+				assert.NotContains(t, drainSnapshotIDs(t, provider, 0), cfg.id("snap-copysnap-unsup-dst"), "a refused copy must leave no snapshot behind")
+			}
+		})
+
+		if !capabilities.SnapshotCopy {
+			return
+		}
+
 		t.Run("success", func(t *testing.T) {
 			provider := newProvider(t)
 			ctx := context.Background()
@@ -864,14 +918,6 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			})
 			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
 		})
-
-		t.Run("unsupported_version", func(t *testing.T) {
-			provider := newProvider(t)
-			_, err := provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{
-				SourceSnapshotID: cfg.id("snap-a"), DestinationSnapshotID: cfg.id("snap-b"), VolumeID: cfg.id("vol-a"),
-			})
-			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
-		})
 	})
 
 	t.Run("ListSnapshots", func(t *testing.T) {
@@ -1046,7 +1092,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-ok"), NodeID: cfg.node()})
+			pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.id("vol-pub-ok"), NodeID: cfg.node()})
 			require.NoError(t, err)
 			require.NotNil(t, pub)
 			assert.Equal(t, cfg.id("vol-pub-ok"), pub.VolumeID)
@@ -1067,9 +1113,13 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 					volumeID := cfg.id(fmt.Sprintf("vol-pub-idem-v%d", version))
 					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 					require.NoError(t, err)
-					first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 					require.NoError(t, err)
 					second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node()})
+					if unreadableAt(first, version) {
+						requireUnreadableRefused(t, err, version)
+						return
+					}
 					require.NoError(t, err)
 					assert.Equal(t, first, second)
 				})
@@ -1084,9 +1134,9 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-conflict"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-conflict"), NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.id("vol-pub-conflict"), NodeID: cfg.node()})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-conflict"), NodeID: cfg.otherNode()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.id("vol-pub-conflict"), NodeID: cfg.otherNode()})
 			require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
 		})
 
@@ -1101,10 +1151,14 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 					volumeID := cfg.id(fmt.Sprintf("vol-pub-held-v%d", version))
 					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 					require.NoError(t, err)
-					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 					require.NoError(t, err)
 
 					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.otherNode()})
+					if unreadableAt(first, version) {
+						requireUnreadableRefused(t, err, version)
+						return
+					}
 					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "every version must still read the refusal as in use")
 					if version >= ebsprovider.HeldElsewhereSchemaVersion {
 						assert.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
@@ -1127,10 +1181,14 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 					volumeID := cfg.id(fmt.Sprintf("vol-pub-mode-v%d", version))
 					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 					require.NoError(t, err)
-					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 					require.NoError(t, err)
 
 					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node(), ReadOnly: true})
+					if unreadableAt(first, version) {
+						requireUnreadableRefused(t, err, version)
+						return
+					}
 					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
 					assert.NotErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere,
 						"this node holds the volume, so a caller told it is held elsewhere would wait for a node that does not exist")
@@ -1148,14 +1206,14 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			volumeID := cfg.id("vol-pub-dead")
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 			firstURI := dialURI(t, publishedNBDURI(t, first))
 
 			cfg.KillExport(t, first)
 			require.False(t, nbdReachable(firstURI), "the export still answers, so the kill did not take and this case proves nothing")
 
-			second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err, "a dead export must not make its volume unpublishable")
 			nbdInfoRetry(t, dialURI(t, publishedNBDURI(t, second)))
 		})
@@ -1169,7 +1227,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			volumeID := cfg.id("vol-pub-revoked")
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 
 			notice := cfg.RevokeExport(t, published)
@@ -1180,13 +1238,13 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 
 		t.Run("not_found on absent volume", func(t *testing.T) {
 			provider := newProvider(t)
-			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})
+			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})
 			require.ErrorIs(t, err, ebsprovider.ErrNotFound)
 		})
 
 		t.Run("invalid_argument on missing ids", func(t *testing.T) {
 			provider := newProvider(t)
-			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned()})
+			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned()})
 			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
 		})
 	})
@@ -1197,7 +1255,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()})
 			require.NoError(t, err)
 			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()}))
 		})
@@ -1217,7 +1275,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			v3 := ebsprovider.Versioned{SchemaVersion: ebsprovider.NoSealSchemaVersion}
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 
 			first, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node()})
@@ -1239,7 +1297,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 					volumeID := cfg.id(fmt.Sprintf("vol-unpub-noseal-v%d", version))
 					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 					require.NoError(t, err)
-					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 					require.NoError(t, err)
 
 					err = unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
@@ -1258,7 +1316,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			v3 := ebsprovider.Versioned{SchemaVersion: ebsprovider.NoSealSchemaVersion}
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 
 			result, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
@@ -1270,7 +1328,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			again, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
 			require.NoError(t, err)
 			assert.False(t, again.TornDown)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err, "a volume dropped without sealing must still be publishable")
 		})
 
@@ -1288,7 +1346,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			volumeID := cfg.id("vol-unsealed")
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 
 			asked := []string{volumeID, cfg.id("vol-unsealed-never")}
@@ -1343,17 +1401,17 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			volumeID := cfg.id("vol-excl-second")
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()})
 			require.ErrorIsf(t, err, ebsprovider.ErrVolumeInUse,
 				"scope %q promises a second writer is refused; letting this through is the two-writer case the guarantee exists to prevent", exclusion.Scope)
 
 			// Releasing the claim must make the volume publishable again, or
 			// the guarantee is indistinguishable from a permanent lock.
 			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()})
 			require.NoError(t, err, "a released claim must be reclaimable, or an unpublish strands the volume")
 		})
 
@@ -1378,7 +1436,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 				wg.Go(func() {
 					<-start
 					_, errs[i] = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
-						Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: node,
+						Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: node,
 					})
 				})
 			}
@@ -1406,7 +1464,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			volumeID := cfg.id("vol-excl-delrace")
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
 			require.NoError(t, err)
 
 			err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID})

spinifex/providers/ebs/conformance/nbdclient.go

@@ -191,7 +191,7 @@ func publishForNBD(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClient
 	t.Cleanup(func() { cleanupVolume(t, provider, cfg, volumeID) })
 
 	pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned: ebsprovider.NewLatestVersioned(),
 		VolumeID:  volumeID,
 		NodeID:    cfg.NodeID,
 		ReadOnly:  readOnly,
@@ -337,7 +337,7 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
 			t.Cleanup(func() { cleanupVolume(t, provider, cfg, cfg.VolumePrefix+"-readonly") })
 
 			_, err = provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{
-				Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.VolumePrefix + "-readonly",
+				Versioned: ebsprovider.NewLatestVersioned(), VolumeID: cfg.VolumePrefix + "-readonly",
 				NodeID: cfg.NodeID, ReadOnly: true,
 			})
 			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability)

spinifex/providers/ebs/errors.go

@@ -110,7 +110,8 @@ func (e *ProviderError) Unwrap() error {
 	}
 }
 
-func checkVersion(version uint16) error {
+// CheckVersion refuses a schema version outside the served window.
+func CheckVersion(version uint16) error {
 	if !SupportsSchemaVersion(version) {
 		return fmt.Errorf("%w: got %d, %s", ErrUnsupportedVersion, version, SupportedVersionsText())
 	}

spinifex/providers/ebs/errors_test.go

@@ -72,8 +72,8 @@ func TestErrorCodeRetryable(t *testing.T) {
 }
 
 func TestCheckVersion(t *testing.T) {
-	assert.NoError(t, checkVersion(LegacySchemaVersion))
-	assert.NoError(t, checkVersion(SchemaVersion))
-	assert.ErrorIs(t, checkVersion(SchemaVersion+1), ErrUnsupportedVersion)
-	assert.ErrorIs(t, checkVersion(0), ErrUnsupportedVersion)
+	assert.NoError(t, CheckVersion(LegacySchemaVersion))
+	assert.NoError(t, CheckVersion(SchemaVersion))
+	assert.ErrorIs(t, CheckVersion(SchemaVersion+1), ErrUnsupportedVersion)
+	assert.ErrorIs(t, CheckVersion(0), ErrUnsupportedVersion)
 }

spinifex/providers/ebs/memory.go

@@ -64,7 +64,7 @@ func NewMemoryProvider(capabilities Capabilities) *MemoryProvider {
 }
 
 func (m *MemoryProvider) GetCapabilities(_ context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	return &GetCapabilitiesResponse{
@@ -75,11 +75,10 @@ func (m *MemoryProvider) GetCapabilities(_ context.Context, req GetCapabilitiesR
 }
 
 func (m *MemoryProvider) CreateVolume(_ context.Context, req CreateVolumeRequest) (*Volume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
-	if req.VolumeID == "" || req.CapacityRange.RequiredBytes <= 0 ||
-		(req.CapacityRange.LimitBytes > 0 && req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes) {
+	if req.VolumeID == "" || !req.CapacityRange.Valid() {
 		return nil, fmt.Errorf("%w: invalid volume ID or capacity range", ErrInvalidArgument)
 	}
 	if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" {
@@ -155,7 +154,7 @@ func (m *MemoryProvider) SeedData(volumeID string) ([]byte, bool) {
 }
 
 func (m *MemoryProvider) GetVolume(_ context.Context, req GetVolumeRequest) (*Volume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	if req.VolumeID == "" {
@@ -174,7 +173,7 @@ func (m *MemoryProvider) GetVolume(_ context.Context, req GetVolumeRequest) (*Vo
 // a token stays meaningful across calls. The token is the ID to resume after,
 // which keeps paging correct when volumes are created or deleted mid-walk.
 func (m *MemoryProvider) ListVolumes(_ context.Context, req ListVolumesRequest) (*ListVolumesResponse, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	if !m.capabilities.VolumeEnumeration {
@@ -194,11 +193,10 @@ func (m *MemoryProvider) ListVolumes(_ context.Context, req ListVolumesRequest)
 }
 
 func (m *MemoryProvider) ExpandVolume(_ context.Context, req ExpandVolumeRequest) (*Volume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
-	if req.VolumeID == "" || req.CapacityRange.RequiredBytes <= 0 ||
-		(req.CapacityRange.LimitBytes > 0 && req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes) {
+	if req.VolumeID == "" || !req.CapacityRange.Valid() {
 		return nil, fmt.Errorf("%w: invalid volume ID or capacity range", ErrInvalidArgument)
 	}
 	m.mu.Lock()
@@ -218,7 +216,7 @@ func (m *MemoryProvider) ExpandVolume(_ context.Context, req ExpandVolumeRequest
 }
 
 func (m *MemoryProvider) DeleteVolume(_ context.Context, req DeleteVolumeRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return err
 	}
 	if req.VolumeID == "" {
@@ -241,7 +239,7 @@ func (m *MemoryProvider) DeleteVolume(_ context.Context, req DeleteVolumeRequest
 }
 
 func (m *MemoryProvider) CreateSnapshot(_ context.Context, req CreateSnapshotRequest) (*Snapshot, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	if req.SnapshotID == "" || req.VolumeID == "" {
@@ -341,7 +339,7 @@ func (m *MemoryProvider) Snapshot(snapshotID string) (*Snapshot, bool) {
 }
 
 func (m *MemoryProvider) DeleteSnapshot(_ context.Context, req DeleteSnapshotRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return err
 	}
 	if req.SnapshotID == "" {
@@ -362,7 +360,7 @@ func (m *MemoryProvider) DeleteSnapshot(_ context.Context, req DeleteSnapshotReq
 // destination, an empty or equal ID pair, and a snapshot that does not
 // belong to VolumeID (the caller's claimed source volume).
 func (m *MemoryProvider) CopySnapshot(_ context.Context, req CopySnapshotRequest) (*Snapshot, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	if req.SourceSnapshotID == "" || req.DestinationSnapshotID == "" || req.VolumeID == "" {
@@ -371,6 +369,9 @@ func (m *MemoryProvider) CopySnapshot(_ context.Context, req CopySnapshotRequest
 	if req.SourceSnapshotID == req.DestinationSnapshotID {
 		return nil, fmt.Errorf("%w: destination snapshot must differ from the source", ErrInvalidArgument)
 	}
+	if !m.capabilities.SnapshotCopy {
+		return nil, fmt.Errorf("%w: snapshot copy", ErrUnsupportedCapability)
+	}
 
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -442,7 +443,7 @@ func memorySnapshotRef(version uint16, snapshot *Snapshot) SnapshotRef {
 }
 
 func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeRequest) (*PublishedVolume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	if req.VolumeID == "" || req.NodeID == "" {

spinifex/providers/ebs/memory_test.go

@@ -111,7 +111,7 @@ func TestProviderErrorUnwrap(t *testing.T) {
 	assert.Equal(t, "volume disappeared", err.Error())
 }
 
-// TestMemoryProviderChecksVersionOnEveryMethod covers the checkVersion guard
+// TestMemoryProviderChecksVersionOnEveryMethod covers the CheckVersion guard
 // on every one of the nine EBSProvider methods directly on MemoryProvider,
 // independent of NATSProvider's own client-side pre-check.
 func TestMemoryProviderChecksVersionOnEveryMethod(t *testing.T) {
@@ -364,7 +364,7 @@ func TestMemoryProviderListSnapshotsCarriesSourceVolume(t *testing.T) {
 // case where copying would either lose data or attribute it to the wrong
 // volume.
 func TestMemoryProviderCopySnapshot(t *testing.T) {
-	provider := NewMemoryProvider(Capabilities{})
+	provider := NewMemoryProvider(Capabilities{SnapshotCopy: true})
 	copiedAt := time.Date(2026, 8, 5, 6, 0, 0, 0, time.UTC)
 	provider.now = func() time.Time { return copiedAt }
 	ctx := context.Background()

spinifex/providers/ebs/nats.go

@@ -55,18 +55,23 @@ var dataMovementVerbs = map[string]bool{
 	verbVolumeUnpublish: true,
 }
 
+// PublishTimeout bounds a publish, which on a remote block backend waits for
+// the backend to attach the volume and for its device to appear. A provider
+// must answer within it, or the caller cannot tell whether the volume is exported.
+const PublishTimeout = 5 * time.Minute
+
 // timeoutFor sizes a request to the work its subject names.
 func (p *NATSProvider) timeoutFor(subject string) time.Duration {
-	if movesData(subject) {
+	verb := parseSubject(subject).verb
+	switch {
+	case dataMovementVerbs[verb]:
 		return max(p.requestTimeout, dataMovementTimeout)
+	case verb == verbVolumePublish:
+		return max(p.requestTimeout, PublishTimeout)
 	}
 	return p.requestTimeout
 }
 
-func movesData(subject string) bool {
-	return dataMovementVerbs[parseSubject(subject).verb]
-}
-
 // SnapshotSubject addresses a create by source volume so the owning node can
 // serve it. The create verb is its own token: a bare volume ID here would
 // collide with DeleteSnapshotSubject and make the wildcard unsubscribable.
@@ -256,7 +261,8 @@ var (
 )
 
 // NewNATSProvider returns a provider client over conn. A non-positive
-// requestTimeout takes the 30s default; data-moving subjects get at least 180s.
+// requestTimeout takes the 30s default; data-moving subjects get at least 180s
+// and a publish at least PublishTimeout.
 func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvider {
 	if requestTimeout <= 0 {
 		requestTimeout = defaultRequestTimeout
@@ -265,7 +271,7 @@ func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvide
 }
 
 func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response GetCapabilitiesResponse
@@ -279,7 +285,7 @@ func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesR
 }
 
 func (p *NATSProvider) CreateVolume(ctx context.Context, req CreateVolumeRequest) (*Volume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	if err := ValidateSeedData(req.SeedData); err != nil {
@@ -299,7 +305,7 @@ func (p *NATSProvider) CreateVolume(ctx context.Context, req CreateVolumeRequest
 }
 
 func (p *NATSProvider) GetVolume(ctx context.Context, req GetVolumeRequest) (*Volume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	ownerSubject, err := GetVolumeOwnerSubject(req.VolumeID)
@@ -323,7 +329,7 @@ func (p *NATSProvider) GetVolume(ctx context.Context, req GetVolumeRequest) (*Vo
 // question is about the whole provider, not one volume, so it goes to the
 // queue group rather than to any single volume's mounting node.
 func (p *NATSProvider) ListVolumes(ctx context.Context, req ListVolumesRequest) (*ListVolumesResponse, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response ListVolumesResponse
@@ -337,7 +343,7 @@ func (p *NATSProvider) ListVolumes(ctx context.Context, req ListVolumesRequest)
 }
 
 func (p *NATSProvider) ExpandVolume(ctx context.Context, req ExpandVolumeRequest) (*Volume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	ownerSubject, err := ExpandVolumeOwnerSubject(req.VolumeID)
@@ -358,7 +364,7 @@ func (p *NATSProvider) ExpandVolume(ctx context.Context, req ExpandVolumeRequest
 }
 
 func (p *NATSProvider) DeleteVolume(ctx context.Context, req DeleteVolumeRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return err
 	}
 	var response DeleteVolumeResponse
@@ -369,7 +375,7 @@ func (p *NATSProvider) DeleteVolume(ctx context.Context, req DeleteVolumeRequest
 }
 
 func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotRequest) (*Snapshot, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	requestSubject, err := SnapshotSubject(req.VolumeID)
@@ -441,7 +447,7 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 }
 
 func (p *NATSProvider) DeleteSnapshot(ctx context.Context, req DeleteSnapshotRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return err
 	}
 	var response DeleteSnapshotResponse
@@ -456,7 +462,7 @@ func (p *NATSProvider) DeleteSnapshot(ctx context.Context, req DeleteSnapshotReq
 // of small object writes with no flush/upload step slow enough to warrant a
 // completion subject.
 func (p *NATSProvider) CopySnapshot(ctx context.Context, req CopySnapshotRequest) (*Snapshot, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	ownerSubject, err := SnapshotCopyOwnerSubject(req.VolumeID)
@@ -504,7 +510,7 @@ func snapshotRefIDs(refs []SnapshotRef) []string {
 }
 
 func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeRequest) (*PublishedVolume, error) {
-	if err := checkVersion(req.SchemaVersion); err != nil {
+	if err := CheckVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	subject, err := PublishSubject(req.NodeID)
@@ -654,7 +660,7 @@ func (p *NATSProvider) requestOwnerFirst(ctx context.Context, ownerSubject, queu
 }
 
 func responseError(version uint16, providerErr *ProviderError) error {
-	if err := checkVersion(version); err != nil {
+	if err := CheckVersion(version); err != nil {
 		return fmt.Errorf("%w: %w", ErrUnreadableReply, err)
 	}
 	if providerErr != nil {

spinifex/providers/ebs/nats_test.go

@@ -230,7 +230,7 @@ func TestSubjectBuildersRejectUnsafeTokens(t *testing.T) {
 
 // TestNATSProviderChecksVersionBeforeTransport confirms every EBSProvider
 // method on NATSProvider rejects an unsupported SchemaVersion via
-// checkVersion before it ever touches the connection. A nil *nats.Conn would
+// CheckVersion before it ever touches the connection. A nil *nats.Conn would
 // panic or return nats.ErrConnectionClosed if any method reached transport,
 // so seeing ErrUnsupportedVersion here proves the ordering.
 func TestNATSProviderChecksVersionBeforeTransport(t *testing.T) {
@@ -794,12 +794,15 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	fast := []string{
 		CapabilitiesSubject, GetVolumeSubject, ListVolumesSubject,
 		ListSnapshotsSubject, ownerDescribe,
-		publish, unsealed,
+		unsealed,
 	}
 	for _, subject := range fast {
 		assert.Equal(t, callerTimeout, p.timeoutFor(subject),
 			"%s answers from state already held, so it keeps the caller's timeout", subject)
 	}
+
+	assert.Equal(t, PublishTimeout, p.timeoutFor(publish),
+		"a publish waits for the backend to attach the volume, so it must not inherit a describe's budget")
 }
 
 // TestTimeoutForNeverShortensTheCallersBudget covers a caller that already asks

spinifex/providers/ebs/natsserve/handlers_test.go

@@ -200,10 +200,11 @@ func TestHandlersReplyInRequestVersion(t *testing.T) {
 }
 
 // seededProvider holds what every wireVerbs request needs to succeed: vol-1
-// and snap-1 taken from it, with enumeration advertised so listing succeeds.
+// and snap-1 taken from it, with enumeration and copy advertised so listing
+// and copying succeed.
 func seededProvider(t *testing.T) ebsprovider.EBSProvider {
 	t.Helper()
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true, SnapshotEnumeration: true})
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true, SnapshotEnumeration: true, SnapshotCopy: true})
 	ctx := context.Background()
 	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 	require.NoError(t, err)
@@ -353,6 +354,7 @@ func TestServedProviderRoundTrip(t *testing.T) {
 	capabilities := ebsprovider.Capabilities{
 		VolumeEnumeration:   true,
 		SnapshotEnumeration: true,
+		SnapshotCopy:        true,
 		OnlineExpansion:     true,
 	}
 	conn := serveProvider(t, ebsprovider.NewMemoryProvider(capabilities), natsserve.Options{NoQueueGroup: true})

spinifex/providers/ebs/nullprovider/nullprovider.go

@@ -29,6 +29,7 @@ var Capabilities = ebsprovider.Capabilities{
 	ReadOnlyPublish:         true,
 	VolumeEnumeration:       true,
 	SnapshotEnumeration:     true,
+	SnapshotCopy:            true,
 	Exclusion:               ebsprovider.ExclusionSemantics{Scope: ebsprovider.ExclusionScopeNode},
 }
 

spinifex/providers/ebs/ocibsd/budget_test.go

@@ -0,0 +1,18 @@
+package ocibsd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+)
+
+// A publish's attach and device waits must end inside its own bound, and that
+// bound inside the client's, or a slow attach reaches the caller as a timeout
+// that leaves unknown whether the volume was exported.
+func TestPublishWaitsFitTheClientsPublishBudget(t *testing.T) {
+	waits := defaultAttachTimeout + defaultPollInterval + defaultDeviceTimeout
+	assert.Less(t, waits, defaultPublishTimeout, "the attach and device waits must fit the publish bound")
+	assert.Less(t, defaultPublishTimeout, ebsprovider.PublishTimeout, "the publish bound must answer before the client gives up")
+}

spinifex/providers/ebs/ocibsd/conformance_test.go

@@ -0,0 +1,37 @@
+package ocibsd_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/natsserve"
+)
+
+func TestConformanceInProcess(t *testing.T) {
+	conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		c := newCluster(t)
+		return settling{EBSProvider: c.router(), fake: c.fake}
+	})
+}
+
+// Each node serves its own per-node subjects and joins the shared queue, as
+// the per-node service role does.
+func TestConformanceOverNATS(t *testing.T) {
+	conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		c := newCluster(t)
+		_, conn := testutil.StartTestNATS(t)
+		for node, p := range c.nodes {
+			stop, err := natsserve.Serve(t.Context(), conn, settling{EBSProvider: p, fake: c.fake}, natsserve.Options{NodeID: node})
+			require.NoError(t, err)
+			t.Cleanup(stop)
+		}
+		return ebsprovider.NewNATSProvider(conn, 5*time.Second)
+	})
+}

spinifex/providers/ebs/ocibsd/export_test.go

@@ -0,0 +1,5 @@
+package ocibsd
+
+// ExitCode lets the live test read a remote command's exit status as the
+// iscsiadm initiator does.
+var ExitCode = exitCode

spinifex/providers/ebs/ocibsd/helpers_test.go

@@ -0,0 +1,147 @@
+package ocibsd_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/ocibsd"
+)
+
+const (
+	testCluster     = "cluster-1"
+	testCompartment = "ocid1.compartment.oc1..test"
+	testAD          = "Uocm:US-ASHBURN-AD-1"
+	testAZ          = "ap-southeast-2a"
+)
+
+// cluster is two nodes' providers over one OCI tenancy, each with its own
+// instance and iSCSI initiator.
+type cluster struct {
+	fake       *oci.BlockFake
+	nodes      map[string]*ocibsd.Provider
+	initiators map[string]*ocibsd.FakeInitiator
+}
+
+func instanceOf(nodeID string) string { return "ocid1.instance.oc1..of-" + nodeID }
+
+func testConfig(nodeID string, client oci.BlockClient, initiator ocibsd.Initiator) ocibsd.Config {
+	return ocibsd.Config{
+		ClusterID: testCluster, CompartmentID: testCompartment,
+		AvailabilityDomain: testAD, AvailabilityZone: testAZ,
+		NodeID: nodeID, InstanceID: instanceOf(nodeID), InstanceAvailabilityDomain: testAD,
+		Client: client, Initiator: initiator,
+		PollInterval: time.Millisecond, VolumeTimeout: time.Second, AttachTimeout: time.Second, DetachTimeout: time.Second,
+	}
+}
+
+// newNode builds node-1's provider over client, unstarted, with opts applied
+// to its config.
+func newNode(t *testing.T, client oci.BlockClient, initiator ocibsd.Initiator, opts ...func(*ocibsd.Config)) *ocibsd.Provider {
+	t.Helper()
+	cfg := testConfig("node-1", client, initiator)
+	for _, opt := range opts {
+		opt(&cfg)
+	}
+	p, err := ocibsd.New(cfg)
+	require.NoError(t, err)
+	return p
+}
+
+// onNode places a provider on nodeID and its instance.
+func onNode(nodeID string) func(*ocibsd.Config) {
+	return func(cfg *ocibsd.Config) { cfg.NodeID, cfg.InstanceID = nodeID, instanceOf(nodeID) }
+}
+
+func newCluster(t *testing.T) *cluster {
+	t.Helper()
+	return newClusterOver(t, oci.NewBlockFake(), nil)
+}
+
+// newClusterOver builds the cluster over fake, reached through client when
+// one is given, so a test can interpose on the OCI calls.
+func newClusterOver(t *testing.T, fake *oci.BlockFake, client oci.BlockClient) *cluster {
+	t.Helper()
+	if client == nil {
+		client = fake
+	}
+	c := &cluster{fake: fake, nodes: map[string]*ocibsd.Provider{}, initiators: map[string]*ocibsd.FakeInitiator{}}
+	for _, node := range []string{"node-1", "node-2"} {
+		initiator := ocibsd.NewFakeInitiator()
+		p := newNode(t, client, initiator, onNode(node))
+		require.NoError(t, p.Start(context.Background()))
+		c.nodes[node], c.initiators[node] = p, initiator
+	}
+	return c
+}
+
+// router answers the verbs any node can answer on node-1, and sends each
+// per-node verb to the node it names, as the NATS subjects do.
+type router struct {
+	*ocibsd.Provider
+
+	c *cluster
+}
+
+var _ ebsprovider.EBSProvider = router{}
+
+func (c *cluster) router() router { return router{Provider: c.nodes["node-1"], c: c} }
+
+func (r router) node(nodeID string) *ocibsd.Provider {
+	if p, ok := r.c.nodes[nodeID]; ok {
+		return p
+	}
+	return r.Provider
+}
+
+func (r router) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
+	return r.node(req.NodeID).PublishVolume(ctx, req)
+}
+
+func (r router) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	return r.node(req.NodeID).UnpublishVolume(ctx, req)
+}
+
+func (r router) HoldsUnsealedWrites(ctx context.Context, nodeID string, volumeIDs []string) ([]string, error) {
+	return r.node(nodeID).HoldsUnsealedWrites(ctx, nodeID, volumeIDs)
+}
+
+// settling finishes a backup once a filtered read has seen it pending, so a
+// wait for a pending snapshot is exercised and still ends.
+type settling struct {
+	ebsprovider.EBSProvider
+
+	fake *oci.BlockFake
+}
+
+func (s settling) ListSnapshots(ctx context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
+	resp, err := s.EBSProvider.ListSnapshots(ctx, req)
+	if err == nil && req.SnapshotID != "" && len(resp.Snapshots) == 1 && resp.Snapshots[0].State == ebsprovider.SnapshotStatePending {
+		if err := s.fake.SettleBackup(resp.Snapshots[0].Handle, oci.BackupAvailable); err != nil {
+			return nil, err
+		}
+	}
+	return resp, err
+}
+
+var latest = ebsprovider.NewLatestVersioned()
+
+func createVolume(t *testing.T, p ebsprovider.EBSProvider, volumeID string, capacity int64) *ebsprovider.Volume {
+	t.Helper()
+	v, err := p.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: capacity},
+	})
+	require.NoError(t, err)
+	return v
+}
+
+func ociVolume(t *testing.T, fake *oci.BlockFake, handle string) oci.BlockVolume {
+	t.Helper()
+	v, err := fake.GetVolume(context.Background(), handle)
+	require.NoError(t, err)
+	return v
+}

spinifex/providers/ebs/ocibsd/initiator.go

@@ -0,0 +1,113 @@
+package ocibsd
+
+import (
+	"context"
+	"fmt"
+	"sync"
+)
+
+// Target is the iSCSI target an OCI attachment exposes.
+type Target struct {
+	IQN  string
+	IPv4 string
+	Port int
+}
+
+// ByPathDevice is the udev path of the target's one LUN.
+func ByPathDevice(t Target) string {
+	return fmt.Sprintf("/dev/disk/by-path/ip-%s:%d-iscsi-%s-lun-1", t.IPv4, t.Port, t.IQN)
+}
+
+// Initiator manages this node's iSCSI sessions.
+type Initiator interface {
+	// Login opens a session to t, or keeps the one open, and returns the
+	// block device once it exists.
+	Login(ctx context.Context, t Target) (devicePath string, err error)
+	// Logout closes any session to t and forgets it. With none it succeeds.
+	Logout(ctx context.Context, t Target) error
+	// Sessions lists the targets this node holds a session to.
+	Sessions(ctx context.Context) ([]Target, error)
+}
+
+// FakeInitiator is an in-memory Initiator that records the sessions it holds
+// and the logins and logouts it was asked for, in order.
+type FakeInitiator struct {
+	mu           sync.Mutex
+	sessions     map[Target]bool
+	failLogin    error
+	failSessions error
+	events       []string
+}
+
+var _ Initiator = (*FakeInitiator)(nil)
+
+// NewFakeInitiator returns a FakeInitiator holding no sessions.
+func NewFakeInitiator() *FakeInitiator {
+	return &FakeInitiator{sessions: map[Target]bool{}}
+}
+
+// FailNextLogin makes the next Login return err and open nothing.
+func (f *FakeInitiator) FailNextLogin(err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.failLogin = err
+}
+
+// FailSessions makes every Sessions call return err.
+func (f *FakeInitiator) FailSessions(err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.failSessions = err
+}
+
+// SeedSession opens a session no Login made, as one left by an earlier run.
+func (f *FakeInitiator) SeedSession(t Target) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.sessions[t] = true
+}
+
+// Open reports the targets with an open session.
+func (f *FakeInitiator) Open() []Target {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]Target, 0, len(f.sessions))
+	for t := range f.sessions {
+		out = append(out, t)
+	}
+	return out
+}
+
+// Events is every login and logout, as "login <iqn>" or "logout <iqn>".
+func (f *FakeInitiator) Events() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.events...)
+}
+
+func (f *FakeInitiator) Login(_ context.Context, t Target) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.events = append(f.events, "login "+t.IQN)
+	if err := f.failLogin; err != nil {
+		f.failLogin = nil
+		return "", err
+	}
+	f.sessions[t] = true
+	return ByPathDevice(t), nil
+}
+
+func (f *FakeInitiator) Logout(_ context.Context, t Target) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.events = append(f.events, "logout "+t.IQN)
+	delete(f.sessions, t)
+	return nil
+}
+
+func (f *FakeInitiator) Sessions(context.Context) ([]Target, error) {
+	if err := func() error { f.mu.Lock(); defer f.mu.Unlock(); return f.failSessions }(); err != nil {
+		return nil, err
+	}
+	return f.Open(), nil
+}

spinifex/providers/ebs/ocibsd/iscsiadm.go

@@ -0,0 +1,218 @@
+package ocibsd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ReplacementTimeoutSeconds is the node.session.timeo.replacement_timeout set
+// on every node record: a cut session fails its queued writes after this long
+// instead of holding them for the image's default.
+const ReplacementTimeoutSeconds = 120
+
+// iscsiadm exit codes (open-iscsi iscsi_err.h) that mean the work is done.
+const (
+	iscsiErrSessExists  = 15
+	iscsiErrNoObjsFound = 21
+)
+
+const (
+	defaultDeviceTimeout = 30 * time.Second
+	defaultDevicePoll    = 100 * time.Millisecond
+)
+
+// CommandRunner runs a host command with no shell between, returning its
+// combined output. A failed command's error must expose ExitCode().
+type CommandRunner interface {
+	Run(ctx context.Context, name string, args ...string) ([]byte, error)
+}
+
+// ExecRunner is the CommandRunner over os/exec.
+type ExecRunner struct{}
+
+var _ CommandRunner = ExecRunner{}
+
+func (ExecRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
+	out, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
+	if err != nil {
+		return out, fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
+	}
+	return out, nil
+}
+
+// ISCSIAdm is the Initiator over open-iscsi's iscsiadm.
+type ISCSIAdm struct {
+	Runner CommandRunner
+	// Binary defaults to "iscsiadm"; Stat to os.Stat. Zero durations take
+	// the defaults above.
+	Binary        string
+	Stat          func(path string) (os.FileInfo, error)
+	DeviceTimeout time.Duration
+	DevicePoll    time.Duration
+}
+
+var _ Initiator = (*ISCSIAdm)(nil)
+
+// NewISCSIAdm returns an ISCSIAdm that runs iscsiadm through runner.
+func NewISCSIAdm(runner CommandRunner) *ISCSIAdm {
+	return &ISCSIAdm{Runner: runner}
+}
+
+func (a *ISCSIAdm) binary() string {
+	if a.Binary == "" {
+		return "iscsiadm"
+	}
+	return a.Binary
+}
+
+func portal(t Target) string { return t.IPv4 + ":" + strconv.Itoa(t.Port) }
+
+func (a *ISCSIAdm) node(ctx context.Context, t Target, args ...string) error {
+	full := append([]string{"-m", "node", "-T", t.IQN, "-p", portal(t)}, args...)
+	_, err := a.Runner.Run(ctx, a.binary(), full...)
+	return err
+}
+
+// Login writes the node record, with manual startup so a reboot does not log
+// back in behind the daemon and the short replacement timeout, then logs in.
+// No login is attempted unless both settings were written.
+func (a *ISCSIAdm) Login(ctx context.Context, t Target) (string, error) {
+	if err := a.node(ctx, t, "-o", "new"); err != nil {
+		return "", fmt.Errorf("create node record: %w", err)
+	}
+	settings := []struct{ name, value string }{
+		{"node.startup", "manual"},
+		{"node.session.timeo.replacement_timeout", strconv.Itoa(ReplacementTimeoutSeconds)},
+	}
+	for _, s := range settings {
+		if err := a.node(ctx, t, "-o", "update", "-n", s.name, "-v", s.value); err != nil {
+			return "", fmt.Errorf("set %s: %w", s.name, err)
+		}
+	}
+	if err := a.node(ctx, t, "-l"); err != nil && exitCode(err) != iscsiErrSessExists {
+		return "", fmt.Errorf("log in: %w", err)
+	}
+	return a.awaitDevice(ctx, ByPathDevice(t))
+}
+
+// Logout closes any session to t and deletes its node record. A session still
+// listed after the logout is an error naming the logout's exit code, since
+// iscsiadm can report success or no match and leave it logged in.
+func (a *ISCSIAdm) Logout(ctx context.Context, t Target) error {
+	logoutErr := a.node(ctx, t, "-u")
+	if logoutErr != nil && exitCode(logoutErr) != iscsiErrNoObjsFound {
+		return fmt.Errorf("log out: %w", logoutErr)
+	}
+	sessions, err := a.Sessions(ctx)
+	if err != nil {
+		return fmt.Errorf("confirm log out: %w", err)
+	}
+	if slices.Contains(sessions, t) {
+		code := 0
+		if logoutErr != nil {
+			code = exitCode(logoutErr)
+		}
+		return fmt.Errorf("log out: session to %s at %s is still logged in after iscsiadm -u exited %d", t.IQN, portal(t), code)
+	}
+	if err := a.node(ctx, t, "-o", "delete"); err != nil && exitCode(err) != iscsiErrNoObjsFound {
+		return fmt.Errorf("delete node record: %w", err)
+	}
+	return nil
+}
+
+// Sessions lists this node's sessions. A line it cannot read is skipped, so
+// a session it cannot identify is never acted on.
+func (a *ISCSIAdm) Sessions(ctx context.Context) ([]Target, error) {
+	out, err := a.Runner.Run(ctx, a.binary(), "-m", "session")
+	// iscsiadm drops a session whose attributes it cannot read and may then
+	// report none, so an unread attribute fails the listing.
+	if bytes.Contains(out, []byte("iSCSI ERROR")) {
+		return nil, fmt.Errorf("list sessions: iscsiadm could not read every session: %s", strings.TrimSpace(string(out)))
+	}
+	if err != nil {
+		if exitCode(err) == iscsiErrNoObjsFound {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("list sessions: %w", err)
+	}
+	var sessions []Target
+	for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		t, ok := parseSession(line)
+		if !ok {
+			slog.WarnContext(ctx, "ocibsd: skipping an iscsiadm session line it cannot read", "line", line)
+			continue
+		}
+		sessions = append(sessions, t)
+	}
+	return sessions, nil
+}
+
+// parseSession reads one `iscsiadm -m session` line, such as
+// "tcp: [1] 169.254.2.2:3260,1 iqn.2015-12.com.oracleiaas:abc (non-flash)".
+func parseSession(line string) (Target, bool) {
+	fields := strings.Fields(line)
+	if len(fields) < 4 || fields[0] != "tcp:" {
+		return Target{}, false
+	}
+	addr, _, _ := strings.Cut(fields[2], ",")
+	i := strings.LastIndexByte(addr, ':')
+	if i <= 0 {
+		return Target{}, false
+	}
+	port, err := strconv.Atoi(addr[i+1:])
+	if err != nil || port <= 0 {
+		return Target{}, false
+	}
+	return Target{IQN: fields[3], IPv4: addr[:i], Port: port}, true
+}
+
+func (a *ISCSIAdm) awaitDevice(ctx context.Context, path string) (string, error) {
+	stat, timeout, interval := a.Stat, a.DeviceTimeout, a.DevicePoll
+	if stat == nil {
+		stat = os.Stat
+	}
+	if timeout <= 0 {
+		timeout = defaultDeviceTimeout
+	}
+	if interval <= 0 {
+		interval = defaultDevicePoll
+	}
+	err := poll(ctx, timeout, interval, "device "+path+" appearing", func(context.Context) (bool, error) {
+		_, err := stat(path)
+		if err == nil {
+			return true, nil
+		}
+		if !errors.Is(err, os.ErrNotExist) {
+			return false, fmt.Errorf("stat %s: %w", path, err)
+		}
+		return false, nil
+	})
+	if errors.Is(err, errGaveUp) {
+		return "", fmt.Errorf("device %s did not appear", path)
+	}
+	if err != nil {
+		return "", err
+	}
+	return path, nil
+}
+
+// exitCode is the exit status err carries, or -1 when it carries none.
+func exitCode(err error) int {
+	var coded interface{ ExitCode() int }
+	if errors.As(err, &coded) {
+		return coded.ExitCode()
+	}
+	return -1
+}

spinifex/providers/ebs/ocibsd/iscsiadm_test.go

@@ -0,0 +1,245 @@
+package ocibsd_test
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/ocibsd"
+)
+
+// exitError is a failed command's error, carrying its exit status as an
+// *exec.ExitError does.
+type exitError int
+
+func (e exitError) Error() string { return fmt.Sprintf("exit status %d", int(e)) }
+func (e exitError) ExitCode() int { return int(e) }
+
+// scriptedRunner records each argv and answers from replies, keyed by the
+// argv's first matching substring; an argv with no reply succeeds empty.
+type scriptedRunner struct {
+	calls   []string
+	replies map[string]reply
+}
+
+type reply struct {
+	out string
+	err error
+}
+
+func (r *scriptedRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) {
+	argv := name + " " + strings.Join(args, " ")
+	r.calls = append(r.calls, argv)
+	for match, rep := range r.replies {
+		if strings.Contains(argv, match) {
+			return []byte(rep.out), rep.err
+		}
+	}
+	return nil, nil
+}
+
+var target = ocibsd.Target{IQN: "iqn.2015-12.com.oracleiaas:vol-1", IPv4: "169.254.2.2", Port: 3260}
+
+const nodeArgs = "iscsiadm -m node -T iqn.2015-12.com.oracleiaas:vol-1 -p 169.254.2.2:3260"
+
+const (
+	targetSession = "tcp: [1] 169.254.2.2:3260,1 iqn.2015-12.com.oracleiaas:vol-1 (non-flash)\n"
+	otherSession  = "tcp: [2] 169.254.2.2:3260,1 iqn.2015-12.com.oracleiaas:vol-2 (non-flash)\n"
+
+	unreadableSession = "iSCSI ERROR: Failed to read 'username': permission deny when reading " +
+		"'/sys/class/iscsi_session/session1/username' # ../../libopeniscsiusr/sysfs.c:_sysfs_prop_get_str():184\n"
+)
+
+func present(string) (os.FileInfo, error) { return nil, nil }
+
+func newISCSIAdm(r *scriptedRunner, stat func(string) (os.FileInfo, error)) *ocibsd.ISCSIAdm {
+	a := ocibsd.NewISCSIAdm(r)
+	a.Stat, a.DeviceTimeout, a.DevicePoll = stat, 50*time.Millisecond, time.Millisecond
+	return a
+}
+
+func TestLoginWritesTheNodeRecordBeforeLoggingIn(t *testing.T) {
+	cases := []struct {
+		name    string
+		replies map[string]reply
+	}{
+		{name: "a new session"},
+		{name: "a session already open", replies: map[string]reply{" -l": {err: exitError(15)}}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := &scriptedRunner{replies: tc.replies}
+			var statted string
+			device, err := newISCSIAdm(r, func(path string) (os.FileInfo, error) { statted = path; return nil, nil }).Login(context.Background(), target)
+			require.NoError(t, err)
+			assert.Equal(t, ocibsd.ByPathDevice(target), device)
+			assert.Equal(t, device, statted)
+			assert.Equal(t, []string{
+				nodeArgs + " -o new",
+				nodeArgs + " -o update -n node.startup -v manual",
+				nodeArgs + " -o update -n node.session.timeo.replacement_timeout -v 120",
+				nodeArgs + " -l",
+			}, r.calls)
+		})
+	}
+}
+
+func TestLoginFailures(t *testing.T) {
+	cases := []struct {
+		name      string
+		replies   map[string]reply
+		stat      func(string) (os.FileInfo, error)
+		want      string
+		wantCalls int
+	}{
+		{
+			name:      "a record that cannot be created logs nothing in",
+			replies:   map[string]reply{"-o new": {err: exitError(6)}},
+			want:      "create node record",
+			wantCalls: 1,
+		},
+		{
+			name:      "no login without the short replacement timeout",
+			replies:   map[string]reply{"replacement_timeout": {err: exitError(6)}},
+			want:      "set node.session.timeo.replacement_timeout",
+			wantCalls: 3,
+		},
+		{
+			name:      "a refused login",
+			replies:   map[string]reply{" -l": {err: exitError(24)}},
+			want:      "log in",
+			wantCalls: 4,
+		},
+		{
+			name:      "a device that never appears",
+			stat:      func(string) (os.FileInfo, error) { return nil, fs.ErrNotExist },
+			want:      "did not appear",
+			wantCalls: 4,
+		},
+		{
+			name:      "a device that cannot be read",
+			stat:      func(string) (os.FileInfo, error) { return nil, fs.ErrPermission },
+			want:      "permission denied",
+			wantCalls: 4,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := &scriptedRunner{replies: tc.replies}
+			stat := tc.stat
+			if stat == nil {
+				stat = present
+			}
+			_, err := newISCSIAdm(r, stat).Login(context.Background(), target)
+			require.ErrorContains(t, err, tc.want)
+			assert.Len(t, r.calls, tc.wantCalls)
+		})
+	}
+}
+
+func TestLogout(t *testing.T) {
+	cases := []struct {
+		name    string
+		replies map[string]reply
+		want    string
+		deleted bool // the record delete ran before the error
+	}{
+		{name: "an open session"},
+		{name: "no session and no record", replies: map[string]reply{" -u": {err: exitError(21)}, "-o delete": {err: exitError(21)}}},
+		{name: "a failed logout", replies: map[string]reply{" -u": {err: exitError(10)}}, want: "log out"},
+		{name: "a record that cannot be deleted", replies: map[string]reply{"-o delete": {err: exitError(6)}}, want: "delete node record", deleted: true},
+		{name: "another target's session is left alone", replies: map[string]reply{"-m session": {out: otherSession}}},
+		{
+			name:    "a session still listed after a successful logout",
+			replies: map[string]reply{"-m session": {out: targetSession}},
+			want:    "still logged in after iscsiadm -u exited 0",
+		},
+		{
+			name:    "a session still listed after a logout that matched nothing",
+			replies: map[string]reply{" -u": {err: exitError(21)}, "-m session": {out: targetSession}},
+			want:    "still logged in after iscsiadm -u exited 21",
+		},
+		{name: "sessions that cannot be listed", replies: map[string]reply{"-m session": {err: exitError(6)}}, want: "confirm log out"},
+		{
+			name: "a logout and listing blind to the session without CAP_SYS_ADMIN",
+			replies: map[string]reply{
+				" -u":        {out: "iscsiadm: No matching sessions found", err: exitError(21)},
+				"-m session": {out: unreadableSession + "iscsiadm: No active sessions.\n", err: exitError(21)},
+			},
+			want: "could not read every session",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := &scriptedRunner{replies: tc.replies}
+			err := newISCSIAdm(r, present).Logout(context.Background(), target)
+			if tc.want != "" {
+				require.ErrorContains(t, err, tc.want)
+				if !tc.deleted {
+					assert.NotContains(t, r.calls, nodeArgs+" -o delete", "a failed logout must keep the record")
+				}
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, []string{nodeArgs + " -u", "iscsiadm -m session", nodeArgs + " -o delete"}, r.calls)
+		})
+	}
+}
+
+func TestSessions(t *testing.T) {
+	cases := []struct {
+		name string
+		rep  reply
+		want []ocibsd.Target
+		err  string
+	}{
+		{
+			name: "every readable session",
+			rep: reply{out: "tcp: [1] 169.254.2.2:3260,1 iqn.2015-12.com.oracleiaas:vol-1 (non-flash)\n" +
+				"garbage\n" +
+				"tcp: [2] 10.0.0.5:3261,1 iqn.2003-01.org.linux-iscsi.san:lun0 (non-flash)\n" +
+				"tcp: [3] 10.0.0.6:notaport,1 iqn.2003-01.org.linux-iscsi.san:lun1 (non-flash)\n"},
+			want: []ocibsd.Target{target, {IQN: "iqn.2003-01.org.linux-iscsi.san:lun0", IPv4: "10.0.0.5", Port: 3261}},
+		},
+		{name: "no sessions", rep: reply{out: "iscsiadm: No active sessions.", err: exitError(21)}},
+		{name: "iscsid unreachable", rep: reply{err: exitError(20)}, err: "list sessions"},
+		{
+			// What iscsiadm prints without CAP_SYS_ADMIN while a session is open.
+			name: "an unreadable session reported as none",
+			rep:  reply{out: unreadableSession + "iscsiadm: No active sessions.\n", err: exitError(21)},
+			err:  "could not read every session",
+		},
+		{
+			name: "an unreadable session beside a readable one",
+			rep:  reply{out: unreadableSession + targetSession},
+			err:  "could not read every session",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := &scriptedRunner{replies: map[string]reply{"-m session": tc.rep}}
+			got, err := newISCSIAdm(r, present).Sessions(context.Background())
+			if tc.err != "" {
+				require.ErrorContains(t, err, tc.err)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+			assert.Equal(t, []string{"iscsiadm -m session"}, r.calls)
+		})
+	}
+}
+
+func TestExecRunnerKeepsTheExitCode(t *testing.T) {
+	_, err := ocibsd.ExecRunner{}.Run(context.Background(), "false")
+	var coded interface{ ExitCode() int }
+	require.ErrorAs(t, err, &coded)
+	assert.Equal(t, 1, coded.ExitCode())
+}

spinifex/providers/ebs/ocibsd/live_test.go

@@ -0,0 +1,257 @@
+package ocibsd_test
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/natsserve"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/ocibsd"
+)
+
+// The live run reads API-key credentials from the local OCI CLI config and
+// drives iscsiadm on the instance over SSH; see docs/oci-integration/README.md.
+const (
+	liveEnv          = "SPINIFEX_OCIBSD_LIVE"
+	liveCompartment  = "SPINIFEX_OCIBSD_COMPARTMENT_ID"
+	liveAD           = "SPINIFEX_OCIBSD_AVAILABILITY_DOMAIN"
+	liveInstance     = "SPINIFEX_OCIBSD_INSTANCE_ID"
+	liveSSHHost      = "SPINIFEX_OCIBSD_SSH_HOST"
+	liveSSHUser      = "SPINIFEX_OCIBSD_SSH_USER"
+	liveSSHKey       = "SPINIFEX_OCIBSD_SSH_KEY"
+	liveKnownHosts   = "SPINIFEX_OCIBSD_SSH_KNOWN_HOSTS"
+	liveClusterID    = "SPINIFEX_OCIBSD_CLUSTER_ID"
+	liveOCIProfile   = "SPINIFEX_OCIBSD_OCI_PROFILE"
+	liveNodeID       = "ocibsd-live"
+	liveAZ           = "live-1a"
+	liveNATSDeadline = 10 * time.Minute
+)
+
+type liveEnvironment struct {
+	compartment, ad, instance, clusterID string
+	client                               oci.BlockClient
+	ssh                                  *sshRunner
+}
+
+// liveSetup skips unless the live run is asked for, and fails on any input it
+// is missing rather than running against a half-configured tenancy.
+func liveSetup(t *testing.T) *liveEnvironment {
+	t.Helper()
+	if os.Getenv(liveEnv) == "" {
+		t.Skipf("skipping: set %s=1, with the inputs in docs/oci-integration/README.md, to run against real OCI", liveEnv)
+	}
+	need := func(name string) string {
+		value := os.Getenv(name)
+		require.NotEmptyf(t, value, "%s is required for the live run", name)
+		return value
+	}
+	env := &liveEnvironment{
+		compartment: need(liveCompartment), ad: need(liveAD), instance: need(liveInstance),
+		clusterID: os.Getenv(liveClusterID),
+	}
+	if env.clusterID == "" {
+		b := make([]byte, 16)
+		_, err := rand.Read(b)
+		require.NoError(t, err)
+		env.clusterID = hex.EncodeToString(b)
+	}
+	t.Logf("live run: cluster ID %s, instance %s, availability domain %s", env.clusterID, env.instance, env.ad)
+
+	client, err := oci.NewConfigFileBlockClient("", os.Getenv(liveOCIProfile))
+	require.NoError(t, err)
+	env.client = client
+	env.ssh = newSSHRunner(t, need(liveSSHHost), os.Getenv(liveSSHUser), need(liveSSHKey), os.Getenv(liveKnownHosts))
+	return env
+}
+
+// provider builds and starts a Provider for the instance, with iscsiadm and
+// the device check run on it over SSH.
+func (e *liveEnvironment) provider(t *testing.T) *ocibsd.Provider {
+	t.Helper()
+	initiator := ocibsd.NewISCSIAdm(e.ssh)
+	initiator.Stat = e.ssh.statDevice
+	p, err := ocibsd.New(ocibsd.Config{
+		ClusterID: e.clusterID, CompartmentID: e.compartment,
+		AvailabilityDomain: e.ad, AvailabilityZone: liveAZ,
+		NodeID: liveNodeID, InstanceID: e.instance, InstanceAvailabilityDomain: e.ad,
+		Client: e.client, Initiator: initiator,
+	})
+	require.NoError(t, err)
+	require.NoError(t, p.Start(t.Context()))
+	return p
+}
+
+func runPrefix(kind string) string {
+	return kind + strconv.FormatInt(time.Now().Unix(), 36) + "-"
+}
+
+func TestLiveConformanceInProcess(t *testing.T) {
+	env := liveSetup(t)
+	conformance.RunSuiteWithConfig(t, func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		return env.provider(t)
+	}, conformance.SuiteConfig{NamePrefix: runPrefix("lp"), NodeID: liveNodeID})
+}
+
+func TestLiveConformanceOverNATS(t *testing.T) {
+	env := liveSetup(t)
+	conformance.RunSuiteWithConfig(t, func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		_, conn := testutil.StartTestNATS(t)
+		// t.Context is cancelled before cleanups run, and the suite's teardown
+		// is served through this server, so it needs a context that outlives it.
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+		stop, err := natsserve.Serve(ctx, conn, env.provider(t), natsserve.Options{NodeID: liveNodeID})
+		require.NoError(t, err)
+		t.Cleanup(stop)
+		return ebsprovider.NewNATSProvider(conn, liveNATSDeadline)
+	}, conformance.SuiteConfig{NamePrefix: runPrefix("ln"), NodeID: liveNodeID})
+}
+
+// TestLiveSizeUnits checks that OCI's GB is 2^30 bytes: the device the
+// instance sees must be SizeGB gibibytes and hold the requested capacity.
+func TestLiveSizeUnits(t *testing.T) {
+	env := liveSetup(t)
+	p := env.provider(t)
+	ctx := t.Context()
+	volumeID := "vol-" + runPrefix("lsz") + "units"
+	capacity := 51*int64(1<<30) + 1
+
+	v, err := p.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: capacity},
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		assert.NoError(t, p.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: volumeID, Handle: v.Handle}))
+	})
+	assert.Equal(t, capacity, v.CapacityBytes)
+
+	got, err := env.client.GetVolume(ctx, v.Handle)
+	require.NoError(t, err)
+	t.Logf("requested %d bytes; OCI reports %d GB; provider reports %d bytes", capacity, got.SizeGB, v.CapacityBytes)
+	assert.Equal(t, ocibsd.SizeGB(capacity), got.SizeGB)
+
+	published, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: volumeID, Handle: v.Handle, NodeID: liveNodeID})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_, err := p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: volumeID, Handle: v.Handle, NodeID: liveNodeID})
+		assert.NoError(t, err)
+	})
+	require.NotNil(t, published.Export.Block)
+	assert.Equal(t, uint64(capacity), published.Export.Block.SizeBytes)
+
+	out, err := env.ssh.Run(ctx, "blockdev", "--getsize64", published.Export.Block.DevicePath)
+	require.NoError(t, err)
+	deviceBytes, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64)
+	require.NoError(t, err)
+	t.Logf("device %s on the instance is %d bytes", published.Export.Block.DevicePath, deviceBytes)
+	assert.Equal(t, got.SizeGB*int64(1<<30), deviceBytes, "OCI's GB must be 2^30 bytes")
+	assert.GreaterOrEqual(t, deviceBytes, capacity, "the device must hold the size the guest is told")
+}
+
+// TestLiveRetryTokenReplay recreates a deleted volume under the same ID, and
+// so the same retry token, which OCI replays for a day: the provider must
+// never answer with the deleted volume.
+func TestLiveRetryTokenReplay(t *testing.T) {
+	env := liveSetup(t)
+	p := env.provider(t)
+	ctx := t.Context()
+	volumeID := "vol-" + runPrefix("lrt") + "replay"
+	req := ebsprovider.CreateVolumeRequest{Versioned: latest, VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}
+
+	first, err := p.CreateVolume(ctx, req)
+	require.NoError(t, err)
+	require.NoError(t, p.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: volumeID, Handle: first.Handle}))
+	deleted, err := env.client.GetVolume(ctx, first.Handle)
+	require.NoError(t, err)
+	t.Logf("first volume %s is %s after delete", first.Handle, deleted.State)
+
+	second, err := p.CreateVolume(ctx, req)
+	if err != nil {
+		t.Logf("recreate refused: %v", err)
+		assert.Contains(t, err.Error(), first.Handle, "the refusal should name the replayed volume")
+		return
+	}
+	t.Cleanup(func() {
+		assert.NoError(t, p.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: volumeID, Handle: second.Handle}))
+	})
+	t.Logf("recreate answered volume %s", second.Handle)
+	assert.NotEqual(t, first.Handle, second.Handle, "the provider answered with the deleted volume")
+	live, err := env.client.GetVolume(ctx, second.Handle)
+	require.NoError(t, err)
+	assert.False(t, live.State.Gone(), "the provider answered with a %s volume", live.State)
+}
+
+// sshRunner is a CommandRunner that runs each command under sudo on the
+// instance. ssh exits with the remote command's status, so ExitCode survives.
+type sshRunner struct {
+	base []string
+}
+
+var _ ocibsd.CommandRunner = (*sshRunner)(nil)
+
+func newSSHRunner(t *testing.T, host, user, key, knownHosts string) *sshRunner {
+	t.Helper()
+	if user == "" {
+		user = "opc"
+	}
+	if knownHosts == "" {
+		knownHosts = filepath.Join(t.TempDir(), "known_hosts")
+	}
+	control := t.TempDir()
+	t.Cleanup(func() {
+		_ = exec.Command("ssh", "-o", "ControlPath="+filepath.Join(control, "c"), "-O", "exit", user+"@"+host).Run()
+	})
+	return &sshRunner{base: []string{
+		"-i", key, "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", "-o", "ConnectTimeout=15",
+		"-o", "StrictHostKeyChecking=accept-new", "-o", "UserKnownHostsFile=" + knownHosts,
+		"-o", "ControlMaster=auto", "-o", "ControlPath=" + filepath.Join(control, "c"), "-o", "ControlPersist=120",
+		user + "@" + host,
+	}}
+}
+
+func (r *sshRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
+	words := []string{"sudo", shellQuote(name)}
+	for _, a := range args {
+		words = append(words, shellQuote(a))
+	}
+	argv := append(append([]string(nil), r.base...), "--", strings.Join(words, " "))
+	out, err := exec.CommandContext(ctx, "ssh", argv...).CombinedOutput()
+	if err != nil {
+		return out, fmt.Errorf("ssh %s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
+	}
+	return out, nil
+}
+
+// statDevice is ISCSIAdm's device check run on the instance: a missing block
+// device reads as fs.ErrNotExist, as os.Stat reports it locally.
+func (r *sshRunner) statDevice(path string) (os.FileInfo, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	_, err := r.Run(ctx, "test", "-b", path)
+	if err != nil && ocibsd.ExitCode(err) == 1 {
+		return nil, &fs.PathError{Op: "stat", Path: path, Err: fs.ErrNotExist}
+	}
+	return nil, err
+}
+
+func shellQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}

spinifex/providers/ebs/ocibsd/provider.go

@@ -0,0 +1,291 @@
+// Package ocibsd is an EBSProvider that stores every volume as an OCI Block
+// Volume, every snapshot as an incremental OCI volume backup, and attaches a
+// published volume to its node over iSCSI.
+//
+// A Provider runs on one node and answers for it alone: PublishVolume and
+// UnpublishVolume act on that node's instance. Every other verb can be
+// answered by any node, because OCI holds all of the state.
+package ocibsd
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log/slog"
+	"slices"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+)
+
+// Freeform tags ocibsd sets on the OCI resources it creates. They carry a
+// resource's EBS identity and requested size, so no other store is needed.
+const (
+	TagCluster       = "spinifex-cluster"
+	TagVolumeID      = "spinifex-volume-id"
+	TagSnapshotID    = "spinifex-snapshot-id"
+	TagCapacityBytes = "spinifex-capacity-bytes"
+)
+
+const gib = int64(1) << 30
+
+// VPUs per GB for OCI's Balanced and Higher Performance tiers.
+const (
+	BalancedVPUsPerGB          int64 = 10
+	HigherPerformanceVPUsPerGB int64 = 20
+)
+
+const (
+	defaultPollInterval   = 2 * time.Second
+	defaultVolumeTimeout  = 5 * time.Minute
+	defaultAttachTimeout  = 3 * time.Minute
+	defaultDetachTimeout  = 3 * time.Minute
+	retryTokenHashLetters = 48
+
+	// defaultPublishTimeout bounds a whole publish short of the client's
+	// budget, so a slow attach is answered as retry-later rather than abandoned.
+	defaultPublishTimeout = ebsprovider.PublishTimeout - 30*time.Second
+)
+
+// Config is what a Provider needs to reach OCI and to act for its node.
+type Config struct {
+	// ClusterID tags every resource, so one compartment can hold several
+	// clusters without one enumerating or deleting another's.
+	ClusterID     string
+	CompartmentID string
+
+	// AvailabilityDomain is where every volume is created, and
+	// AvailabilityZone is the EC2 name of the cluster's one zone.
+	AvailabilityDomain string
+	AvailabilityZone   string
+
+	// NodeID, InstanceID and InstanceAvailabilityDomain describe the node this
+	// provider runs on; the last comes from the instance metadata service.
+	NodeID                     string
+	InstanceID                 string
+	InstanceAvailabilityDomain string
+
+	Client    oci.BlockClient
+	Initiator Initiator
+
+	// Zero values take the defaults above.
+	PollInterval  time.Duration
+	VolumeTimeout time.Duration
+	AttachTimeout time.Duration
+	DetachTimeout time.Duration
+	// PublishTimeout bounds a whole publish, attach and login included.
+	PublishTimeout time.Duration
+}
+
+// Provider is the ocibsd EBSProvider for one node.
+type Provider struct {
+	cfg Config
+
+	// started is set once Start has logged out stale sessions.
+	started atomic.Bool
+
+	mu    sync.Mutex
+	locks map[string]*volumeLock
+}
+
+type volumeLock struct {
+	mu      sync.Mutex
+	holders int
+}
+
+var _ ebsprovider.EBSProvider = (*Provider)(nil)
+
+// New returns a Provider for cfg, refusing a node outside the cluster's AD:
+// a volume created there could never be attached to it.
+func New(cfg Config) (*Provider, error) {
+	for name, value := range map[string]string{
+		"cluster ID": cfg.ClusterID, "compartment ID": cfg.CompartmentID,
+		"availability domain": cfg.AvailabilityDomain, "availability zone": cfg.AvailabilityZone,
+		"node ID": cfg.NodeID, "instance ID": cfg.InstanceID,
+	} {
+		if value == "" {
+			return nil, fmt.Errorf("ocibsd: %s is required", name)
+		}
+	}
+	if cfg.Client == nil || cfg.Initiator == nil {
+		return nil, errors.New("ocibsd: an OCI block client and an iSCSI initiator are required")
+	}
+	if cfg.InstanceAvailabilityDomain != cfg.AvailabilityDomain {
+		return nil, fmt.Errorf("ocibsd: instance %s is in availability domain %q, but the cluster's volumes are in %q",
+			cfg.InstanceID, cfg.InstanceAvailabilityDomain, cfg.AvailabilityDomain)
+	}
+	if cfg.PollInterval <= 0 {
+		cfg.PollInterval = defaultPollInterval
+	}
+	if cfg.VolumeTimeout <= 0 {
+		cfg.VolumeTimeout = defaultVolumeTimeout
+	}
+	if cfg.AttachTimeout <= 0 {
+		cfg.AttachTimeout = defaultAttachTimeout
+	}
+	if cfg.DetachTimeout <= 0 {
+		cfg.DetachTimeout = defaultDetachTimeout
+	}
+	if cfg.PublishTimeout <= 0 {
+		cfg.PublishTimeout = defaultPublishTimeout
+	}
+	return &Provider{cfg: cfg, locks: map[string]*volumeLock{}}, nil
+}
+
+// Capabilities is what ocibsd advertises. OCI refuses a second attachment of
+// a volume that is not shareable wherever it is asked for, so exclusion is
+// cluster-wide; an attachment never lapses, and a cut session is not fenced.
+func Capabilities() ebsprovider.Capabilities {
+	return ebsprovider.Capabilities{
+		CrashConsistentSnapshot: true,
+		ReadOnlyPublish:         true,
+		VolumeEnumeration:       true,
+		SnapshotEnumeration:     true,
+		Exclusion: ebsprovider.ExclusionSemantics{
+			Scope:           ebsprovider.ExclusionScopeCluster,
+			ClaimTTLSeconds: 0,
+			FencesLostClaim: false,
+		},
+	}
+}
+
+func (p *Provider) GetCapabilities(_ context.Context, req ebsprovider.GetCapabilitiesRequest) (*ebsprovider.GetCapabilitiesResponse, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	return &ebsprovider.GetCapabilitiesResponse{
+		Versioned:        ebsprovider.ReplyVersioned(req.SchemaVersion),
+		Capabilities:     Capabilities(),
+		ContractVersions: ebsprovider.SupportedSchemaVersions(),
+	}, nil
+}
+
+// HoldsUnsealedWrites answers that none are held: QEMU opens the device with
+// cache.direct=on, and the iSCSI target acknowledges a write once OCI has it.
+func (p *Provider) HoldsUnsealedWrites(_ context.Context, nodeID string, volumeIDs []string) ([]string, error) {
+	if nodeID == "" {
+		return nil, fmt.Errorf("%w: node ID is required", ebsprovider.ErrInvalidArgument)
+	}
+	if slices.Contains(volumeIDs, "") {
+		return nil, fmt.Errorf("%w: empty volume ID", ebsprovider.ErrInvalidArgument)
+	}
+	return nil, nil
+}
+
+// RetryToken is the opc-retry-token for creating the resource kind named id
+// in cluster, so a create repeated after a lost reply makes nothing new.
+func RetryToken(kind, cluster, id string) string {
+	sum := sha256.Sum256([]byte(kind + "\x00" + cluster + "\x00" + id))
+	return "spx-" + hex.EncodeToString(sum[:])[:retryTokenHashLetters]
+}
+
+// SizeGB is the OCI size of a volume of capacityBytes: whole GB rounded up,
+// and never below OCI's floor. OCI's GB is taken to be 2^30 bytes.
+func SizeGB(capacityBytes int64) int64 {
+	return max((capacityBytes+gib-1)/gib, oci.MinVolumeSizeGB)
+}
+
+// VPUsPerGB maps an EBS volume type to an OCI performance tier. The
+// provisioned-IOPS types get Higher Performance and every other type Balanced.
+func VPUsPerGB(volumeType string) (int64, error) {
+	switch volumeType {
+	case "", "gp2", "gp3", "st1", "sc1":
+		return BalancedVPUsPerGB, nil
+	case "io1", "io2":
+		return HigherPerformanceVPUsPerGB, nil
+	default:
+		return 0, fmt.Errorf("%w: volume type %q has no OCI performance tier", ebsprovider.ErrInvalidArgument, volumeType)
+	}
+}
+
+// capacityOf reads the recorded EBS size from a resource's tags.
+func capacityOf(tags map[string]string) (int64, error) {
+	raw, ok := tags[TagCapacityBytes]
+	if !ok {
+		return 0, fmt.Errorf("ocibsd: resource has no %s tag", TagCapacityBytes)
+	}
+	capacity, err := strconv.ParseInt(raw, 10, 64)
+	if err != nil || capacity <= 0 {
+		return 0, fmt.Errorf("ocibsd: resource has an invalid %s tag %q", TagCapacityBytes, raw)
+	}
+	return capacity, nil
+}
+
+// ociError translates an OCI failure no verb handles itself. A conflict takes
+// the onConflict sentinel when the verb gives one, a quota or throttling
+// refusal is retry-later, and anything else is passed on as is.
+func ociError(op string, err error, onConflict ...error) error {
+	if len(onConflict) > 0 && errors.Is(err, oci.ErrConflict) {
+		return fmt.Errorf("%w: %s: %w", onConflict[0], op, err)
+	}
+	if errors.Is(err, oci.ErrLimitExceeded) {
+		// The contract has no resource-exhausted code, and the same request
+		// succeeds once the limit frees, as when another volume detaches.
+		return fmt.Errorf("%w: %s: %w", ebsprovider.ErrUnavailable, op, err)
+	}
+	return fmt.Errorf("ocibsd: %s: %w", op, err)
+}
+
+// errGaveUp is what poll returns once its timeout passes.
+var errGaveUp = errors.New("ocibsd: gave up waiting")
+
+// poll calls done every interval until it reports true or fails, returning
+// errGaveUp once timeout passes and ctx's error if ctx ends first.
+func poll(ctx context.Context, timeout, interval time.Duration, what string, done func(context.Context) (bool, error)) error {
+	start := time.Now()
+	for {
+		ok, err := done(ctx)
+		if err != nil || ok {
+			return err
+		}
+		if time.Since(start) >= timeout {
+			slog.WarnContext(ctx, "ocibsd: gave up waiting", "what", what, "waited_ms", otelsetup.Millis(time.Since(start)))
+			return errGaveUp
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(interval):
+		}
+	}
+}
+
+// waitFor polls an OCI operation, failing as retry-later once timeout passes,
+// since the same request may then find the work finished.
+func (p *Provider) waitFor(ctx context.Context, timeout time.Duration, what string, done func(context.Context) (bool, error)) error {
+	err := poll(ctx, timeout, p.cfg.PollInterval, what, done)
+	if errors.Is(err, errGaveUp) {
+		return fmt.Errorf("%w: %s did not finish in time", ebsprovider.ErrUnavailable, what)
+	}
+	return err
+}
+
+// lockVolume serialises this node's publish and unpublish of one volume, so
+// two concurrent publishes here do not both attach it.
+func (p *Provider) lockVolume(volumeID string) func() {
+	p.mu.Lock()
+	lock, ok := p.locks[volumeID]
+	if !ok {
+		lock = &volumeLock{}
+		p.locks[volumeID] = lock
+	}
+	lock.holders++
+	p.mu.Unlock()
+
+	lock.mu.Lock()
+	return func() {
+		lock.mu.Unlock()
+		p.mu.Lock()
+		if lock.holders--; lock.holders == 0 {
+			delete(p.locks, volumeID)
+		}
+		p.mu.Unlock()
+	}
+}

spinifex/providers/ebs/ocibsd/provider_test.go

@@ -0,0 +1,626 @@
+package ocibsd_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/ocibsd"
+)
+
+func TestNewRefusesANodeOutsideTheClustersAD(t *testing.T) {
+	cfg := testConfig("node-1", oci.NewBlockFake(), ocibsd.NewFakeInitiator())
+	cfg.InstanceAvailabilityDomain = "Uocm:US-ASHBURN-AD-2"
+	_, err := ocibsd.New(cfg)
+	require.ErrorContains(t, err, "AD-2")
+
+	cfg = testConfig("node-1", oci.NewBlockFake(), ocibsd.NewFakeInitiator())
+	cfg.CompartmentID = ""
+	_, err = ocibsd.New(cfg)
+	require.ErrorContains(t, err, "compartment ID is required")
+}
+
+func TestCapabilities(t *testing.T) {
+	p := newCluster(t).nodes["node-1"]
+	resp, err := p.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: latest})
+	require.NoError(t, err)
+	caps := resp.Capabilities
+	assert.Equal(t, ebsprovider.ExclusionSemantics{Scope: ebsprovider.ExclusionScopeCluster}, caps.Exclusion,
+		"OCI refuses a second attachment anywhere, never lapses one, and does not fence a cut session")
+	assert.False(t, caps.SnapshotCopy, "OCI copies a backup only to another region")
+	assert.False(t, caps.VolumeSeeding)
+	assert.False(t, caps.OnlineExpansion)
+	assert.True(t, caps.ReadOnlyPublish)
+	assert.True(t, caps.CrashConsistentSnapshot)
+	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
+
+	_, err = p.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{Versioned: latest, SourceSnapshotID: "snap-a", DestinationSnapshotID: "snap-b", VolumeID: "vol-a"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability)
+}
+
+func TestCreateVolumeSizesAtTheFloorAndReportsTheRequest(t *testing.T) {
+	cases := []struct {
+		name     string
+		capacity int64
+		ociGB    int64
+	}{
+		{"below the floor", 1 << 30, oci.MinVolumeSizeGB},
+		{"a part GB rounds up", 60<<30 + 1, 61},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			c := newCluster(t)
+			p := c.router()
+			v := createVolume(t, p, "vol-size", tc.capacity)
+			assert.Equal(t, tc.capacity, v.CapacityBytes)
+			assert.Equal(t, tc.ociGB, ociVolume(t, c.fake, v.Handle).SizeGB)
+
+			got, err := p.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: latest, VolumeID: "vol-size"})
+			require.NoError(t, err)
+			assert.Equal(t, tc.capacity, got.CapacityBytes, "the requested size outlives the provider in a tag")
+
+			pub, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-size", NodeID: "node-1"})
+			require.NoError(t, err)
+			assert.Equal(t, uint64(tc.capacity), pub.Export.Block.SizeBytes, "the guest sees the requested size, not OCI's")
+		})
+	}
+}
+
+func TestCreateVolumeIsPlacedInTheClustersAD(t *testing.T) {
+	c := newCluster(t)
+	ctx := context.Background()
+
+	v, err := c.nodes["node-2"].CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: "vol-placed", AvailabilityZone: testAZ, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+	})
+	require.NoError(t, err)
+	assert.Equal(t, testAD, ociVolume(t, c.fake, v.Handle).AvailabilityDomain)
+	assert.Equal(t, testAZ, v.AvailabilityZone)
+
+	_, err = c.nodes["node-1"].CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: "vol-elsewhere", AvailabilityZone: "ap-southeast-2b", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+	})
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+	assert.Len(t, c.fake.Volumes(), 1, "no volume is created where no node can attach it")
+}
+
+func TestCreateVolumeMapsTheVolumeTypeToVPUs(t *testing.T) {
+	cases := map[string]int64{"": 10, "gp2": 10, "gp3": 10, "st1": 10, "sc1": 10, "io1": 20, "io2": 20}
+	for volumeType, vpus := range cases {
+		t.Run(fmt.Sprintf("%q", volumeType), func(t *testing.T) {
+			c := newCluster(t)
+			v, err := c.router().CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
+				Versioned: latest, VolumeID: "vol-tier", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+				Parameters: volumeParams(t, volumeType),
+			})
+			require.NoError(t, err)
+			assert.Equal(t, vpus, ociVolume(t, c.fake, v.Handle).VPUsPerGB)
+		})
+	}
+
+	c := newCluster(t)
+	ctx := context.Background()
+	_, err := c.router().CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: "vol-magnetic", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+		Parameters: volumeParams(t, "standard"),
+	})
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+
+	createVolume(t, c.router(), "vol-retier", 1<<30)
+	_, err = c.router().CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: "vol-retier", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+		Parameters: volumeParams(t, "io1"),
+	})
+	require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists, "a repeat asking for another tier is not the same volume")
+}
+
+func TestCreateVolumeAfterALostReplyReturnsTheFirstVolume(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	req := ebsprovider.CreateVolumeRequest{Versioned: latest, VolumeID: "vol-lost", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}
+
+	c.fake.LoseReply("CreateVolume")
+	_, err := p.CreateVolume(ctx, req)
+	require.ErrorIs(t, err, oci.ErrReplyLost)
+
+	v, err := c.nodes["node-2"].CreateVolume(ctx, req)
+	require.NoError(t, err, "the repeat may reach another worker")
+	require.Len(t, c.fake.Volumes(), 1)
+	assert.Equal(t, c.fake.Volumes()[0].ID, v.Handle)
+	assert.Equal(t, ocibsd.RetryToken("volume", testCluster, "vol-lost"), c.fake.RetryToken(v.Handle))
+}
+
+func TestCreateVolumeRefusesARetryTokenReplayOfADeletedVolume(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-again", 1<<30)
+	require.NoError(t, p.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: "vol-again"}))
+
+	_, err := p.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: latest, VolumeID: "vol-again", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+	require.ErrorContains(t, err, "TERMINATED", "a terminated volume must never be answered as created")
+}
+
+func TestCreateVolumeFromASnapshot(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-src", 70<<30)
+	snap, err := p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: latest, SnapshotID: "snap-src", VolumeID: "vol-src"})
+	require.NoError(t, err)
+	restore := ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: "vol-restored", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 70 << 30},
+		SourceSnapshotID: "snap-src", SourceSnapshotVolumeID: "vol-src",
+	}
+
+	_, err = p.CreateVolume(ctx, restore)
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "a backup still being taken cannot be restored yet")
+
+	require.NoError(t, c.fake.SettleBackup(snap.Handle, oci.BackupAvailable))
+	small := restore
+	small.CapacityRange.RequiredBytes = 1 << 30
+	_, err = p.CreateVolume(ctx, small)
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument, "a volume smaller than its snapshot is refused")
+
+	v, err := p.CreateVolume(ctx, restore)
+	require.NoError(t, err)
+	restored := ociVolume(t, c.fake, v.Handle)
+	assert.Equal(t, snap.Handle, restored.SourceBackupID)
+	assert.Equal(t, int64(70), restored.SizeGB)
+
+	again, err := p.CreateVolume(ctx, restore)
+	require.NoError(t, err)
+	assert.Equal(t, v.Handle, again.Handle)
+	blank := restore
+	blank.SourceSnapshotID, blank.SourceSnapshotVolumeID = "", ""
+	_, err = p.CreateVolume(ctx, blank)
+	require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists, "a blank create is not a repeat of a restore")
+}
+
+func TestCreateVolumeFromAFailedSnapshotIsRefused(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-src", 1<<30)
+	snap, err := p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: latest, SnapshotID: "snap-faulty", VolumeID: "vol-src"})
+	require.NoError(t, err)
+	require.NoError(t, c.fake.SettleBackup(snap.Handle, oci.BackupFaulty))
+
+	_, err = p.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned: latest, VolumeID: "vol-from-faulty", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+		SourceSnapshotID: "snap-faulty", SourceSnapshotVolumeID: "vol-src",
+	})
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+}
+
+func TestAVolumeOfAnotherClusterIsNotFound(t *testing.T) {
+	fake := oci.NewBlockFake()
+	theirs := newNode(t, fake, ocibsd.NewFakeInitiator(), func(cfg *ocibsd.Config) { cfg.ClusterID = "cluster-2" })
+	v := createVolume(t, theirs, "vol-shared-name", 1<<30)
+
+	ours := newClusterOver(t, fake, nil).router()
+	ctx := context.Background()
+	_, err := ours.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: latest, VolumeID: "vol-shared-name"})
+	require.ErrorIs(t, err, ebsprovider.ErrNotFound)
+	_, err = ours.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: latest, VolumeID: "vol-shared-name", Handle: v.Handle})
+	require.ErrorIs(t, err, ebsprovider.ErrNotFound, "a handle is honoured only for this cluster's volume")
+	require.NoError(t, ours.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: "vol-shared-name", Handle: v.Handle}))
+	assert.Equal(t, oci.VolumeAvailable, ociVolume(t, fake, v.Handle).State, "another cluster's volume is never deleted")
+}
+
+func TestExpandVolume(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	v := createVolume(t, p, "vol-grow", 1<<30)
+
+	grown, err := p.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: latest, VolumeID: "vol-grow", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}})
+	require.NoError(t, err)
+	assert.Equal(t, int64(2<<30), grown.CapacityBytes)
+	assert.Equal(t, int64(oci.MinVolumeSizeGB), ociVolume(t, c.fake, v.Handle).SizeGB, "within the floor only the recorded size changes")
+
+	grown, err = p.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: latest, VolumeID: "vol-grow", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 80 << 30}})
+	require.NoError(t, err)
+	assert.Equal(t, int64(80<<30), grown.CapacityBytes)
+	assert.Equal(t, int64(80), ociVolume(t, c.fake, v.Handle).SizeGB)
+
+	same, err := p.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: latest, VolumeID: "vol-grow", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 80 << 30}})
+	require.NoError(t, err)
+	assert.Equal(t, int64(80<<30), same.CapacityBytes)
+
+	_, err = p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-grow", NodeID: "node-1"})
+	require.NoError(t, err)
+	_, err = p.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: latest, VolumeID: "vol-grow", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 90 << 30}})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
+}
+
+// raceClient runs race before each UpdateVolume it forwards, as another
+// worker's write landing between this worker's read and its write.
+type raceClient struct {
+	oci.BlockClient
+
+	race func()
+}
+
+func (r *raceClient) UpdateVolume(ctx context.Context, volumeID string, in oci.UpdateVolumeInput) (oci.BlockVolume, error) {
+	if r.race != nil {
+		r.race()
+	}
+	return r.BlockClient.UpdateVolume(ctx, volumeID, in)
+}
+
+func TestConcurrentExpansionsNeverShrinkTheVolume(t *testing.T) {
+	fake := oci.NewBlockFake()
+	other := newClusterOver(t, fake, nil).router()
+	rc := &raceClient{BlockClient: fake}
+	p := newClusterOver(t, fake, rc).router()
+	ctx := context.Background()
+	v := createVolume(t, p, "vol-race", 1<<30)
+
+	rc.race = func() {
+		rc.race = nil
+		_, err := other.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: latest, VolumeID: "vol-race", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 3 << 30}})
+		require.NoError(t, err)
+	}
+	_, err := p.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: latest, VolumeID: "vol-race", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}})
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument, "re-read after the 412, 2 GiB would shrink the 3 GiB volume")
+
+	got, err := p.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: latest, VolumeID: "vol-race"})
+	require.NoError(t, err)
+	assert.Equal(t, int64(3<<30), got.CapacityBytes)
+	assert.Equal(t, strconv.Itoa(3<<30), ociVolume(t, fake, v.Handle).FreeformTags[ocibsd.TagCapacityBytes])
+}
+
+func TestExpansionGivesUpWhileTheVolumeKeepsChanging(t *testing.T) {
+	fake := oci.NewBlockFake()
+	rc := &raceClient{BlockClient: fake}
+	p := newClusterOver(t, fake, rc).router()
+	ctx := context.Background()
+	v := createVolume(t, p, "vol-busy", 1<<30)
+
+	rc.race = func() {
+		_, err := fake.UpdateVolume(ctx, v.Handle, oci.UpdateVolumeInput{FreeformTags: ociVolume(t, fake, v.Handle).FreeformTags})
+		require.NoError(t, err)
+	}
+	_, err := p.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: latest, VolumeID: "vol-busy", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+	assert.Equal(t, strconv.Itoa(1<<30), ociVolume(t, fake, v.Handle).FreeformTags[ocibsd.TagCapacityBytes])
+}
+
+func TestDeleteVolumeWhileABackupIsTakenIsInUse(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-backing-up", 1<<30)
+	_, err := p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: latest, SnapshotID: "snap-taking", VolumeID: "vol-backing-up"})
+	require.NoError(t, err)
+
+	err = p.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: "vol-backing-up"})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "OCI's 409 while a backup is taken")
+	require.NoError(t, p.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: "vol-never"}))
+}
+
+func TestSnapshots(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-snap", 3<<30)
+	createVolume(t, p, "vol-other", 1<<30)
+
+	c.fake.LoseReply("CreateVolumeBackup")
+	_, err := p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: latest, SnapshotID: "snap-1", VolumeID: "vol-snap"})
+	require.ErrorIs(t, err, oci.ErrReplyLost)
+	snap, err := c.nodes["node-2"].CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: latest, SnapshotID: "snap-1", VolumeID: "vol-snap"})
+	require.NoError(t, err)
+	require.Len(t, c.fake.Backups(), 1, "a repeat after a lost reply takes no second backup")
+	assert.Equal(t, ebsprovider.SnapshotStatePending, snap.State, "CreateSnapshot answers once OCI accepts the backup")
+	assert.Equal(t, int64(3<<30), snap.SizeBytes)
+	assert.Equal(t, "vol-snap", snap.SourceVolumeID)
+	assert.Equal(t, ocibsd.RetryToken("snapshot", testCluster, "snap-1"), c.fake.RetryToken(snap.Handle))
+	backup := c.fake.Backups()[0]
+	assert.Equal(t, map[string]string{
+		ocibsd.TagCluster: testCluster, ocibsd.TagSnapshotID: "snap-1", ocibsd.TagVolumeID: "vol-snap", ocibsd.TagCapacityBytes: strconv.Itoa(3 << 30),
+	}, backup.FreeformTags)
+
+	_, err = p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: latest, SnapshotID: "snap-1", VolumeID: "vol-other"})
+	require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists)
+
+	err = p.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: latest, SnapshotID: "snap-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "OCI's 409 on a backup still being taken is retry-later")
+
+	read := func() ebsprovider.SnapshotState {
+		resp, err := p.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: latest, SnapshotID: "snap-1"})
+		require.NoError(t, err)
+		require.Len(t, resp.Snapshots, 1)
+		return resp.Snapshots[0].State
+	}
+	assert.Equal(t, ebsprovider.SnapshotStatePending, read(), "CREATING")
+	require.NoError(t, c.fake.SettleBackup(snap.Handle, oci.BackupRequestReceived))
+	assert.Equal(t, ebsprovider.SnapshotStatePending, read(), "REQUEST_RECEIVED")
+	require.NoError(t, c.fake.SettleBackup(snap.Handle, oci.BackupAvailable))
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, read(), "AVAILABLE")
+
+	require.NoError(t, p.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: latest, SnapshotID: "snap-1"}))
+	require.NoError(t, p.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: latest, SnapshotID: "snap-1"}), "a deleted snapshot deletes again")
+
+	faulty, err := p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: latest, SnapshotID: "snap-2", VolumeID: "vol-snap"})
+	require.NoError(t, err)
+	require.NoError(t, c.fake.SettleBackup(faulty.Handle, oci.BackupFaulty))
+	resp, err := p.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: latest})
+	require.NoError(t, err)
+	require.Len(t, resp.Snapshots, 1, "a deleted backup is not enumerated")
+	assert.Equal(t, ebsprovider.SnapshotStateError, resp.Snapshots[0].State, "FAULTY")
+}
+
+func TestPublishRepublishesOnlyInTheSameAccessMode(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-pub", 1<<30)
+
+	first, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-pub", NodeID: "node-1"})
+	require.NoError(t, err)
+	require.NoError(t, first.Export.Validate())
+	require.Equal(t, ebsprovider.ExportKindBlock, first.Export.Kind)
+	sessions := c.initiators["node-1"].Open()
+	require.Len(t, sessions, 1)
+	assert.Equal(t, ocibsd.ByPathDevice(sessions[0]), first.Export.Block.DevicePath)
+	assert.Regexp(t, `^/dev/disk/by-path/ip-169\.254\.2\.\d+:3260-iscsi-iqn\.2015-12\.com\.oracleiaas:.+-lun-1$`, first.Export.Block.DevicePath)
+
+	again, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-pub", NodeID: "node-1"})
+	require.NoError(t, err)
+	assert.Equal(t, first, again)
+	attachments, err := c.fake.ListVolumeAttachments(ctx, oci.AttachmentFilter{InstanceID: instanceOf("node-1")})
+	require.NoError(t, err)
+	assert.Len(t, attachments, 1, "a republish reuses the attachment")
+
+	_, err = p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-pub", NodeID: "node-1", ReadOnly: true})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
+	assert.NotErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
+}
+
+func TestPublishIsRefusedWhileAnotherInstanceHoldsTheVolume(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	v := createVolume(t, p, "vol-held", 1<<30)
+	c.fake.SeedAttachment(oci.VolumeAttachment{ID: "ocid1.volumeattachment.oc1..seeded", InstanceID: "ocid1.instance.oc1..elsewhere", VolumeID: v.Handle, State: oci.AttachmentAttached})
+
+	_, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-held", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
+}
+
+func TestPublishBelowTheDescriptorVersionIsRefusedWithoutAttaching(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-v1", 1<<30)
+
+	_, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-v1", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+	require.NoError(t, p.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: latest, VolumeID: "vol-v1"}),
+		"a refused publish must leave the volume unattached, and so deletable")
+}
+
+// racingClient attaches the volume to another instance just before this
+// node's attach reaches OCI, as a publish racing on another node does.
+type racingClient struct{ *oci.BlockFake }
+
+func (r racingClient) AttachVolume(ctx context.Context, in oci.AttachVolumeInput) (oci.VolumeAttachment, error) {
+	r.SeedAttachment(oci.VolumeAttachment{ID: "ocid1.volumeattachment.oc1..raced", InstanceID: "ocid1.instance.oc1..racer", VolumeID: in.VolumeID, State: oci.AttachmentAttaching})
+	return r.BlockFake.AttachVolume(ctx, in)
+}
+
+func TestPublishThatLosesAnAttachRaceIsHeldElsewhere(t *testing.T) {
+	fake := oci.NewBlockFake()
+	c := newClusterOver(t, fake, racingClient{fake})
+	createVolume(t, c.router(), "vol-raced", 1<<30)
+
+	_, err := c.router().PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-raced", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
+}
+
+func TestPublishRetryLaterRefusals(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-a", 1<<30)
+	createVolume(t, p, "vol-b", 1<<30)
+
+	c.fake.FailOp("AttachVolume", fmt.Errorf("staged: %w", oci.ErrConflict))
+	_, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-a", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "a conflict no attachment explains is retry-later")
+
+	c.initiators["node-1"].FailNextLogin(errors.New("no session"))
+	_, err = p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-a", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+	_, err = p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-a", NodeID: "node-1"})
+	require.NoError(t, err, "the retry logs in over the attachment the failed login left")
+
+	c.fake.AttachmentCap = 1
+	_, err = p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-b", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "past the attachment cap a publish waits for a detach")
+	_, err = p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-b", NodeID: "node-2"})
+	require.NoError(t, err, "the cap is per instance")
+}
+
+func TestPublishOfADetachingAttachmentIsRetryLater(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	v := createVolume(t, p, "vol-detaching", 1<<30)
+	c.fake.SeedAttachment(oci.VolumeAttachment{ID: "ocid1.volumeattachment.oc1..leaving", InstanceID: instanceOf("node-1"), VolumeID: v.Handle, State: oci.AttachmentDetaching})
+
+	_, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-detaching", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+}
+
+// A publish that outlasts its own bound is retry-later, and the retry reuses
+// the attachment it left rather than attaching the volume again.
+func TestPublishPastItsBoundIsRetryLaterAndTheRetryReusesTheAttachment(t *testing.T) {
+	fake := oci.NewBlockFake()
+	fake.AttachAfter = 50
+	p := newNode(t, fake, ocibsd.NewFakeInitiator(), func(cfg *ocibsd.Config) {
+		cfg.AttachTimeout = time.Minute
+		cfg.PublishTimeout = 20 * time.Millisecond
+	})
+	ctx := context.Background()
+	require.NoError(t, p.Start(ctx))
+	createVolume(t, p, "vol-slow", 1<<30)
+
+	req := ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-slow", NodeID: "node-1"}
+	var published *ebsprovider.PublishedVolume
+	var err error
+	timedOut := 0
+	for published == nil {
+		published, err = p.PublishVolume(ctx, req)
+		if err != nil {
+			require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+			require.NotErrorIs(t, err, context.DeadlineExceeded, "a spent bound is the provider's answer, not a lost caller")
+			timedOut++
+			require.Less(t, timedOut, 1000, "the attachment never settled")
+		}
+	}
+	require.Positive(t, timedOut, "the first publish must have outlasted its bound")
+	require.NoError(t, published.Export.Validate())
+	attachments, err := fake.ListVolumeAttachments(ctx, oci.AttachmentFilter{InstanceID: instanceOf("node-1")})
+	require.NoError(t, err)
+	assert.Len(t, attachments, 1, "every retry reuses the attachment the first publish made")
+}
+
+func TestPerNodeVerbsRefuseAnotherNode(t *testing.T) {
+	p := newCluster(t).nodes["node-1"]
+	ctx := context.Background()
+	_, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-x", NodeID: "node-2"})
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+	_, err = p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: "vol-x", NodeID: "node-2"})
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+}
+
+func TestUnpublish(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	v := createVolume(t, p, "vol-unpub", 1<<30)
+
+	result, err := p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: "vol-unpub", NodeID: "node-1"})
+	require.NoError(t, err, "a volume with no attachment here is already unpublished")
+	assert.False(t, result.TornDown)
+	result, err = p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: "vol-absent", NodeID: "node-1"})
+	require.NoError(t, err)
+	assert.False(t, result.TornDown)
+
+	_, err = p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-unpub", NodeID: "node-1"})
+	require.NoError(t, err)
+	result, err = p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: "vol-unpub", NodeID: "node-2"})
+	require.NoError(t, err, "node-2 holds no attachment of it")
+	assert.False(t, result.TornDown)
+
+	result, err = p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: "vol-unpub", NodeID: "node-1"})
+	require.NoError(t, err)
+	assert.True(t, result.TornDown)
+	assert.Empty(t, c.initiators["node-1"].Open(), "the session is logged out")
+	attachments, err := c.fake.ListVolumeAttachments(ctx, oci.AttachmentFilter{VolumeID: v.Handle})
+	require.NoError(t, err)
+	require.Len(t, attachments, 1)
+	assert.Equal(t, oci.AttachmentDetached, attachments[0].State, "unpublish returns only once OCI reports DETACHED")
+}
+
+func TestUnpublishFinishesADetachAlreadyUnderWay(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	v := createVolume(t, p, "vol-half", 1<<30)
+	c.fake.SeedAttachment(oci.VolumeAttachment{ID: "ocid1.volumeattachment.oc1..half", InstanceID: instanceOf("node-1"), VolumeID: v.Handle, State: oci.AttachmentDetaching})
+
+	result, err := p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: "vol-half", NodeID: "node-1"})
+	require.NoError(t, err)
+	assert.True(t, result.TornDown)
+}
+
+func TestHoldsUnsealedWritesAnswersNone(t *testing.T) {
+	p := newCluster(t).nodes["node-1"]
+	held, err := p.HoldsUnsealedWrites(context.Background(), "node-1", []string{"vol-a"})
+	require.NoError(t, err)
+	assert.Empty(t, held)
+	_, err = p.HoldsUnsealedWrites(context.Background(), "node-1", []string{""})
+	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+}
+
+// provisioningClient reports a volume PROVISIONING for its first reads, as
+// OCI does for the seconds after a create.
+type provisioningClient struct {
+	*oci.BlockFake
+
+	reads *int
+	state oci.VolumeState
+}
+
+func (p provisioningClient) GetVolume(ctx context.Context, volumeID string) (oci.BlockVolume, error) {
+	v, err := p.BlockFake.GetVolume(ctx, volumeID)
+	if *p.reads > 0 && err == nil {
+		*p.reads--
+		v.State = p.state
+	}
+	return v, err
+}
+
+func TestCreateVolumeWaitsForAvailable(t *testing.T) {
+	fake := oci.NewBlockFake()
+	reads := 0
+	c := newClusterOver(t, fake, provisioningClient{BlockFake: fake, reads: &reads, state: oci.VolumeProvisioning})
+	p := c.router()
+	ctx := context.Background()
+
+	reads = 3
+	v := createVolume(t, p, "vol-slow", 1<<30)
+	assert.Equal(t, ebsprovider.VolumeStateAvailable, v.State)
+	assert.Zero(t, reads, "the create read the volume until it was available")
+
+	reads = 1
+	got, err := p.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: latest, VolumeID: "vol-slow", Handle: v.Handle})
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.VolumeStateCreating, got.State)
+
+	faulty := newClusterOver(t, fake, provisioningClient{BlockFake: fake, reads: &reads, state: oci.VolumeFaulty})
+	reads = 1
+	_, err = faulty.router().CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: latest, VolumeID: "vol-faulty", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+	require.ErrorContains(t, err, "FAULTY")
+}
+
+func TestOCIQuotaRefusalsAreRetryLater(t *testing.T) {
+	c := newCluster(t)
+	c.fake.FailOp("CreateVolume", fmt.Errorf("staged: %w", oci.ErrLimitExceeded))
+	_, err := c.router().CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{Versioned: latest, VolumeID: "vol-quota", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+}
+
+func TestAttachmentsThatDoNotSettleAreRetryLater(t *testing.T) {
+	c := newCluster(t)
+	p := c.router()
+	ctx := context.Background()
+	createVolume(t, p, "vol-stuck", 1<<30)
+
+	c.fake.AttachAfter = 1 << 20
+	_, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-stuck", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "an attachment still ATTACHING at the timeout")
+
+	c.fake.DetachAfter = 1 << 20
+	_, err = p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: latest, VolumeID: "vol-stuck", NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "an attachment still DETACHING at the timeout")
+}
+
+func volumeParams(t *testing.T, volumeType string) json.RawMessage {
+	t.Helper()
+	raw, err := ebsprovider.NewVolumeParameters(volumeType)
+	require.NoError(t, err)
+	return raw
+}

spinifex/providers/ebs/ocibsd/publish.go

@@ -0,0 +1,234 @@
+package ocibsd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+)
+
+func targetOf(a oci.VolumeAttachment) Target {
+	return Target{IQN: a.IQN, IPv4: a.IPv4, Port: a.Port}
+}
+
+// checkNode refuses a request for another node: this provider can attach a
+// volume only to its own instance.
+func (p *Provider) checkNode(nodeID string) error {
+	if nodeID != p.cfg.NodeID {
+		return fmt.Errorf("%w: node %s is served by its own provider, not %s's", ebsprovider.ErrInvalidArgument, nodeID, p.cfg.NodeID)
+	}
+	return nil
+}
+
+// errPublishTimeout is the cause of a publish that ran out of its own budget.
+var errPublishTimeout = errors.New("ocibsd: publish budget spent")
+
+// PublishVolume attaches the volume to this node's instance over iSCSI and
+// returns its block device. An attachment here in the same access mode is
+// reused; any other attachment refuses the publish.
+func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
+	publishCtx, cancel := context.WithTimeoutCause(ctx, p.cfg.PublishTimeout, errPublishTimeout)
+	defer cancel()
+	published, err := p.publishVolume(publishCtx, req)
+	if err != nil && ctx.Err() == nil && errors.Is(context.Cause(publishCtx), errPublishTimeout) {
+		slog.WarnContext(ctx, "ocibsd: publish ran out of its budget", "volume", req.VolumeID, "budget_ms", otelsetup.Millis(p.cfg.PublishTimeout))
+		// Retry-later, since a retry reuses any attachment left behind. The cause is
+		// text so a sentinel inside it cannot override that classification.
+		return nil, fmt.Errorf("%w: publish of volume %s did not finish in time: %s", ebsprovider.ErrUnavailable, req.VolumeID, err.Error())
+	}
+	return published, err
+}
+
+func (p *Provider) publishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	// A block export has no form below the descriptor version, so refuse
+	// before attaching rather than attach a volume the caller cannot read.
+	if req.SchemaVersion < ebsprovider.DescriptorSchemaVersion {
+		return nil, fmt.Errorf("%w: a block export requires schema version %d, got %d", ebsprovider.ErrUnsupportedVersion, ebsprovider.DescriptorSchemaVersion, req.SchemaVersion)
+	}
+	if req.VolumeID == "" || req.NodeID == "" {
+		return nil, fmt.Errorf("%w: volume and node IDs are required", ebsprovider.ErrInvalidArgument)
+	}
+	if err := p.checkNode(req.NodeID); err != nil {
+		return nil, err
+	}
+	if err := p.checkStarted(); err != nil {
+		return nil, err
+	}
+	unlock := p.lockVolume(req.VolumeID)
+	defer unlock()
+
+	v, ok, err := p.findVolume(ctx, req.VolumeID, req.Handle)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+	}
+	capacity, err := capacityOf(v.FreeformTags)
+	if err != nil {
+		return nil, err
+	}
+
+	attached, err := p.attachments(ctx, v.ID)
+	if err != nil {
+		return nil, err
+	}
+	if err := p.heldElsewhere(req, attached); err != nil {
+		return nil, err
+	}
+	var mine *oci.VolumeAttachment
+	if len(attached) > 0 {
+		mine = &attached[len(attached)-1]
+		// OCI changes an attachment's access mode only by detaching it.
+		if mine.ReadOnly != req.ReadOnly {
+			return nil, fmt.Errorf("%w: volume %s is already attached here read_only=%t", ebsprovider.ErrVolumeInUse, req.VolumeID, mine.ReadOnly)
+		}
+		if mine.State == oci.AttachmentDetaching {
+			return nil, fmt.Errorf("%w: volume %s is still detaching from this node", ebsprovider.ErrUnavailable, req.VolumeID)
+		}
+	} else {
+		a, err := p.attach(ctx, req, v.ID)
+		if err != nil {
+			return nil, err
+		}
+		mine = &a
+	}
+
+	ready, err := p.awaitAttached(ctx, mine.ID)
+	if err != nil {
+		return nil, err
+	}
+	device, err := p.cfg.Initiator.Login(ctx, targetOf(ready))
+	if err != nil {
+		return nil, fmt.Errorf("%w: log in to volume %s: %w", ebsprovider.ErrUnavailable, req.VolumeID, err)
+	}
+	return &ebsprovider.PublishedVolume{
+		VolumeID: req.VolumeID,
+		NodeID:   req.NodeID,
+		Export: ebsprovider.VolumeExport{
+			Kind:  ebsprovider.ExportKindBlock,
+			Block: &ebsprovider.BlockExport{DevicePath: device, SizeBytes: uint64(capacity)}, //nolint:gosec // G115: capacityOf refuses a capacity below 1
+		},
+	}, nil
+}
+
+// attach asks OCI for a new attachment to this node's instance. A conflict is
+// another attachment that appeared since the list, so it is read again.
+func (p *Provider) attach(ctx context.Context, req ebsprovider.PublishVolumeRequest, volumeOCID string) (oci.VolumeAttachment, error) {
+	a, err := p.cfg.Client.AttachVolume(ctx, oci.AttachVolumeInput{
+		InstanceID: p.cfg.InstanceID, VolumeID: volumeOCID, DisplayName: req.VolumeID, ReadOnly: req.ReadOnly,
+	})
+	switch {
+	case err == nil:
+		return a, nil
+	case errors.Is(err, oci.ErrNotFound):
+		return oci.VolumeAttachment{}, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+	case !errors.Is(err, oci.ErrConflict):
+		return oci.VolumeAttachment{}, ociError("attach volume "+req.VolumeID, err)
+	}
+	attached, listErr := p.attachments(ctx, volumeOCID)
+	if listErr != nil {
+		return oci.VolumeAttachment{}, listErr
+	}
+	if err := p.heldElsewhere(req, attached); err != nil {
+		return oci.VolumeAttachment{}, err
+	}
+	return oci.VolumeAttachment{}, ociError("attach volume "+req.VolumeID, err, ebsprovider.ErrUnavailable)
+}
+
+// heldElsewhere refuses a publish of a volume attached to another instance.
+func (p *Provider) heldElsewhere(req ebsprovider.PublishVolumeRequest, attached []oci.VolumeAttachment) error {
+	for _, a := range attached {
+		if a.InstanceID != p.cfg.InstanceID {
+			return fmt.Errorf("%w: volume %s is attached to instance %s", ebsprovider.HeldElsewhereError(req.SchemaVersion), req.VolumeID, a.InstanceID)
+		}
+	}
+	return nil
+}
+
+func (p *Provider) awaitAttached(ctx context.Context, attachmentID string) (oci.VolumeAttachment, error) {
+	var a oci.VolumeAttachment
+	err := p.waitFor(ctx, p.cfg.AttachTimeout, "attachment "+attachmentID+" attaching", func(ctx context.Context) (bool, error) {
+		var err error
+		if a, err = p.cfg.Client.GetVolumeAttachment(ctx, attachmentID); err != nil {
+			return false, ociError("get volume attachment", err)
+		}
+		switch a.State {
+		case oci.AttachmentAttached:
+			return true, nil
+		case oci.AttachmentAttaching:
+			return false, nil
+		default:
+			return false, fmt.Errorf("%w: attachment %s is %s", ebsprovider.ErrUnavailable, attachmentID, a.State)
+		}
+	})
+	return a, err
+}
+
+// UnpublishVolume logs out of the volume's session on this node, detaches it
+// from this node's instance, and waits for OCI to report it DETACHED, the
+// point after which the old session can write nothing. NoSeal changes nothing.
+func (p *Provider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	if err := ebsprovider.CheckUnpublishVersion(req); err != nil {
+		return nil, err
+	}
+	if req.VolumeID == "" || req.NodeID == "" {
+		return nil, fmt.Errorf("%w: volume and node IDs are required", ebsprovider.ErrInvalidArgument)
+	}
+	if err := p.checkNode(req.NodeID); err != nil {
+		return nil, err
+	}
+	unlock := p.lockVolume(req.VolumeID)
+	defer unlock()
+
+	result := &ebsprovider.UnpublishedVolume{}
+	v, ok, err := p.findVolume(ctx, req.VolumeID, req.Handle)
+	if err != nil || !ok {
+		return result, err
+	}
+	attached, err := p.attachments(ctx, v.ID)
+	if err != nil {
+		return nil, err
+	}
+	tornDown := false
+	for _, a := range attached {
+		if a.InstanceID != p.cfg.InstanceID {
+			continue
+		}
+		if err := p.cfg.Initiator.Logout(ctx, targetOf(a)); err != nil {
+			return nil, fmt.Errorf("%w: log out of volume %s: %w", ebsprovider.ErrUnavailable, req.VolumeID, err)
+		}
+		if a.State != oci.AttachmentDetaching {
+			err := p.cfg.Client.DetachVolume(ctx, a.ID)
+			if err != nil && !errors.Is(err, oci.ErrNotFound) && !errors.Is(err, oci.ErrConflict) {
+				return nil, ociError("detach volume "+req.VolumeID, err)
+			}
+		}
+		if err := p.awaitDetached(ctx, a.ID); err != nil {
+			return nil, err
+		}
+		tornDown = true
+	}
+	result.TornDown = tornDown && req.SchemaVersion >= ebsprovider.NoSealSchemaVersion
+	return result, nil
+}
+
+func (p *Provider) awaitDetached(ctx context.Context, attachmentID string) error {
+	return p.waitFor(ctx, p.cfg.DetachTimeout, "attachment "+attachmentID+" detaching", func(ctx context.Context) (bool, error) {
+		a, err := p.cfg.Client.GetVolumeAttachment(ctx, attachmentID)
+		if errors.Is(err, oci.ErrNotFound) {
+			return true, nil
+		}
+		if err != nil {
+			return false, ociError("get volume attachment", err)
+		}
+		return a.State == oci.AttachmentDetached, nil
+	})
+}

spinifex/providers/ebs/ocibsd/snapshots.go

@@ -0,0 +1,169 @@
+package ocibsd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"slices"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+)
+
+// snapshotState is an OCI backup state as an EBS snapshot state. A state this
+// build does not know is reported pending, so a caller keeps reading it.
+func snapshotState(state oci.BackupState) ebsprovider.SnapshotState {
+	switch state {
+	case oci.BackupAvailable:
+		return ebsprovider.SnapshotStateCompleted
+	case oci.BackupFaulty:
+		return ebsprovider.SnapshotStateError
+	default:
+		return ebsprovider.SnapshotStatePending
+	}
+}
+
+// findBackup resolves an EBS snapshot to its live OCI backup.
+func (p *Provider) findBackup(ctx context.Context, snapshotID, handle string) (oci.VolumeBackup, bool, error) {
+	return findTagged(ctx, p, "volume backup", TagSnapshotID, "snapshot ID", snapshotID, handle, p.cfg.Client.GetVolumeBackup,
+		func(ctx context.Context, tags map[string]string) ([]oci.VolumeBackup, error) {
+			return p.cfg.Client.ListVolumeBackups(ctx, oci.BackupFilter{CompartmentID: p.cfg.CompartmentID, Tags: tags})
+		})
+}
+
+func snapshotOf(b oci.VolumeBackup) (*ebsprovider.Snapshot, error) {
+	capacity, err := capacityOf(b.FreeformTags)
+	if err != nil {
+		return nil, err
+	}
+	return &ebsprovider.Snapshot{
+		ID:             b.FreeformTags[TagSnapshotID],
+		SourceVolumeID: b.FreeformTags[TagVolumeID],
+		SizeBytes:      capacity,
+		CreatedAt:      b.TimeCreated,
+		State:          snapshotState(b.State),
+		Handle:         b.ID,
+	}, nil
+}
+
+// CreateSnapshot starts an incremental backup and answers pending as soon as
+// OCI accepts it. A repeat is answered from the backup the first call started,
+// which its tags find even once the source volume is gone.
+func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	if req.SnapshotID == "" || req.VolumeID == "" {
+		return nil, fmt.Errorf("%w: snapshot and volume IDs are required", ebsprovider.ErrInvalidArgument)
+	}
+	existing, ok, err := p.findBackup(ctx, req.SnapshotID, "")
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		if existing.FreeformTags[TagVolumeID] != req.VolumeID {
+			return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrAlreadyExists, req.SnapshotID)
+		}
+		return snapshotOf(existing)
+	}
+
+	v, ok, err := p.findVolume(ctx, req.VolumeID, req.VolumeHandle)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+	}
+	capacity, ok := v.FreeformTags[TagCapacityBytes]
+	if !ok {
+		return nil, fmt.Errorf("ocibsd: volume %s has no %s tag", req.VolumeID, TagCapacityBytes)
+	}
+	created, err := p.cfg.Client.CreateVolumeBackup(ctx, oci.CreateVolumeBackupInput{
+		VolumeID:    v.ID,
+		DisplayName: req.SnapshotID,
+		FreeformTags: map[string]string{
+			TagCluster: p.cfg.ClusterID, TagSnapshotID: req.SnapshotID, TagVolumeID: req.VolumeID, TagCapacityBytes: capacity,
+		},
+		RetryToken: RetryToken("snapshot", p.cfg.ClusterID, req.SnapshotID),
+	})
+	switch {
+	case errors.Is(err, oci.ErrNotFound):
+		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+	case err != nil:
+		return nil, ociError("create snapshot "+req.SnapshotID, err, ebsprovider.ErrUnavailable)
+	}
+	if created.State.Gone() {
+		return nil, fmt.Errorf("ocibsd: create snapshot %s answered with %s backup %s", req.SnapshotID, created.State, created.ID)
+	}
+	return snapshotOf(created)
+}
+
+// DeleteSnapshot deletes the backup. OCI refuses one still being taken, which
+// is answered retry-later; a backup that does not exist is deleted.
+func (p *Provider) DeleteSnapshot(ctx context.Context, req ebsprovider.DeleteSnapshotRequest) error {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return err
+	}
+	if req.SnapshotID == "" {
+		return fmt.Errorf("%w: snapshot ID is required", ebsprovider.ErrInvalidArgument)
+	}
+	b, ok, err := p.findBackup(ctx, req.SnapshotID, req.Handle)
+	if err != nil || !ok {
+		return err
+	}
+	if err := p.cfg.Client.DeleteVolumeBackup(ctx, b.ID); err != nil && !errors.Is(err, oci.ErrNotFound) {
+		return ociError("delete snapshot "+req.SnapshotID, err, ebsprovider.ErrUnavailable)
+	}
+	return nil
+}
+
+// CopySnapshot is refused: OCI copies a backup only to another region.
+func (p *Provider) CopySnapshot(_ context.Context, req ebsprovider.CopySnapshotRequest) (*ebsprovider.Snapshot, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	return nil, fmt.Errorf("%w: snapshot copy", ebsprovider.ErrUnsupportedCapability)
+}
+
+// ListSnapshots lists this cluster's backups by EBS snapshot ID, or reads one.
+func (p *Provider) ListSnapshots(ctx context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
+	if err := ebsprovider.CheckListSnapshotsVersion(req); err != nil {
+		return nil, err
+	}
+	refOf := func(b oci.VolumeBackup) ebsprovider.SnapshotRef {
+		return ebsprovider.SnapshotRef{
+			ID:             b.FreeformTags[TagSnapshotID],
+			SourceVolumeID: b.FreeformTags[TagVolumeID],
+			Handle:         b.ID,
+			State:          ebsprovider.RefState(req.SchemaVersion, snapshotState(b.State)),
+		}
+	}
+	if req.SnapshotID != "" {
+		b, ok, err := p.findBackup(ctx, req.SnapshotID, "")
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			return ebsprovider.FilteredSnapshots(req, nil), nil
+		}
+		ref := refOf(b)
+		return ebsprovider.FilteredSnapshots(req, &ref), nil
+	}
+
+	found, err := p.cfg.Client.ListVolumeBackups(ctx, oci.BackupFilter{CompartmentID: p.cfg.CompartmentID, Tags: map[string]string{TagCluster: p.cfg.ClusterID}})
+	if err != nil {
+		return nil, ociError("list volume backups", err)
+	}
+	backups := make(map[string]oci.VolumeBackup, len(found))
+	for _, b := range found {
+		if id := b.FreeformTags[TagSnapshotID]; id != "" && !b.State.Gone() {
+			backups[id] = b
+		}
+	}
+	ids := slices.Sorted(maps.Keys(backups))
+	response := &ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion)}
+	response.Snapshots, response.NextToken = ebsprovider.Page(ids, req.StartingToken, int(req.PageSize()),
+		func(id string) ebsprovider.SnapshotRef { return refOf(backups[id]) })
+	return response, nil
+}

spinifex/providers/ebs/ocibsd/startup.go

@@ -0,0 +1,113 @@
+package ocibsd
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+)
+
+// RequiredPolicy is what the tenancy must grant the node's credential in the
+// compartment ocibsd keeps its volumes in.
+const RequiredPolicy = "manage volume-family and use instances"
+
+// CheckCredential makes the read ocibsd makes at start: it proves client
+// authenticates and can read volumes in the compartment, not that every grant
+// in RequiredPolicy is present, so an inspect-only credential passes.
+func CheckCredential(ctx context.Context, client oci.BlockClient, compartmentID, availabilityDomain, clusterID string) error {
+	if _, err := client.ListVolumes(ctx, oci.VolumeFilter{
+		CompartmentID: compartmentID, AvailabilityDomain: availabilityDomain,
+		Tags: map[string]string{TagCluster: clusterID},
+	}); err != nil {
+		return fmt.Errorf("ocibsd: read volumes in compartment %s: %w; the tenancy needs a policy granting this node's credential %s in that compartment, as \"Allow dynamic-group <group> to manage volume-family in compartment id %[1]s\" and \"Allow dynamic-group <group> to use instances in compartment id %[1]s\" (or group <group> for an API key)",
+			compartmentID, err, RequiredPolicy)
+	}
+	return nil
+}
+
+// Start readies the provider on node start, before it serves anything. It
+// checks the credential as CheckCredential does, then logs out stale
+// sessions; PublishVolume is refused until it ends.
+func (p *Provider) Start(ctx context.Context) error {
+	if err := CheckCredential(ctx, p.cfg.Client, p.cfg.CompartmentID, p.cfg.AvailabilityDomain, p.cfg.ClusterID); err != nil {
+		return err
+	}
+	if err := p.logoutStaleSessions(ctx); err != nil {
+		return err
+	}
+	p.started.Store(true)
+	return nil
+}
+
+func (p *Provider) checkStarted() error {
+	if !p.started.Load() {
+		return fmt.Errorf("%w: ocibsd on node %s has not finished starting", ebsprovider.ErrUnavailable, p.cfg.NodeID)
+	}
+	return nil
+}
+
+// logoutStaleSessions logs out a session only when its target is an OCI
+// attachment of a volume tagged with this cluster that OCI no longer attaches
+// to this node. A session it cannot tie to such a volume is left alone.
+func (p *Provider) logoutStaleSessions(ctx context.Context) error {
+	sessions, err := p.cfg.Initiator.Sessions(ctx)
+	if err != nil {
+		return fmt.Errorf("ocibsd: list iSCSI sessions: %w", err)
+	}
+	var candidates []Target
+	for _, s := range sessions {
+		if strings.HasPrefix(s.IQN, oci.IQNPrefix) {
+			candidates = append(candidates, s)
+		}
+	}
+	if len(candidates) == 0 {
+		return nil
+	}
+	attachments, err := p.cfg.Client.ListVolumeAttachments(ctx, oci.AttachmentFilter{CompartmentID: p.cfg.CompartmentID})
+	if err != nil {
+		return fmt.Errorf("ocibsd: list volume attachments: %w", err)
+	}
+	for _, s := range candidates {
+		stale, reason := p.isStale(ctx, s, attachments)
+		if !stale {
+			slog.InfoContext(ctx, "ocibsd: keeping iSCSI session", "iqn", s.IQN, "portal", portal(s), "reason", reason)
+			continue
+		}
+		if err := p.cfg.Initiator.Logout(ctx, s); err != nil {
+			return fmt.Errorf("ocibsd: log out of stale session %s: %w", s.IQN, err)
+		}
+		slog.InfoContext(ctx, "ocibsd: logged out stale iSCSI session", "iqn", s.IQN, "portal", portal(s))
+	}
+	return nil
+}
+
+// isStale decides one session, and says why a session it keeps is kept.
+func (p *Provider) isStale(ctx context.Context, s Target, attachments []oci.VolumeAttachment) (bool, string) {
+	volumes := map[string]bool{}
+	for _, a := range attachments {
+		if a.IQN == s.IQN {
+			volumes[a.VolumeID] = true
+		}
+	}
+	if len(volumes) == 0 {
+		return false, "no OCI attachment has this IQN"
+	}
+	for volumeID := range volumes {
+		v, err := p.cfg.Client.GetVolume(ctx, volumeID)
+		if err != nil {
+			return false, "volume " + volumeID + " cannot be read: " + err.Error()
+		}
+		if v.FreeformTags[TagCluster] != p.cfg.ClusterID {
+			return false, "volume " + volumeID + " is not this cluster's"
+		}
+		for _, a := range attachments {
+			if a.VolumeID == volumeID && a.InstanceID == p.cfg.InstanceID && a.State != oci.AttachmentDetached {
+				return false, "volume " + volumeID + " is attached to this node"
+			}
+		}
+	}
+	return true, ""
+}

spinifex/providers/ebs/ocibsd/startup_test.go

@@ -0,0 +1,183 @@
+package ocibsd_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/ocibsd"
+)
+
+func seedVolume(t *testing.T, fake *oci.BlockFake, cluster string) string {
+	t.Helper()
+	v, err := fake.CreateVolume(context.Background(), oci.CreateVolumeInput{
+		CompartmentID: testCompartment, AvailabilityDomain: testAD, SizeGB: oci.MinVolumeSizeGB,
+		FreeformTags: map[string]string{ocibsd.TagCluster: cluster, ocibsd.TagVolumeID: "vol-seeded", ocibsd.TagCapacityBytes: "1073741824"},
+	})
+	require.NoError(t, err)
+	return v.ID
+}
+
+func ociTarget(name string) ocibsd.Target {
+	return ocibsd.Target{IQN: oci.IQNPrefix + name, IPv4: "169.254.2.2", Port: 3260}
+}
+
+func seedAttachment(fake *oci.BlockFake, id, instance, volume string, state oci.AttachmentState, target ocibsd.Target) {
+	fake.SeedAttachment(oci.VolumeAttachment{
+		ID: id, InstanceID: instance, VolumeID: volume, State: state,
+		IQN: target.IQN, IPv4: target.IPv4, Port: target.Port,
+	})
+}
+
+func TestStartLogsOutOnlyThisClustersDetachedSessions(t *testing.T) {
+	cases := []struct {
+		name      string
+		arrange   func(t *testing.T, fake *oci.BlockFake, target ocibsd.Target)
+		target    ocibsd.Target
+		loggedOut bool
+	}{
+		{
+			name:   "a tagged volume no longer attached here is logged out",
+			target: ociTarget("detached"),
+			arrange: func(t *testing.T, fake *oci.BlockFake, target ocibsd.Target) {
+				seedAttachment(fake, "att-old", instanceOf("node-1"), seedVolume(t, fake, testCluster), oci.AttachmentDetached, target)
+			},
+			loggedOut: true,
+		},
+		{
+			name:   "a tagged volume now attached to another node is logged out here",
+			target: ociTarget("moved"),
+			arrange: func(t *testing.T, fake *oci.BlockFake, target ocibsd.Target) {
+				volume := seedVolume(t, fake, testCluster)
+				seedAttachment(fake, "att-old", instanceOf("node-1"), volume, oci.AttachmentDetached, target)
+				seedAttachment(fake, "att-new", instanceOf("node-2"), volume, oci.AttachmentAttached, target)
+			},
+			loggedOut: true,
+		},
+		{
+			name:   "a volume still attached to this node is kept",
+			target: ociTarget("attached"),
+			arrange: func(t *testing.T, fake *oci.BlockFake, target ocibsd.Target) {
+				seedAttachment(fake, "att", instanceOf("node-1"), seedVolume(t, fake, testCluster), oci.AttachmentAttached, target)
+			},
+		},
+		{
+			name:   "another cluster's volume is kept",
+			target: ociTarget("foreign"),
+			arrange: func(t *testing.T, fake *oci.BlockFake, target ocibsd.Target) {
+				seedAttachment(fake, "att", instanceOf("node-1"), seedVolume(t, fake, "cluster-2"), oci.AttachmentDetached, target)
+			},
+		},
+		{
+			name:   "a volume that cannot be read is kept",
+			target: ociTarget("unreadable"),
+			arrange: func(t *testing.T, fake *oci.BlockFake, target ocibsd.Target) {
+				seedAttachment(fake, "att", instanceOf("node-1"), seedVolume(t, fake, testCluster), oci.AttachmentDetached, target)
+				fake.FailOp("GetVolume", errors.New("staged: service unavailable"))
+			},
+		},
+		{
+			name:    "an OCI IQN with no attachment on record is kept",
+			target:  ociTarget("unknown"),
+			arrange: func(*testing.T, *oci.BlockFake, ocibsd.Target) {},
+		},
+		{
+			name:   "a target that is not an OCI volume is kept without asking OCI",
+			target: ocibsd.Target{IQN: "iqn.2003-01.org.linux-iscsi.san:lun0", IPv4: "10.0.0.5", Port: 3260},
+			arrange: func(_ *testing.T, fake *oci.BlockFake, _ ocibsd.Target) {
+				fake.FailOp("ListVolumeAttachments", errors.New("staged: must not be called"))
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			fake := oci.NewBlockFake()
+			tc.arrange(t, fake, tc.target)
+			initiator := ocibsd.NewFakeInitiator()
+			initiator.SeedSession(tc.target)
+			p := newNode(t, fake, initiator)
+
+			require.NoError(t, p.Start(context.Background()))
+			if tc.loggedOut {
+				assert.Equal(t, []string{"logout " + tc.target.IQN}, initiator.Events())
+				assert.Empty(t, initiator.Open())
+			} else {
+				assert.Empty(t, initiator.Events())
+				assert.Equal(t, []ocibsd.Target{tc.target}, initiator.Open())
+			}
+		})
+	}
+}
+
+func TestPublishWaitsForTheStaleSessionLogout(t *testing.T) {
+	fake := oci.NewBlockFake()
+	stale := ociTarget("stale")
+	seedAttachment(fake, "att-old", instanceOf("node-1"), seedVolume(t, fake, testCluster), oci.AttachmentDetached, stale)
+	initiator := ocibsd.NewFakeInitiator()
+	initiator.SeedSession(stale)
+	p := newNode(t, fake, initiator)
+	ctx := context.Background()
+	createVolume(t, p, "vol-after", 1<<30)
+	publish := ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-after", NodeID: "node-1"}
+
+	_, err := p.PublishVolume(ctx, publish)
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "no publish before the stale sessions are gone")
+	assert.Empty(t, initiator.Events(), "and no login")
+
+	require.NoError(t, p.Start(ctx))
+	_, err = p.PublishVolume(ctx, publish)
+	require.NoError(t, err)
+	events := initiator.Events()
+	require.Len(t, events, 2)
+	assert.Equal(t, "logout "+stale.IQN, events[0])
+	assert.Regexp(t, `^login `, events[1])
+}
+
+func TestStartFailures(t *testing.T) {
+	cases := []struct {
+		name    string
+		arrange func(fake *oci.BlockFake, initiator *ocibsd.FakeInitiator)
+		want    string
+	}{
+		{
+			name: "a credential that cannot read volumes fails the start",
+			arrange: func(fake *oci.BlockFake, _ *ocibsd.FakeInitiator) {
+				fake.FailOp("ListVolumes", errors.New("staged: 404 NotAuthorizedOrNotFound"))
+			},
+			want: "manage volume-family",
+		},
+		{
+			name: "sessions that cannot be listed fail the start",
+			arrange: func(_ *oci.BlockFake, initiator *ocibsd.FakeInitiator) {
+				initiator.FailSessions(errors.New("iscsid is not running"))
+			},
+			want: "list iSCSI sessions",
+		},
+		{
+			name: "attachments that cannot be listed fail the start",
+			arrange: func(fake *oci.BlockFake, initiator *ocibsd.FakeInitiator) {
+				initiator.SeedSession(ociTarget("any"))
+				fake.FailOp("ListVolumeAttachments", errors.New("staged: 500"))
+			},
+			want: "list volume attachments",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			fake := oci.NewBlockFake()
+			initiator := ocibsd.NewFakeInitiator()
+			tc.arrange(fake, initiator)
+			p := newNode(t, fake, initiator)
+
+			require.ErrorContains(t, p.Start(context.Background()), tc.want)
+			_, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: latest, VolumeID: "vol-x", NodeID: "node-1"})
+			require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "a node that did not start publishes nothing")
+			assert.Empty(t, initiator.Events())
+		})
+	}
+}

spinifex/providers/ebs/ocibsd/volumes.go

@@ -0,0 +1,381 @@
+package ocibsd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"slices"
+	"strconv"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/oci"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+)
+
+// tagged is an OCI resource ocibsd finds by its tags.
+type tagged interface {
+	Tags() map[string]string
+	IsGone() bool
+}
+
+// findTagged resolves an EBS ID to the live OCI resource whose idTag carries
+// it, by handle when the caller has one and by tag otherwise. A resource
+// another cluster tagged, or one OCI is deleting, is not found.
+func findTagged[T tagged](ctx context.Context, p *Provider, kind, idTag, idName, id, handle string,
+	get func(context.Context, string) (T, error), list func(context.Context, map[string]string) ([]T, error),
+) (T, bool, error) {
+	var none T
+	if handle != "" {
+		r, err := get(ctx, handle)
+		if errors.Is(err, oci.ErrNotFound) {
+			return none, false, nil
+		}
+		if err != nil {
+			return none, false, ociError("get "+kind, err)
+		}
+		if r.IsGone() || r.Tags()[TagCluster] != p.cfg.ClusterID || r.Tags()[idTag] != id {
+			return none, false, nil
+		}
+		return r, true, nil
+	}
+	found, err := list(ctx, map[string]string{TagCluster: p.cfg.ClusterID, idTag: id})
+	if err != nil {
+		return none, false, ociError("list "+kind+"s", err)
+	}
+	live := slices.DeleteFunc(found, T.IsGone)
+	switch len(live) {
+	case 0:
+		return none, false, nil
+	case 1:
+		return live[0], true, nil
+	default:
+		return none, false, fmt.Errorf("ocibsd: %d OCI %ss carry %s %s", len(live), kind, idName, id)
+	}
+}
+
+// findVolume resolves an EBS volume to its live OCI volume.
+func (p *Provider) findVolume(ctx context.Context, volumeID, handle string) (oci.BlockVolume, bool, error) {
+	return findTagged(ctx, p, "volume", TagVolumeID, "volume ID", volumeID, handle, p.cfg.Client.GetVolume,
+		func(ctx context.Context, tags map[string]string) ([]oci.BlockVolume, error) {
+			return p.cfg.Client.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: p.cfg.CompartmentID, Tags: tags})
+		})
+}
+
+// attachments lists the volume's attachments in any state but DETACHED.
+func (p *Provider) attachments(ctx context.Context, volumeOCID string) ([]oci.VolumeAttachment, error) {
+	all, err := p.cfg.Client.ListVolumeAttachments(ctx, oci.AttachmentFilter{CompartmentID: p.cfg.CompartmentID, VolumeID: volumeOCID})
+	if err != nil {
+		return nil, ociError("list volume attachments", err)
+	}
+	return slices.DeleteFunc(all, func(a oci.VolumeAttachment) bool { return a.State == oci.AttachmentDetached }), nil
+}
+
+func (p *Provider) volumeOf(ctx context.Context, v oci.BlockVolume) (*ebsprovider.Volume, error) {
+	capacity, err := capacityOf(v.FreeformTags)
+	if err != nil {
+		return nil, err
+	}
+	state := ebsprovider.VolumeStateAvailable
+	switch v.State {
+	case oci.VolumeProvisioning, oci.VolumeRestoring:
+		state = ebsprovider.VolumeStateCreating
+	default:
+		attached, err := p.attachments(ctx, v.ID)
+		if err != nil {
+			return nil, err
+		}
+		if len(attached) > 0 {
+			state = ebsprovider.VolumeStateInUse
+		}
+	}
+	return &ebsprovider.Volume{
+		ID:               v.FreeformTags[TagVolumeID],
+		CapacityBytes:    capacity,
+		State:            state,
+		Handle:           v.ID,
+		AvailabilityZone: p.cfg.AvailabilityZone,
+	}, nil
+}
+
+func (p *Provider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	required := req.CapacityRange.RequiredBytes
+	if req.VolumeID == "" || !req.CapacityRange.Valid() {
+		return nil, fmt.Errorf("%w: volume ID and a positive capacity range are required", ebsprovider.ErrInvalidArgument)
+	}
+	if len(req.SeedData) > 0 {
+		return nil, fmt.Errorf("%w: volume seeding", ebsprovider.ErrUnsupportedCapability)
+	}
+	if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" {
+		return nil, fmt.Errorf("%w: a source snapshot needs its source volume ID", ebsprovider.ErrInvalidArgument)
+	}
+	if req.AvailabilityZone != "" && req.AvailabilityZone != p.cfg.AvailabilityZone {
+		return nil, fmt.Errorf("%w: availability zone %q is not this cluster's %q", ebsprovider.ErrInvalidArgument, req.AvailabilityZone, p.cfg.AvailabilityZone)
+	}
+	params, err := ebsprovider.ParseVolumeParameters(req.Parameters)
+	if err != nil {
+		return nil, err
+	}
+	vpus, err := VPUsPerGB(params.VolumeType)
+	if err != nil {
+		return nil, err
+	}
+
+	// A repeat, including one after a lost reply, is answered from the volume
+	// the first call created, which its tags find.
+	existing, ok, err := p.findVolume(ctx, req.VolumeID, "")
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		if err := p.sameVolume(ctx, existing, req, vpus); err != nil {
+			return nil, err
+		}
+		return p.awaitVolume(ctx, existing.ID)
+	}
+
+	in := oci.CreateVolumeInput{
+		CompartmentID:      p.cfg.CompartmentID,
+		AvailabilityDomain: p.cfg.AvailabilityDomain,
+		DisplayName:        req.VolumeID,
+		SizeGB:             SizeGB(required),
+		VPUsPerGB:          vpus,
+		FreeformTags: map[string]string{
+			TagCluster: p.cfg.ClusterID, TagVolumeID: req.VolumeID, TagCapacityBytes: strconv.FormatInt(required, 10),
+		},
+		RetryToken: RetryToken("volume", p.cfg.ClusterID, req.VolumeID),
+	}
+	if req.SourceSnapshotID != "" {
+		backup, err := p.restorableBackup(ctx, req)
+		if err != nil {
+			return nil, err
+		}
+		in.SourceBackupID = backup.ID
+		in.SizeGB = max(in.SizeGB, backup.SizeGB)
+	}
+	created, err := p.cfg.Client.CreateVolume(ctx, in)
+	if err != nil {
+		return nil, ociError("create volume "+req.VolumeID, err, ebsprovider.ErrUnavailable)
+	}
+	// OCI replays a retry token for a day, so a volume deleted and created
+	// again under the same ID inside that window comes back terminated.
+	if created.State.Gone() {
+		return nil, fmt.Errorf("ocibsd: create volume %s answered with %s volume %s", req.VolumeID, created.State, created.ID)
+	}
+	return p.awaitVolume(ctx, created.ID)
+}
+
+// sameVolume refuses a repeat create whose request differs from the one that
+// made existing, as CSI's idempotency rule requires.
+func (p *Provider) sameVolume(ctx context.Context, existing oci.BlockVolume, req ebsprovider.CreateVolumeRequest, vpus int64) error {
+	conflict := fmt.Errorf("%w: volume %s", ebsprovider.ErrAlreadyExists, req.VolumeID)
+	capacity, err := capacityOf(existing.FreeformTags)
+	if err != nil {
+		return err
+	}
+	if capacity != req.CapacityRange.RequiredBytes || existing.VPUsPerGB != vpus {
+		return conflict
+	}
+	if (existing.SourceBackupID == "") != (req.SourceSnapshotID == "") {
+		return conflict
+	}
+	if existing.SourceBackupID == "" {
+		return nil
+	}
+	source, err := p.cfg.Client.GetVolumeBackup(ctx, existing.SourceBackupID)
+	if err != nil && !errors.Is(err, oci.ErrNotFound) {
+		return ociError("get volume backup", err)
+	}
+	if err != nil || source.FreeformTags[TagSnapshotID] != req.SourceSnapshotID {
+		return conflict
+	}
+	return nil
+}
+
+// restorableBackup is the backup a create from req's snapshot restores.
+func (p *Provider) restorableBackup(ctx context.Context, req ebsprovider.CreateVolumeRequest) (oci.VolumeBackup, error) {
+	backup, ok, err := p.findBackup(ctx, req.SourceSnapshotID, "")
+	if err != nil {
+		return oci.VolumeBackup{}, err
+	}
+	if !ok {
+		return oci.VolumeBackup{}, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrNotFound, req.SourceSnapshotID)
+	}
+	if origin := backup.FreeformTags[TagVolumeID]; origin != req.SourceSnapshotVolumeID {
+		return oci.VolumeBackup{}, fmt.Errorf("%w: snapshot %s was taken from volume %s, not %s",
+			ebsprovider.ErrInvalidArgument, req.SourceSnapshotID, origin, req.SourceSnapshotVolumeID)
+	}
+	if capacity, err := capacityOf(backup.FreeformTags); err == nil && req.CapacityRange.RequiredBytes < capacity {
+		return oci.VolumeBackup{}, fmt.Errorf("%w: requested capacity is smaller than snapshot %s", ebsprovider.ErrInvalidArgument, req.SourceSnapshotID)
+	}
+	switch snapshotState(backup.State) {
+	case ebsprovider.SnapshotStatePending:
+		return oci.VolumeBackup{}, fmt.Errorf("%w: snapshot %s is still pending", ebsprovider.ErrUnavailable, req.SourceSnapshotID)
+	case ebsprovider.SnapshotStateError:
+		return oci.VolumeBackup{}, fmt.Errorf("%w: snapshot %s failed", ebsprovider.ErrInvalidArgument, req.SourceSnapshotID)
+	}
+	return backup, nil
+}
+
+// awaitVolume waits for a volume to leave PROVISIONING or RESTORING.
+func (p *Provider) awaitVolume(ctx context.Context, volumeOCID string) (*ebsprovider.Volume, error) {
+	var v oci.BlockVolume
+	err := p.waitFor(ctx, p.cfg.VolumeTimeout, "volume "+volumeOCID+" becoming available", func(ctx context.Context) (bool, error) {
+		var err error
+		if v, err = p.cfg.Client.GetVolume(ctx, volumeOCID); err != nil {
+			return false, ociError("get volume", err)
+		}
+		switch v.State {
+		case oci.VolumeAvailable:
+			return true, nil
+		case oci.VolumeProvisioning, oci.VolumeRestoring:
+			return false, nil
+		default:
+			return false, fmt.Errorf("ocibsd: volume %s is %s", volumeOCID, v.State)
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+	return p.volumeOf(ctx, v)
+}
+
+func (p *Provider) GetVolume(ctx context.Context, req ebsprovider.GetVolumeRequest) (*ebsprovider.Volume, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	if req.VolumeID == "" {
+		return nil, fmt.Errorf("%w: volume ID is required", ebsprovider.ErrInvalidArgument)
+	}
+	v, ok, err := p.findVolume(ctx, req.VolumeID, req.Handle)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+	}
+	return p.volumeOf(ctx, v)
+}
+
+// ListVolumes lists this cluster's volumes, by EBS volume ID.
+func (p *Provider) ListVolumes(ctx context.Context, req ebsprovider.ListVolumesRequest) (*ebsprovider.ListVolumesResponse, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	found, err := p.cfg.Client.ListVolumes(ctx, oci.VolumeFilter{CompartmentID: p.cfg.CompartmentID, Tags: map[string]string{TagCluster: p.cfg.ClusterID}})
+	if err != nil {
+		return nil, ociError("list volumes", err)
+	}
+	handles := make(map[string]string, len(found))
+	for _, v := range found {
+		if id := v.FreeformTags[TagVolumeID]; id != "" && !v.State.Gone() {
+			handles[id] = v.ID
+		}
+	}
+	ids := slices.Sorted(maps.Keys(handles))
+	response := &ebsprovider.ListVolumesResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion)}
+	response.Volumes, response.NextToken = ebsprovider.Page(ids, req.StartingToken, int(req.PageSize()),
+		func(id string) ebsprovider.VolumeRef { return ebsprovider.VolumeRef{ID: id, Handle: handles[id]} })
+	return response, nil
+}
+
+// expandAttempts bounds ExpandVolume's re-reads when a concurrent update moves
+// the volume's ETag between its read and its write.
+const expandAttempts = 5
+
+// ExpandVolume within the OCI floor changes only the recorded capacity; past
+// it, OCI resizes the volume too. A volume attached anywhere is refused. The
+// update names the ETag it read, so a concurrent expansion is never undone.
+func (p *Provider) ExpandVolume(ctx context.Context, req ebsprovider.ExpandVolumeRequest) (*ebsprovider.Volume, error) {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return nil, err
+	}
+	if req.VolumeID == "" || !req.CapacityRange.Valid() {
+		return nil, fmt.Errorf("%w: volume ID and a positive capacity range are required", ebsprovider.ErrInvalidArgument)
+	}
+	found, ok, err := p.findVolume(ctx, req.VolumeID, req.Handle)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+	}
+	for attempt := 1; ; attempt++ {
+		v, err := p.cfg.Client.GetVolume(ctx, found.ID)
+		if errors.Is(err, oci.ErrNotFound) || (err == nil && v.IsGone()) {
+			return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+		}
+		if err != nil {
+			return nil, ociError("read volume "+req.VolumeID, err)
+		}
+		out, err := p.expandFrom(ctx, req, v)
+		if oci.PreconditionFailed(err) && attempt < expandAttempts {
+			continue
+		}
+		return out, err
+	}
+}
+
+// expandFrom applies one expansion to v as read, conditional on v's ETag.
+func (p *Provider) expandFrom(ctx context.Context, req ebsprovider.ExpandVolumeRequest, v oci.BlockVolume) (*ebsprovider.Volume, error) {
+	required := req.CapacityRange.RequiredBytes
+	capacity, err := capacityOf(v.FreeformTags)
+	if err != nil {
+		return nil, err
+	}
+	if required < capacity {
+		return nil, fmt.Errorf("%w: volume %s cannot shrink from %d to %d bytes", ebsprovider.ErrInvalidArgument, req.VolumeID, capacity, required)
+	}
+	if required == capacity {
+		return p.volumeOf(ctx, v)
+	}
+	attached, err := p.attachments(ctx, v.ID)
+	if err != nil {
+		return nil, err
+	}
+	if len(attached) > 0 {
+		return nil, fmt.Errorf("%w: volume %s is attached, and online expansion is not supported", ebsprovider.ErrVolumeInUse, req.VolumeID)
+	}
+	tags := maps.Clone(v.FreeformTags)
+	tags[TagCapacityBytes] = strconv.FormatInt(required, 10)
+	in := oci.UpdateVolumeInput{FreeformTags: tags, IfMatch: v.ETag}
+	if size := SizeGB(required); size > v.SizeGB {
+		in.SizeGB = size
+	}
+	updated, err := p.cfg.Client.UpdateVolume(ctx, v.ID, in)
+	if errors.Is(err, oci.ErrNotFound) {
+		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
+	}
+	if err != nil {
+		return nil, ociError("expand volume "+req.VolumeID, err, ebsprovider.ErrUnavailable)
+	}
+	return p.volumeOf(ctx, updated)
+}
+
+// DeleteVolume refuses a volume attached anywhere, and OCI's refusal while a
+// backup of it is being taken. A volume that does not exist is deleted.
+func (p *Provider) DeleteVolume(ctx context.Context, req ebsprovider.DeleteVolumeRequest) error {
+	if err := ebsprovider.CheckVersion(req.SchemaVersion); err != nil {
+		return err
+	}
+	if req.VolumeID == "" {
+		return fmt.Errorf("%w: volume ID is required", ebsprovider.ErrInvalidArgument)
+	}
+	v, ok, err := p.findVolume(ctx, req.VolumeID, req.Handle)
+	if err != nil || !ok {
+		return err
+	}
+	attached, err := p.attachments(ctx, v.ID)
+	if err != nil {
+		return err
+	}
+	if len(attached) > 0 {
+		return fmt.Errorf("%w: volume %s is attached to %s", ebsprovider.ErrVolumeInUse, req.VolumeID, attached[0].InstanceID)
+	}
+	if err := p.cfg.Client.DeleteVolume(ctx, v.ID); err != nil && !errors.Is(err, oci.ErrNotFound) {
+		return ociError("delete volume "+req.VolumeID, err, ebsprovider.ErrVolumeInUse)
+	}
+	return nil
+}

spinifex/providers/ebs/parameters.go

@@ -0,0 +1,36 @@
+package ebsprovider
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// VolumeParameters is what the control plane sends in
+// CreateVolumeRequest.Parameters. A provider that has no use for a field
+// ignores it; one that maps it, such as a performance tier, reads it here.
+type VolumeParameters struct {
+	// VolumeType is the EC2 volume type the caller asked for, such as gp3.
+	VolumeType string `json:"volume_type,omitempty"`
+}
+
+// NewVolumeParameters encodes the parameters for a volume of volumeType.
+func NewVolumeParameters(volumeType string) (json.RawMessage, error) {
+	raw, err := json.Marshal(VolumeParameters{VolumeType: volumeType})
+	if err != nil {
+		return nil, fmt.Errorf("encode volume parameters: %w", err)
+	}
+	return raw, nil
+}
+
+// ParseVolumeParameters decodes raw, which may be empty. Malformed parameters
+// are refused as invalid rather than read as an absent volume type.
+func ParseVolumeParameters(raw json.RawMessage) (VolumeParameters, error) {
+	var params VolumeParameters
+	if len(raw) == 0 {
+		return params, nil
+	}
+	if err := json.Unmarshal(raw, &params); err != nil {
+		return VolumeParameters{}, fmt.Errorf("%w: volume parameters: %w", ErrInvalidArgument, err)
+	}
+	return params, nil
+}

spinifex/providers/ebs/parameters_test.go

@@ -0,0 +1,31 @@
+package ebsprovider
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVolumeParameters(t *testing.T) {
+	t.Run("round trip", func(t *testing.T) {
+		raw, err := NewVolumeParameters("io1")
+		require.NoError(t, err)
+		assert.JSONEq(t, `{"volume_type":"io1"}`, string(raw))
+		params, err := ParseVolumeParameters(raw)
+		require.NoError(t, err)
+		assert.Equal(t, "io1", params.VolumeType)
+	})
+
+	t.Run("absent parameters have no volume type", func(t *testing.T) {
+		params, err := ParseVolumeParameters(nil)
+		require.NoError(t, err)
+		assert.Empty(t, params.VolumeType)
+	})
+
+	t.Run("malformed parameters are invalid", func(t *testing.T) {
+		_, err := ParseVolumeParameters(json.RawMessage(`{"volume_type":`))
+		require.ErrorIs(t, err, ErrInvalidArgument)
+	})
+}