fix(lambda): bound ListFunctions server work per request

Walk the account stream by exact sequence, up to the end a plain stream
info reports, so each request examines a fixed number of sequences
whatever they hold. Add an account-isolation test over two real
account buckets.

.github/workflows/publish-system-ami.yml

@@ -1,20 +1,21 @@
 name: Publish System AMIs
 
-# Builds the system AMIs and publishes each compressed qcow2 + .sha256 sidecar
-# to the Cloudflare R2 bucket that backs iso.mulgadc.com/system-ami/. The Alpine
-# images build with libguestfs (build-system-image.sh); the Ubuntu GPU images
-# build with mkosi, which installs from resolute-updates and resolute-security,
-# so each release ships the newest kernel. Eight images ship:
-#   spinifex-eks-node               Alpine EKS K3s node (server|agent, role at boot)
-#   spinifex-ecs-node               Alpine ECS container instance
-#   spinifex-rds-postgres           Alpine RDS PostgreSQL instance
-#   spinifex-rds-mariadb            Alpine RDS MariaDB instance
-#   spinifex-eks-node-gpu           Ubuntu EKS K3s GPU worker (mkosi)
-#   spinifex-ecs-node-gpu           Ubuntu ECS GPU container instance (mkosi)
-#   ubuntu-26.04-nvidia-gpu-x86_64  Ubuntu NVIDIA GPU base image (mkosi)
-#   ubuntu-26.04-amd-gpu-x86_64     Ubuntu AMD ROCm GPU base image (mkosi)
+# Builds the system AMIs (libguestfs, on the runner) and publishes each
+# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
+# iso.mulgadc.com/system-ami/. Six images ship:
+#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node       Alpine ECS container instance
+#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
+#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
 # Operators then install them on any cluster with:
-#   spx admin images import --name <catalog key>
+#   spx admin images import --name spinifex-eks-node
+#   spx admin images import --name spinifex-ecs-node
+#   spx admin images import --name spinifex-eks-node-gpu
+#   spx admin images import --name spinifex-ecs-node-gpu
+#   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
 #
 # Publishing overwrites the mirror artifact, so it runs only from release.yml
 # once a tag has passed its gates, with siblings at the tags go.mod requires.
@@ -53,12 +54,9 @@ jobs:
           VARIANT: ${{ inputs.variant }}
         run: |
           set -euo pipefail
-          # builder=legacy entries carry a manifest for build-system-image.sh;
-          # builder=mkosi entries carry the mkosi-build.sh --image name and the
-          # R2 object base, which must equal the catalog URL basename.
-          cpu='{"name":"spinifex-eks-node","builder":"legacy","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","builder":"legacy","manifest":"scripts/images/ecs-agent/manifest.conf"}'
-          gpu='{"name":"spinifex-eks-node-gpu","builder":"mkosi","image":"spinifex-eks-node-gpu","object":"spinifex-eks-node-gpu-x86_64"},{"name":"spinifex-ecs-node-gpu","builder":"mkosi","image":"spinifex-ecs-node-gpu","object":"spinifex-ecs-node-gpu-x86_64"},{"name":"ubuntu-26.04-nvidia-gpu","builder":"mkosi","image":"ubuntu-gpu-nvidia","object":"ubuntu-26.04-nvidia-gpu-x86_64"},{"name":"ubuntu-26.04-amd-gpu","builder":"mkosi","image":"ubuntu-gpu-amd","object":"ubuntu-26.04-amd-gpu-x86_64"}'
-          rds='{"name":"spinifex-rds-postgres","builder":"legacy","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","builder":"legacy","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
+          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
@@ -134,8 +132,6 @@ jobs:
           go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Install build + upload tooling
-        # mkosi legs build and upload inside the pinned builder container.
-        if: matrix.builder == 'legacy'
         run: |
           set -euo pipefail
           sudo apt-get update -qq
@@ -148,7 +144,6 @@ jobs:
           fi
 
       - name: Build + publish ${{ matrix.name }} to R2
-        if: matrix.builder == 'legacy'
         working-directory: spinifex
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
@@ -173,34 +168,3 @@ jobs:
             FLAGS+=(--dry-run)
           fi
           ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"
-
-      - name: Build + publish ${{ matrix.name }} to R2 (mkosi)
-        if: matrix.builder == 'mkosi'
-        working-directory: spinifex
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
-          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
-          R2_BUCKET: spinifex-iso
-          DRY_RUN: ${{ inputs.dry_run }}
-          IMAGE: ${{ matrix.image }}
-          OBJECT: ${{ matrix.object }}
-          # On-disk scratch for the same reason as the legacy legs: the GPU raw
-          # and its qcow2 do not fit a RAM-backed /tmp.
-          MKOSI_OUTPUT_DIR: ${{ runner.temp }}/mkosi-output
-        run: |
-          set -euo pipefail
-          mkdir -p "${MKOSI_OUTPUT_DIR}"
-          ./scripts/mkosi-build.sh --image "${IMAGE}" -- --manifest-format=json
-          # The kernel this image boots, from mkosi's package manifest.
-          KERNEL=$(jq -r '.packages[] | select(.name | test("^linux-image-[0-9]")) | "\(.name) \(.version)"' "${MKOSI_OUTPUT_DIR}/${IMAGE}.manifest")
-          if [[ -z "${KERNEL}" ]]; then
-            echo "::error::no linux-image-<version> package in ${IMAGE}.manifest" >&2
-            exit 1
-          fi
-          echo "kernel: ${KERNEL}"
-          FLAGS=(--image "${IMAGE}" --name "${OBJECT}")
-          if [[ "${DRY_RUN}" == "true" ]]; then
-            FLAGS+=(--dry-run)
-          fi
-          ./scripts/mkosi-publish.sh "${FLAGS[@]}"

Makefile

@@ -515,9 +515,9 @@ install-system:
 	fi
 
 install-go:
-	@echo -e "\n....Installing Go 1.27.4 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.27.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.27.4-bookworm@sha256:4fa7c968ba93f1635670522dbb01a2c78150443e1385746924566079f36570ea AS builder
+FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS builder
 
 ARG VERSION
 

docs/terraform-workbooks/demo-app/Dockerfile

@@ -4,7 +4,7 @@
 # distroless base (no shell, no package manager) for a small, production-shaped
 # image suitable for pushing to ECR and pulling onto EKS workers.
 
-FROM golang:1.27.4-alpine@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS build
+FROM golang:1.27.3-alpine@sha256:d200d2dd76b03a99b52651fd1325aff7dcd35422c35c01bac56571b0507f3deb AS build
 WORKDIR /src
 COPY go.mod ./
 COPY main.go ./

docs/terraform-workbooks/demo-app/go.mod

@@ -1,3 +1,3 @@
 module github.com/mulgadc/spinifex-demo
 
-go 1.27.4
+go 1.27.3

docs/testing/e2e-coverage.md

@@ -140,7 +140,6 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 ### Storage and durability
 
 - **`storagefault`** — what a guest does when its storage backend stops answering, and what stop/start does when the seal failed. **The claim is about corruption, not availability.** SIGSTOPs a cluster-wide service, so it always runs last and in its own cell. Three-node, because the RS(2,1) shard floor and cross-node takeover after a failed seal do not exist on one node.
-- **No suite reaches a `pending` snapshot.** Every e2e cell runs the default `viperblockd` provider, which finishes a snapshot before `CreateSnapshot` answers, so `single`'s `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch` and `TestCreateImage`, and `rds`'s `TestSnapshotRestore` and `TestDelete`, run the create that writes the snapshot document `pending` before the provider call and settle it `completed` within the request, and the pending-snapshot reaper finds nothing to settle. The `pending` paths are unit-tested against the memory provider answering `pending`: the reaper settling snapshots and AMIs and carrying out deferred deletes, the request's rollback and its ambiguous calls, and `DeleteSnapshot` of a pending snapshot in `domains/ec2/snapshot`; `CreateImage` and the readiness gates in `domains/ec2/image`, `domains/ec2/instance` and `domains/ec2/volume`; DB snapshots and the final snapshot waiting in `creating` in `domains/rds/instance`; and the import's wait in `operator/admin`. The contract's `pending` answer, its filtered `ListSnapshots` and the `state` it reports are in the conformance suite (`providers/ebs/conformance`), which every provider runs.
 - **`partialblock`** — concurrent sub-block writes to the same block surviving a cold reread through NBD from a real guest.
 - **`storagegrowth`** — volume growth and image pull. Requires `external_mode=pool`; skips on a nat env, which has no EIP and therefore no guest SSH.
 - **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest forced off and relaunched once its budget expires. Proves a real reboot by requiring a *new* boot ID.

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex
 
-go 1.27.4
+go 1.27.3
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8

spec/EBSProvider/README.md

@@ -17,7 +17,6 @@ The contract that every implementation of the `EBSProvider` interface must meet.
   A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
 - Is the schema version part of the contract?
   Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
-  A snapshot's `state` in a ListSnapshots ref, and the `snapshot_id` filter, arrive at contract version 4: a ref carries no state below it, and a filtered call below it is refused with `unsupported_version` rather than answered with every snapshot.
 - Are `internal` and `unavailable` the only errors that a failed provider may cause?
 - Must a present but malformed ID be refused with `invalid_argument`?
   IDs are opaque here, so only an omitted one is refused.
@@ -101,10 +100,13 @@ The contract that every implementation of the `EBSProvider` interface must meet.
 
 ### Snapshots
 
+- Should a pending snapshot's move to `completed` or `error` be modelled, as a step that happens without a call?
+  Without it a pending snapshot never completes, and no snapshot is ever in `error`.
+
 #### CreateSnapshot
 
-- May a provider forget a snapshot's source?
-  The control plane mints every snapshot ID and records it before the first call, so the ID alone names the request. A provider that no longer knows a snapshot's source, such as qemu-nbd after a restart, answers a retry with the snapshot, and refuses with `already_exists` only when it knows the source differs. The specification models a provider that always knows the source.
+- May success return a `pending` snapshot?
+  AWS starts a snapshot as `pending`, while the memory implementation returns `completed` and the NATS implementation waits for completion and refuses anything else.
 - Must a retry compare the volume handle?
   It is ignored, as in the memory implementation, so a volume deleted and recreated under the same ID returns the old snapshot.
 - Should a copy's ID count as a retry?
@@ -131,7 +133,7 @@ The contract that every implementation of the `EBSProvider` interface must meet.
 - Is the handle check part of the contract?
   The Viperblock implementation ignores the handle and deletes anyway.
 - May a pending snapshot be deleted?
-  AWS allows it, while OCI refuses to delete a backup that is still being created, so the control plane never asks: it defers a customer's delete of a pending snapshot until the snapshot settles.
+  AWS allows it.
 - What does success promise?
   The Go docs say the snapshot is gone, but a snapshot whose handle did not match is left in place.
 - Should a snapshot that does not exist be refused with `not_found`?

spec/EBSProvider/Snapshots.tla

@@ -13,9 +13,8 @@ CreateSnapshotRequests == [snapshot_id   : Optional(SnapshotIDs),
 
 CreateSnapshot(req, val, err) ==
   (*******************************************************************************************************************)
-  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. The snapshot holds the         *)
-  (* volume's data as it was when the call returned, even while it is pending, and a pending snapshot settles later  *)
-  (* without a call (SettleSnapshot). A retry of a call that already succeeded returns the same snapshot.            *)
+  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. A retry of a call that         *)
+  (* already succeeded returns the same snapshot.                                                                    *)
   (*******************************************************************************************************************)
   LET snapID == req.snapshot_id
       volID  == req.volume_id
@@ -98,45 +97,27 @@ CopySnapshot(req, val, err) ==
 
 
 ListSnapshotsRequests == [max_results    : -1..(MaxListResults + 1),
-                          starting_token : Optional(ListTokens),
-                          snapshot_id    : Optional(SnapshotIDs)]
-
-\* The ref that ListSnapshots returns for a snapshot, carrying its current state.
-SnapshotRef(id) == [id               |-> id,
-                    source_volume_id |-> snapshots[id].source_volume_id,
-                    handle           |-> snapshots[id].handle,
-                    state            |-> snapshots[id].state]
+                          starting_token : Optional(ListTokens)]
 
 ListSnapshots(req, val, err) ==
   (*******************************************************************************************************************)
   (* Some of the snapshots are listed, a page at a time, and nothing changes. A call with no starting token that     *)
-  (* returns no next token lists every snapshot. A call that names a snapshot returns that snapshot in its current   *)
-  (* state, or nothing when it does not exist, so an accepted snapshot is never reported absent while it is pending. *)
+  (* returns no next token lists every snapshot.                                                                     *)
   (*******************************************************************************************************************)
-  LET snapID == req.snapshot_id
-
-      Errors ==
-        \* A call that names a snapshot reads only that one, so it needs no enumeration.
-        LET checks == [unsupported_capability |-> snapID = None /\ ~Capabilities.snapshot_enumeration]
-        IN {e \in DOMAIN checks : checks[e]}
+  LET Errors ==
+    LET checks == [unsupported_capability |-> ~Capabilities.snapshot_enumeration]
+    IN {e \in DOMAIN checks : checks[e]}
 
      \* The request was rejected, or the provider failed to answer.
   IN \/ /\ val = None
         /\ err \in Errors \union ProviderErrors
         /\ UNCHANGED vars
 
-     \* The request named one snapshot, so paging is ignored.
-     \/ /\ Errors = {}
-        /\ snapID /= None
-        /\ val = [snapshots  |-> IF KeyExists(snapshots, snapID) THEN {SnapshotRef(snapID)} ELSE {},
-                  next_token |-> None]
-        /\ err = None
-        /\ UNCHANGED vars
-
      \* The request was accepted.
      \/ /\ Errors = {}
-        /\ snapID = None
-        /\ LET refs == {SnapshotRef(id) : id \in DOMAIN snapshots}
+        /\ LET refs == {[id               |-> id,
+                         source_volume_id |-> snapshots[id].source_volume_id,
+                         handle           |-> snapshots[id].handle] : id \in DOMAIN snapshots}
            \* The page and the next token are the implementation's choice.
            IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
                 /\ Cardinality(page) <= PageSize(req.max_results)
@@ -175,18 +156,4 @@ DeleteSnapshot(req, err) ==
         /\ err = None
         /\ UNCHANGED <<volumes, publications>>
 
------------------------------------------------------------------------------------------------------------------------
-\* Steps without a call
-
-SettleSnapshot(snapID) ==
-  (*******************************************************************************************************************)
-  (* A pending snapshot settles as completed or error, as the provider finishes it. Its data is unchanged, since it  *)
-  (* was fixed when CreateSnapshot returned, and a snapshot that has settled never changes state again.              *)
-  (*******************************************************************************************************************)
-  /\ KeyExists(snapshots, snapID)
-  /\ snapshots[snapID].state = "pending"
-  /\ \E s \in {"completed", "error"} :
-       snapshots' = [snapshots EXCEPT ![snapID].state = s]
-  /\ UNCHANGED <<volumes, publications>>
-
 =======================================================================================================================

spinifex/domains/ec2/ebs/metadata/metadata.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"strings"
 	"time"
@@ -83,48 +82,6 @@ type Snapshot struct {
 	AvailabilityZone string            `json:"availability_zone"`
 	Tags             map[string]string `json:"tags,omitempty"`
 	ProviderHandle   string            `json:"provider_handle,omitempty"`
-	// DeleteRequested is a DeleteSnapshot accepted before the snapshot could be
-	// deleted; the snapshot is hidden from then on and the reaper deletes it.
-	DeleteRequested bool `json:"delete_requested,omitempty"`
-	// AmbiguousCreate is set before every provider CreateSnapshot call and kept
-	// while any call on the document went unanswered, as one may still be accepted.
-	AmbiguousCreate bool `json:"ambiguous_create,omitempty"`
-	// TombstonedAt is when a deleted AmbiguousCreate snapshot's document became
-	// a tombstone, under which a late-accepted snapshot of its ID is deleted.
-	TombstonedAt time.Time `json:"tombstoned_at,omitzero"`
-}
-
-// Hidden reports a snapshot a customer has deleted: it is absent to every
-// request, and only the reaper still acts on its document.
-func (s Snapshot) Hidden() bool {
-	return s.DeleteRequested || !s.TombstonedAt.IsZero()
-}
-
-// Tombstoned reports a document kept only to catch a late-accepted create.
-func (s Snapshot) Tombstoned() bool {
-	return !s.TombstonedAt.IsZero()
-}
-
-// SnapshotStateCompleted is the one snapshot state a consumer may use.
-const SnapshotStateCompleted = "completed"
-
-// The states of an AMI document. A CreateImage's AMI is pending until its
-// snapshot settles; available is the one state an instance may launch from.
-const (
-	AMIStatePending   = "pending"
-	AMIStateAvailable = "available"
-	AMIStateFailed    = "failed"
-)
-
-// Ready reports a snapshot whose contents a consumer may read.
-func (s Snapshot) Ready() bool {
-	return s.State == SnapshotStateCompleted
-}
-
-// NotReadyError is the IncorrectState a consumer answers for a snapshot that
-// is not Ready.
-func (s Snapshot) NotReadyError() error {
-	return awserrors.Errorf(awserrors.ErrorIncorrectState, "Snapshot is in invalid state - %s", s.State)
 }
 
 // RootDeviceName is the device every image this platform serves reports as its
@@ -158,18 +115,6 @@ type AMI struct {
 	UEFIData *UEFIDataRef `json:"uefi_data,omitempty"`
 }
 
-// Available reports an AMI an instance may launch from or a copy may read. An
-// empty State predates the field, and every such image is complete.
-func (a AMI) Available() bool {
-	return a.State == "" || a.State == AMIStateAvailable
-}
-
-// UnavailableError is the InvalidAMIID.Unavailable answer for imageID, whose
-// document is a, when a is not Available.
-func (a AMI) UnavailableError(imageID string) error {
-	return awserrors.Errorf(awserrors.ErrorInvalidAMIIDUnavailable, "The image '%s' is %s, not available.", imageID, a.State)
-}
-
 // UEFIDataRef names an image's UEFI variable store object and the firmware it
 // was written under, so a launch can check the firmware without reading it.
 type UEFIDataRef struct {

spinifex/domains/ec2/image/account_termination_current_behaviour_test.go

@@ -57,11 +57,6 @@ func newStorageServices(t *testing.T) storageServices {
 	snapshots.SetEBSProvider(provider)
 	images := ec2image.NewImageServiceImplWithConfig(cfg, store, nil)
 	images.SetEBSProvider(provider)
-	// CreateImage snapshots through a snapshot service with no index handle.
-	imageSnapshots := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, store, nc)
-	imageSnapshots.SetEBSProvider(provider)
-	images.SetSnapshotCreator(imageSnapshots)
-	images.SetImageClaims(imageSnapshots)
 
 	return storageServices{
 		volumes: volumes, snapshots: snapshots, images: images,
@@ -235,9 +230,9 @@ func TestCurrentBehaviour_AccountTermination_DeleteSnapshotDuringCloneCreationSt
 	assert.EqualError(t, err, awserrors.ErrorInvalidSnapshotNotFound, "a clone started after the delete is refused")
 }
 
-// CreateImage on an instance reads the image the instance launched from before
-// it snapshots the root volume, so once that image is deregistered the call
-// fails and the provider holds no snapshot for it.
+// CreateImage on an instance re-reads the image the instance launched from,
+// after the provider has taken the snapshot, so once that image is
+// deregistered the call fails and leaves a provider snapshot with no document.
 func TestCurrentBehaviour_AccountTermination_CreateImageOfRetainedInstanceNeedsItsLaunchImage(t *testing.T) {
 	s := newStorageServices(t)
 	ctx := context.Background()
@@ -257,7 +252,17 @@ func TestCurrentBehaviour_AccountTermination_CreateImageOfRetainedInstanceNeedsI
 
 	listed, err := s.provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewVersioned()})
 	require.NoError(t, err)
+	var orphans []string
 	for _, ref := range listed.Snapshots {
-		assert.NotEqual(t, "vol-retained-root", ref.SourceVolumeID, "the provider took a snapshot for an image that failed")
+		if ref.SourceVolumeID == "vol-retained-root" {
+			orphans = append(orphans, ref.ID)
+		}
+	}
+	require.Len(t, orphans, 1, "the provider kept the snapshot it took before the image read failed")
+
+	docs, err := s.metadata.ListSnapshotsStrict(ctx, ctAccountID)
+	require.NoError(t, err)
+	for _, doc := range docs {
+		assert.NotEqual(t, orphans[0], doc.SnapshotID, "no document names that snapshot")
 	}
 }

spinifex/domains/ec2/image/create_image_pending_test.go

@@ -1,308 +0,0 @@
-package image
-
-import (
-	"context"
-	"strings"
-	"sync"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// pendingImageFixture is an image service wired to a snapshot service over the
-// same store and memory provider, whose reaper settles what CreateImage leaves.
-type pendingImageFixture struct {
-	svc       *ImageServiceImpl
-	snapshots *ec2snapshot.SnapshotServiceImpl
-	mem       *ebsprovider.MemoryProvider
-	volume    ebsmetadata.Volume
-}
-
-func newPendingImageFixture(t *testing.T) *pendingImageFixture {
-	t.Helper()
-	svc, _, provider := setupProviderSnapshotImageService(t)
-	mem, ok := provider.(*ebsprovider.MemoryProvider)
-	require.True(t, ok)
-	snapshots := useProvider(svc, mem)
-	return &pendingImageFixture{
-		svc:       svc,
-		snapshots: snapshots,
-		mem:       mem,
-		volume:    seedProviderVolume(t, svc, provider, "vol-createimage01", 8),
-	}
-}
-
-func (f *pendingImageFixture) createImage(t *testing.T) (*ec2.CreateImageOutput, error) {
-	t.Helper()
-	return f.svc.CreateImageFromInstance(CreateImageParams{
-		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-createimage01"), Name: aws.String("pending-image")},
-		RootVolumeID: f.volume.VolumeID,
-	}, testAccountID)
-}
-
-// onlyAMI is the one AMI document in the store, and its snapshot's document.
-func (f *pendingImageFixture) onlyAMI(t *testing.T) (ebsmetadata.AMI, ebsmetadata.Snapshot) {
-	t.Helper()
-	amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
-	require.NoError(t, err)
-	require.Len(t, amis, 1)
-	snap, err := f.svc.MetadataStore().GetSnapshot(context.Background(), testAccountID, amis[0].SnapshotID)
-	require.NoError(t, err)
-	return amis[0], snap
-}
-
-func (f *pendingImageFixture) sweep(t *testing.T) {
-	t.Helper()
-	_, err := f.snapshots.NewPendingSnapshotReaper().Sweep(context.Background())
-	require.NoError(t, err)
-}
-
-// recordingCreateProvider reads the image's documents when the provider is called.
-type recordingCreateProvider struct {
-	ebsprovider.EBSProvider
-
-	store     *ebsmetadata.Store
-	amis      []ebsmetadata.AMI
-	snapshot  ebsmetadata.Snapshot
-	readError error
-}
-
-func (p *recordingCreateProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
-	p.amis, p.readError = p.store.ListAMIs(ctx)
-	if p.readError == nil {
-		p.snapshot, p.readError = p.store.GetSnapshot(ctx, testAccountID, req.SnapshotID)
-	}
-	return p.EBSProvider.CreateSnapshot(ctx, req)
-}
-
-func TestCreateImage_WritesBothDocumentsPendingBeforeTheProviderCall(t *testing.T) {
-	f := newPendingImageFixture(t)
-	recording := &recordingCreateProvider{EBSProvider: f.mem, store: f.svc.MetadataStore()}
-	f.snapshots.SetEBSProvider(recording)
-
-	_, err := f.createImage(t)
-	require.NoError(t, err)
-
-	require.NoError(t, recording.readError)
-	require.Len(t, recording.amis, 1, "the AMI document exists when the provider is called")
-	assert.Equal(t, ebsmetadata.AMIStatePending, recording.amis[0].State)
-	assert.Equal(t, recording.snapshot.SnapshotID, recording.amis[0].SnapshotID)
-	assert.Equal(t, "pending", recording.snapshot.State, "the snapshot document is pending when the provider is called")
-}
-
-func TestCreateImage_DefiniteRefusalRemovesBothDocuments(t *testing.T) {
-	f := newPendingImageFixture(t)
-	f.snapshots.SetEBSProvider(&createInUseProvider{EBSProvider: f.mem})
-
-	_, err := f.createImage(t)
-	code, _, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok, "unregistered error: %v", err)
-	assert.Equal(t, awserrors.ErrorIncorrectState, code)
-
-	amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, amis, "a refused image leaves no AMI document")
-	snaps, err := f.svc.MetadataStore().ListAllSnapshots(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, snaps, "a refused image leaves no snapshot document")
-}
-
-func TestCreateImage_SettlesTheSnapshotAndTheAMI(t *testing.T) {
-	tests := []struct {
-		name    string
-		arrange func(f *pendingImageFixture)
-		wantErr string
-		// settle is how the provider finishes a snapshot it answered pending; empty: it did not.
-		settle      ebsprovider.SnapshotState
-		wantPending bool
-		wantSnap    string
-		wantAMI     string
-	}{
-		{name: "completed in the request", arrange: func(*pendingImageFixture) {}, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable},
-		{
-			name: "answered pending, then completed", arrange: func(f *pendingImageFixture) { f.mem.AnswerSnapshotsPending(true) },
-			wantPending: true, settle: ebsprovider.SnapshotStateCompleted, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable,
-		},
-		{
-			name: "answered pending, then error", arrange: func(f *pendingImageFixture) { f.mem.AnswerSnapshotsPending(true) },
-			wantPending: true, settle: ebsprovider.SnapshotStateError, wantSnap: "error", wantAMI: ebsmetadata.AMIStateFailed,
-		},
-		{
-			name: "both answers lost", arrange: func(f *pendingImageFixture) { f.mem.FailCreateSnapshotsAfterAccept(2, nil) },
-			wantErr: awserrors.ErrorServerInternal, wantPending: true, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable,
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingImageFixture(t)
-			tc.arrange(f)
-
-			_, err := f.createImage(t)
-			if tc.wantErr != "" {
-				require.Error(t, err)
-				assert.Equal(t, tc.wantErr, err.Error())
-			} else {
-				require.NoError(t, err)
-			}
-
-			ami, snap := f.onlyAMI(t)
-			if tc.wantPending {
-				assert.Equal(t, ebsmetadata.AMIStatePending, ami.State)
-				assert.Equal(t, "pending", snap.State)
-				if tc.settle != "" {
-					require.NoError(t, f.mem.SettleSnapshot(snap.SnapshotID, tc.settle))
-				}
-				f.sweep(t)
-				ami, snap = f.onlyAMI(t)
-			}
-			assert.Equal(t, tc.wantSnap, snap.State)
-			assert.Equal(t, tc.wantAMI, ami.State)
-		})
-	}
-}
-
-// DeregisterImage is allowed on an image whose snapshot has not completed.
-func TestDeregisterImage_AllowsAPendingAMI(t *testing.T) {
-	f := newPendingImageFixture(t)
-	f.mem.AnswerSnapshotsPending(true)
-	out, err := f.createImage(t)
-	require.NoError(t, err)
-
-	_, err = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: out.ImageId}, testAccountID)
-	require.NoError(t, err)
-	_, err = f.svc.MetadataStore().GetAMI(context.Background(), aws.StringValue(out.ImageId))
-	assert.Error(t, err, "the pending AMI is deregistered")
-}
-
-// putHookStore runs onPut once, before the write to a matching key that
-// follows skip earlier matching writes reaches the store.
-type putHookStore struct {
-	objectstore.ObjectStore
-
-	match func(key string) bool
-	skip  int
-	onPut func()
-
-	mu   sync.Mutex
-	seen int
-}
-
-func (s *putHookStore) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
-	if s.match(aws.StringValue(input.Key)) {
-		s.mu.Lock()
-		fire := s.seen == s.skip
-		s.seen++
-		s.mu.Unlock()
-		if fire {
-			s.onPut()
-		}
-	}
-	return s.ObjectStore.PutObject(ctx, input)
-}
-
-// A DeregisterImage that lands between the reaper's reread of a pending AMI
-// and its write either waits the write out or is refused retryably; one that
-// succeeds is never undone by the reaper writing the settled AMI back.
-func TestDeregisterImage_DuringTheReapersAMIWriteStaysDeleted(t *testing.T) {
-	f := newPendingImageFixture(t)
-	f.mem.AnswerSnapshotsPending(true)
-	out, err := f.createImage(t)
-	require.NoError(t, err)
-	imageID := aws.StringValue(out.ImageId)
-	ami, snap := f.onlyAMI(t)
-	require.Equal(t, ebsmetadata.AMIStatePending, ami.State)
-	require.NoError(t, f.mem.SettleSnapshot(snap.SnapshotID, ebsprovider.SnapshotStateCompleted))
-
-	key, err := ebsmetadata.AMIKey(imageID)
-	require.NoError(t, err)
-	var deregErr error
-	deregistered := false
-	hooked := &putHookStore{ObjectStore: f.svc.store, match: func(k string) bool { return k == key }}
-	hooked.onPut = func() {
-		deregistered = true
-		_, deregErr = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: out.ImageId}, testAccountID)
-	}
-	reaping := ec2snapshot.NewSnapshotServiceImplWithStore(f.svc.config, hooked, nil)
-	reaping.SetEBSProvider(f.mem)
-	f.svc.SetImageClaims(reaping)
-
-	_, err = reaping.NewPendingSnapshotReaper().Sweep(context.Background())
-	require.NoError(t, err)
-	require.True(t, deregistered, "the reaper wrote the settled AMI")
-
-	assertDeregistrationStaysDeleted(t, f, imageID, deregErr, func() {
-		_, err := reaping.NewPendingSnapshotReaper().Sweep(context.Background())
-		require.NoError(t, err)
-	})
-}
-
-// assertDeregistrationStaysDeleted checks a DeregisterImage of imageID that
-// raced a settling write, and answered deregErr: one that succeeded left the
-// AMI gone, and one refused retryably succeeds on retry and stays gone after sweep.
-func assertDeregistrationStaysDeleted(t *testing.T, f *pendingImageFixture, imageID string, deregErr error, sweep func()) {
-	t.Helper()
-	_, getErr := f.svc.MetadataStore().GetAMI(context.Background(), imageID)
-	if deregErr == nil {
-		require.True(t, objectstore.IsNoSuchKeyError(getErr), "a deregistered AMI was written back: %v", getErr)
-		return
-	}
-	code, _, ok := awserrors.ResolveErrorDetail(deregErr)
-	require.True(t, ok, "unregistered error: %v", deregErr)
-	assert.Equal(t, awserrors.ErrorIncorrectState, code, "a refused deregistration is retryable")
-	require.NoError(t, getErr)
-
-	_, err := f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
-	require.NoError(t, err, "the retry succeeds once the settling write is done")
-	sweep()
-	_, getErr = f.svc.MetadataStore().GetAMI(context.Background(), imageID)
-	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the retried deregistration stays deleted: %v", getErr)
-}
-
-// A DeregisterImage that lands between CreateImage's reread of its pending AMI
-// and the write making it available is refused retryably, never undone.
-func TestDeregisterImage_DuringCreateImagesAMIWriteStaysDeleted(t *testing.T) {
-	f := newPendingImageFixture(t)
-	sample, err := ebsmetadata.AMIKey("ami-0")
-	require.NoError(t, err)
-	amiKeyPrefix := sample[:strings.LastIndex(sample, "ami-0")]
-	var imageID string
-	var deregErr error
-	deregistered := false
-	// The Recorded hook's pending write is skipped; the Completed hook's is the one.
-	hooked := &putHookStore{
-		ObjectStore: f.svc.store,
-		skip:        1,
-		match:       func(k string) bool { return strings.HasPrefix(k, amiKeyPrefix) },
-	}
-	hooked.onPut = func() {
-		amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
-		if err != nil || len(amis) != 1 {
-			deregErr = err
-			return
-		}
-		imageID = amis[0].ImageID
-		deregistered = true
-		_, deregErr = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
-	}
-	f.svc.metadata = ebsmetadata.NewStore(hooked, testBucket)
-	settling := ec2snapshot.NewSnapshotServiceImplWithStore(f.svc.config, hooked, nil)
-	settling.SetEBSProvider(f.mem)
-	f.svc.SetSnapshotCreator(settling)
-	f.svc.SetImageClaims(settling)
-
-	out, err := f.createImage(t)
-	require.NoError(t, err)
-	require.True(t, deregistered, "CreateImage wrote the available AMI")
-	require.Equal(t, imageID, aws.StringValue(out.ImageId))
-
-	assertDeregistrationStaysDeleted(t, f, imageID, deregErr, func() { f.sweep(t) })
-}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -41,42 +40,10 @@ func setupProviderSnapshotImageService(t *testing.T) (*ImageServiceImpl, *object
 		},
 	}
 	svc := NewImageServiceImplWithConfig(cfg, store, nil)
-	useProvider(svc, provider)
+	svc.SetEBSProvider(provider)
 	return svc, store, provider
 }
 
-// useProvider makes p svc's provider and wires a snapshot service over svc's
-// store and p, as the daemon wires its own, for CreateImage to snapshot through.
-func useProvider(svc *ImageServiceImpl, p ebsprovider.EBSProvider) *ec2snapshot.SnapshotServiceImpl {
-	svc.SetEBSProvider(p)
-	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(svc.config, svc.store, nil)
-	snapshots.SetEBSProvider(p)
-	svc.SetSnapshotCreator(snapshots)
-	svc.SetImageClaims(snapshots)
-	return snapshots
-}
-
-// createImageOf runs CreateImage of a stopped instance whose root volume is
-// volumeID, and returns its AMI's snapshot ID when one was recorded.
-func createImageOf(t *testing.T, svc *ImageServiceImpl, volumeID, name string) (string, error) {
-	t.Helper()
-	out, err := svc.CreateImageFromInstance(CreateImageParams{
-		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-" + name), Name: aws.String(name)},
-		RootVolumeID: volumeID,
-	}, testAccountID)
-	amis, listErr := svc.MetadataStore().ListAMIs(context.Background())
-	require.NoError(t, listErr)
-	for _, ami := range amis {
-		if ami.Name == name {
-			return ami.SnapshotID, err
-		}
-	}
-	if err == nil {
-		t.Fatalf("CreateImage %s returned %s but recorded no AMI", name, aws.StringValue(out.ImageId))
-	}
-	return "", err
-}
-
 // seedProviderVolume creates a provider-backed volume plus its ebsmetadata
 // document, wiring the document's ProviderHandle to the volume's real handle
 // so the fixture cannot drift from what the provider actually issued.
@@ -133,15 +100,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestCreateImageFromInstance_Provider_CreatesThroughProvider locks that a
+// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
 // root-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestCreateImageFromInstance_Provider_CreatesThroughProvider(t *testing.T) {
+func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	snapshotID, err := createImageOf(t, svc, doc.VolumeID, "stopped001")
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -149,28 +116,28 @@ func TestCreateImageFromInstance_Provider_CreatesThroughProvider(t *testing.T) {
 	// confirms the snapshot genuinely exists in the provider (sourced from
 	// the right volume) instead of the call having silently no-opped.
 	snap, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: doc.VolumeID,
+		Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-stopped001", VolumeID: doc.VolumeID,
 	})
 	require.NoError(t, err)
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestCreateImageFromInstance_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestCreateImageFromInstance_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	_, err := createImageOf(t, svc, doc.VolumeID, "running001")
+	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	_, err = createImageOf(t, svc, doc.VolumeID, "running002")
+	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -269,21 +236,16 @@ func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.
 	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
 }
 
-// TestCreateImageFromInstance_Provider_UnavailableLeavesThePendingSnapshot
-// locks that a CreateImage snapshot whose call and repeat both go unanswered
-// returns InternalError and leaves its document pending for the reaper.
-func TestCreateImageFromInstance_Provider_UnavailableLeavesThePendingSnapshot(t *testing.T) {
+// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
+// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
+func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
-	useProvider(svc, &createUnavailableProvider{EBSProvider: provider})
+	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
 
-	snapshotID, err := createImageOf(t, svc, doc.VolumeID, "stopped-busy")
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	snap, err := svc.metadata.GetSnapshot(context.Background(), testAccountID, snapshotID)
-	require.NoError(t, err)
-	assert.Equal(t, "pending", snap.State)
-	assert.True(t, snap.AmbiguousCreate)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
 }
 
 // createInUseProvider refuses every CreateSnapshot as volume in use.
@@ -295,14 +257,14 @@ func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.Create
 	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
 }
 
-// TestCreateImageFromInstance_Provider_VolumeInUseIsIncorrectState locks that
-// the CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
-func TestCreateImageFromInstance_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
+// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
+// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
+func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
-	useProvider(svc, &createInUseProvider{EBSProvider: provider})
+	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
 
-	_, err := createImageOf(t, svc, doc.VolumeID, "attached-inuse")
+	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
 
 	code, message, ok := awserrors.ResolveErrorDetail(err)
 	require.True(t, ok, "unregistered error: %v", err)
@@ -333,7 +295,7 @@ func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *t
 	svc.natsConn = nc
 	svc.config.DataDir = testutil.SocketTempDir(t)
 	counting := &countingSnapshotProvider{EBSProvider: provider}
-	useProvider(svc, counting)
+	svc.SetEBSProvider(counting)
 
 	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
 	doc.State, doc.AttachedInstance = "in-use", instanceID

spinifex/domains/ec2/image/readiness_gate_test.go

@@ -1,78 +0,0 @@
-package image
-
-import (
-	"context"
-	"testing"
-
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestRegisterImage_RefusesASnapshotThatIsNotCompleted(t *testing.T) {
-	tests := []struct {
-		name     string
-		snapshot ebsmetadata.Snapshot
-		wantCode string
-	}{
-		{name: "pending", snapshot: ebsmetadata.Snapshot{State: "pending"}, wantCode: awserrors.ErrorIncorrectState},
-		{name: "error", snapshot: ebsmetadata.Snapshot{State: "error"}, wantCode: awserrors.ErrorIncorrectState},
-		{name: "deleted while pending", snapshot: ebsmetadata.Snapshot{State: "pending", DeleteRequested: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
-		{name: "deleted after completing", snapshot: ebsmetadata.Snapshot{State: "completed", DeleteRequested: true, AmbiguousCreate: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			svc, store := setupTestImageService(t)
-			snap := tc.snapshot
-			snap.SnapshotID, snap.VolumeID, snap.VolumeSize, snap.OwnerID = "snap-gate000001", "vol-gate000001", 8, testAccountID
-			putTestSnapshotDoc(t, store, snap)
-
-			_, err := svc.RegisterImage(context.Background(), validRegisterImageServiceInput(snap.SnapshotID), testAccountID)
-			code, _, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok, "unregistered error: %v", err)
-			assert.Equal(t, tc.wantCode, code)
-			amis, err := svc.MetadataStore().ListAMIs(context.Background())
-			require.NoError(t, err)
-			assert.Empty(t, amis, "a refused registration writes no AMI")
-		})
-	}
-}
-
-// An AMI whose snapshot is gone reads as gone, as an AMI with no snapshot document does.
-func TestCopyImage_RefusesASourceThatIsNotAvailable(t *testing.T) {
-	tests := []struct {
-		name     string
-		amiState string
-		hideSnap bool
-		wantCode string
-	}{
-		{name: "pending", amiState: ebsmetadata.AMIStatePending, wantCode: awserrors.ErrorInvalidAMIIDUnavailable},
-		{name: "failed", amiState: ebsmetadata.AMIStateFailed, wantCode: awserrors.ErrorInvalidAMIIDUnavailable},
-		{name: "snapshot deleted", amiState: "available", hideSnap: true, wantCode: awserrors.ErrorInvalidAMIIDNotFound},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			svc, store := setupTestImageService(t)
-			seedCopyableAMI(t, store, "ami-gate000001", "gated", testAccountID, "snap-gate000001", "vol-gate000001", 8)
-			ctx := context.Background()
-			ami, err := svc.MetadataStore().GetAMI(ctx, "ami-gate000001")
-			require.NoError(t, err)
-			ami.State = tc.amiState
-			require.NoError(t, svc.MetadataStore().PutAMI(ctx, ami))
-			if tc.hideSnap {
-				snap := readTestSnapshotDoc(t, store, testAccountID, "snap-gate000001")
-				snap.DeleteRequested = true
-				putTestSnapshotDoc(t, store, snap)
-			}
-
-			_, err = svc.CopyImage(ctx, validCopyImageServiceInput("ami-gate000001", "copy"), testAccountID)
-			code, _, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok, "unregistered error: %v", err)
-			assert.Equal(t, tc.wantCode, code)
-			amis, err := svc.MetadataStore().ListAMIs(ctx)
-			require.NoError(t, err)
-			assert.Len(t, amis, 1, "a refused copy writes no AMI")
-		})
-	}
-}

spinifex/domains/ec2/image/service_impl.go

@@ -55,24 +55,6 @@ type ImageServiceImpl struct {
 
 	uefiRecords UEFIStoreRecords
 	nodeID      string
-
-	// snapshots takes an image's snapshot through the snapshot service, so the
-	// claim and reference are the ones CreateSnapshot uses.
-	snapshots ec2snapshot.ImageSnapshotCreator
-	// imageClaims is the claim DeregisterImage shares with whatever settles a
-	// pending AMI; nil excludes nothing.
-	imageClaims ec2snapshot.ImageClaims
-}
-
-// SetSnapshotCreator wires the snapshot service CreateImage snapshots through.
-func (s *ImageServiceImpl) SetSnapshotCreator(snapshots ec2snapshot.ImageSnapshotCreator) {
-	s.snapshots = snapshots
-}
-
-// SetImageClaims wires the image claims of the snapshot service whose reaper
-// settles this service's pending AMIs.
-func (s *ImageServiceImpl) SetImageClaims(claims ec2snapshot.ImageClaims) {
-	s.imageClaims = claims
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane.
@@ -479,14 +461,12 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		}
 	}
 
-	// Step 2: Read source volume config for size
-	if s.config == nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	if s.snapshots == nil || s.imageClaims == nil {
-		slog.Error("CreateImageFromInstance: no snapshot service wired", "amiId", amiID)
-		return nil, errors.New(awserrors.ErrorServerInternal)
+	// Step 2: Snapshot root volume
+	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
+		return nil, err
 	}
+
+	// Step 3: Read source volume config for size
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, params.RootVolumeID)
 	if err != nil {
 		slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err)
@@ -494,7 +474,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}
 	volumeSizeGiB := volMeta.CapacityGiB
 
-	// Step 3: Read source AMI config for architecture, platform, etc.
+	// Step 4: Read source AMI config for architecture, platform, etc.
 	sourceAMI := ebsmetadata.AMI{
 		Architecture:    "x86_64",
 		PlatformDetails: "Linux/UNIX",
@@ -509,6 +489,13 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		sourceAMI = srcCfg
 	}
 
+	// Step 5: Store snapshot metadata
+	if err := s.putSnapshotMetadata(context.Background(), snapshotID, params.RootVolumeID, volumeSizeGiB, accountID); err != nil {
+		slog.Error("CreateImageFromInstance: failed to write snapshot metadata", "snapshotId", snapshotID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	// Step 6: Build and store AMI config
 	meta := ebsmetadata.AMI{
 		ImageID:         amiID,
 		Name:            name,
@@ -526,41 +513,57 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		EnaSupport:   aws.Bool(true),
 		Distro:       sourceAMI.Distro,
 		DistroFamily: sourceAMI.DistroFamily,
-		// Pending until its snapshot completes, here or in the snapshot reaper.
-		State:    ebsmetadata.AMIStatePending,
+		// Snapshot succeeded before this point, so the image is complete.
+		State:    "available",
 		UEFIData: uefiData,
 	}
 
-	// Step 4: Snapshot the root volume. Its document and reference, then the
-	// AMI's document, are written pending before the provider is called, so an
-	// accepted snapshot whose answer is lost is still settled by the reaper.
-	ctx := context.Background()
-	snap, err := s.snapshots.CreateImageSnapshot(ctx, volMeta, snapshotID, accountID, "Created by CreateImage("+aws.StringValue(input.InstanceId)+") for "+amiID,
-		ec2snapshot.ImageSnapshotHooks{
-			Recorded:  func(ctx context.Context) error { return s.putAMIConfig(ctx, amiID, meta) },
-			Refused:   func(ctx context.Context) { s.removePendingAMI(ctx, amiID) },
-			Completed: func(ctx context.Context) { s.imageClaims.SettlePendingAMI(ctx, amiID, ebsmetadata.AMIStateAvailable) },
-		})
-	if err != nil {
+	if err := s.putAMIConfig(context.Background(), amiID, meta); err != nil {
 		// A UEFI object left under the image's prefix is in the orphan report.
-		slog.Error("CreateImageFromInstance: snapshot failed", "amiId", amiID, "snapshotId", snapshotID, "err", err)
-		return nil, err
+		slog.Error("CreateImageFromInstance: failed to store AMI config", "amiId", amiID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	slog.Info("CreateImageFromInstance completed", "amiId", amiID, "snapshotId", snapshotID, "snapshotState", aws.StringValue(snap.State))
+	slog.Info("CreateImageFromInstance completed", "amiId", amiID, "snapshotId", snapshotID)
 
 	return &ec2.CreateImageOutput{
 		ImageId: aws.String(amiID),
 	}, nil
 }
 
-// removePendingAMI deletes the AMI document of a CreateImage whose snapshot
-// the provider refused, before the snapshot's own document goes.
-func (s *ImageServiceImpl) removePendingAMI(ctx context.Context, amiID string) {
-	if err := s.metadata.DeleteAMI(ctx, amiID); err != nil && !objectstore.IsNoSuchKeyError(err) {
-		slog.ErrorContext(ctx, "CreateImageFromInstance: failed to remove a refused image's AMI document; the reaper marks it failed",
-			"amiId", amiID, "err", err)
+// snapshotVolume snapshots the root volume through the EBS provider, whether
+// or not the instance is running. A provider that buffers writes drains the
+// volume inside its own CreateSnapshot, so no drain is issued here.
+func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
+	if s.config == nil {
+		return errors.New(awserrors.ErrorServerInternal)
 	}
+
+	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
+	if err != nil {
+		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
+}
+
+// createSnapshot delegates snapshot creation for volMeta to the EBS provider.
+// op names the calling path so a failure is attributable in the log.
+func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetadata.Volume, snapshotID, op string) error {
+	if s.provider == nil {
+		slog.ErrorContext(ctx, "no EBS provider configured", "op", op)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volMeta.VolumeID, VolumeHandle: volMeta.ProviderHandle,
+	})
+	if err != nil || created == nil {
+		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
+		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
+	}
+	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
+	return nil
 }
 
 // getVolumeMetadata reads a volume's control-plane document, remapping a
@@ -645,9 +648,6 @@ func (s *ImageServiceImpl) GetAMISourceVolumeID(ctx context.Context, imageID str
 			"imageId", imageID, "snapshotId", meta.SnapshotID, "err", err)
 		return "", errors.New(awserrors.ErrorServerInternal)
 	}
-	if snapCfg.Hidden() {
-		return "", errors.New(awserrors.ErrorInvalidSnapshotNotFound)
-	}
 	if snapCfg.VolumeID == "" {
 		slog.ErrorContext(ctx, "GetAMISourceVolumeID: snapshot metadata has no source volume",
 			"imageId", imageID, "snapshotId", meta.SnapshotID)
@@ -815,10 +815,6 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 	if !callerCanReadAMI(srcMeta, accountID) {
 		return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
 	}
-	// The copy shares the source's snapshot, so it is no readier than it.
-	if !srcMeta.Available() {
-		return nil, srcMeta.UnavailableError(sourceImageID)
-	}
 
 	// Orphaned source (missing SnapshotID, or snapshot gone/corrupt) is
 	// reported as NotFound — don't leak the half-broken state.
@@ -846,9 +842,6 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 	}
-	if srcSnap.Hidden() {
-		return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
-	}
 
 	if exists, err := s.amiNameExists(ctx, name); err != nil {
 		slog.ErrorContext(ctx, "CopyImage: failed to check AMI name uniqueness", "name", name, "err", err)
@@ -1070,14 +1063,6 @@ func (s *ImageServiceImpl) RegisterImage(ctx context.Context, input *ec2.Registe
 		slog.ErrorContext(ctx, "RegisterImage: failed to read snapshot metadata", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if snapCfg.Hidden() {
-		return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
-	}
-	// An image registered on a snapshot that is not completed could never be
-	// launched from, and none is written pending outside CreateImage.
-	if !snapCfg.Ready() {
-		return nil, snapCfg.NotReadyError()
-	}
 
 	snapSizeGiB := uint64(0)
 	if snapCfg.VolumeSize > 0 {
@@ -1190,16 +1175,6 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
-	// Held across both deletes, so the reaper cannot write back a settled AMI
-	// it read before them.
-	if s.imageClaims != nil {
-		unlock, err := s.imageClaims.LockImage(ctx, imageID)
-		if err != nil {
-			return nil, err
-		}
-		defer unlock()
-	}
-
 	// Only the image's own prefix: a copy or a source instance has its own.
 	if n, err := uefistore.DeleteObjects(ctx, s.store, s.bucketName, imageID); err != nil {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete the UEFI variable store", "imageId", imageID, "deleted", n, "err", err)

spinifex/domains/ec2/image/service_impl_test.go

@@ -6,6 +6,8 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"io"
 	"log/slog"
+	"net/http"
+	"net/http/httptest"
 	"strconv"
 	"strings"
 	"testing"
@@ -152,24 +154,6 @@ func TestCreateImageFromInstance_RunningInstance_NoNATS(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-// An image service the daemon gave no snapshot service fails CreateImage with
-// InternalError before it records anything.
-func TestCreateImageFromInstance_NoSnapshotServiceIsInternalError(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := NewImageServiceImplWithConfig(&config.Config{Predastore: config.PredastoreConfig{Bucket: testBucket}}, store, nil)
-	createTestVolumeConfig(t, store, "vol-root123", 10)
-
-	_, err := svc.CreateImageFromInstance(CreateImageParams{
-		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-test123"), Name: aws.String("my-image")},
-		RootVolumeID: "vol-root123",
-	}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	amis, err := svc.MetadataStore().ListAMIs(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, amis)
-}
-
 func TestCreateImageFromInstance_StoppedInstance_NoConfig(t *testing.T) {
 	svc, _ := setupTestImageService(t)
 
@@ -2566,3 +2550,114 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
+
+func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config:     &config.Config{},
+		store:      store,
+		bucketName: testBucket,
+	}
+	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+func TestSnapshotVolume_BackendInitFails(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	defer srv.Close()
+
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config: &config.Config{
+			Predastore: config.PredastoreConfig{
+				Host:   srv.URL,
+				Region: "us-east-1",
+				Bucket: testBucket,
+			},
+		},
+		store:      store,
+		bucketName: testBucket,
+	}
+	createTestVolumeConfig(t, store, "vol-stopped1", 10)
+
+	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+// imageS3MockServer passes ListObjectsV2 (Backend.Init) but returns 404 for everything else.
+func imageS3MockServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.Contains(r.URL.RawQuery, "list-type=2") {
+			w.Header().Set("Content-Type", "application/xml")
+			w.WriteHeader(http.StatusOK)
+			io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?>`+
+				`<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
+				`<Name>test-bucket</Name><KeyCount>0</KeyCount>`+
+				`<MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>`)
+			return
+		}
+		w.Header().Set("Content-Type", "application/xml")
+		w.WriteHeader(http.StatusNotFound)
+		io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?>`+
+			`<Error><Code>NoSuchKey</Code>`+
+			`<Message>The specified key does not exist.</Message></Error>`)
+	}))
+}
+
+func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config: &config.Config{
+			Predastore: config.PredastoreConfig{
+				Host:   srv.URL,
+				Region: "us-east-1",
+				Bucket: testBucket,
+			},
+			Viperblock: config.ViperblockConfig{
+				EncryptionKeyFile: "/nonexistent-image-stop-key.bin",
+			},
+		},
+		store:      store,
+		bucketName: testBucket,
+	}
+	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
+
+	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+func TestSnapshotVolume_LoadStateFails(t *testing.T) {
+	srv := imageS3MockServer(t)
+	defer srv.Close()
+
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config: &config.Config{
+			Predastore: config.PredastoreConfig{
+				Host:      srv.URL,
+				Region:    "us-east-1",
+				Bucket:    testBucket,
+				AccessKey: "test-key",
+				SecretKey: "test-secret",
+			},
+		},
+		store:      store,
+		bucketName: testBucket,
+	}
+	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
+
+	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}

spinifex/domains/ec2/image/uefi_test.go

@@ -55,7 +55,7 @@ func newUEFIImageFixture(t *testing.T) *uefiImageFixture {
 	svc.SetUEFIStores(records, "node-a")
 	volume := seedProviderVolume(t, svc, provider, "vol-uefi001", 8)
 	snapshots := &countingSnapshots{EBSProvider: provider}
-	useProvider(svc, snapshots)
+	svc.SetEBSProvider(snapshots)
 	return &uefiImageFixture{svc: svc, snapshots: snapshots, objects: objects, records: records, volume: volume}
 }
 

spinifex/domains/ec2/instance/service_impl.go

@@ -697,12 +697,6 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec
 		slog.WarnContext(ctx, "PrepareRunInstances: AMI not owned by caller", "imageId", *input.ImageId, "amiOwner", amiOwner, "accountID", accountID)
 		return nil, nil, nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
 	}
-	// A CreateImage AMI is pending until its snapshot completes, and failed if
-	// it errors; neither has a root volume to clone.
-	if !amiMeta.Available() {
-		slog.WarnContext(ctx, "PrepareRunInstances: AMI is not available", "imageId", *input.ImageId, "state", amiMeta.State)
-		return nil, nil, nil, amiMeta.UnavailableError(*input.ImageId)
-	}
 	if err := checkImageArchitecture(instanceType, amiMeta.Architecture); err != nil {
 		slog.WarnContext(ctx, "PrepareRunInstances: AMI architecture does not match instance type", "imageId", *input.ImageId, "InstanceType", *input.InstanceType, "amiArchitecture", amiMeta.Architecture)
 		return nil, nil, nil, err

spinifex/domains/ec2/instance/service_impl_test.go

@@ -3169,30 +3169,6 @@ func TestPrepareRunInstances_AMINotOwnedByCaller(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error())
 }
 
-func TestPrepareRunInstances_AMINotAvailable(t *testing.T) {
-	for _, state := range []string{ebsmetadata.AMIStatePending, ebsmetadata.AMIStateFailed} {
-		t.Run(state, func(t *testing.T) {
-			types, _ := defaultPrepareInstanceTypes()
-			svc := &InstanceServiceImpl{
-				instanceTypes: types,
-				amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{
-					"ami-1": {ImageOwnerAlias: "acc", State: state},
-				}},
-			}
-			_, _, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
-				InstanceType: aws.String("t3.micro"),
-				ImageId:      aws.String("ami-1"),
-				MinCount:     aws.Int64(1),
-				MaxCount:     aws.Int64(1),
-			}, "acc", "")
-			code, message, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok, "unregistered error: %v", err)
-			assert.Equal(t, awserrors.ErrorInvalidAMIIDUnavailable, code)
-			assert.Contains(t, message, state)
-		})
-	}
-}
-
 func TestPrepareRunInstances_KeyPairNotFound(t *testing.T) {
 	types, _ := defaultPrepareInstanceTypes()
 	svc := &InstanceServiceImpl{

spinifex/domains/ec2/snapshot/claims.go

@@ -1,197 +0,0 @@
-package snapshot
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-	"time"
-
-	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
-	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-)
-
-// ClaimPrefix is the kvsync prefix of every snapshot's claim, ClaimPrefix+ID,
-// naming the node that alone may call the provider for it or delete its
-// document. An AMI's claim, keyed by image ID, shares the prefix.
-const ClaimPrefix = "snapshot."
-
-// claimReleaseTimeout bounds the unlock after the last write, which runs even
-// when the request that held the claim has been cancelled.
-const claimReleaseTimeout = 5 * time.Second
-
-// NewClaimGroup returns node's group of snapshot claims over store. The daemon
-// builds it once per process, as kvsync requires of every prefix.
-func NewClaimGroup(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
-	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
-}
-
-// NodeLiveness reports when a node last heartbeated. found is false only for a
-// node with no heartbeat; a read that fails is an error, never a dead node.
-type NodeLiveness interface {
-	LastSeen(ctx context.Context, node string) (seen time.Time, found bool, err error)
-}
-
-// errClaimHeld is a snapshot claim another holder has: a request of this
-// process, or a node whose heartbeat is still fresh.
-var errClaimHeld = errors.New("snapshot claim is held")
-
-// Claims decides who may act on a pending snapshot. A kvsync claim names a
-// node, not a process, so the claims this process holds are also kept here:
-// the reaper on the node of a running request must not adopt that request's.
-type Claims struct {
-	group    *kvsync.ClaimGroup
-	node     string
-	liveness NodeLiveness
-	now      func() time.Time
-
-	mu    sync.Mutex
-	local map[string]struct{}
-}
-
-// NewClaims returns the claims of node over group. A nil liveness never judges
-// a holder dead, so a claim another node holds is never taken over.
-func NewClaims(group *kvsync.ClaimGroup, node string, liveness NodeLiveness) *Claims {
-	return &Claims{group: group, node: node, liveness: liveness, now: time.Now, local: map[string]struct{}{}}
-}
-
-// localClaims excludes only within this process: a service built without the
-// daemon's claim group has no other process to exclude.
-func localClaims() *Claims {
-	return NewClaims(nil, "", nil)
-}
-
-// heldClaim is one acquisition of a snapshot's claim.
-type heldClaim struct {
-	claims     *Claims
-	snapshotID string
-	claim      *kvsync.Claim
-	token      kvsync.Token
-}
-
-// reserve marks snapshotID held by this process, and is false when it already is.
-func (c *Claims) reserve(snapshotID string) bool {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if _, held := c.local[snapshotID]; held {
-		return false
-	}
-	c.local[snapshotID] = struct{}{}
-	return true
-}
-
-func (c *Claims) unreserve(snapshotID string) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	delete(c.local, snapshotID)
-}
-
-// lock takes snapshotID's claim for a creating request. It never takes over:
-// the ID is new, so any holder is a fault worth refusing the request for.
-func (c *Claims) lock(ctx context.Context, snapshotID string) (*heldClaim, error) {
-	return c.acquire(ctx, snapshotID, false)
-}
-
-// take acquires snapshotID's claim for the reaper. A claim another node holds
-// is taken over only once that node's heartbeat is LivenessWindow old or
-// absent; while it is fresh, or cannot be read, the claim is refused.
-func (c *Claims) take(ctx context.Context, snapshotID string) (*heldClaim, error) {
-	return c.acquire(ctx, snapshotID, true)
-}
-
-// acquire reserves snapshotID in this process and locks its claim, taking it
-// over from a dead holder only when takeOver allows.
-func (c *Claims) acquire(ctx context.Context, snapshotID string, takeOver bool) (*heldClaim, error) {
-	if !c.reserve(snapshotID) {
-		return nil, fmt.Errorf("%w: %s by this process", errClaimHeld, snapshotID)
-	}
-	held, err := c.acquireReserved(ctx, snapshotID, takeOver)
-	if err != nil {
-		c.unreserve(snapshotID)
-		return nil, err
-	}
-	return held, nil
-}
-
-func (c *Claims) acquireReserved(ctx context.Context, snapshotID string, takeOver bool) (*heldClaim, error) {
-	held := &heldClaim{claims: c, snapshotID: snapshotID}
-	if c.group == nil {
-		return held, nil
-	}
-	held.claim = c.group.Get(snapshotID)
-	tok, err := held.claim.TryLock(ctx)
-	if err == nil {
-		held.token = tok
-		return held, nil
-	}
-	he, ok := errors.AsType[*kvsync.HeldError](err)
-	if !takeOver || !ok || he.Current.Holder == "" {
-		return nil, fmt.Errorf("take snapshot claim %s: %w", snapshotID, err)
-	}
-	holder := he.Current
-	if c.liveness == nil {
-		return nil, fmt.Errorf("%w: %s by %s, and no liveness is wired to judge it", errClaimHeld, snapshotID, holder.Holder)
-	}
-	seen, found, err := c.liveness.LastSeen(ctx, holder.Holder)
-	if err != nil {
-		return nil, fmt.Errorf("%w: %s by %s, whose heartbeat cannot be read: %w", errClaimHeld, snapshotID, holder.Holder, err)
-	}
-	now := c.now()
-	if found && clusterv1.LiveAt(seen, now) {
-		return nil, fmt.Errorf("%w: %s by %s, which is still heartbeating", errClaimHeld, snapshotID, holder.Holder)
-	}
-	tok, err = held.claim.TakeOver(ctx, holder)
-	if err != nil {
-		return nil, fmt.Errorf("take over snapshot claim %s from %s: %w", snapshotID, holder.Holder, err)
-	}
-	attrs := []any{"snapshotId", snapshotID, "holder", holder.Holder, "fromGeneration", holder.Generation, "generation", tok.Generation}
-	if found {
-		attrs = append(attrs, "heartbeat_age_ms", otelsetup.Millis(now.Sub(seen)))
-	}
-	slog.WarnContext(ctx, "Took over a pending snapshot's claim from a node whose heartbeat has stopped", attrs...)
-	held.token = tok
-	return held, nil
-}
-
-// release gives the claim up after the holder's last write. A failed unlock
-// leaves the claim with this node until a restart or a takeover frees it, so
-// it is logged and not returned.
-func (h *heldClaim) release(ctx context.Context) {
-	defer h.claims.unreserve(h.snapshotID)
-	if h.claim == nil {
-		return
-	}
-	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), claimReleaseTimeout)
-	defer cancel()
-	if err := h.claim.Unlock(ctx, h.token); err != nil {
-		slog.WarnContext(ctx, "Failed to release a snapshot claim",
-			"snapshotId", h.snapshotID, "generation", h.token.Generation, "err", err)
-	}
-}
-
-// releaseOwn unlocks snapshotID's claim when it names this node and no request
-// of this process holds it: a claim left by an earlier process of this node,
-// whose requests did not survive its restart.
-func (c *Claims) releaseOwn(ctx context.Context, snapshotID string) (bool, error) {
-	if c.group == nil || !c.reserve(snapshotID) {
-		return false, nil
-	}
-	defer c.unreserve(snapshotID)
-	claim := c.group.Get(snapshotID)
-	tok, ok, err := claim.Holder(ctx)
-	if err != nil {
-		return false, fmt.Errorf("read snapshot claim %s: %w", snapshotID, err)
-	}
-	if !ok || tok.Holder != c.node {
-		return false, nil
-	}
-	if err := claim.Unlock(ctx, tok); err != nil {
-		if errors.Is(err, kvsync.ErrLost) {
-			return false, nil
-		}
-		return false, fmt.Errorf("release snapshot claim %s: %w", snapshotID, err)
-	}
-	return true, nil
-}

spinifex/domains/ec2/snapshot/delete_pending_test.go

@@ -1,581 +0,0 @@
-package snapshot
-
-import (
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// fastClaims makes svc's writers retry a held claim without the real wait.
-func fastClaims(svc *SnapshotServiceImpl) *SnapshotServiceImpl {
-	svc.claimRetryDelay = time.Millisecond
-	return svc
-}
-
-func (f *pendingFixture) deleteSnapshot(svc *SnapshotServiceImpl, id string) error {
-	_, err := svc.DeleteSnapshot(f.t.Context(), &ec2.DeleteSnapshotInput{SnapshotId: aws.String(id)}, testAccountID)
-	return err
-}
-
-// assertHidden requires that every request answers id as a deleted snapshot.
-func (f *pendingFixture) assertHidden(svc *SnapshotServiceImpl, id string) {
-	f.t.Helper()
-	listed, err := svc.DescribeSnapshots(f.t.Context(), &ec2.DescribeSnapshotsInput{}, testAccountID)
-	require.NoError(f.t, err)
-	for _, snap := range listed.Snapshots {
-		assert.NotEqual(f.t, id, aws.StringValue(snap.SnapshotId), "DescribeSnapshots lists a deleted snapshot")
-	}
-	_, err = svc.DescribeSnapshots(f.t.Context(), &ec2.DescribeSnapshotsInput{SnapshotIds: aws.StringSlice([]string{id})}, testAccountID)
-	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
-	_, err = svc.DescribeSnapshotsStrict(f.t.Context(), &ec2.DescribeSnapshotsInput{SnapshotIds: aws.StringSlice([]string{id})}, testAccountID)
-	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
-	_, err = svc.CopySnapshot(f.t.Context(), &ec2.CopySnapshotInput{SourceSnapshotId: aws.String(id)}, testAccountID)
-	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
-	assert.EqualError(f.t, f.deleteSnapshot(svc, id), awserrors.ErrorInvalidSnapshotNotFound, "a repeated delete")
-}
-
-// DeleteSnapshot marks a pending or AmbiguousCreate snapshot and returns,
-// leaving the provider alone; any other is deleted as it always was.
-func TestDeleteSnapshot_MarksWhatTheProviderMayStillHold(t *testing.T) {
-	const id = "snap-markdelete0001"
-	cases := []struct {
-		name       string
-		doc        ebsmetadata.Snapshot
-		wantMarked bool
-	}{
-		{name: "pending", doc: ebsmetadata.Snapshot{State: snapshotStatePending}, wantMarked: true},
-		{name: "pending with an unanswered create", doc: ebsmetadata.Snapshot{State: snapshotStatePending, AmbiguousCreate: true}, wantMarked: true},
-		{name: "completed with an unanswered create", doc: ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted, AmbiguousCreate: true}, wantMarked: true},
-		{name: "errored with an unanswered create", doc: ebsmetadata.Snapshot{State: snapshotStateError, AmbiguousCreate: true}, wantMarked: true},
-		{name: "completed", doc: ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted}},
-		{name: "errored", doc: ebsmetadata.Snapshot{State: snapshotStateError}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			svc := f.node(nodeA)
-			tc.doc.SnapshotID = id
-			f.seedDoc(tc.doc, true)
-			providerSnapshot(f, id)
-
-			require.NoError(t, f.deleteSnapshot(svc, id))
-
-			doc, ok := f.doc(id)
-			if !tc.wantMarked {
-				assert.False(t, ok, "a settled snapshot is deleted as it always was")
-				assert.Equal(t, []string{"snapshot " + id}, f.provider.deleteCalls())
-				assert.Empty(t, f.refsOf(pendingVolumeID))
-				return
-			}
-			require.True(t, ok)
-			assert.True(t, doc.DeleteRequested)
-			assert.Equal(t, tc.doc.State, doc.State, "the mark changes nothing else")
-			assert.Empty(t, f.provider.deleteCalls(), "the provider is not asked while the snapshot may be in progress")
-			_, held := f.providerHolds(id)
-			assert.True(t, held)
-			assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID), "the reference stays for the reaper")
-			assert.Empty(t, f.claimHolder(id), "the claim is released once the mark is written")
-			f.assertHidden(svc, id)
-		})
-	}
-}
-
-// The InUse checks run first, unchanged, on a pending snapshot too.
-func TestDeleteSnapshot_PendingInUseIsStillRefused(t *testing.T) {
-	const id = "snap-markinuse00001"
-	cases := []struct {
-		name string
-		seed func(f *pendingFixture)
-	}{
-		{name: "a volume was created from it", seed: func(f *pendingFixture) {
-			require.NoError(f.t, f.metadata.PutVolume(f.t.Context(), ebsmetadata.Volume{
-				VolumeID: "vol-fromsnap000001", TenantID: testAccountID, SnapshotID: id, State: "available",
-			}))
-		}},
-		{name: "a registered AMI names it", seed: func(f *pendingFixture) {
-			require.NoError(f.t, f.metadata.PutAMI(f.t.Context(), ebsmetadata.AMI{ImageID: "ami-0aaaaaaaaaaaaaaa1", SnapshotID: id, State: ebsmetadata.AMIStatePending}))
-		}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			svc := f.node(nodeA)
-			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
-			tc.seed(f)
-
-			err := f.deleteSnapshot(svc, id)
-			require.Error(t, err)
-			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidSnapshotInUse), "got %v", err)
-			doc, ok := f.doc(id)
-			require.True(t, ok)
-			assert.False(t, doc.DeleteRequested)
-		})
-	}
-}
-
-// A claim another live node holds is retried three times and then answered
-// IncorrectState; one a stopped node holds is taken over.
-func TestDeleteSnapshot_ClaimContention(t *testing.T) {
-	const id = "snap-markheld000001"
-	cases := []struct {
-		name        string
-		holder      string
-		holderSeen  time.Time
-		wantErr     string
-		wantRefused int
-	}{
-		{name: "held by a live node", holder: nodeC, holderSeen: time.Now(), wantErr: awserrors.ErrorIncorrectState, wantRefused: claimRetries + 1},
-		{name: "held by a node whose heartbeat stopped", holder: nodeC, holderSeen: time.Now().Add(-2 * clusterv1.LivenessWindow)},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			svc := fastClaims(f.node(nodeA))
-			refused := 0
-			svc.claimRefused = func(string) { refused++ }
-			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
-			f.holdClaim(tc.holder, id)
-			f.liveness.beat(tc.holder, tc.holderSeen)
-
-			err := f.deleteSnapshot(svc, id)
-
-			assert.Equal(t, tc.wantRefused, refused)
-			doc, ok := f.doc(id)
-			require.True(t, ok)
-			if tc.wantErr != "" {
-				require.Error(t, err)
-				assert.True(t, awserrors.IsErrorCode(err, tc.wantErr), "got %v", err)
-				assert.False(t, doc.DeleteRequested)
-				assert.Equal(t, tc.holder, f.claimHolder(id))
-				return
-			}
-			require.NoError(t, err)
-			assert.True(t, doc.DeleteRequested)
-			assert.Empty(t, f.claimHolder(id))
-		})
-	}
-}
-
-// Each row of the marked column: the reaper deletes a marked snapshot once
-// the provider settles it, never repeats one the provider does not hold, and
-// keeps a tombstone in place of a document that had an unanswered create.
-func TestPendingSnapshotReaper_MarkedColumn(t *testing.T) {
-	const id = "snap-markedrow00001"
-	cases := []struct {
-		name      string
-		ambiguous bool
-		setup     func(f *pendingFixture)
-		// wantKept: the document stays marked, its reference and provider snapshot untouched.
-		wantKept      bool
-		wantTombstone bool
-		wantDeletes   int // provider DeleteSnapshot calls
-	}{
-		{name: "provider pending", setup: func(f *pendingFixture) {
-			f.mem.AnswerSnapshotsPending(true)
-			providerSnapshot(f, id)
-		}, wantKept: true},
-		{name: "provider completed", setup: func(f *pendingFixture) { providerSnapshot(f, id) }, wantDeletes: 1},
-		{name: "provider error", setup: func(f *pendingFixture) {
-			f.mem.AnswerSnapshotsPending(true)
-			providerSnapshot(f, id)
-			require.NoError(f.t, f.mem.SettleSnapshot(id, ebsprovider.SnapshotStateError))
-		}, wantDeletes: 1},
-		{name: "provider completed, create unanswered", ambiguous: true, setup: func(f *pendingFixture) { providerSnapshot(f, id) },
-			wantTombstone: true, wantDeletes: 1},
-		{name: "provider delete answers not_found", setup: func(f *pendingFixture) {
-			providerSnapshot(f, id)
-			f.provider.onDelete = func(ebsprovider.DeleteSnapshotRequest) error { return ebsprovider.ErrNotFound }
-		}, wantDeletes: 1},
-		{name: "absent", wantDeletes: 0},
-		{name: "absent, create unanswered", ambiguous: true, wantTombstone: true},
-		{name: "absent, claim held by a live node", setup: func(f *pendingFixture) { f.holdClaim(nodeC, id) }, wantKept: true},
-		{name: "provider read fails", setup: func(f *pendingFixture) {
-			f.provider.onList = func(ebsprovider.ListSnapshotsRequest) (bool, *ebsprovider.ListSnapshotsResponse, error) {
-				return true, nil, ebsprovider.ErrUnavailable
-			}
-		}, wantKept: true},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			reaper := f.node(nodeB)
-			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending, DeleteRequested: true, AmbiguousCreate: tc.ambiguous}, true)
-			if tc.setup != nil {
-				tc.setup(f)
-			}
-			_, heldBefore := f.providerHolds(id)
-
-			f.sweep(reaper)
-
-			assert.Zero(t, f.provider.createCalls(), "a marked snapshot is never repeated")
-			assert.Len(t, f.provider.deleteCalls(), tc.wantDeletes)
-			doc, ok := f.doc(id)
-			switch {
-			case tc.wantKept:
-				require.True(t, ok)
-				assert.True(t, doc.DeleteRequested)
-				assert.False(t, doc.Tombstoned())
-				assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID))
-				_, held := f.providerHolds(id)
-				assert.Equal(t, heldBefore, held)
-			case tc.wantTombstone:
-				require.True(t, ok, "a document that had an unanswered create is kept as a tombstone")
-				assert.True(t, doc.Tombstoned())
-				assert.Empty(t, f.refsOf(pendingVolumeID))
-				f.assertHidden(f.node(nodeA), id)
-			default:
-				assert.False(t, ok)
-				assert.Empty(t, f.refsOf(pendingVolumeID))
-			}
-			if !tc.wantKept && tc.wantDeletes > 0 && f.provider.onDelete == nil {
-				_, held := f.providerHolds(id)
-				assert.False(t, held)
-			}
-		})
-	}
-}
-
-// A failure or interruption between the provider delete, the reference's
-// removal and the document's removal leaves the marked document for the next
-// sweep, which finishes; no reference is ever left without its document.
-func TestPendingSnapshotReaper_CleanupInterrupted(t *testing.T) {
-	const id = "snap-cleanupstep001"
-	cases := []struct {
-		name      string
-		ambiguous bool
-		break_    func(f *pendingFixture, reaper *SnapshotServiceImpl)
-		heal      func(f *pendingFixture, reaper *SnapshotServiceImpl)
-		wantRef   bool // the reference is still there after the broken sweep
-	}{
-		{name: "the provider delete fails", break_: func(f *pendingFixture, _ *SnapshotServiceImpl) {
-			f.provider.onDelete = func(ebsprovider.DeleteSnapshotRequest) error { return ebsprovider.ErrUnavailable }
-		}, heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.provider.onDelete = nil }, wantRef: true},
-		{name: "interrupted after the provider delete", break_: func(_ *pendingFixture, r *SnapshotServiceImpl) {
-			r.reapAt = func(step reapStep, _ string) bool { return step == reapProviderDeleted }
-		}, heal: func(_ *pendingFixture, r *SnapshotServiceImpl) { r.reapAt = nil }, wantRef: true},
-		{name: "the reference removal fails", break_: func(f *pendingFixture, r *SnapshotServiceImpl) {
-			r.snapRefs = f.unreachableRefs()
-		}, heal: func(f *pendingFixture, r *SnapshotServiceImpl) { r.snapRefs = f.node(nodeB).snapRefs }, wantRef: true},
-		{name: "interrupted after the reference removal", break_: func(_ *pendingFixture, r *SnapshotServiceImpl) {
-			r.reapAt = func(step reapStep, _ string) bool { return step == reapRefRemoved }
-		}, heal: func(_ *pendingFixture, r *SnapshotServiceImpl) { r.reapAt = nil }},
-		{name: "the document removal fails", break_: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.failDelete = true },
-			heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.failDelete = false }},
-		{name: "the tombstone write fails", ambiguous: true, break_: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.putsBefore = 0 },
-			heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.putsBefore = -1 }},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			reaper := f.node(nodeB)
-			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending, DeleteRequested: true, AmbiguousCreate: tc.ambiguous}, true)
-			providerSnapshot(f, id)
-
-			tc.break_(f, reaper)
-			f.sweep(reaper)
-
-			doc, ok := f.doc(id)
-			require.True(t, ok, "the document is the record the next sweep finds the work from")
-			assert.True(t, doc.DeleteRequested)
-			assert.False(t, doc.Tombstoned())
-			if tc.wantRef {
-				assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID))
-			} else {
-				assert.Empty(t, f.refsOf(pendingVolumeID))
-			}
-
-			tc.heal(f, reaper)
-			f.sweep(reaper)
-
-			_, held := f.providerHolds(id)
-			assert.False(t, held)
-			assert.Empty(t, f.refsOf(pendingVolumeID))
-			doc, ok = f.doc(id)
-			if tc.ambiguous {
-				require.True(t, ok)
-				assert.True(t, doc.Tombstoned())
-			} else {
-				assert.False(t, ok)
-			}
-		})
-	}
-}
-
-// A DeleteSnapshot that lands while the reaper holds the claim, between its
-// reread and its write, waits on the claim and ends with the snapshot deleted,
-// whether it runs on another node or on the reaper's own.
-func TestDeleteSnapshot_LandsBetweenTheReapersRereadAndWrite(t *testing.T) {
-	const id = "snap-deleteraced001"
-	for _, sameNode := range []bool{false, true} {
-		name := "another node"
-		if sameNode {
-			name = "the reaper's node"
-		}
-		t.Run(name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			reaper := fastClaims(f.node(nodeB))
-			deleter := fastClaims(f.node(nodeA))
-			if sameNode {
-				deleter = reaper
-			}
-			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
-			providerSnapshot(f, id)
-
-			refused := make(chan struct{}, claimRetries+1)
-			deleter.claimRefused = func(string) { refused <- struct{}{} }
-			var wg sync.WaitGroup
-			var deleteErr error
-			reaper.reapAt = func(step reapStep, _ string) bool {
-				if step == reapReread {
-					wg.Go(func() { deleteErr = f.deleteSnapshot(deleter, id) })
-					<-refused
-				}
-				return false
-			}
-			deleter.claimRetryDelay = 50 * time.Millisecond
-
-			f.sweep(reaper)
-			wg.Wait()
-
-			require.NoError(t, deleteErr, "the delete waits out the reaper's claim")
-			_, ok := f.doc(id)
-			assert.False(t, ok, "the delete is not undone by the reaper's settle")
-			_, held := f.providerHolds(id)
-			assert.False(t, held)
-			assert.Empty(t, f.refsOf(pendingVolumeID))
-		})
-	}
-}
-
-// A request whose provider call returns after its claim was taken over and the
-// snapshot marked leaves the document marked and pending, for the reaper.
-func TestCreateSnapshot_AnswerAfterTheMarkIsNotWritten(t *testing.T) {
-	f := newPendingFixture(t)
-	svc := f.node(nodeA)
-	f.liveness.beat(nodeA, time.Now().Add(-2*clusterv1.LivenessWindow))
-	deleter := fastClaims(f.node(nodeB))
-	f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
-		if n == 1 {
-			require.NoError(t, f.deleteSnapshot(deleter, req.SnapshotID))
-		}
-		return false, nil, nil
-	}
-
-	_, err := f.create(svc)
-	require.NoError(t, err, "the provider answered, so the request succeeds")
-
-	doc := f.onlyDoc()
-	assert.True(t, doc.DeleteRequested)
-	assert.Equal(t, snapshotStatePending, doc.State, "a marked document is never settled by the request")
-	assert.True(t, doc.AmbiguousCreate)
-
-	f.sweep(f.node(nodeC))
-	assert.True(t, f.onlyDoc().Tombstoned())
-	_, held := f.providerHolds(doc.SnapshotID)
-	assert.False(t, held)
-}
-
-// AmbiguousCreate is on the document whenever the provider is called, and is
-// cleared only by a definite answer to a document with no earlier call left
-// unanswered.
-func TestAmbiguousCreate_SetBeforeEveryCall(t *testing.T) {
-	const reaped = "snap-ambiguousreap1"
-	cases := []struct {
-		name string
-		// reaper seeds a pending document with the flag clear and sweeps it,
-		// instead of a CreateSnapshot.
-		reaper  bool
-		pending bool              // the memory provider answers pending
-		onCall  func(n int) error // nil error: the memory provider answers
-		crash   createStep
-		want    bool
-		wantErr bool
-	}{
-		{name: "request answered completed", want: false},
-		{name: "request answered pending", pending: true, want: false},
-		{name: "request first call unanswered, repeat completed", onCall: firstFails(errAmbiguous), want: true},
-		{name: "request crashed after sending", crash: stepAnswered, want: true, wantErr: true},
-		{name: "request repeat refused for good", onCall: func(n int) error {
-			if n == 1 {
-				return errAmbiguous
-			}
-			return ebsprovider.ErrAlreadyExists
-		}, want: true, wantErr: true},
-		{name: "reaper repeat answered", reaper: true, want: false},
-		{name: "reaper repeat unanswered", reaper: true, onCall: firstFails(errAmbiguous), want: true},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			svc := f.node(nodeA)
-			f.mem.AnswerSnapshotsPending(tc.pending)
-			var flagged []bool
-			f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
-				doc, _ := f.doc(req.SnapshotID)
-				flagged = append(flagged, doc.AmbiguousCreate)
-				if tc.onCall != nil {
-					if err := tc.onCall(n); err != nil {
-						return true, nil, err
-					}
-				}
-				return false, nil, nil
-			}
-			if tc.crash != 0 {
-				svc.crashAt = func(step createStep) bool { return step == tc.crash }
-			}
-
-			var id string
-			if tc.reaper {
-				id = reaped
-				f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
-				f.sweep(f.node(nodeB))
-			} else {
-				_, err := f.create(svc)
-				assert.Equal(t, tc.wantErr, err != nil, "err %v", err)
-				id = f.onlyDoc().SnapshotID
-			}
-
-			require.NotEmpty(t, flagged)
-			for i, set := range flagged {
-				assert.True(t, set, "call %d was sent with AmbiguousCreate clear", i+1)
-			}
-			doc, ok := f.doc(id)
-			require.True(t, ok)
-			assert.Equal(t, tc.want, doc.AmbiguousCreate)
-		})
-	}
-}
-
-// firstFails answers the first call with err and lets the rest through.
-func firstFails(err error) func(int) error {
-	return func(n int) error {
-		if n == 1 {
-			return err
-		}
-		return nil
-	}
-}
-
-// The original call of a snapshot that a repeat completed and the customer
-// then deleted is accepted late: the snapshot it recreates is deleted under
-// the tombstone once it settles, and the tombstone is dropped after 7 days.
-func TestTombstone_DeletesALateAcceptedSnapshotAndExpires(t *testing.T) {
-	f := newPendingFixture(t)
-	svc := f.node(nodeA)
-	reaper := f.node(nodeB)
-	now := time.Now()
-	reaper.now = func() time.Time { return now }
-	var late ebsprovider.CreateSnapshotRequest
-	f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
-		if n <= 2 {
-			late = req
-			return true, nil, errAmbiguous
-		}
-		return false, nil, nil
-	}
-	_, err := f.create(svc)
-	require.EqualError(t, err, awserrors.ErrorServerInternal)
-	id := f.onlyDoc().SnapshotID
-
-	// The reaper's repeat completes it, and the customer deletes it.
-	f.sweep(reaper)
-	doc := f.onlyDoc()
-	require.Equal(t, ebsmetadata.SnapshotStateCompleted, doc.State)
-	require.True(t, doc.AmbiguousCreate, "the request's calls were never answered")
-	require.NoError(t, f.deleteSnapshot(svc, id))
-	f.sweep(reaper)
-	require.True(t, f.onlyDoc().Tombstoned())
-	_, held := f.providerHolds(id)
-	require.False(t, held)
-
-	// The request's original call is accepted now, and is still in progress.
-	f.mem.AnswerSnapshotsPending(true)
-	_, err = f.mem.CreateSnapshot(t.Context(), late)
-	require.NoError(t, err)
-	f.sweep(reaper)
-	_, held = f.providerHolds(id)
-	assert.True(t, held, "a snapshot still pending is not deleted")
-	require.NoError(t, f.mem.SettleSnapshot(id, ebsprovider.SnapshotStateCompleted))
-	f.sweep(reaper)
-	_, held = f.providerHolds(id)
-	assert.False(t, held, "the late snapshot is deleted under the tombstone")
-	assert.True(t, f.onlyDoc().Tombstoned())
-	f.assertHidden(svc, id)
-	assert.Equal(t, 3, f.provider.createCalls(), "the request's two calls and one repeat; nothing repeats a tombstone's create")
-
-	now = now.Add(tombstoneRetention - time.Minute)
-	f.sweep(reaper)
-	assert.Len(t, f.docs(), 1, "the tombstone is kept for its retention")
-	now = now.Add(2 * time.Minute)
-	f.sweep(reaper)
-	assert.Empty(t, f.docs(), "the tombstone is dropped after its retention")
-	assert.Empty(t, f.refsOf(pendingVolumeID))
-}
-
-// The AMI pass settles each pending AMI from its snapshot's document.
-func TestPendingSnapshotReaper_SettlesPendingAMIs(t *testing.T) {
-	const snapID = "snap-amisettle00001"
-	cases := []struct {
-		name     string
-		amiState string
-		snap     *ebsmetadata.Snapshot // nil: no snapshot document
-		want     string
-	}{
-		{name: "snapshot completed", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted}, want: ebsmetadata.AMIStateAvailable},
-		{name: "snapshot errored", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: snapshotStateError}, want: ebsmetadata.AMIStateFailed},
-		{name: "snapshot document missing", amiState: ebsmetadata.AMIStatePending, want: ebsmetadata.AMIStateFailed},
-		{name: "snapshot pending", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: snapshotStatePending}, want: ebsmetadata.AMIStatePending},
-		{name: "AMI already available", amiState: ebsmetadata.AMIStateAvailable, want: ebsmetadata.AMIStateAvailable},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			f := newPendingFixture(t)
-			if tc.snap != nil {
-				tc.snap.SnapshotID = snapID
-				f.seedDoc(*tc.snap, true)
-				if tc.snap.State == snapshotStatePending {
-					f.mem.AnswerSnapshotsPending(true)
-					providerSnapshot(f, snapID)
-				}
-			}
-			const amiID = "ami-0bbbbbbbbbbbbbbb1"
-			require.NoError(t, f.metadata.PutAMI(t.Context(), ebsmetadata.AMI{
-				ImageID: amiID, SnapshotID: snapID, ImageOwnerAlias: testAccountID, State: tc.amiState,
-			}))
-
-			f.sweep(f.node(nodeB))
-
-			ami, err := f.metadata.GetAMI(t.Context(), amiID)
-			require.NoError(t, err)
-			assert.Equal(t, tc.want, ami.State)
-		})
-	}
-}
-
-func TestCopySnapshot_RefusesASourceThatIsNotCompleted(t *testing.T) {
-	for _, state := range []string{snapshotStatePending, "error"} {
-		t.Run(state, func(t *testing.T) {
-			svc, _ := providerSnapshotService(t, "vol-copygate")
-			require.NoError(t, svc.MetadataStore().PutSnapshot(t.Context(), ebsmetadata.Snapshot{
-				SnapshotID: "snap-copygate00001", VolumeID: "vol-copygate", VolumeSize: 4, State: state, OwnerID: testAccountID,
-			}))
-
-			_, err := svc.CopySnapshot(t.Context(), &ec2.CopySnapshotInput{SourceSnapshotId: aws.String("snap-copygate00001")}, testAccountID)
-			code, _, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok, "unregistered error: %v", err)
-			assert.Equal(t, awserrors.ErrorIncorrectState, code)
-			snaps, err := svc.MetadataStore().ListAllSnapshots(t.Context())
-			require.NoError(t, err)
-			assert.Len(t, snaps, 1, "a refused copy writes no snapshot document")
-		})
-	}
-}

spinifex/domains/ec2/snapshot/pending.go

@@ -1,113 +0,0 @@
-package snapshot
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-// The snapshot document states CreateSnapshot and the reaper move between.
-const (
-	snapshotStatePending = string(ebsprovider.SnapshotStatePending)
-	snapshotStateError   = string(ebsprovider.SnapshotStateError)
-)
-
-// definiteRefusal reports a CreateSnapshot error that is the provider's answer
-// that it holds no snapshot for the call. Anything else, a timeout, a lost
-// connection or unavailable, may follow an accepted call.
-func definiteRefusal(err error) bool {
-	return errors.Is(err, ebsprovider.ErrInvalidArgument) || errors.Is(err, ebsprovider.ErrNotFound) ||
-		errors.Is(err, ebsprovider.ErrAlreadyExists) || errors.Is(err, ebsprovider.ErrVolumeInUse)
-}
-
-// refusalSettlesError reports a refusal no later call can change: the source
-// volume is gone, or the ID is another volume's snapshot. Only these make a
-// pending document error once an earlier call may still be accepted.
-func refusalSettlesError(err error) bool {
-	return errors.Is(err, ebsprovider.ErrNotFound) || errors.Is(err, ebsprovider.ErrAlreadyExists)
-}
-
-// applyProviderAnswer moves a document to what the provider answered: completed
-// at 100%, or pending with the provider's handle for the reaper to follow. The
-// call was answered, so AmbiguousCreate stays only if an earlier one was not.
-func applyProviderAnswer(created *ebsprovider.Snapshot, earlierUnanswered bool) func(*ebsmetadata.Snapshot) {
-	return func(doc *ebsmetadata.Snapshot) {
-		doc.AmbiguousCreate = earlierUnanswered
-		if created.Handle != "" {
-			doc.ProviderHandle = created.Handle
-		}
-		if created.State == ebsprovider.SnapshotStateCompleted {
-			doc.State = ebsmetadata.SnapshotStateCompleted
-			doc.Progress = "100%"
-		}
-	}
-}
-
-// repeatOutcome is the write a CreateSnapshot answer after an unanswered call
-// makes: answered, the provider's answer; a refusal no later call can change,
-// error; anything else, nil, leaving the document pending.
-func repeatOutcome(created *ebsprovider.Snapshot, err error, earlierUnanswered bool) (mutate func(*ebsmetadata.Snapshot), answered bool) {
-	switch {
-	case err == nil && created != nil:
-		return applyProviderAnswer(created, earlierUnanswered), true
-	case refusalSettlesError(err):
-		return func(d *ebsmetadata.Snapshot) {
-			markSnapshotError(d)
-			d.AmbiguousCreate = earlierUnanswered
-		}, false
-	}
-	return nil, false
-}
-
-func markSnapshotCompleted(doc *ebsmetadata.Snapshot) {
-	doc.State = ebsmetadata.SnapshotStateCompleted
-	doc.Progress = "100%"
-}
-
-func markSnapshotError(doc *ebsmetadata.Snapshot) {
-	doc.State = snapshotStateError
-}
-
-// updatePending rereads a snapshot document and applies mutate only while it
-// is still pending, so a delete or settlement since the caller read it is not
-// undone. It returns the document as read or written, zero when it is gone.
-func (s *SnapshotServiceImpl) updatePending(ctx context.Context, accountID, snapshotID string, mutate func(*ebsmetadata.Snapshot)) (ebsmetadata.Snapshot, bool, error) {
-	return s.rewrite(ctx, accountID, snapshotID, func(doc *ebsmetadata.Snapshot) bool {
-		// A marked document is the reaper's to delete, never to settle.
-		if doc.State != snapshotStatePending || doc.Hidden() {
-			return false
-		}
-		mutate(doc)
-		return true
-	})
-}
-
-// rewrite rereads snapshotID's document and writes it back when mutate asks
-// to. It returns the document as read or written and whether it wrote; one
-// that is gone is zero, and mutate never sees it.
-func (s *SnapshotServiceImpl) rewrite(ctx context.Context, accountID, snapshotID string, mutate func(*ebsmetadata.Snapshot) bool) (ebsmetadata.Snapshot, bool, error) {
-	doc, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return ebsmetadata.Snapshot{}, false, nil
-		}
-		return ebsmetadata.Snapshot{}, false, fmt.Errorf("read snapshot %s: %w", snapshotID, err)
-	}
-	if !mutate(&doc) {
-		return doc, false, nil
-	}
-	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
-		return ebsmetadata.Snapshot{}, false, fmt.Errorf("write snapshot %s: %w", snapshotID, err)
-	}
-	return doc, true, nil
-}
-
-// writeContext bounds a write made after the provider answered, which has to
-// land even when the request that asked has been cancelled.
-func writeContext(ctx context.Context) (context.Context, context.CancelFunc) {
-	return context.WithTimeout(context.WithoutCancel(ctx), snapshotCleanupTimeout)
-}

spinifex/domains/ec2/snapshot/pending_reaper.go

@@ -1,464 +0,0 @@
-package snapshot
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-// PendingSnapshotReaper settles every pending snapshot document from the
-// provider. CreateSnapshot returns once the provider answers, and a snapshot
-// that settles later, or a request that crashed or failed ambiguously, leaves
-// its document pending; this sweep is where that work converges (ADR-0003:S8).
-//
-// It deletes nothing a customer did not ask to delete (ADR-0003:S7): only a
-// snapshot whose document DeleteSnapshot marked, its reference and document,
-// and a late-accepted snapshot under the tombstone that delete left.
-type PendingSnapshotReaper struct {
-	svc *SnapshotServiceImpl
-}
-
-// tombstoneRetention is how long a deleted AmbiguousCreate snapshot's document
-// stays, so a create accepted that late is still found and deleted.
-const tombstoneRetention = 7 * 24 * time.Hour
-
-// reapStep is a point in the reaper's work on one document, for a test to
-// interrupt it at.
-type reapStep int
-
-const (
-	reapReread reapStep = iota + 1
-	reapProviderDeleted
-	reapRefRemoved
-)
-
-// NewPendingSnapshotReaper builds the reaper over this service's documents,
-// provider and claims.
-func (s *SnapshotServiceImpl) NewPendingSnapshotReaper() *PendingSnapshotReaper {
-	return &PendingSnapshotReaper{svc: s}
-}
-
-func (r *PendingSnapshotReaper) Class() string { return "pending-snapshot" }
-
-// Sweep reads every snapshot document and settles each pending one it can,
-// carries out each accepted delete, and rereads each tombstone; then it settles
-// each pending AMI from its snapshot. It reports how many documents it wrote.
-func (r *PendingSnapshotReaper) Sweep(ctx context.Context) (int, error) {
-	if r.svc.provider == nil {
-		return 0, nil
-	}
-	docs, err := r.svc.metadata.ListAllSnapshotsStrict(ctx)
-	if err != nil {
-		return 0, fmt.Errorf("list snapshot documents: %w", err)
-	}
-	written := 0
-	for _, doc := range docs {
-		if err := ctx.Err(); err != nil {
-			return written, err
-		}
-		var wrote bool
-		switch {
-		case doc.Tombstoned():
-			wrote = r.sweepTombstone(ctx, doc)
-		case doc.DeleteRequested, doc.State == snapshotStatePending:
-			wrote = r.settle(ctx, doc)
-		default:
-			continue
-		}
-		if wrote {
-			written++
-		}
-	}
-	amis, err := r.settleAMIs(ctx)
-	return written + amis, err
-}
-
-// interrupted reports a test stopping the reaper at step for snapshotID.
-func (r *PendingSnapshotReaper) interrupted(step reapStep, snapshotID string) bool {
-	return r.svc.reapAt != nil && r.svc.reapAt(step, snapshotID)
-}
-
-// providerSnapshot reads snapshotID from the provider. found is false when the
-// provider holds no snapshot by that ID.
-func (r *PendingSnapshotReaper) providerSnapshot(ctx context.Context, doc ebsmetadata.Snapshot) (ebsprovider.SnapshotRef, bool, error) {
-	ref, found, err := ebsprovider.ReadSnapshot(ctx, r.svc.provider, doc.SnapshotID)
-	if err != nil {
-		slog.WarnContext(ctx, "pending-snapshot: provider read failed, retrying next sweep",
-			"snapshotId", doc.SnapshotID, "accountId", doc.OwnerID, "err", err)
-	}
-	return ref, found, err
-}
-
-// settle follows the provider's state for doc, carries out its accepted
-// delete, or repeats its CreateSnapshot when the provider holds no snapshot by
-// that ID. It reports whether it wrote.
-func (r *PendingSnapshotReaper) settle(ctx context.Context, doc ebsmetadata.Snapshot) bool {
-	// A marked document is only cleaned up, from a provider read under its claim.
-	if doc.DeleteRequested {
-		return r.underClaim(ctx, doc, nil)
-	}
-	ref, found, err := r.providerSnapshot(ctx, doc)
-	if err != nil {
-		return false
-	}
-	if found {
-		return r.follow(ctx, doc, ref)
-	}
-	return r.underClaim(ctx, doc, func(current ebsmetadata.Snapshot) bool { return r.repeat(ctx, current) })
-}
-
-// providerSettled reports a provider snapshot that is completed or errored,
-// the states in which a provider deletes it.
-func providerSettled(ref ebsprovider.SnapshotRef) bool {
-	return ref.State == ebsprovider.SnapshotStateCompleted || ref.State == ebsprovider.SnapshotStateError
-}
-
-// withClaim runs fn under id's claim, which take acquires, and reports false
-// without running it when the claim is not taken.
-func withClaim(ctx context.Context, id string, take func(context.Context, string) (*heldClaim, error), fn func() bool) bool {
-	held, err := take(ctx, id)
-	if err != nil {
-		level := slog.LevelWarn
-		if errors.Is(err, errClaimHeld) {
-			level = slog.LevelInfo
-		}
-		slog.Log(ctx, level, "pending-snapshot: claim not taken, left for the next sweep", "id", id, "err", err)
-		return false
-	}
-	defer held.release(ctx)
-	return fn()
-}
-
-// underClaim takes doc's claim, rereads its document and runs act on it while
-// it is unmarked and pending. A marked document, however it was read before,
-// is cleaned up instead.
-func (r *PendingSnapshotReaper) underClaim(ctx context.Context, doc ebsmetadata.Snapshot, act func(ebsmetadata.Snapshot) bool) bool {
-	return withClaim(ctx, doc.SnapshotID, r.svc.claims.take, func() bool {
-		// The claim may have been taken after the creating request rolled back or
-		// settled the document, so it is read again before anything is written.
-		current, err := r.svc.metadata.GetSnapshot(ctx, doc.OwnerID, doc.SnapshotID)
-		if err != nil || current.Tombstoned() {
-			return false
-		}
-		if r.interrupted(reapReread, doc.SnapshotID) {
-			return false
-		}
-		if current.DeleteRequested {
-			return r.cleanUpIfSettled(ctx, current)
-		}
-		if current.State != snapshotStatePending {
-			return false
-		}
-		return act(current)
-	})
-}
-
-// cleanUpIfSettled reads doc's snapshot from the provider and carries out its
-// accepted delete once the provider no longer holds it pending.
-func (r *PendingSnapshotReaper) cleanUpIfSettled(ctx context.Context, doc ebsmetadata.Snapshot) bool {
-	ref, found, err := r.providerSnapshot(ctx, doc)
-	if err != nil || (found && !providerSettled(ref)) {
-		return false
-	}
-	return r.cleanUp(ctx, doc, ref, found)
-}
-
-// deleteAtProvider deletes snapshotID at the provider; one already gone there
-// is deleted.
-func (r *PendingSnapshotReaper) deleteAtProvider(ctx context.Context, snapshotID, handle string) error {
-	err := r.svc.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: handle,
-	})
-	if errors.Is(err, ebsprovider.ErrNotFound) {
-		return nil
-	}
-	return err
-}
-
-// follow writes the state the provider reports for a snapshot it holds.
-func (r *PendingSnapshotReaper) follow(ctx context.Context, doc ebsmetadata.Snapshot, ref ebsprovider.SnapshotRef) bool {
-	var mutate func(*ebsmetadata.Snapshot)
-	switch ref.State {
-	case ebsprovider.SnapshotStateCompleted:
-		mutate = func(d *ebsmetadata.Snapshot) {
-			markSnapshotCompleted(d)
-			if d.ProviderHandle == "" {
-				d.ProviderHandle = ref.Handle
-			}
-		}
-	case ebsprovider.SnapshotStateError:
-		mutate = markSnapshotError
-	case ebsprovider.SnapshotStatePending:
-		return false
-	default:
-		slog.WarnContext(ctx, "pending-snapshot: provider reported no state for a held snapshot, retrying next sweep",
-			"snapshotId", doc.SnapshotID, "state", ref.State)
-		return false
-	}
-	return r.underClaim(ctx, doc, func(current ebsmetadata.Snapshot) bool { return r.write(ctx, current, mutate) })
-}
-
-// repeat asks the provider again, under the snapshot's claim, for a pending
-// snapshot it does not hold. Absence alone settles nothing, because a call the
-// creating request sent may still be accepted; the repeat returns that snapshot or makes it.
-func (r *PendingSnapshotReaper) repeat(ctx context.Context, current ebsmetadata.Snapshot) bool {
-	handle, ok := r.volumeHandle(ctx, current)
-	if !ok {
-		return false
-	}
-	// The flag goes on before the call, so a crash once it is sent leaves it.
-	earlierUnanswered := current.AmbiguousCreate
-	if !earlierUnanswered {
-		if _, ok, err := r.svc.updatePending(ctx, current.OwnerID, current.SnapshotID, func(d *ebsmetadata.Snapshot) {
-			d.AmbiguousCreate = true
-		}); err != nil || !ok {
-			slog.WarnContext(ctx, "pending-snapshot: could not record the repeat before sending it, retrying next sweep",
-				"snapshotId", current.SnapshotID, "err", err)
-			return false
-		}
-	}
-	created, err := r.svc.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: current.SnapshotID, VolumeID: current.VolumeID, VolumeHandle: handle,
-	})
-	mutate, answered := repeatOutcome(created, err, earlierUnanswered)
-	switch {
-	case answered:
-		return r.write(ctx, current, mutate)
-	case mutate != nil:
-		slog.WarnContext(ctx, "pending-snapshot: provider refused the repeat for good, marking the snapshot error",
-			"snapshotId", current.SnapshotID, "volumeId", current.VolumeID, "err", err)
-		return r.write(ctx, current, mutate)
-	default:
-		slog.WarnContext(ctx, "pending-snapshot: repeat not answered, retrying next sweep",
-			"snapshotId", current.SnapshotID, "volumeId", current.VolumeID, "err", err)
-		return false
-	}
-}
-
-// cleanUp carries out an accepted delete, under the snapshot's claim: the
-// provider's delete, then the reference, then the document, or a tombstone if
-// a create on it went unanswered. Each step must succeed before the next.
-func (r *PendingSnapshotReaper) cleanUp(ctx context.Context, doc ebsmetadata.Snapshot, ref ebsprovider.SnapshotRef, found bool) bool {
-	// A marked snapshot the provider does not hold is never repeated: the
-	// customer asked for it gone, and a tombstone catches a late acceptance.
-	if found {
-		handle := ref.Handle
-		if handle == "" {
-			handle = doc.ProviderHandle
-		}
-		if err := r.deleteAtProvider(ctx, doc.SnapshotID, handle); err != nil {
-			slog.WarnContext(ctx, "pending-snapshot: provider delete of a deleted snapshot failed, retrying next sweep",
-				"snapshotId", doc.SnapshotID, "err", err)
-			return false
-		}
-		if r.interrupted(reapProviderDeleted, doc.SnapshotID) {
-			return false
-		}
-	}
-	// The document goes last: it is the only record this sweep discovers work
-	// from, and a reference left without it would block DeleteVolume for good.
-	if err := r.svc.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
-		slog.WarnContext(ctx, "pending-snapshot: reference of a deleted snapshot not removed, retrying next sweep",
-			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
-		return false
-	}
-	if r.interrupted(reapRefRemoved, doc.SnapshotID) {
-		return false
-	}
-	if doc.AmbiguousCreate {
-		doc.TombstonedAt = r.svc.clock().UTC()
-		if err := r.svc.metadata.PutSnapshot(ctx, doc); err != nil {
-			slog.WarnContext(ctx, "pending-snapshot: tombstone of a deleted snapshot not written, retrying next sweep",
-				"snapshotId", doc.SnapshotID, "err", err)
-			return false
-		}
-		slog.InfoContext(ctx, "pending-snapshot: deleted, keeping a tombstone for a create that may still be accepted",
-			"snapshotId", doc.SnapshotID)
-		return true
-	}
-	if err := r.svc.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
-		slog.WarnContext(ctx, "pending-snapshot: document of a deleted snapshot not removed, retrying next sweep",
-			"snapshotId", doc.SnapshotID, "err", err)
-		return false
-	}
-	slog.InfoContext(ctx, "pending-snapshot: deleted", "snapshotId", doc.SnapshotID)
-	return true
-}
-
-// sweepTombstone reads the provider for a tombstone's ID and deletes a snapshot a late
-// create made there once it is no longer pending. A tombstone past its
-// retention, with nothing pending under it, is dropped.
-func (r *PendingSnapshotReaper) sweepTombstone(ctx context.Context, doc ebsmetadata.Snapshot) bool {
-	return withClaim(ctx, doc.SnapshotID, r.svc.claims.take, func() bool {
-		ref, found, err := r.providerSnapshot(ctx, doc)
-		if err != nil {
-			return false
-		}
-		if found {
-			if !providerSettled(ref) {
-				return false
-			}
-			if err := r.deleteAtProvider(ctx, doc.SnapshotID, ref.Handle); err != nil {
-				slog.WarnContext(ctx, "pending-snapshot: late-accepted snapshot of a deleted ID not deleted, retrying next sweep",
-					"snapshotId", doc.SnapshotID, "err", err)
-				return false
-			}
-			slog.WarnContext(ctx, "pending-snapshot: deleted a snapshot accepted after the customer deleted it",
-				"snapshotId", doc.SnapshotID)
-		}
-		age := r.svc.clock().Sub(doc.TombstonedAt)
-		if age < tombstoneRetention {
-			return found
-		}
-		if err := r.svc.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
-			slog.WarnContext(ctx, "pending-snapshot: expired tombstone not dropped, retrying next sweep",
-				"snapshotId", doc.SnapshotID, "err", err)
-			return found
-		}
-		slog.InfoContext(ctx, "pending-snapshot: dropped an expired tombstone", "snapshotId", doc.SnapshotID, "age_ms", otelsetup.Millis(age))
-		return true
-	})
-}
-
-// settleAMIs moves each pending AMI to its snapshot's state: completed makes it
-// available, and error or a missing or deleted snapshot makes it failed.
-func (r *PendingSnapshotReaper) settleAMIs(ctx context.Context) (int, error) {
-	amis, err := r.svc.metadata.ListAMIs(ctx)
-	if err != nil {
-		return 0, fmt.Errorf("list AMI documents: %w", err)
-	}
-	written := 0
-	for _, ami := range amis {
-		if err := ctx.Err(); err != nil {
-			return written, err
-		}
-		if ami.State != ebsmetadata.AMIStatePending || ami.SnapshotID == "" {
-			continue
-		}
-		state, ok := r.amiStateFromSnapshot(ctx, ami)
-		if !ok {
-			continue
-		}
-		if r.svc.settlePendingAMI(ctx, ami.ImageID, state, r.svc.claims.take) {
-			written++
-		}
-	}
-	return written, nil
-}
-
-// amiStateFromSnapshot is the state ami's snapshot document gives it, and
-// false while that snapshot is pending or cannot be read.
-func (r *PendingSnapshotReaper) amiStateFromSnapshot(ctx context.Context, ami ebsmetadata.AMI) (string, bool) {
-	owner := awsidentifiers.GlobalAccountID
-	if awsidentifiers.IsAccountID(ami.ImageOwnerAlias) {
-		owner = ami.ImageOwnerAlias
-	}
-	snap, err := r.svc.metadata.GetSnapshot(ctx, owner, ami.SnapshotID)
-	switch {
-	case objectstore.IsNoSuchKeyError(err):
-		return ebsmetadata.AMIStateFailed, true
-	case err != nil:
-		slog.WarnContext(ctx, "pending-snapshot: snapshot of a pending AMI unreadable, retrying next sweep",
-			"imageId", ami.ImageID, "snapshotId", ami.SnapshotID, "err", err)
-		return "", false
-	case snap.Hidden() || snap.State == snapshotStateError:
-		return ebsmetadata.AMIStateFailed, true
-	case snap.State == ebsmetadata.SnapshotStateCompleted:
-		return ebsmetadata.AMIStateAvailable, true
-	default:
-		return "", false
-	}
-}
-
-// volumeHandle is the provider handle of doc's source volume, which the
-// repeat sends as the request did. A volume document that is gone gives no
-// handle, and the provider answers for the volume; one unreadable is retried.
-func (r *PendingSnapshotReaper) volumeHandle(ctx context.Context, doc ebsmetadata.Snapshot) (string, bool) {
-	volume, err := r.svc.metadata.GetVolume(ctx, doc.OwnerID, doc.VolumeID)
-	if err == nil {
-		return volume.ProviderHandle, true
-	}
-	if objectstore.IsNoSuchKeyError(err) {
-		return "", true
-	}
-	slog.WarnContext(ctx, "pending-snapshot: source volume document unreadable, retrying next sweep",
-		"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
-	return "", false
-}
-
-// write applies mutate to doc's current document while it is still pending,
-// under a claim the caller holds. A request that crashed between its document
-// and its reference left the reference missing, so it is restored first.
-func (r *PendingSnapshotReaper) write(ctx context.Context, doc ebsmetadata.Snapshot, mutate func(*ebsmetadata.Snapshot)) bool {
-	if err := r.svc.ensureSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
-		slog.WarnContext(ctx, "pending-snapshot: volume reference not restored, retrying next sweep",
-			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
-		return false
-	}
-	defer r.dropRefIfDeleted(ctx, doc)
-	written, ok, err := r.svc.updatePending(ctx, doc.OwnerID, doc.SnapshotID, mutate)
-	if err != nil {
-		slog.WarnContext(ctx, "pending-snapshot: document write failed, retrying next sweep",
-			"snapshotId", doc.SnapshotID, "accountId", doc.OwnerID, "err", err)
-		return false
-	}
-	if ok && written.State != snapshotStatePending {
-		slog.InfoContext(ctx, "pending-snapshot: settled", "snapshotId", doc.SnapshotID, "state", written.State)
-	}
-	return ok
-}
-
-// dropRefIfDeleted removes doc's reference again when its document is gone: a
-// rollback or DeleteSnapshot may have removed both before write restored it.
-func (r *PendingSnapshotReaper) dropRefIfDeleted(ctx context.Context, doc ebsmetadata.Snapshot) {
-	_, err := r.svc.metadata.GetSnapshot(ctx, doc.OwnerID, doc.SnapshotID)
-	if !objectstore.IsNoSuchKeyError(err) {
-		return
-	}
-	if err := r.svc.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
-		slog.WarnContext(ctx, "pending-snapshot: reference of a deleted snapshot not removed",
-			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
-	}
-}
-
-// ReleaseNodeClaims releases every snapshot and image claim that names this
-// node and no request of this process holds, whatever its document's state. A
-// daemon runs it as it starts and before it serves requests: none survived.
-func (s *SnapshotServiceImpl) ReleaseNodeClaims(ctx context.Context) (int, error) {
-	docs, err := s.metadata.ListAllSnapshotsStrict(ctx)
-	if err != nil {
-		return 0, fmt.Errorf("list snapshot documents: %w", err)
-	}
-	amis, err := s.metadata.ListAMIsStrict(ctx)
-	if err != nil {
-		return 0, fmt.Errorf("list AMI documents: %w", err)
-	}
-	ids := make([]string, 0, len(docs)+len(amis))
-	for _, doc := range docs {
-		ids = append(ids, doc.SnapshotID)
-	}
-	for _, ami := range amis {
-		ids = append(ids, ami.ImageID)
-	}
-	released := 0
-	var errs []error
-	for _, id := range ids {
-		ok, err := s.claims.releaseOwn(ctx, id)
-		if err != nil {
-			errs = append(errs, err)
-			continue
-		}
-		if ok {
-			released++
-		}
-	}
-	return released, errors.Join(errs...)
-}

spinifex/domains/ec2/snapshot/service_impl.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
-	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -53,32 +52,6 @@ type SnapshotServiceImpl struct {
 	mutex    sync.RWMutex
 	metadata *ebsmetadata.Store
 	provider ebsprovider.EBSProvider
-	claims   *Claims
-
-	// crashAt stops CreateSnapshot after a step, as a crash there would. Tests only.
-	crashAt func(createStep) bool
-	// reapAt interrupts the reaper at a step for snapshotID. Tests only.
-	reapAt func(step reapStep, snapshotID string) bool
-	// now is the reaper's clock for tombstones; nil is time.Now. Tests only.
-	now func() time.Time
-	// claimRetryDelay overrides the wait between claim attempts. Tests only.
-	claimRetryDelay time.Duration
-	// claimRefused observes each refused claim attempt of a writer. Tests only.
-	claimRefused func(snapshotID string)
-}
-
-// clock is the time the reaper judges a tombstone's age by.
-func (s *SnapshotServiceImpl) clock() time.Time {
-	if s.now != nil {
-		return s.now()
-	}
-	return time.Now()
-}
-
-// SetClaims wires the snapshot claims shared by CreateSnapshot and the
-// pending-snapshot reaper. Without it they exclude only within this process.
-func (s *SnapshotServiceImpl) SetClaims(claims *Claims) {
-	s.claims = claims
 }
 
 // SetEBSProvider injects the provider boundary for snapshot operations.
@@ -129,7 +102,6 @@ func NewSnapshotServiceImplWithNATS(ctx context.Context, cfg *config.Config, nat
 		natsConn: natsConn,
 		snapRefs: refs,
 		metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket),
-		claims:   localClaims(),
 	}, kv, nil
 }
 
@@ -141,7 +113,6 @@ func NewSnapshotServiceImplWithStore(cfg *config.Config, store objectstore.Objec
 		store:    store,
 		natsConn: natsConn,
 		metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket),
-		claims:   localClaims(),
 	}
 	if len(snapshotKV) > 0 {
 		svc.snapRefs = kvstore.Over[[]string](nil, snapshotKV[0], snapshotRefsConfig())
@@ -164,8 +135,7 @@ func snapshotRefsConfig() kvstore.Config {
 }
 
 // getSnapshotConfig reads the caller's own snapshot document. A snapshot
-// outside the caller's prefix is absent, so ownership needs no second check,
-// and so is one whose delete was accepted.
+// outside the caller's prefix is absent, so ownership needs no second check.
 func (s *SnapshotServiceImpl) getSnapshotConfig(ctx context.Context, accountID, snapshotID string) (ebsmetadata.Snapshot, error) {
 	cfg, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
 	if err != nil {
@@ -174,9 +144,6 @@ func (s *SnapshotServiceImpl) getSnapshotConfig(ctx context.Context, accountID,
 		}
 		return ebsmetadata.Snapshot{}, err
 	}
-	if cfg.Hidden() {
-		return ebsmetadata.Snapshot{}, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
-	}
 	return cfg, nil
 }
 
@@ -236,288 +203,47 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
-	doc := pendingDocument(volume, snapshotID, accountID, aws.StringValue(input.Description))
-	doc.Tags = awstags.Extract(input.TagSpecifications, "snapshot")
-	return s.createSnapshot(ctx, doc, providerCreateRequest(volume, snapshotID), createHooks{})
-}
-
-// pendingDocument is the document a snapshot of volume is recorded as before
-// the provider is called. AmbiguousCreate is set because that call is next.
-func pendingDocument(volume ebsmetadata.Volume, snapshotID, accountID, description string) ebsmetadata.Snapshot {
-	return ebsmetadata.Snapshot{
-		SnapshotID: snapshotID, VolumeID: volume.VolumeID,
+	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
+	})
+	if err != nil || created == nil {
+		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, ProviderCreateError(volumeID, err)
+	}
+	now := time.Now()
+	snapshotCfg := ebsmetadata.Snapshot{
+		SnapshotID: snapshotID, VolumeID: volumeID,
 		VolumeSize: safecast.Uint64ToInt64(volume.CapacityGiB),
-		State:      snapshotStatePending, Progress: "0%", StartTime: time.Now(),
-		Description: description, Encrypted: volume.Encrypted,
-		OwnerID: accountID, AvailabilityZone: volume.AvailabilityZone,
-		AmbiguousCreate: true,
+		State:      string(created.State), Progress: "100%", StartTime: now,
+		Encrypted: volume.Encrypted,
+		OwnerID:   accountID, AvailabilityZone: volume.AvailabilityZone,
+		Tags:           awstags.Extract(input.TagSpecifications, "snapshot"),
+		ProviderHandle: created.Handle,
 	}
-}
-
-func providerCreateRequest(volume ebsmetadata.Volume, snapshotID string) ebsprovider.CreateSnapshotRequest {
-	return ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: snapshotID, VolumeID: volume.VolumeID, VolumeHandle: volume.ProviderHandle,
+	if snapshotCfg.State == "" {
+		snapshotCfg.State = "completed"
 	}
-}
-
-// ImageSnapshotHooks are the AMI document's writes around an image's snapshot.
-// Recorded runs after the snapshot's document and reference and before the
-// provider call; its error rolls the snapshot back.
-type ImageSnapshotHooks struct {
-	Recorded func(ctx context.Context) error
-	// Refused removes the AMI document when the provider refused the first call.
-	Refused func(ctx context.Context)
-	// Completed runs once the snapshot's document is written completed.
-	Completed func(ctx context.Context)
-}
-
-// ImageSnapshotCreator takes the snapshot behind a CreateImage through the
-// same document, reference, claim and provider path as CreateSnapshot.
-type ImageSnapshotCreator interface {
-	CreateImageSnapshot(ctx context.Context, volume ebsmetadata.Volume, snapshotID, accountID, description string, hooks ImageSnapshotHooks) (*ec2.Snapshot, error)
-}
-
-var _ ImageSnapshotCreator = (*SnapshotServiceImpl)(nil)
-
-// ImageClaims serialises the settling of a pending AMI with DeregisterImage:
-// each holds the image's claim, in the snapshot claim group keyed by image ID.
-type ImageClaims interface {
-	LockImage(ctx context.Context, imageID string) (unlock func(), err error)
-	SettlePendingAMI(ctx context.Context, imageID, state string)
-}
-
-var _ ImageClaims = (*SnapshotServiceImpl)(nil)
-
-// LockImage takes imageID's claim for a request, as writeUnderClaim takes a
-// snapshot's; one still held after the retries is a retryable IncorrectState.
-func (s *SnapshotServiceImpl) LockImage(ctx context.Context, imageID string) (func(), error) {
-	held, err := s.takeClaimWithRetry(ctx, "image", imageID)
-	if err != nil {
-		return nil, err
-	}
-	return func() { held.release(ctx) }, nil
-}
-
-// SettlePendingAMI moves imageID's AMI to state for a request, retrying a held
-// claim as LockImage does; an AMI it leaves pending is settled by the reaper.
-func (s *SnapshotServiceImpl) SettlePendingAMI(ctx context.Context, imageID, state string) {
-	s.settlePendingAMI(ctx, imageID, state, func(ctx context.Context, id string) (*heldClaim, error) {
-		return s.takeClaimWithRetry(ctx, "image", id)
-	})
-}
-
-// settlePendingAMI writes state to imageID's AMI under the claim take returns,
-// which DeregisterImage also takes, and only while a reread finds it pending.
-func (s *SnapshotServiceImpl) settlePendingAMI(ctx context.Context, imageID, state string, take func(context.Context, string) (*heldClaim, error)) bool {
-	return withClaim(ctx, imageID, take, func() bool {
-		current, err := s.metadata.GetAMI(ctx, imageID)
-		if err != nil || current.State != ebsmetadata.AMIStatePending {
-			return false
-		}
-		current.State = state
-		if err := s.metadata.PutAMI(ctx, current); err != nil {
-			slog.WarnContext(ctx, "pending AMI's state not written, left for the reaper", "imageId", imageID, "err", err)
-			return false
-		}
-		slog.InfoContext(ctx, "pending AMI settled", "imageId", imageID, "snapshotId", current.SnapshotID, "state", state)
-		return true
-	})
-}
-
-// CreateImageSnapshot snapshots an image's root volume as CreateSnapshot does,
-// running hooks to write the AMI document beside the snapshot's.
-func (s *SnapshotServiceImpl) CreateImageSnapshot(ctx context.Context, volume ebsmetadata.Volume, snapshotID, accountID, description string, hooks ImageSnapshotHooks) (*ec2.Snapshot, error) {
-	if s.provider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", "CreateImageSnapshot")
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	doc := pendingDocument(volume, snapshotID, accountID, description)
-	return s.createSnapshot(ctx, doc, providerCreateRequest(volume, snapshotID), createHooks(hooks))
-}
-
-// createHooks are the writes a CreateImage adds around createSnapshot's.
-type createHooks ImageSnapshotHooks
-
-// createStep is a point in createSnapshot after which a crash leaves its
-// writes so far, for a test to stop the request at.
-type createStep int
-
-const (
-	stepClaimed createStep = iota + 1
-	stepRecorded
-	stepReferenced
-	stepAnswered
-	stepSettled
-	stepRollbackMarked
-	stepRollbackUnreferenced
-)
-
-// createAttempts bounds createSnapshot's provider calls: the first, and one
-// repeat of a call that may have been accepted.
-const createAttempts = 2
-
-// errCreateCrashed is what createSnapshot returns when a test stops it at a step.
-var errCreateCrashed = errors.New("snapshot create stopped at a test crash point")
-
-// createSnapshot records doc pending and then asks the provider for it, so a
-// provider snapshot never exists without a document. Only a refusal of the
-// first call removes the document; anything else leaves it for the reaper.
-func (s *SnapshotServiceImpl) createSnapshot(ctx context.Context, doc ebsmetadata.Snapshot, req ebsprovider.CreateSnapshotRequest, hooks createHooks) (*ec2.Snapshot, error) {
-	snapshotID, volumeID, accountID := doc.SnapshotID, doc.VolumeID, doc.OwnerID
-
-	held, err := s.claims.lock(ctx, snapshotID)
-	if err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot could not take the snapshot claim", "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	crashed := false
-	defer func() {
-		if !crashed {
-			held.release(ctx)
-		}
-	}()
-	crash := func(step createStep) bool {
-		crashed = s.crashAt != nil && s.crashAt(step)
-		return crashed
-	}
-	if crash(stepClaimed) {
-		return nil, errCreateCrashed
+	if input.Description != nil {
+		snapshotCfg.Description = *input.Description
 	}
 
-	// The document comes first because it is what the reaper discovers work
-	// from; a reference with no document would block DeleteVolume for good.
-	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot failed to write the pending snapshot document", "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	if crash(stepRecorded) {
-		return nil, errCreateCrashed
-	}
+	// Track the volume→snapshot dependency in KV before persisting to S3.
+	// This ensures we never have an untracked snapshot in S3.
 	if err := s.addSnapshotRef(ctx, volumeID, snapshotID); err != nil {
 		slog.ErrorContext(ctx, "CreateSnapshot failed to add snapshot ref to KV", "snapshotId", snapshotID, "volumeId", volumeID, "err", err)
-		if s.rollBackCreate(ctx, doc, crash) {
-			return nil, errCreateCrashed
-		}
+		_ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle})
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if crash(stepReferenced) {
-		return nil, errCreateCrashed
-	}
-	if hooks.Recorded != nil {
-		if err := hooks.Recorded(ctx); err != nil {
-			slog.ErrorContext(ctx, "CreateSnapshot failed to record the image beside its snapshot", "snapshotId", snapshotID, "err", err)
-			if s.rollBackCreate(ctx, doc, crash) {
-				return nil, errCreateCrashed
-			}
-			return nil, errors.New(awserrors.ErrorServerInternal)
-		}
+	if err := s.metadata.PutSnapshot(ctx, snapshotCfg); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to write config", "snapshotId", snapshotID, "err", err)
+		_ = s.removeSnapshotRefForCleanup(ctx, volumeID, snapshotID)
+		_ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle})
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// The document carries AmbiguousCreate from its first write, so a crash
-	// once a call is sent leaves the flag set. A call that may have been
-	// accepted is repeated once: the provider answers it with what it holds.
-	var created *ebsprovider.Snapshot
-	earlierUnanswered := false
-	for range createAttempts {
-		created, err = s.provider.CreateSnapshot(ctx, req)
-		if crash(stepAnswered) {
-			return nil, errCreateCrashed
-		}
-		if (err == nil && created != nil) || earlierUnanswered {
-			break
-		}
-		if definiteRefusal(err) {
-			// The request holds the claim, so no other call for this ID exists that
-			// a provider could still accept: the document can go.
-			slog.WarnContext(ctx, "CreateSnapshot refused by the provider", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-			if hooks.Refused != nil {
-				hooks.Refused(ctx)
-			}
-			if s.rollBackCreate(ctx, doc, crash) {
-				return nil, errCreateCrashed
-			}
-			return nil, ProviderCreateError(volumeID, err)
-		}
-		slog.WarnContext(ctx, "CreateSnapshot provider call failed ambiguously, repeating it", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		earlierUnanswered = true
-	}
+	slog.InfoContext(ctx, "CreateSnapshot completed", "snapshotId", snapshotID, "volumeId", volumeID)
 
-	mutate, answered := repeatOutcome(created, err, earlierUnanswered)
-	switch {
-	case answered:
-		return s.answerCreate(ctx, doc, mutate, hooks, crash)
-	case mutate != nil:
-		slog.WarnContext(ctx, "CreateSnapshot repeat refused for good, marking the snapshot error", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		wctx, cancel := writeContext(ctx)
-		defer cancel()
-		if _, _, werr := s.updatePending(wctx, accountID, snapshotID, mutate); werr != nil {
-			slog.ErrorContext(ctx, "CreateSnapshot failed to mark the snapshot error", "snapshotId", snapshotID, "err", werr)
-		}
-		return nil, ProviderCreateError(volumeID, err)
-	case definiteRefusal(err):
-		// The first call may still be accepted once this refusal clears.
-		slog.WarnContext(ctx, "CreateSnapshot repeat refused, leaving the snapshot pending", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, ProviderCreateError(volumeID, err)
-	}
-	slog.ErrorContext(ctx, "CreateSnapshot provider call failed ambiguously twice, leaving the snapshot pending", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-	return nil, errors.New(awserrors.ErrorServerInternal)
-}
-
-// answerCreate writes the provider's answer onto the pending document and
-// returns the snapshot as written. The answer fixed the contents, so a failed
-// write still reports success; the reaper settles the document.
-func (s *SnapshotServiceImpl) answerCreate(ctx context.Context, doc ebsmetadata.Snapshot, answer func(*ebsmetadata.Snapshot),
-	hooks createHooks, crash func(createStep) bool) (*ec2.Snapshot, error) {
-	wctx, cancel := writeContext(ctx)
-	defer cancel()
-	written, ok, err := s.updatePending(wctx, doc.OwnerID, doc.SnapshotID, answer)
-	completed := ok && written.State == ebsmetadata.SnapshotStateCompleted
-	switch {
-	case err != nil:
-		slog.ErrorContext(ctx, "CreateSnapshot failed to record the provider's answer", "snapshotId", doc.SnapshotID, "err", err)
-		written = doc
-	case !ok && written.SnapshotID == "":
-		// Deleted while the provider answered; report what the answer was.
-		written = doc
-		answer(&written)
-	}
-	if crash(stepSettled) {
-		return nil, errCreateCrashed
-	}
-	if completed && hooks.Completed != nil {
-		hooks.Completed(wctx)
-	}
-	slog.InfoContext(ctx, "CreateSnapshot completed", "snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "state", written.State)
-	return snapshotConfigToEC2(written), nil
-}
-
-// rollBackCreate removes a snapshot the provider refused or never saw, in the
-// reaper's order: the document is marked deleted with no call outstanding, then
-// the reference and the document go. It reports a test crash.
-func (s *SnapshotServiceImpl) rollBackCreate(ctx context.Context, doc ebsmetadata.Snapshot, crash func(createStep) bool) bool {
-	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), snapshotCleanupTimeout)
-	defer cancel()
-	// A marked document a failure below leaves is the reaper's to delete, and it
-	// never repeats the create of one, so the refusal the caller saw stands.
-	doc.DeleteRequested, doc.AmbiguousCreate = true, false
-	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot failed to mark a refused snapshot's document deleted", "snapshotId", doc.SnapshotID, "err", err)
-	}
-	if crash(stepRollbackMarked) {
-		return true
-	}
-	if err := s.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot failed to remove a refused snapshot's ref, leaving its document for the reaper",
-			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
-		return false
-	}
-	if crash(stepRollbackUnreferenced) {
-		return true
-	}
-	if err := s.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
-		slog.ErrorContext(ctx, "CreateSnapshot failed to remove a refused snapshot's document, leaving it for the reaper", "snapshotId", doc.SnapshotID, "err", err)
-	}
-	return false
+	return snapshotConfigToEC2(snapshotCfg), nil
 }
 
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
@@ -590,9 +316,6 @@ func (s *SnapshotServiceImpl) describeSnapshots(ctx context.Context, input *ec2.
 
 	var snapshots []*ec2.Snapshot
 	for _, cfg := range configs {
-		if cfg.Hidden() {
-			continue
-		}
 		if len(parsedFilters) > 0 && !snapshotMatchesFilters(cfg, parsedFilters) {
 			continue
 		}
@@ -690,7 +413,7 @@ func (s *SnapshotServiceImpl) describeSnapshotsByIDs(ctx context.Context, input
 
 		// The read was keyed by the ID inside the caller's prefix, so the
 		// document is theirs and is the one named. Both stay as assertions.
-		if result.Document.SnapshotID != snapshotIDs[i] || result.Document.OwnerID != accountID || result.Document.Hidden() {
+		if result.Document.SnapshotID != snapshotIDs[i] || result.Document.OwnerID != accountID {
 			return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
 		}
 
@@ -926,20 +649,6 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 		slog.ErrorContext(ctx, "no EBS provider configured", "op", "DeleteSnapshot")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-
-	// A provider may refuse to delete a snapshot still in progress, and an
-	// unanswered create may still be accepted, so either is marked for the reaper.
-	if deferDelete(cfg) {
-		marked, current, err := s.requestDelete(ctx, cfg)
-		if err != nil {
-			return nil, err
-		}
-		if marked {
-			slog.InfoContext(ctx, "DeleteSnapshot accepted; the snapshot is deleted once it settles", "snapshotId", snapshotID)
-			return &ec2.DeleteSnapshotOutput{}, nil
-		}
-		cfg = current
-	}
 	if err := s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: cfg.ProviderHandle}); err != nil {
 		slog.ErrorContext(ctx, "DeleteSnapshot provider deletion failed", "snapshotId", snapshotID, "err", err)
 		return nil, providerDeleteError(snapshotID, err)
@@ -964,104 +673,6 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 	return &ec2.DeleteSnapshotOutput{}, nil
 }
 
-// deferDelete reports a snapshot DeleteSnapshot marks rather than deletes.
-func deferDelete(doc ebsmetadata.Snapshot) bool {
-	return doc.State == snapshotStatePending || doc.AmbiguousCreate
-}
-
-// requestDelete marks doc DeleteRequested under its claim. A document that
-// settled with no call outstanding since it was read is returned unmarked,
-// for the caller to delete as any other.
-func (s *SnapshotServiceImpl) requestDelete(ctx context.Context, doc ebsmetadata.Snapshot) (bool, ebsmetadata.Snapshot, error) {
-	marked := false
-	var current ebsmetadata.Snapshot
-	err := s.writeUnderClaim(ctx, doc.OwnerID, doc.SnapshotID, func(d *ebsmetadata.Snapshot) (bool, error) {
-		if d.Hidden() {
-			return false, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
-		}
-		current = *d
-		if !deferDelete(*d) {
-			return false, nil
-		}
-		d.DeleteRequested = true
-		marked = true
-		return true, nil
-	})
-	if err != nil {
-		return false, ebsmetadata.Snapshot{}, err
-	}
-	if current.SnapshotID == "" {
-		return false, ebsmetadata.Snapshot{}, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
-	}
-	return marked, current, nil
-}
-
-// Contention on a snapshot's claim is bounded: three retries over two seconds.
-const (
-	claimRetries    = 3
-	claimRetryDelay = 2 * time.Second / claimRetries
-)
-
-// writeUnderClaim rereads snapshotID's document under its claim and writes it
-// when mutate asks to, so no write lands between another writer's reread and
-// its write. A document that is gone is not mutated, and mutate never sees it.
-func (s *SnapshotServiceImpl) writeUnderClaim(ctx context.Context, accountID, snapshotID string,
-	mutate func(*ebsmetadata.Snapshot) (bool, error)) error {
-	held, err := s.takeClaimWithRetry(ctx, "snapshot", snapshotID)
-	if err != nil {
-		return err
-	}
-	defer held.release(ctx)
-	var mutateErr error
-	_, _, err = s.rewrite(ctx, accountID, snapshotID, func(doc *ebsmetadata.Snapshot) bool {
-		var write bool
-		write, mutateErr = mutate(doc)
-		return write && mutateErr == nil
-	})
-	if mutateErr != nil {
-		return mutateErr
-	}
-	if err != nil {
-		return awserrors.Errorf(awserrors.ErrorServerInternal, "%v", err)
-	}
-	return nil
-}
-
-// takeClaimWithRetry takes the claim on resource id, of kind, as the
-// reaper does, retrying a holder that is live; one still held after the
-// retries is IncorrectState, which the caller retries.
-func (s *SnapshotServiceImpl) takeClaimWithRetry(ctx context.Context, kind, id string) (*heldClaim, error) {
-	delay := claimRetryDelay
-	if s.claimRetryDelay > 0 {
-		delay = s.claimRetryDelay
-	}
-	for attempt := 0; ; attempt++ {
-		held, err := s.claims.take(ctx, id)
-		if err == nil {
-			return held, nil
-		}
-		if !errors.Is(err, errClaimHeld) {
-			slog.ErrorContext(ctx, "Snapshot claim could not be taken", "kind", kind, "id", id, "err", err)
-			return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "take the claim of %s %s: %v", kind, id, err)
-		}
-		if s.claimRefused != nil {
-			s.claimRefused(id)
-		}
-		if attempt == claimRetries {
-			slog.InfoContext(ctx, "Snapshot claim still held after retries", "kind", kind, "id", id, "err", err)
-			return nil, awserrors.Errorf(awserrors.ErrorIncorrectState,
-				"The %s '%s' is being updated. Retry the request.", kind, id)
-		}
-		timer := time.NewTimer(delay)
-		select {
-		case <-ctx.Done():
-			timer.Stop()
-			return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "take the claim of %s %s: %v", kind, id, ctx.Err())
-		case <-timer.C:
-		}
-	}
-}
-
 // CopySnapshot duplicates a snapshot under a new ID owned by the caller.
 // Served only through the provider: the copy needs its own snapshot metadata
 // on the backend, and on an encrypted volume that metadata is AEAD-sealed with
@@ -1092,9 +703,6 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS
 		slog.WarnContext(ctx, "CopySnapshot: account does not own source snapshot", "snapshotId", sourceSnapshotID, "accountID", accountID, "ownerID", sourceCfg.OwnerID)
 		return nil, errors.New(awserrors.ErrorUnauthorizedOperation)
 	}
-	if !sourceCfg.Ready() {
-		return nil, sourceCfg.NotReadyError()
-	}
 
 	// Without a provider the only reachable path writes the control-plane
 	// config alone. That copy described and created volumes fine, then failed
@@ -1175,25 +783,6 @@ func (s *SnapshotServiceImpl) addSnapshotRef(ctx context.Context, volumeID, snap
 	return nil
 }
 
-// ensureSnapshotRef adds snapshotID to the volume's snapshot list unless it
-// is already there, restoring a reference a crashed request did not write.
-func (s *SnapshotServiceImpl) ensureSnapshotRef(ctx context.Context, volumeID, snapshotID string) error {
-	if s.snapRefs == nil {
-		return nil
-	}
-	err := s.snapRefs.Upsert(ctx, volumeID, func(snapshots *[]string) (bool, error) {
-		if slices.Contains(*snapshots, snapshotID) {
-			return false, nil
-		}
-		*snapshots = append(*snapshots, snapshotID)
-		return true, nil
-	})
-	if err != nil {
-		return fmt.Errorf("ensureSnapshotRef: failed to record %s against volume %s: %w", snapshotID, volumeID, err)
-	}
-	return nil
-}
-
 // removeSnapshotRefForCleanup removes a reference with a bounded context that
 // survives request cancellation after the snapshot operation has committed.
 func (s *SnapshotServiceImpl) removeSnapshotRefForCleanup(ctx context.Context, volumeID, snapshotID string) error {
@@ -1325,31 +914,10 @@ func (s *SnapshotServiceImpl) mirrorSnapshotTags(resources []*string, accountID
 			}
 			return err
 		}
-		if cfg.Hidden() {
-			continue
+		if cfg.Tags == nil {
+			cfg.Tags = map[string]string{}
 		}
-		apply := func(d *ebsmetadata.Snapshot) {
-			if d.Tags == nil {
-				d.Tags = map[string]string{}
-			}
-			mut(d.Tags)
-		}
-		// A document the reaper or a DeleteSnapshot may still write is written
-		// under its claim, so the tags cannot undo their write or it theirs.
-		if deferDelete(cfg) {
-			err := s.writeUnderClaim(context.Background(), accountID, *res, func(d *ebsmetadata.Snapshot) (bool, error) {
-				if d.Hidden() {
-					return false, nil
-				}
-				apply(d)
-				return true, nil
-			})
-			if err != nil {
-				return err
-			}
-			continue
-		}
-		apply(&cfg)
+		mut(cfg.Tags)
 		if err := s.metadata.PutSnapshot(context.Background(), cfg); err != nil {
 			return err
 		}

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1246,9 +1246,10 @@ func (s *snapshotMetadataFailingObjectStore) PutObject(ctx context.Context, inpu
 	return s.ObjectStore.PutObject(ctx, input)
 }
 
-// TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure: the pending
-// document is written before the provider is called, so a failed write leaves
-// no provider snapshot behind to orphan.
+// TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure covers
+// CreateSnapshot's rollback path: when persisting the snapshot config fails
+// after the provider has already created the snapshot, CreateSnapshot must
+// delete the just-created provider snapshot rather than orphan it.
 func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 	store := &snapshotMetadataFailingObjectStore{ObjectStore: objectstore.NewMemoryObjectStore()}
 	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
@@ -1272,10 +1273,13 @@ func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 
 	snapshotID := strings.TrimSuffix(path.Base(store.attemptedKey), ".json")
 	_, err = provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback"})
-	require.NoError(t, err, "the source volume must be unaffected by the failed create")
+	require.NoError(t, err, "the source volume must be unaffected by the snapshot rollback")
 
-	// The ID is free on another source volume only if the provider never took
-	// it: MemoryProvider refuses a held ID on a different volume.
+	// Recreating the same snapshot ID against a different source volume only
+	// succeeds if the original snapshot no longer exists (MemoryProvider
+	// returns already_exists for a conflicting source volume on a live
+	// snapshot ID), proving the rollback actually deleted it rather than
+	// merely leaving it orphaned but reachable.
 	_, err = provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback-check", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 * 1024 * 1024 * 1024},
 	})
@@ -1283,7 +1287,7 @@ func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 	_, err = provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: "vol-rollback-check",
 	})
-	require.NoError(t, err, "a failed document write must leave no provider snapshot")
+	require.NoError(t, err, "rollback must have deleted the just-created provider snapshot")
 }
 
 // providerSnapshotService builds a snapshot service backed by the memory
@@ -1543,17 +1547,17 @@ func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
 	assert.Contains(t, message, "Detach")
 }
 
-// A failure that is not a refusal may follow an accepted call, so after its
-// repeat fails the same way the request is internal and the snapshot is left
-// pending for the reaper rather than removed.
+// A provider that cannot answer now, such as one refusing a volume another
+// node holds, is retry-later for the client; any other failure is internal.
 func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
 	cases := []struct {
-		name string
-		err  error
+		name     string
+		err      error
+		wantCode string
 	}{
-		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}},
-		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}},
-		{"transport failure", errors.New("connection reset")},
+		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
+		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
+		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -1563,11 +1567,7 @@ func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
 
 			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-			docs, err := svc.metadata.ListAllSnapshotsStrict(context.Background())
-			require.NoError(t, err)
-			require.Len(t, docs, 1)
-			assert.Equal(t, "pending", docs[0].State)
+			assert.Equal(t, tc.wantCode, err.Error())
 		})
 	}
 }

spinifex/domains/ec2/volume/provider_snapshot_source_test.go

@@ -51,7 +51,6 @@ func writeSnapshotMetadata(t *testing.T, store objectstore.ObjectStore, snapshot
 	t.Helper()
 	require.NoError(t, ebsmetadata.NewStore(store, "test-bucket").PutSnapshot(context.Background(), ebsmetadata.Snapshot{
 		SnapshotID: snapshotID, VolumeID: sourceVolumeID, VolumeSize: sizeGiB, OwnerID: ownerID,
-		State: ebsmetadata.SnapshotStateCompleted,
 	}))
 }
 

spinifex/domains/ec2/volume/service_impl.go

@@ -229,15 +229,10 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 		// The ownership check is the key: a snapshot outside the caller's prefix
 		// is not found, so the endpoint never confirms another account's IDs.
 		snapMeta, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
-		if err != nil || snapMeta.Hidden() {
+		if err != nil {
 			slog.ErrorContext(ctx, "CreateVolume: snapshot not found", "snapshotId", snapshotID, "err", err)
 			return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
 		}
-		// A pending snapshot's contents are not readable yet, and an errored
-		// one's never are.
-		if !snapMeta.Ready() {
-			return nil, snapMeta.NotReadyError()
-		}
 		sourceVolumeName = snapMeta.VolumeID
 		snapshotSizeGiB = snapMeta.VolumeSize
 	}

spinifex/domains/ec2/volume/service_impl_test.go

@@ -743,7 +743,6 @@ func putTestSnapshotMetadata(t *testing.T, store *objectstore.MemoryObjectStore,
 		VolumeID:   "vol-source",
 		VolumeSize: sizeGiB,
 		OwnerID:    ownerID,
-		State:      ebsmetadata.SnapshotStateCompleted,
 	}))
 }
 

spinifex/domains/ec2/volume/snapshot_readiness_test.go

@@ -1,48 +0,0 @@
-package volume
-
-import (
-	"context"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCreateVolume_FromSnapshot_RefusesASnapshotThatIsNotCompleted(t *testing.T) {
-	tests := []struct {
-		name     string
-		snapshot ebsmetadata.Snapshot
-		wantCode string
-	}{
-		{name: "pending", snapshot: ebsmetadata.Snapshot{State: "pending"}, wantCode: awserrors.ErrorIncorrectState},
-		{name: "error", snapshot: ebsmetadata.Snapshot{State: "error"}, wantCode: awserrors.ErrorIncorrectState},
-		{name: "deleted while pending", snapshot: ebsmetadata.Snapshot{State: "pending", DeleteRequested: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
-		{name: "deleted after completing", snapshot: ebsmetadata.Snapshot{State: "completed", DeleteRequested: true, AmbiguousCreate: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			store := objectstore.NewMemoryObjectStore()
-			svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
-			metadata := ebsmetadata.NewStore(store, "test-bucket")
-			snap := tc.snapshot
-			snap.SnapshotID, snap.VolumeID, snap.VolumeSize, snap.OwnerID = "snap-gate000001", "vol-source", 8, testVolAccountID
-			require.NoError(t, metadata.PutSnapshot(context.Background(), snap))
-
-			_, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{
-				AvailabilityZone: aws.String("ap-southeast-2a"),
-				SnapshotId:       aws.String(snap.SnapshotID),
-			}, testVolAccountID)
-			code, _, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok, "unregistered error: %v", err)
-			assert.Equal(t, tc.wantCode, code)
-			volumes, err := metadata.ListAllVolumes(context.Background())
-			require.NoError(t, err)
-			assert.Empty(t, volumes, "a refused CreateVolume writes no volume")
-		})
-	}
-}

spinifex/domains/lambda/awsapi/get.go

@@ -56,6 +56,27 @@ func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (functi
 	return rec, revision, err
 }
 
+// render opens the record's environment and renders its configuration; an
+// environment that cannot be opened fails rather than being left out.
+func (d Deps) render(ctx context.Context, rec function.Record, revision uint64) (*lambda.FunctionConfiguration, error) {
+	var env map[string]string
+	if rec.EnvCipher != "" {
+		if d.AEAD == nil {
+			return nil, serviceError("environment decryption is unavailable")
+		}
+		var err error
+		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
+			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
+			return nil, serviceError("could not read the function environment")
+		}
+	}
+	cfg, err := configuration(rec, revision, env)
+	if err != nil {
+		return nil, serviceError("could not build the function configuration")
+	}
+	return cfg, nil
+}
+
 // loadConfiguration resolves the requested $LATEST function and renders its
 // configuration; it is a metadata read and never touches the code store.
 func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (function.Record, *lambda.FunctionConfiguration, error) {
@@ -64,19 +85,9 @@ func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (fun
 		return function.Record{}, nil, err
 	}
 
-	var env map[string]string
-	if rec.EnvCipher != "" {
-		if d.AEAD == nil {
-			return function.Record{}, nil, serviceError("environment decryption is unavailable")
-		}
-		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
-			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return function.Record{}, nil, serviceError("could not read the function environment")
-		}
-	}
-	cfg, err := configuration(rec, revision, env)
+	cfg, err := d.render(ctx, rec, revision)
 	if err != nil {
-		return function.Record{}, nil, serviceError("could not build the function configuration")
+		return function.Record{}, nil, err
 	}
 	return rec, cfg, nil
 }

spinifex/domains/lambda/awsapi/handler_test.go

@@ -60,6 +60,14 @@ type localClient struct {
 	configErr error
 	// configCalls counts UpdateConfiguration requests sent to the owner.
 	configCalls int
+	// deleteCalls counts Delete requests sent to the owner.
+	deleteCalls int
+	// deleteErr fails Delete before the owner sees it; loseDeleteReply
+	// reports a transport error after the owner committed it.
+	deleteErr       error
+	loseDeleteReply bool
+	// listErr fails List before the owner sees it.
+	listErr error
 }
 
 var _ function.Client = (*localClient)(nil)
@@ -169,6 +177,42 @@ func (c *localClient) UpdateConfiguration(ctx context.Context, account, region s
 	return r.Record, r.Revision, nil
 }
 
+func (c *localClient) List(ctx context.Context, account, region, marker string, maxItems int) ([]function.ListedFunction, string, error) {
+	c.mu.Lock()
+	fail := c.listErr
+	c.mu.Unlock()
+	if fail != nil {
+		return nil, "", fail
+	}
+	r, err := c.svc.List(ctx, &function.ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, account)
+	if err != nil {
+		return nil, "", err
+	}
+	return r.Functions, r.NextMarker, nil
+}
+
+func (c *localClient) Delete(ctx context.Context, account, region string, rec function.Record, rev uint64) error {
+	c.mu.Lock()
+	hook, fail, lose := c.beforeCommit, c.deleteErr, c.loseDeleteReply
+	c.beforeCommit = nil
+	c.deleteCalls++
+	c.mu.Unlock()
+	if fail != nil {
+		return fail
+	}
+	if hook != nil {
+		hook()
+	}
+	if _, err := c.svc.Delete(ctx, &function.DeleteRequest{Region: region, FunctionName: rec.FunctionName,
+		FunctionID: rec.FunctionID, Revision: rev}, account); err != nil {
+		return err
+	}
+	if lose {
+		return errors.New("nats: timeout")
+	}
+	return nil
+}
+
 // memStore is an in-memory code store; objects is keyed by Ref.
 type memStore struct {
 	mu      sync.Mutex
@@ -864,7 +908,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "DeleteFunction", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListFunctions", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
 	assert.Equal(t, "UpdateFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "DeleteEventSourceMapping", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/event-source-mappings/x", nil)))
@@ -873,7 +917,10 @@ func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
 	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "UpdateFunctionCode", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/code", nil)))
-	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "DeleteFunction", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions", nil)))
+	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/", nil)))
+	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x/configuration", nil)))
 }
 
 func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {

spinifex/domains/lambda/awsapi/invoke.go

@@ -57,13 +57,13 @@ func (r *RefusalError) Error() string { return "invocation refused before dispat
 
 func (r *RefusalError) Unwrap() error { return r.Err }
 
-// invokeTarget is the function and qualifier an Invoke request names, once
+// functionTarget is the function and qualifier a request names, once
 // every deterministic check on the request has passed.
-type invokeTarget struct{ Name, Qualifier string }
+type functionTarget struct{ Name, Qualifier string }
 
 // validateInvoke runs every check that needs no stored state: options, size,
 // payload JSON, function name and qualifier. Nothing here reads metadata.
-func validateInvoke(c *call) (*invokeTarget, error) {
+func validateInvoke(c *call) (*functionTarget, error) {
 	if err := checkInvokeOptions(c.Header); err != nil {
 		return nil, err
 	}
@@ -76,6 +76,12 @@ func validateInvoke(c *call) (*invokeTarget, error) {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidRequestContentException,
 			"Could not parse request body into json")
 	}
+	return qualifiedTarget(c)
+}
+
+// qualifiedTarget resolves the function name and the qualifier from either
+// its suffix or the Qualifier query, which c.Query must hold strictly parsed.
+func qualifiedTarget(c *call) (*functionTarget, error) {
 	if len(c.Params[0]) > maxFunctionNameLen {
 		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must have length less than or equal to 256")
 	}
@@ -101,7 +107,7 @@ func validateInvoke(c *call) (*invokeTarget, error) {
 		}
 		q = query[0]
 	}
-	return &invokeTarget{Name: name, Qualifier: q}, nil
+	return &functionTarget{Name: name, Qualifier: q}, nil
 }
 
 // invoke implements Invoke for RequestResponse on $LATEST. serve has already

spinifex/domains/lambda/awsapi/listdelete.go

@@ -0,0 +1,107 @@
+package awsapi
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// singleQuery returns the one value of a query member; present but repeated
+// or empty is refused.
+func singleQuery(c *call, member, field string) (string, bool, error) {
+	v, ok := c.Query[member]
+	if !ok {
+		return "", false, nil
+	}
+	if len(v) != 1 || v[0] == "" {
+		return "", true, invalid("1 validation error detected: Value at '%s' failed to satisfy constraint: Member must have length greater than or equal to 1", field)
+	}
+	return v[0], true, nil
+}
+
+// listFunctions implements ListFunctions over $LATEST. Pages hold at most
+// function.MaxListPage functions, and may be short or empty while a
+// NextMarker remains, since each owner request also bounds its scan.
+func (d Deps) listFunctions(ctx context.Context, c *call) (reply, error) {
+	if c.Query.Has("MasterRegion") {
+		return reply{}, unsupported("MasterRegion")
+	}
+	if v, ok, err := singleQuery(c, "FunctionVersion", "functionVersion"); err != nil {
+		return reply{}, err
+	} else if ok && v != lambda.FunctionVersionAll {
+		return reply{}, invalid("1 validation error detected: Value '%s' at 'functionVersion' failed to satisfy constraint: Member must satisfy enum value set: [ALL]", v)
+	}
+	maxItems := function.MaxListPage
+	if v, ok, err := singleQuery(c, "MaxItems", "maxItems"); err != nil {
+		return reply{}, err
+	} else if ok {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 1 || n > maxListItems {
+			return reply{}, invalid("MaxItems must be between 1 and %d", maxListItems)
+		}
+		maxItems = min(n, function.MaxListPage)
+	}
+	marker, _, err := singleQuery(c, "Marker", "marker")
+	if err != nil {
+		return reply{}, err
+	}
+
+	listed, next, err := d.Meta.List(ctx, c.AccountID, c.Region, marker, maxItems)
+	if err != nil {
+		if _, coded := awserrors.ResolveErrorCode(err); coded {
+			return reply{}, err
+		}
+		slog.ErrorContext(ctx, "Lambda: list owner request failed", "err", err)
+		return reply{}, serviceError("the functions could not be listed; retry the request")
+	}
+	out := &lambda.ListFunctionsOutput{Functions: make([]*lambda.FunctionConfiguration, 0, len(listed))}
+	for _, l := range listed {
+		cfg, err := d.render(ctx, l.Record, l.Revision)
+		if err != nil {
+			return reply{}, err
+		}
+		// ListFunctions returns a subset of the configuration: AWS leaves the
+		// state and last-update members to GetFunction.
+		cfg.State, cfg.StateReason, cfg.StateReasonCode = nil, nil, nil
+		cfg.LastUpdateStatus, cfg.LastUpdateStatusReason, cfg.LastUpdateStatusReasonCode = nil, nil, nil
+		out.Functions = append(out.Functions, cfg)
+	}
+	if next != "" {
+		out.NextMarker = aws.String(next)
+	}
+	return reply{Status: http.StatusOK, Body: out}, nil
+}
+
+// deleteFunction implements DeleteFunction for a whole function. Any explicit
+// qualifier is refused before anything is read, so a request scoped to a
+// version can never delete the function.
+func (d Deps) deleteFunction(ctx context.Context, c *call) (reply, error) {
+	if c.Delete == nil {
+		return reply{}, serviceError("delete request was not validated")
+	}
+	if q := c.Delete.Qualifier; q != "" {
+		if q == "$LATEST" {
+			return reply{}, invalid("$LATEST version cannot be deleted without deleting the function.")
+		}
+		return reply{}, notFound(c.Region, c.AccountID, c.Delete.Name+":"+q)
+	}
+	rec, revision, err := d.lookupTarget(ctx, c, c.Delete.Name, "")
+	if err != nil {
+		return reply{}, err
+	}
+	if err := d.Meta.Delete(ctx, c.AccountID, c.Region, rec, revision); err != nil {
+		// A lost reply may follow the unbinding; the caller reads the function to learn which.
+		if _, coded := awserrors.ResolveErrorCode(err); coded {
+			return reply{}, err
+		}
+		slog.ErrorContext(ctx, "Lambda: delete owner request failed", "function_id", rec.FunctionID, "err", err)
+		return reply{}, serviceError("the function could not be deleted; retry the request")
+	}
+	return reply{Status: http.StatusNoContent}, nil
+}

spinifex/domains/lambda/awsapi/listdelete_test.go

@@ -0,0 +1,385 @@
+package awsapi_test
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	listPath  = "/2015-03-31/functions"
+	helloPath = "/2015-03-31/functions/hello"
+	helloARN  = "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+)
+
+// listAll walks ListFunctions from path, following NextMarker, and returns the
+// pages it read.
+func listAll(t *testing.T, e *env, path string) []lambda.ListFunctionsOutput {
+	t.Helper()
+	var pages []lambda.ListFunctionsOutput
+	marker := ""
+	for range 100 {
+		p := path
+		if marker != "" {
+			p += "&Marker=" + url.QueryEscape(marker)
+		}
+		rec, err := e.do(http.MethodGet, p, nil)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, rec.Code)
+		var out lambda.ListFunctionsOutput
+		require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
+		pages = append(pages, out)
+		if out.NextMarker == nil {
+			return pages
+		}
+		marker = *out.NextMarker
+	}
+	t.Fatal("the walk does not end")
+	return nil
+}
+
+func listedNames(pages []lambda.ListFunctionsOutput) []string {
+	var names []string
+	for _, p := range pages {
+		for _, f := range p.Functions {
+			names = append(names, *f.FunctionName)
+		}
+	}
+	return names
+}
+
+func createActive(t *testing.T, e *env, names ...string) {
+	t.Helper()
+	for _, name := range names {
+		_, err := e.do(http.MethodPost, listPath, createBody(name, nil))
+		require.NoError(t, err)
+	}
+	e.activate(t)
+}
+
+func TestListFunctions_WalksEveryFunctionAcrossPagesOnBothPaths(t *testing.T) {
+	e := newUpdateEnv(t)
+	createActive(t, e, "second", "third")
+
+	pages := listAll(t, e, listPath+"?MaxItems=1")
+	assert.Len(t, pages, 3, "a page that fills on the last function has no NextMarker")
+	assert.ElementsMatch(t, []string{"hello", "second", "third"}, listedNames(pages))
+	slash := listAll(t, e, listPath+"/?MaxItems=1")
+	assert.Equal(t, listedNames(pages), listedNames(slash))
+
+	rec, err := e.do(http.MethodGet, listPath, nil)
+	require.NoError(t, err)
+	assert.NotContains(t, rec.Body.String(), "NextMarker")
+}
+
+// ListFunctions renders each function as GetFunctionConfiguration does, with
+// its environment opened, less the state and last-update members.
+func TestListFunctions_RendersTheConfigurationSubset(t *testing.T) {
+	e := newUpdateEnv(t)
+	rec, err := e.do(http.MethodGet, helloPath+"/configuration", nil)
+	require.NoError(t, err)
+	want := decodeConfig(t, rec.Body.Bytes())
+	want.State, want.StateReason, want.StateReasonCode = nil, nil, nil
+	want.LastUpdateStatus, want.LastUpdateStatusReason, want.LastUpdateStatusReasonCode = nil, nil, nil
+
+	rec, err = e.do(http.MethodGet, listPath, nil)
+	require.NoError(t, err)
+	for _, member := range []string{`"State"`, `"StateReason`, `"LastUpdateStatus`} {
+		assert.NotContains(t, rec.Body.String(), member)
+	}
+	var out lambda.ListFunctionsOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
+	require.Len(t, out.Functions, 1)
+	assert.Equal(t, want, *out.Functions[0])
+	assert.Equal(t, "prod", *out.Functions[0].Environment.Variables["MODE"])
+}
+
+func TestListFunctions_EmptyAccountReturnsAnEmptyList(t *testing.T) {
+	e := newEnv(t)
+	rec, err := e.do(http.MethodGet, listPath+"?FunctionVersion=ALL", nil)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"Functions":[]}`, rec.Body.String())
+}
+
+func TestListFunctions_PageHoldsAtMostFifty(t *testing.T) {
+	e := newEnv(t)
+	for i := range function.MaxListPage + 1 {
+		_, err := e.do(http.MethodPost, listPath, createBody(fmt.Sprintf("fn-%02d", i), nil))
+		require.NoError(t, err)
+	}
+	pages := listAll(t, e, listPath+"?MaxItems=10000")
+	require.Len(t, pages, 2)
+	assert.Len(t, pages[0].Functions, function.MaxListPage)
+	assert.Len(t, listedNames(pages), function.MaxListPage+1)
+}
+
+// Two accounts holding the same function names see only their own functions,
+// with their own environments, on first and continuation pages, and neither
+// can continue the other's walk.
+func TestListFunctions_AccountsAreIsolated(t *testing.T) {
+	e := newEnv(t)
+	const other = "210987654321"
+	e.deps.Roles = roleSet{roleARN: true, "arn:aws:iam::" + other + ":role/lambda-exec": true}
+	names := map[string][]string{
+		gwtest.AccountID: {"shared-1", "only-a", "shared-2", "shared-3"},
+		other:            {"shared-2", "shared-1", "only-b", "shared-3"},
+	}
+	for _, acct := range []string{gwtest.AccountID, other} {
+		e.h.AccountID = acct
+		for _, name := range names[acct] {
+			rec, err := e.do(http.MethodPost, listPath, createBody(name, func(b map[string]any) {
+				b["Role"] = "arn:aws:iam::" + acct + ":role/lambda-exec"
+				b["Environment"] = map[string]any{"Variables": map[string]string{"OWNER": acct}}
+			}))
+			require.NoError(t, err)
+			require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String())
+		}
+	}
+	e.activate(t)
+
+	markers := map[string]string{}
+	for _, acct := range []string{gwtest.AccountID, other} {
+		e.h.AccountID = acct
+		pages := listAll(t, e, listPath+"?MaxItems=2")
+		require.Len(t, pages, 2, "the walk has a continuation page")
+		markers[acct] = *pages[0].NextMarker
+		assert.Equal(t, names[acct], listedNames(pages))
+		for _, p := range pages {
+			for _, f := range p.Functions {
+				assert.Equal(t, "arn:aws:lambda:"+gwtest.Region+":"+acct+":function:"+*f.FunctionName, *f.FunctionArn)
+				assert.Equal(t, "arn:aws:iam::"+acct+":role/lambda-exec", *f.Role)
+				assert.Equal(t, acct, *f.Environment.Variables["OWNER"])
+			}
+		}
+	}
+	for acct, foreign := range map[string]string{gwtest.AccountID: markers[other], other: markers[gwtest.AccountID]} {
+		e.h.AccountID = acct
+		_, err := e.do(http.MethodGet, listPath+"?Marker="+url.QueryEscape(foreign), nil)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+func TestListFunctions_ParameterValidation(t *testing.T) {
+	e := newUpdateEnv(t)
+	for name, query := range map[string]string{
+		"MaxItems zero":           "MaxItems=0",
+		"MaxItems over 10000":     "MaxItems=10001",
+		"MaxItems not a number":   "MaxItems=x",
+		"MaxItems repeated":       "MaxItems=1&MaxItems=2",
+		"empty Marker":            "Marker=",
+		"malformed Marker":        "Marker=not-a-marker",
+		"MasterRegion":            "MasterRegion=us-east-1",
+		"unknown FunctionVersion": "FunctionVersion=LATEST",
+		"malformed query":         "MaxItems=%zz",
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := e.do(http.MethodGet, listPath+"?"+query, nil)
+			testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		})
+	}
+}
+
+func TestListFunctions_AuthorizesOnEveryResource(t *testing.T) {
+	e := newUpdateEnv(t)
+	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", helloARN))
+	_, err := e.do(http.MethodGet, listPath, nil)
+	gwtest.AssertDenied(t, err)
+	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", "*"))
+	_, err = e.do(http.MethodGet, listPath, nil)
+	require.NoError(t, err)
+}
+
+func TestListFunctions_OwnerFailuresAreRetryable(t *testing.T) {
+	e := newUpdateEnv(t)
+	e.meta.listErr = fmt.Errorf("NATS request to %s: %w", function.SubjectList, nats.ErrNoResponders)
+	_, err := e.do(http.MethodGet, listPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.False(t, awserrors.IsTerminal(err))
+
+	e.meta.listErr = nil
+	e.deps.AEAD = nil
+	_, err = e.do(http.MethodGet, listPath, nil)
+	// An environment that cannot be opened fails the page.
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+}
+
+// deletion reads the retained obligation for functionID.
+func (e *env) deletion(t *testing.T, functionID string) (function.Deletion, bool) {
+	t.Helper()
+	entry, err := e.kv(t).Get(t.Context(), function.KVDeletedKey(gwtest.Region, functionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return function.Deletion{}, false
+	}
+	require.NoError(t, err)
+	var d function.Deletion
+	require.NoError(t, json.Unmarshal(entry.Value(), &d))
+	return d, true
+}
+
+func TestDeleteFunction_RemovesTheFunctionAndRetainsItsRecordsAndCode(t *testing.T) {
+	e := newUpdateEnv(t)
+	before, _ := e.record(t, "hello")
+	copies := e.store.count()
+	// Deletion reads neither the code store nor the environment.
+	codes, aead := e.deps.Codes, e.deps.AEAD
+	e.deps.Codes, e.deps.AEAD = nil, nil
+
+	rec, err := e.do(http.MethodDelete, helloPath, nil)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusNoContent, rec.Code)
+	assert.Empty(t, rec.Body.Bytes())
+
+	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	assert.Empty(t, listedNames(listAll(t, e, listPath+"?MaxItems=1")))
+	_, err = e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+
+	d, ok := e.deletion(t, before.FunctionID)
+	require.True(t, ok)
+	assert.Equal(t, "hello", d.FunctionName)
+	assert.Contains(t, e.uploads(t), before.FunctionID)
+	assert.Equal(t, copies, e.store.count(), "no code copy is removed")
+
+	e.deps.Codes, e.deps.AEAD = codes, aead
+	_, err = e.do(http.MethodPost, listPath, createBody("hello", nil))
+	require.NoError(t, err)
+	again, _ := e.record(t, "hello")
+	assert.NotEqual(t, before.FunctionID, again.FunctionID)
+	_, ok = e.deletion(t, before.FunctionID)
+	assert.True(t, ok, "the old obligation survives the name's reuse")
+}
+
+// Every explicit qualifier, in the query or the name, and every query that
+// cannot be read exactly, is refused before the owner is asked to delete.
+func TestDeleteFunction_RefusesEveryQualifierWithoutMutation(t *testing.T) {
+	cases := map[string]struct {
+		path string
+		code string
+	}{
+		"query $LATEST":         {helloPath + "?Qualifier=%24LATEST", awserrors.ErrorInvalidParameterValueException},
+		"query version":         {helloPath + "?Qualifier=1", awserrors.ErrorResourceNotFoundException},
+		"query alias":           {helloPath + "?Qualifier=live", awserrors.ErrorResourceNotFoundException},
+		"empty query":           {helloPath + "?Qualifier=", awserrors.ErrorInvalidParameterValueException},
+		"bare query key":        {helloPath + "?Qualifier", awserrors.ErrorInvalidParameterValueException},
+		"repeated query":        {helloPath + "?Qualifier=1&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
+		"malformed escape":      {helloPath + "?Qualifier=%zz", awserrors.ErrorInvalidParameterValueException},
+		"malformed other pair":  {helloPath + "?x=%zz&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
+		"semicolon":             {helloPath + "?Qualifier=1;x=2", awserrors.ErrorInvalidParameterValueException},
+		"invalid pattern":       {helloPath + "?Qualifier=a.b", awserrors.ErrorInvalidParameterValueException},
+		"name suffix $LATEST":   {helloPath + ":%24LATEST", awserrors.ErrorInvalidParameterValueException},
+		"name suffix version":   {helloPath + ":1", awserrors.ErrorResourceNotFoundException},
+		"ARN suffix version":    {"/2015-03-31/functions/" + url.PathEscape(helloARN+":1"), awserrors.ErrorResourceNotFoundException},
+		"empty name suffix":     {helloPath + ":", awserrors.ErrorInvalidParameterValueException},
+		"mismatched qualifiers": {helloPath + ":1?Qualifier=2", awserrors.ErrorInvalidParameterValueException},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			e := newUpdateEnv(t)
+			before, beforeRev := e.record(t, "hello")
+			_, err := e.do(http.MethodDelete, tc.path, nil)
+			testutil.RequireAWSCode(t, err, tc.code)
+			assert.Zero(t, e.meta.deleteCalls)
+			requireConfigUnchanged(t, e, before, beforeRev)
+			_, ok := e.deletion(t, before.FunctionID)
+			assert.False(t, ok)
+		})
+	}
+}
+
+func TestDeleteFunction_Authorization(t *testing.T) {
+	allowAll := gwtest.Statement("Allow", "*", "*")
+	t.Run("denied on the function ARN", func(t *testing.T) {
+		e := newUpdateEnv(t)
+		e.h = gwtest.Scoped(allowAll, gwtest.Statement("Deny", "lambda:DeleteFunction", helloARN))
+		before, beforeRev := e.record(t, "hello")
+		_, err := e.do(http.MethodDelete, helloPath, nil)
+		gwtest.AssertDenied(t, err)
+		requireConfigUnchanged(t, e, before, beforeRev)
+	})
+	t.Run("another function's grant does not reach it", func(t *testing.T) {
+		e := newUpdateEnv(t)
+		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN+"-other"))
+		_, err := e.do(http.MethodDelete, helloPath, nil)
+		gwtest.AssertDenied(t, err)
+	})
+	t.Run("allowed on the function ARN", func(t *testing.T) {
+		e := newUpdateEnv(t)
+		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN))
+		rec, err := e.do(http.MethodDelete, helloPath, nil)
+		require.NoError(t, err)
+		assert.Equal(t, http.StatusNoContent, rec.Code)
+	})
+}
+
+func TestDeleteFunction_LostReplyIsAnErrorButTheDeleteStands(t *testing.T) {
+	e := newUpdateEnv(t)
+	e.meta.loseDeleteReply = true
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.False(t, awserrors.IsTerminal(err))
+	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestDeleteFunction_OwnerWithoutTheSubjectIsRetryableAndChangesNothing(t *testing.T) {
+	e := newUpdateEnv(t)
+	before, beforeRev := e.record(t, "hello")
+	e.meta.deleteErr = fmt.Errorf("NATS request to %s: %w", function.SubjectDelete, nats.ErrNoResponders)
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.False(t, awserrors.IsTerminal(err))
+	requireConfigUnchanged(t, e, before, beforeRev)
+}
+
+func TestDeleteFunction_UpdateBetweenReadAndDeleteWins(t *testing.T) {
+	e := newConfigEnv(t)
+	e.meta.beforeCommit = func() {
+		_, err := e.do(http.MethodPut, configPath, map[string]any{"Handler": "handler.winner"})
+		assert.NoError(t, err)
+	}
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	after, _ := e.record(t, "hello")
+	assert.Equal(t, "handler.winner", after.Handler)
+}
+
+func TestDeleteFunction_LeavesMappingsInPlace(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	rec, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	require.NoError(t, err)
+	uuid, _ := decodeMapping(t, rec.Body.Bytes())["UUID"].(string)
+	before, beforeRev, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodDelete, helloPath, nil)
+	require.NoError(t, err)
+	after, afterRev, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, beforeRev, afterRev)
+	assert.Equal(t, before, after)
+	assert.Equal(t, eventsource.StateEnabled, after.State)
+}
+
+func TestDeleteFunction_LaterInvokeIsNotFound(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	require.NoError(t, err)
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	assert.Empty(t, ex.calls)
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -149,6 +149,20 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		}
 	case "GetEventSourceMapping", "DeleteEventSourceMapping":
 		resource = mappingARN(inv.Region, inv.AccountID, params[0])
+	case "ListFunctions":
+		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
+			return invalid("The request query string could not be parsed.")
+		}
+	case "DeleteFunction":
+		// A dropped Qualifier would widen the request to the whole function,
+		// so the query is parsed strictly and the target resolved before anything else.
+		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
+			return invalid("The request query string could not be parsed.")
+		}
+		if c.Delete, err = qualifiedTarget(c); err != nil {
+			return err
+		}
+		resource = qualifiedFunctionARN(inv.Region, inv.AccountID, c.Delete.Name, c.Delete.Qualifier)
 	case "Invoke":
 		// Invoke validates deterministically before authorizing, and its
 		// resource comes from that same validated target. URL.Query drops
@@ -234,6 +248,11 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }
 
 func writeJSON(w http.ResponseWriter, out reply) {
+	if out.Status == http.StatusNoContent {
+		maps.Copy(w.Header(), out.Header)
+		w.WriteHeader(out.Status)
+		return
+	}
 	var body []byte
 	var err error
 	if raw, ok := out.Body.(rawJSON); ok {

spinifex/domains/lambda/awsapi/routes.go

@@ -22,7 +22,10 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 	// Invoke is Invoke's validated target, resolved ahead of the
 	// authorization gate whose resource it names.
-	Invoke *invokeTarget
+	Invoke *functionTarget
+	// Delete is DeleteFunction's validated target, resolved from the strictly
+	// parsed query ahead of the authorization gate.
+	Delete *functionTarget
 	// UpdateConfig is the decoded UpdateFunctionConfiguration body, read
 	// ahead of the PassRole check on a new role.
 	UpdateConfig *lambda.UpdateFunctionConfigurationInput
@@ -49,6 +52,18 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.createFunction(ctx, c)
 		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions", Action: "ListFunctions",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listFunctions(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/", Action: "ListFunctions",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listFunctions(ctx, c)
+		}},
+	{Method: "DELETE", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "DeleteFunction",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.deleteFunction(ctx, c)
+		}},
 	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "GetFunction",
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)

spinifex/domains/lambda/function/client.go

@@ -26,6 +26,10 @@ type Client interface {
 	CodeUpdated(ctx context.Context, accountID, region string, rec Record, revision uint64, codeRevisionID, sha256Hex string, size int64) (Record, uint64, error)
 	// UpdateConfiguration commits cfg for the function at revision and returns the updated record and revision.
 	UpdateConfiguration(ctx context.Context, accountID, region string, rec Record, revision uint64, cfg Configuration) (Record, uint64, error)
+	// List returns up to maxItems of the Region's functions after marker and the next marker, empty at the end.
+	List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error)
+	// Delete unbinds the function read at revision, retaining its cleanup obligation.
+	Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error
 }
 
 // NATSClient is the Client over NATS request/reply to the daemon.
@@ -98,3 +102,19 @@ func (c *NATSClient) UpdateConfiguration(ctx context.Context, accountID, region
 	}
 	return resp.Record, resp.Revision, nil
 }
+
+func (c *NATSClient) List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error) {
+	resp, err := natsmsg.NATSRequest[ListResponse](ctx, c.conn, SubjectList,
+		&ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, metaRequestTimeout, accountID)
+	if err != nil {
+		return nil, "", err
+	}
+	return resp.Functions, resp.NextMarker, nil
+}
+
+func (c *NATSClient) Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error {
+	_, err := natsmsg.NATSRequest[DeleteResponse](ctx, c.conn, SubjectDelete,
+		&DeleteRequest{Region: region, FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, Revision: revision},
+		metaRequestTimeout, accountID)
+	return err
+}

spinifex/domains/lambda/function/delete.go

@@ -0,0 +1,87 @@
+package function
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Delete records the function's cleanup obligation, then unbinds its name by
+// compare-and-swap at the revision the caller read. Code copies and upload
+// records are retained; nothing here removes them.
+func (s *MetaService) Delete(ctx context.Context, req *DeleteRequest, accountID string) (*DeleteResponse, error) {
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	if !ValidName(req.FunctionName) || !idRE.MatchString(req.FunctionID) || accountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "delete request is incomplete")
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	key := KVFunctionKey(req.Region, req.FunctionName)
+	entry, err := kv.Get(ctx, key)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return nil, awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s", req.FunctionName)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("lambda: get function record: %w", err)
+	}
+	rec, err := decode(entry.Value())
+	if err != nil {
+		return nil, err
+	}
+	if entry.Revision() != req.Revision || rec.FunctionID != req.FunctionID {
+		return nil, conflict("Function was modified by another request: %s", req.FunctionName)
+	}
+	if err := s.recordDeletion(ctx, kv, accountID, rec); err != nil {
+		return nil, err
+	}
+	if err := kv.Delete(ctx, key, jetstream.LastRevision(req.Revision)); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return nil, conflict("Function was modified by another request: %s", req.FunctionName)
+		}
+		return nil, fmt.Errorf("lambda: delete function record: %w", err)
+	}
+	return &DeleteResponse{}, nil
+}
+
+// recordDeletion writes the obligation for rec. One already present, from an
+// earlier attempt whose outcome was uncertain, is accepted only for the same
+// function; anything else refuses the delete.
+func (s *MetaService) recordDeletion(ctx context.Context, kv jetstream.KeyValue, accountID string, rec Record) error {
+	want := Deletion{
+		FormatVersion: FormatVersion, AccountID: accountID, Region: rec.Region,
+		FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, RecordedAt: s.now().UTC(),
+	}
+	data, err := json.Marshal(want)
+	if err != nil {
+		return err
+	}
+	key := KVDeletedKey(rec.Region, rec.FunctionID)
+	_, err = kv.Create(ctx, key, data)
+	if err == nil {
+		return nil
+	}
+	if !errors.Is(err, jetstream.ErrKeyExists) {
+		return fmt.Errorf("lambda: record function deletion: %w", err)
+	}
+	entry, err := kv.Get(ctx, key)
+	if err != nil {
+		return fmt.Errorf("lambda: read function deletion: %w", err)
+	}
+	var have Deletion
+	if err := json.Unmarshal(entry.Value(), &have); err != nil {
+		return fmt.Errorf("lambda: function deletion record is unreadable: %w", err)
+	}
+	if have.FormatVersion != FormatVersion || have.AccountID != accountID || have.Region != rec.Region ||
+		have.FunctionName != rec.FunctionName || have.FunctionID != rec.FunctionID {
+		return fmt.Errorf("lambda: function deletion record %s names another function", key)
+	}
+	return nil
+}

spinifex/domains/lambda/function/delete_test.go

@@ -0,0 +1,296 @@
+package function
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// serveAs creates name with stored code under functionID and activates it.
+func serveAs(t *testing.T, svc *MetaService, name, functionID string) *GetResponse {
+	t.Helper()
+	rev := mustCreate(t, svc, name, functionID)
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: name, Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	svc.SetReadiness(func(context.Context, Record) error { return nil })
+	require.NoError(t, svc.ActivatePending(t.Context()))
+	got := mustGet(t, svc, name)
+	require.Equal(t, StateActive, got.Record.State)
+	return got
+}
+
+func deleteReq(got *GetResponse) *DeleteRequest {
+	return &DeleteRequest{Region: testRegion, FunctionName: got.Record.FunctionName, FunctionID: got.Record.FunctionID, Revision: got.Revision}
+}
+
+// deletion reads the retained obligation for functionID, or reports it absent.
+func deletion(t *testing.T, svc *MetaService, functionID string) (Deletion, bool) {
+	t.Helper()
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), KVDeletedKey(testRegion, functionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Deletion{}, false
+	}
+	require.NoError(t, err)
+	var d Deletion
+	require.NoError(t, json.Unmarshal(entry.Value(), &d))
+	return d, true
+}
+
+func requireRetained(t *testing.T, svc *MetaService, name, functionID string) {
+	t.Helper()
+	d, ok := deletion(t, svc, functionID)
+	require.True(t, ok, "deletion obligation for %s is missing", functionID)
+	assert.Equal(t, Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion,
+		FunctionName: name, FunctionID: functionID, RecordedAt: d.RecordedAt}, d)
+	_, ok = upload(t, svc, functionID)
+	assert.True(t, ok, "upload record for %s was removed", functionID)
+}
+
+func TestDelete_UnbindsTheNameAndRetainsTheObligationAndUploads(t *testing.T) {
+	svc, c := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
+	require.NoError(t, err)
+
+	assert.False(t, mustGet(t, svc, "hello").Found)
+	requireRetained(t, svc, "hello", "f1")
+	d, _ := deletion(t, svc, "f1")
+	assert.Equal(t, c.now(), d.RecordedAt)
+	up, _ := upload(t, svc, "f1")
+	assert.Equal(t, UploadAttached, up.State, "the upload record is left exactly as it was")
+}
+
+func TestDelete_MissingFunctionIsNotFound(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: 1}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, ok := deletion(t, svc, "f1")
+	assert.False(t, ok)
+}
+
+func TestDelete_StaleRevisionOrAnotherFunctionIsRefusedWithoutAnObligation(t *testing.T) {
+	svc, _ := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	for name, req := range map[string]*DeleteRequest{
+		"stale revision": {Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: got.Revision - 1},
+		"other function": {Region: testRegion, FunctionName: "hello", FunctionID: "f0", Revision: got.Revision},
+	} {
+		_, err := svc.Delete(t.Context(), req, testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+		assert.Equal(t, got.Revision, mustGet(t, svc, "hello").Revision, name)
+		_, ok := deletion(t, svc, req.FunctionID)
+		assert.False(t, ok, name)
+	}
+}
+
+func TestDelete_RefusesMalformedRequests(t *testing.T) {
+	svc, _ := newTestService(t)
+	for name, req := range map[string]*DeleteRequest{
+		"bad region": {Region: "Bad.Region", FunctionName: "hello", FunctionID: "f1", Revision: 1},
+		"bad name":   {Region: testRegion, FunctionName: "a.b", FunctionID: "f1", Revision: 1},
+		"bad id":     {Region: testRegion, FunctionName: "hello", FunctionID: "f.1", Revision: 1},
+	} {
+		_, err := svc.Delete(t.Context(), req, testAccount)
+		require.Error(t, err, name)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+// The delete passes its read, then pauses before writing while an update
+// commits; the conditional delete loses and the function stays bound. The
+// obligation it left proves nothing was deleted.
+func TestDelete_InterleavedUpdateLosesTheConditionalDelete(t *testing.T) {
+	svc, c := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	paused, release := make(chan struct{}), make(chan struct{})
+	var calls atomic.Int32
+	svc.now = func() time.Time {
+		if calls.Add(1) == 1 {
+			close(paused)
+			<-release
+		}
+		return c.now()
+	}
+	deleteErr := make(chan error, 1)
+	go func() {
+		_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
+		deleteErr <- err
+	}()
+	<-paused
+	cfg := Configuration{Role: got.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
+	won, err := svc.UpdateConfiguration(t.Context(), configReq(got.Revision, cfg), testAccount)
+	require.NoError(t, err)
+	close(release)
+	testutil.RequireAWSCode(t, <-deleteErr, awserrors.ErrorResourceConflictException)
+
+	after := mustGet(t, svc, "hello")
+	require.True(t, after.Found)
+	assert.Equal(t, won.Revision, after.Revision)
+	assert.Equal(t, "f1", after.Record.FunctionID)
+	_, ok := deletion(t, svc, "f1")
+	assert.True(t, ok, "the obligation from the losing attempt is retained")
+}
+
+func TestDelete_ObligationNamingAnotherFunctionRefusesTheDelete(t *testing.T) {
+	svc, _ := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	data, err := json.Marshal(Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion, FunctionName: "other", FunctionID: "f1"})
+	require.NoError(t, err)
+	_, err = kv.Create(t.Context(), KVDeletedKey(testRegion, "f1"), data)
+	require.NoError(t, err)
+
+	_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
+	require.Error(t, err)
+	_, coded := awserrors.ResolveErrorCode(err)
+	assert.False(t, coded, "an inconsistent obligation is an internal fault, retried by the caller")
+	assert.True(t, mustGet(t, svc, "hello").Found)
+}
+
+// A delete of a create still storing its code wins; the late CodeStored and
+// RollbackCreate of that create change nothing, and its upload stays owed.
+func TestDelete_PendingCreateRacingDeletionChangesNothingLater(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: rev}, testAccount)
+	require.NoError(t, err)
+	recreated := serveAs(t, svc, "hello", "f2")
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.Error(t, err)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
+	require.Error(t, err)
+
+	got := mustGet(t, svc, "hello")
+	assert.Equal(t, recreated.Revision, got.Revision)
+	assert.Equal(t, "f2", got.Record.FunctionID)
+	requireOwed(t, svc, "f1")
+	requireRetained(t, svc, "hello", "f1")
+}
+
+func TestDelete_SameNameRecreationKeepsEachObligationApart(t *testing.T) {
+	svc, _ := newTestService(t)
+	first := serveAs(t, svc, "hello", "f1")
+	_, err := svc.Delete(t.Context(), deleteReq(first), testAccount)
+	require.NoError(t, err)
+	second := serveAs(t, svc, "hello", "f2")
+	_, ok := deletion(t, svc, "f2")
+	assert.False(t, ok, "a recreated function has no obligation of its own")
+
+	_, err = svc.Delete(t.Context(), deleteReq(second), testAccount)
+	require.NoError(t, err)
+	requireRetained(t, svc, "hello", "f1")
+	requireRetained(t, svc, "hello", "f2")
+}
+
+// faultKV fails one write to key: before it commits, or after, losing the
+// acknowledgement.
+type faultKV struct {
+	jetstream.KeyValue
+
+	key   string
+	after bool
+	fired atomic.Bool
+}
+
+var errInjected = errors.New("nats: timeout")
+
+func (f *faultKV) trip(key string) bool { return key == f.key && f.fired.CompareAndSwap(false, true) }
+
+func (f *faultKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	if !f.trip(key) {
+		return f.KeyValue.Create(ctx, key, value, opts...)
+	}
+	if f.after {
+		if _, err := f.KeyValue.Create(ctx, key, value, opts...); err != nil {
+			return 0, err
+		}
+	}
+	return 0, errInjected
+}
+
+func (f *faultKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
+	if !f.trip(key) {
+		return f.KeyValue.Delete(ctx, key, opts...)
+	}
+	if f.after {
+		if err := f.KeyValue.Delete(ctx, key, opts...); err != nil {
+			return err
+		}
+	}
+	return errInjected
+}
+
+// Each write of a delete is interrupted before it commits, or commits with its
+// acknowledgement lost. A fresh owner then finds the binding and obligation the
+// stream actually holds, a retry converges, and a recreated name stays apart.
+func TestDelete_InterruptionAndLostAcknowledgementAtEachWrite(t *testing.T) {
+	cases := []struct {
+		name       string
+		key        string
+		after      bool
+		bound      bool
+		obligation bool
+	}{
+		{"obligation write interrupted", KVDeletedKey(testRegion, "f1"), false, true, false},
+		{"obligation acknowledgement lost", KVDeletedKey(testRegion, "f1"), true, true, true},
+		{"name delete interrupted", KVFunctionKey(testRegion, "hello"), false, true, true},
+		{"name delete acknowledgement lost", KVFunctionKey(testRegion, "hello"), true, false, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, nc, _ := testutil.StartTestJetStream(t)
+			js := testutil.NewJetStream(t, nc)
+			svc := NewMetaService(js)
+			got := serveAs(t, svc, "hello", "f1")
+			kv, err := svc.bucket(t.Context(), testAccount)
+			require.NoError(t, err)
+			svc.mu.Lock()
+			svc.buckets[testAccount] = &faultKV{KeyValue: kv, key: tc.key, after: tc.after}
+			svc.mu.Unlock()
+
+			_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
+			require.ErrorIs(t, err, errInjected)
+			_, coded := awserrors.ResolveErrorCode(err)
+			assert.False(t, coded, "an uncertain write is a retryable fault")
+
+			reopened := NewMetaService(js)
+			now := mustGet(t, reopened, "hello")
+			assert.Equal(t, tc.bound, now.Found, "binding")
+			if tc.bound {
+				assert.Equal(t, got.Revision, now.Revision)
+			}
+			_, ok := deletion(t, reopened, "f1")
+			assert.Equal(t, tc.obligation, ok, "obligation")
+			_, ok = upload(t, reopened, "f1")
+			assert.True(t, ok, "upload record retained")
+
+			_, err = reopened.Delete(t.Context(), deleteReq(got), testAccount)
+			if tc.bound {
+				require.NoError(t, err, "the retry completes the delete")
+			} else {
+				testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+			}
+			assert.False(t, mustGet(t, reopened, "hello").Found)
+			requireRetained(t, reopened, "hello", "f1")
+
+			recreated := serveAs(t, reopened, "hello", "f2")
+			assert.Equal(t, "f2", recreated.Record.FunctionID)
+			requireRetained(t, reopened, "hello", "f1")
+			_, ok = deletion(t, reopened, "f2")
+			assert.False(t, ok)
+		})
+	}
+}

spinifex/domains/lambda/function/list.go

@@ -0,0 +1,141 @@
+package function
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"strings"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// List bounds: a page returns at most MaxListPage functions and examines at
+// most maxListScan stream sequences, whatever each one holds.
+const (
+	MaxListPage = 50
+	maxListScan = 200
+)
+
+// listCursor is the decoded marker: the last stream sequence a page examined,
+// bound to the account and Region it was issued for.
+type listCursor struct {
+	Version   int    `json:"v"`
+	AccountID string `json:"a"`
+	Region    string `json:"r"`
+	After     uint64 `json:"s"`
+}
+
+func encodeCursor(c listCursor) (string, error) {
+	data, err := json.Marshal(c)
+	if err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(data), nil
+}
+
+func decodeCursor(marker, accountID, region string) (uint64, error) {
+	bad := awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "Invalid marker: %s", marker)
+	data, err := base64.RawURLEncoding.DecodeString(marker)
+	if err != nil {
+		return 0, bad
+	}
+	var c listCursor
+	if err := json.Unmarshal(data, &c); err != nil {
+		return 0, bad
+	}
+	if c.Version != 1 || c.AccountID != accountID || c.Region != region || c.After == 0 || c.After == math.MaxUint64 {
+		return 0, bad
+	}
+	return c.After, nil
+}
+
+// List returns the Region's functions in stream order after the marker. Each
+// step is one leader get of an exact sequence, bounded by a fast stream-state
+// read, so a page costs at most its scan bound whatever the bucket holds.
+// NextMarker is set unless the page reached the stream end it read.
+func (s *MetaService) List(ctx context.Context, req *ListRequest, accountID string) (*ListResponse, error) {
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	if accountID == "" || req.MaxItems < 1 || req.MaxItems > MaxListPage {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "list request is incomplete")
+	}
+	var after uint64
+	if req.Marker != "" {
+		var err error
+		if after, err = decodeCursor(req.Marker, accountID, req.Region); err != nil {
+			return nil, err
+		}
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	stream := "KV_" + kv.Bucket()
+	// An unfiltered stream info without deleted details reads counters only.
+	info, err := s.js.Stream(ctx, stream)
+	if err != nil {
+		return nil, fmt.Errorf("lambda: list function records: %w", err)
+	}
+	state := info.CachedInfo().State
+	if state.FirstSeq > 0 {
+		after = max(after, state.FirstSeq-1)
+	}
+	keyPrefix := "$KV." + kv.Bucket() + "." + kvFunctionKeyPrefix + req.Region + "."
+	out := &ListResponse{Functions: []ListedFunction{}}
+	for gets := 0; gets < maxListScan && after < state.LastSeq; gets++ {
+		seq := after + 1
+		m, err := kvstore.LeaderMsgGet(ctx, s.js, stream, kvstore.MsgGetRequest{Seq: seq})
+		if errors.Is(err, jetstream.ErrMsgNotFound) {
+			after = seq
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("lambda: list function records: %w", err)
+		}
+		if m.Sequence != seq {
+			return nil, fmt.Errorf("lambda: list function records: %w", jetstream.ErrInvalidJetStreamResponse)
+		}
+		if !strings.HasPrefix(m.Subject, keyPrefix) || removed(m.Header) {
+			after = seq
+			continue
+		}
+		if len(out.Functions) == req.MaxItems {
+			// A full page reached a function it cannot return; the marker
+			// stays before it.
+			break
+		}
+		rec, err := decode(m.Data)
+		if err != nil {
+			return nil, err
+		}
+		out.Functions = append(out.Functions, ListedFunction{Record: rec, Revision: seq})
+		after = seq
+	}
+	if after >= state.LastSeq {
+		return out, nil
+	}
+	if out.NextMarker, err = encodeCursor(listCursor{Version: 1, AccountID: accountID, Region: req.Region, After: after}); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// removed reports a key's delete or purge marker, as the KV Get of nats.go
+// reads one.
+func removed(h nats.Header) bool {
+	if op := h.Get("KV-Operation"); op != "" {
+		return op == "DEL" || op == "PURGE"
+	}
+	switch h.Get(jetstream.MarkerReasonHeader) {
+	case "MaxAge", "Purge", "Remove":
+		return true
+	}
+	return false
+}

spinifex/domains/lambda/function/list_test.go

@@ -0,0 +1,325 @@
+package function
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func listPage(t *testing.T, svc *MetaService, region, marker string, maxItems int) *ListResponse {
+	t.Helper()
+	resp, err := svc.List(t.Context(), &ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, testAccount)
+	require.NoError(t, err)
+	return resp
+}
+
+// walk follows NextMarker to the end and returns the names in order and the
+// number of pages.
+func walk(t *testing.T, svc *MetaService, region string, maxItems int) ([]string, int) {
+	t.Helper()
+	var names []string
+	marker, pages := "", 0
+	for {
+		resp := listPage(t, svc, region, marker, maxItems)
+		pages++
+		require.LessOrEqual(t, len(resp.Functions), maxItems)
+		for _, f := range resp.Functions {
+			names = append(names, f.Record.FunctionName)
+		}
+		if resp.NextMarker == "" {
+			return names, pages
+		}
+		marker = resp.NextMarker
+		require.Less(t, pages, 1000, "the walk does not end")
+	}
+}
+
+func TestList_EmptyAccountHasNoFunctionsAndNoMarker(t *testing.T) {
+	svc, _ := newTestService(t)
+	resp := listPage(t, svc, testRegion, "", MaxListPage)
+	assert.Empty(t, resp.Functions)
+	assert.NotNil(t, resp.Functions)
+	assert.Empty(t, resp.NextMarker)
+}
+
+func TestList_PagesReturnEveryFunctionOnceWithItsRevision(t *testing.T) {
+	svc, _ := newTestService(t)
+	want := make([]string, 0, 7)
+	for i := range 7 {
+		name := fmt.Sprintf("fn-%d", i)
+		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
+		want = append(want, name)
+	}
+	names, pages := walk(t, svc, testRegion, 2)
+	assert.Equal(t, want, names, "stream order, each once")
+	assert.Equal(t, 4, pages)
+
+	page := listPage(t, svc, testRegion, "", 1)
+	require.Len(t, page.Functions, 1)
+	got := mustGet(t, svc, "fn-0")
+	assert.Equal(t, got.Revision, page.Functions[0].Revision)
+	assert.Equal(t, got.Record, page.Functions[0].Record)
+}
+
+// A page that fills on the last function looks ahead, finds the walk over,
+// and returns no marker; one that fills before it keeps the next one.
+func TestList_FullPageAtTheEndHasNoMarker(t *testing.T) {
+	svc, _ := newTestService(t)
+	for i := range 4 {
+		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
+	}
+	_, pages := walk(t, svc, testRegion, 2)
+	assert.Equal(t, 2, pages)
+	assert.Empty(t, listPage(t, svc, testRegion, "", 4).NextMarker)
+
+	first := listPage(t, svc, testRegion, "", 3)
+	require.NotEmpty(t, first.NextMarker)
+	rest := listPage(t, svc, testRegion, first.NextMarker, 3)
+	require.Len(t, rest.Functions, 1, "the record the full page looked at is not skipped")
+	assert.Equal(t, "fn-3", rest.Functions[0].Record.FunctionName)
+	assert.Empty(t, rest.NextMarker)
+}
+
+func TestList_SkipsDeletedFunctionsUploadsObligationsAndOtherRegions(t *testing.T) {
+	svc, _ := newTestService(t)
+	gone := serveAs(t, svc, "gone", "f0")
+	mustCreate(t, svc, "kept", "f1")
+	_, err := svc.Delete(t.Context(), deleteReq(gone), testAccount)
+	require.NoError(t, err)
+	other := createReq("elsewhere", "f2")
+	other.Region = "us-east-1"
+	_, err = svc.Create(t.Context(), other, testAccount)
+	require.NoError(t, err)
+
+	names, _ := walk(t, svc, testRegion, MaxListPage)
+	assert.Equal(t, []string{"kept"}, names)
+	names, _ = walk(t, svc, "us-east-1", MaxListPage)
+	assert.Equal(t, []string{"elsewhere"}, names)
+}
+
+// cursorAfter is the last sequence a marker says its page examined.
+func cursorAfter(t *testing.T, marker string) uint64 {
+	t.Helper()
+	after, err := decodeCursor(marker, testAccount, testRegion)
+	require.NoError(t, err)
+	return after
+}
+
+// walkFrom follows NextMarker from marker to the end, checking that no page
+// examines more than maxListScan sequences, and returns the names it listed.
+func walkFrom(t *testing.T, svc *MetaService, marker string, maxItems int) ([]string, int) {
+	t.Helper()
+	var names []string
+	var after uint64
+	if marker != "" {
+		after = cursorAfter(t, marker)
+	}
+	for pages := 1; ; pages++ {
+		resp := listPage(t, svc, testRegion, marker, maxItems)
+		require.LessOrEqual(t, len(resp.Functions), maxItems)
+		for _, f := range resp.Functions {
+			names = append(names, f.Record.FunctionName)
+		}
+		if resp.NextMarker == "" {
+			return names, pages
+		}
+		next := cursorAfter(t, resp.NextMarker)
+		require.LessOrEqual(t, next-after, uint64(maxListScan), "a page examined more than its scan bound")
+		require.True(t, next > after || len(resp.Functions) > 0, "a page made no progress")
+		marker, after = resp.NextMarker, next
+		require.Less(t, pages, 1000, "the walk does not end")
+	}
+}
+
+// tombstones appends n deleted functions to the stream.
+func tombstones(t *testing.T, svc *MetaService, n int) {
+	t.Helper()
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	for i := range n {
+		name := fmt.Sprintf("tomb-%d", i)
+		mustCreate(t, svc, name, fmt.Sprintf("t%d", i))
+		require.NoError(t, kv.Delete(t.Context(), KVFunctionKey(testRegion, name)))
+	}
+}
+
+// More delete markers than one request may examine give empty pages that
+// still continue, and the walk reaches the live function after them.
+func TestList_ScanBoundGivesShortPagesThatContinue(t *testing.T) {
+	svc, _ := newTestService(t)
+	tombstones(t, svc, maxListScan+5)
+	mustCreate(t, svc, "live", "f1")
+
+	first := listPage(t, svc, testRegion, "", MaxListPage)
+	assert.Empty(t, first.Functions)
+	require.NotEmpty(t, first.NextMarker)
+	assert.Equal(t, uint64(maxListScan), cursorAfter(t, first.NextMarker))
+	names, _ := walkFrom(t, svc, first.NextMarker, MaxListPage)
+	assert.Equal(t, []string{"live"}, names)
+}
+
+// Messages that are not this Region's function records count toward the scan
+// bound like any other sequence: uploads, obligations, other keys and other
+// Regions' functions.
+func TestList_ExcludedRecordsCountTowardTheScanBound(t *testing.T) {
+	svc, _ := newTestService(t)
+	mustCreate(t, svc, "first", "f1")
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	for i := range maxListScan + 30 {
+		var key string
+		switch i % 4 {
+		case 0:
+			key = KVFunctionKey("us-east-1", fmt.Sprintf("x-%d", i))
+		case 1:
+			key = fmt.Sprintf("upload.%s.f9.%d", testRegion, i)
+		case 2:
+			key = KVDeletedKey(testRegion, fmt.Sprintf("d%d", i))
+		default:
+			key = fmt.Sprintf("other.%d", i)
+		}
+		_, err := kv.Put(t.Context(), key, []byte("{}"))
+		require.NoError(t, err)
+	}
+	mustCreate(t, svc, "last", "f2")
+
+	names, pages := walkFrom(t, svc, "", MaxListPage)
+	assert.Equal(t, []string{"first", "last"}, names)
+	assert.GreaterOrEqual(t, pages, 2)
+}
+
+// A key rewritten many times leaves gaps behind its latest revision; missing
+// sequences count toward the bound and the walk crosses them.
+func TestList_SparseSequencesCountTowardTheScanBound(t *testing.T) {
+	svc, _ := newTestService(t)
+	mustCreate(t, svc, "first", "f1")
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	for range 2*maxListScan + 7 {
+		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
+		require.NoError(t, err)
+	}
+	mustCreate(t, svc, "middle", "f2")
+	for range maxListScan {
+		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
+		require.NoError(t, err)
+	}
+	mustCreate(t, svc, "last", "f3")
+
+	page := listPage(t, svc, testRegion, "", MaxListPage)
+	require.Len(t, page.Functions, 1)
+	assert.Equal(t, uint64(maxListScan), cursorAfter(t, page.NextMarker))
+	for _, size := range []int{1, MaxListPage} {
+		names, _ := walkFrom(t, svc, "", size)
+		assert.Equal(t, []string{"first", "middle", "last"}, names, "page size %d", size)
+	}
+}
+
+// A large live set is returned once each, in stream order, in pages that
+// never exceed the requested size.
+func TestList_LargeLiveSetPagesEveryFunctionOnce(t *testing.T) {
+	svc, _ := newTestService(t)
+	want := make([]string, 0, 3*MaxListPage+7)
+	for i := range cap(want) {
+		name := fmt.Sprintf("fn-%03d", i)
+		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
+		want = append(want, name)
+	}
+	for _, size := range []int{1, 7, MaxListPage} {
+		names, _ := walkFrom(t, svc, "", size)
+		assert.Equal(t, want, names, "page size %d", size)
+	}
+}
+
+// A full page looking ahead through delete markers to the end of the stream
+// returns no marker; when the scan bound runs out first, it continues and a
+// later page finds the function beyond them.
+func TestList_LookAheadThroughDeleteMarkers(t *testing.T) {
+	t.Run("reaches the end", func(t *testing.T) {
+		svc, _ := newTestService(t)
+		mustCreate(t, svc, "live", "f1")
+		tombstones(t, svc, 3)
+		page := listPage(t, svc, testRegion, "", 1)
+		require.Len(t, page.Functions, 1)
+		assert.Empty(t, page.NextMarker)
+	})
+	t.Run("scan bound exhausted", func(t *testing.T) {
+		svc, _ := newTestService(t)
+		mustCreate(t, svc, "live", "f1")
+		tombstones(t, svc, maxListScan)
+		mustCreate(t, svc, "beyond", "f2")
+		page := listPage(t, svc, testRegion, "", 1)
+		require.Len(t, page.Functions, 1)
+		assert.Equal(t, "live", page.Functions[0].Record.FunctionName)
+		require.NotEmpty(t, page.NextMarker, "the bound ran out before the walk was known to end")
+		names, _ := walkFrom(t, svc, page.NextMarker, 1)
+		assert.Equal(t, []string{"beyond"}, names)
+	})
+}
+
+// An update moves a function to a later sequence, so this walk, begun before
+// the update, meets it again after the function it had not yet reached.
+func TestList_UpdateDuringWalkMovesFunctionLater(t *testing.T) {
+	svc, _ := newTestService(t)
+	first := serveAs(t, svc, "a", "f1")
+	serveAs(t, svc, "b", "f2")
+	page := listPage(t, svc, testRegion, "", 1)
+	require.Equal(t, "a", page.Functions[0].Record.FunctionName)
+	cfg := Configuration{Role: first.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
+	_, err := svc.UpdateConfiguration(t.Context(), &ConfigurationUpdatedRequest{Region: testRegion, FunctionName: "a",
+		FunctionID: "f1", Revision: first.Revision, Configuration: cfg}, testAccount)
+	require.NoError(t, err)
+
+	var rest []string
+	for marker := page.NextMarker; marker != ""; {
+		p := listPage(t, svc, testRegion, marker, 1)
+		for _, f := range p.Functions {
+			rest = append(rest, f.Record.FunctionName)
+		}
+		marker = p.NextMarker
+	}
+	assert.Equal(t, []string{"b", "a"}, rest)
+}
+
+func TestList_RefusesForeignMalformedAndExhaustedMarkers(t *testing.T) {
+	svc, _ := newTestService(t)
+	for i := range 3 {
+		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
+	}
+	marker := listPage(t, svc, testRegion, "", 1).NextMarker
+	require.NotEmpty(t, marker)
+
+	foreign, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: marker, MaxItems: 1}, "000000000002")
+	assert.Nil(t, foreign)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	_, err = svc.List(t.Context(), &ListRequest{Region: "us-east-1", Marker: marker, MaxItems: 1}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	exhausted, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion, After: math.MaxUint64})
+	require.NoError(t, err)
+	zero, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion})
+	require.NoError(t, err)
+	future, err := encodeCursor(listCursor{Version: 2, AccountID: testAccount, Region: testRegion, After: 1})
+	require.NoError(t, err)
+	for _, bad := range []string{"not base64!", "e30", exhausted, zero, future} {
+		_, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: bad, MaxItems: 1}, testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+func TestList_RefusesMalformedRequests(t *testing.T) {
+	svc, _ := newTestService(t)
+	for _, req := range []*ListRequest{
+		{Region: "Bad.Region", MaxItems: 1},
+		{Region: testRegion, MaxItems: 0},
+		{Region: testRegion, MaxItems: MaxListPage + 1},
+	} {
+		_, err := svc.List(t.Context(), req, testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}

spinifex/domains/lambda/function/record.go

@@ -63,6 +63,7 @@ const (
 	KVBucketAccountHistory = 1
 	kvFunctionKeyPrefix    = "fn."
 	kvUploadKeyPrefix      = "upload."
+	kvDeletedKeyPrefix     = "deleted."
 )
 
 // KVUploadKey returns the key of the record that tracks one code upload. The
@@ -111,6 +112,10 @@ const (
 	SubjectCodeUpdated     = "lambda.function.code-updated"
 	// SubjectConfigurationUpdated commits new settings for an Active function.
 	SubjectConfigurationUpdated = "lambda.function.configuration-updated"
+	// SubjectList pages the function records of one Region.
+	SubjectList = "lambda.function.list"
+	// SubjectDelete unbinds a function name and retains its cleanup obligation.
+	SubjectDelete = "lambda.function.delete"
 )
 
 // CreateRequest carries the record the gateway built; the daemon owns the
@@ -218,3 +223,53 @@ type ConfigurationUpdatedResponse struct {
 	Record   Record `json:"record"`
 	Revision uint64 `json:"revision"`
 }
+
+// KVDeletedKey returns the key of the obligation a deleted function leaves
+// behind. A recreated function receives a new random ID, and an obligation
+// whose identity does not match the deleted function is refused.
+func KVDeletedKey(region, functionID string) string {
+	return kvDeletedKeyPrefix + region + "." + functionID
+}
+
+// Deletion is the retained cleanup obligation of a deleted function: its code
+// copies and upload records are kept until a later owner proves them unused.
+// It is written before the name is unbound, so it alone proves no deletion.
+type Deletion struct {
+	FormatVersion int       `json:"formatVersion"`
+	AccountID     string    `json:"accountId"`
+	Region        string    `json:"region"`
+	FunctionName  string    `json:"functionName"`
+	FunctionID    string    `json:"functionId"`
+	RecordedAt    time.Time `json:"recordedAt"`
+}
+
+// ListRequest asks for one page of a Region's functions after Marker, which
+// is empty for the first page or a NextMarker the owner returned.
+type ListRequest struct {
+	Region   string `json:"region"`
+	Marker   string `json:"marker,omitempty"`
+	MaxItems int    `json:"maxItems"`
+}
+
+// ListedFunction is one function record and its revision.
+type ListedFunction struct {
+	Record   Record `json:"record"`
+	Revision uint64 `json:"revision"`
+}
+
+// ListResponse is one page; NextMarker is empty once the walk has ended.
+type ListResponse struct {
+	Functions  []ListedFunction `json:"functions"`
+	NextMarker string           `json:"nextMarker,omitempty"`
+}
+
+// DeleteRequest unbinds the function the caller read at Revision.
+type DeleteRequest struct {
+	Region       string `json:"region"`
+	FunctionName string `json:"functionName"`
+	FunctionID   string `json:"functionId"`
+	Revision     uint64 `json:"revision"`
+}
+
+// DeleteResponse is empty: success means the name was unbound.
+type DeleteResponse struct{}

spinifex/domains/rds/instance/backup_sweep.go

@@ -193,7 +193,7 @@ func (s *Service) loadAutomatedBackups(ctx context.Context, kv *kvstore.Bucket,
 			continue
 		}
 		var snapshot DBSnapshotRecord
-		rev, found, err := getJSONRevision(ctx, kv, DBSnapshotKey(entry.DBSnapshotIdentifier), &snapshot)
+		found, err = getJSON(ctx, kv, DBSnapshotKey(entry.DBSnapshotIdentifier), &snapshot)
 		if err != nil {
 			return nil, err
 		}
@@ -206,7 +206,7 @@ func (s *Service) loadAutomatedBackups(ctx context.Context, kv *kvstore.Bucket,
 		// The snapshot record is the authority on age, not the index key: the key is
 		// a naming convention, and the record is what a restore reads.
 		entry.CreatedAt = snapshot.CreatedAt
-		entries = append(entries, automatedBackup{AutomatedBackupRecord: entry, stamp: stamp, snapshot: &snapshot, revision: rev})
+		entries = append(entries, automatedBackup{AutomatedBackupRecord: entry, stamp: stamp, snapshot: &snapshot})
 	}
 	return entries, nil
 }
@@ -217,7 +217,6 @@ type automatedBackup struct {
 
 	stamp    string
 	snapshot *DBSnapshotRecord
-	revision uint64
 }
 
 // The newest snapshot retention must not remove. Only an available one counts: a
@@ -243,7 +242,7 @@ func newestAvailableBackup(entries []automatedBackup) string {
 // entry stays so it is not lost from the index.
 func (s *Service) deleteAutomatedBackup(ctx context.Context, kv *kvstore.Bucket,
 	accountID string, entry automatedBackup) (bool, error) {
-	err := s.removeDBSnapshot(ctx, kv, accountID, entry.snapshot, entry.revision)
+	err := s.removeDBSnapshot(ctx, kv, accountID, entry.snapshot)
 	switch {
 	// Both readings of "not now": a snapshot an instance restored from it is still
 	// reading, and one still being cut. Neither is a failure — the next pass

spinifex/domains/rds/instance/delete.go

@@ -19,10 +19,6 @@ import (
 
 var errFinalSnapshotInProgress = errors.New("rds: final DB snapshot creation is still in progress")
 
-// errFinalSnapshotFailed holds a deletion in deleting with its data volume kept:
-// the final snapshot's EC2 snapshot ended in error, so the volume is the only copy.
-var errFinalSnapshotFailed = errors.New("rds: the final DB snapshot failed")
-
 // errModifyInProgress defers a teardown until a modify that already holds the
 // lease, and may be growing the data volume, has let go of it.
 var errModifyInProgress = errors.New("rds: a modification of the DB instance is still in progress")
@@ -107,10 +103,8 @@ func (s *Service) DeleteDBInstance(ctx context.Context, input *rds.DeleteDBInsta
 	teardownCtx, cancel := context.WithDeadline(ctx, deleteClaimEnds(rec))
 	defer cancel()
 	if err := s.teardownDBInstance(teardownCtx, kv, accountID, rec, reservation.creator); err != nil {
-		// The reconciler finishes the delete once the modify lets go, or once
-		// the final snapshot settles.
-		if errors.Is(err, errModifyInProgress) || errors.Is(err, errFinalSnapshotInProgress) ||
-			errors.Is(err, errFinalSnapshotFailed) {
+		// The reconciler finishes the delete once the modify lets go.
+		if errors.Is(err, errModifyInProgress) {
 			return &rds.DeleteDBInstanceOutput{DBInstance: s.projectDBInstance(rec)}, nil
 		}
 		return nil, err
@@ -210,9 +204,7 @@ func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, ac
 
 	// Asked to stop cleanly even though the VM is about to go: it is what makes
 	// the final snapshot a clean checkpoint rather than one needing WAL replay.
-	// Once the final snapshot is cut the VM is already gone, so a resumed
-	// teardown waiting on that snapshot has no engine left to stop.
-	if rec.FinalSnapshotIdentifier != "" && !s.finalSnapshotCut(ctx, kv, rec) {
+	if rec.FinalSnapshotIdentifier != "" {
 		s.stopEngineOrRecordFallback(ctx, accountID, rec, "deleting")
 	}
 
@@ -441,28 +433,30 @@ func (s *Service) takeFinalSnapshot(ctx context.Context, kv *kvstore.Bucket, acc
 		return awserrors.Errorf(awserrors.ErrorDBSnapshotAlreadyExists,
 			"DB snapshot %s already exists", rec.FinalSnapshotIdentifier)
 	}
-	switch {
-	case record.Status == SnapshotStatusAvailable && record.SnapshotID != "":
+	if record.Status == SnapshotStatusAvailable && record.SnapshotID != "" {
 		return nil
-	case record.Status == SnapshotStatusFailed:
-		return fmt.Errorf("%w: %s", errFinalSnapshotFailed, rec.FinalSnapshotIdentifier)
-	case record.Status != SnapshotStatusCreating:
+	}
+	if record.Status != SnapshotStatusCreating {
 		return awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
 			"DB snapshot %s is %s; expected %s", rec.FinalSnapshotIdentifier, record.Status, SnapshotStatusCreating)
-	case record.SnapshotID != "":
-		// Already cut and still settling, which reconcileSnapshots records.
-		return errFinalSnapshotInProgress
 	}
 
 	if !creator {
-		snapshotID, state, err := s.findEC2SnapshotFor(ctx, accountID, rec.FinalSnapshotIdentifier)
+		snapshotID, err := s.findEC2SnapshotFor(ctx, accountID, rec.FinalSnapshotIdentifier)
 		if err != nil {
 			return err
 		}
 		if snapshotID == "" {
 			return errFinalSnapshotInProgress
 		}
-		return s.settleFinalSnapshot(ctx, kv, accountID, rec, snapshotID, state)
+		completed, err := s.completeFinalSnapshot(ctx, kv, rec, snapshotID)
+		if err != nil {
+			return err
+		}
+		if completed {
+			s.recordFinalSnapshotCreated(ctx, accountID, rec, snapshotID)
+		}
+		return nil
 	}
 
 	// System-owned, like the volume it is taken from: the customer addresses it
@@ -489,119 +483,68 @@ func (s *Service) takeFinalSnapshot(ctx context.Context, kv *kvstore.Bucket, acc
 		return fmt.Errorf("rds: take the final snapshot of %s: empty snapshot id", rec.DBInstanceIdentifier)
 	}
 	snapshotID := aws.StringValue(snapshot.SnapshotId)
-	if err := s.settleFinalSnapshot(ctx, kv, accountID, rec, snapshotID, aws.StringValue(snapshot.State)); err != nil {
+	completed, err := s.completeFinalSnapshot(ctx, kv, rec, snapshotID)
+	if err != nil {
 		return fmt.Errorf("rds: record the final snapshot of %s: %w", rec.DBInstanceIdentifier, err)
 	}
-	return nil
-}
-
-// Records the final snapshot's EC2 snapshot and its state, reporting a pending
-// one as errFinalSnapshotInProgress and an errored or gone one as errFinalSnapshotFailed.
-func (s *Service) settleFinalSnapshot(ctx context.Context, kv *kvstore.Bucket, accountID string,
-	rec *DBInstanceRecord, snapshotID, state string) error {
-	written, err := s.completeFinalSnapshot(ctx, kv, rec, snapshotID, state)
-	if err != nil {
-		return err
-	}
-	if written != nil {
-		s.recordSnapshotSettled(ctx, accountID, written, state)
-	}
-	switch status, _ := rdsSnapshotStatus(state); status {
-	case SnapshotStatusCreating:
-		return errFinalSnapshotInProgress
-	case SnapshotStatusFailed:
-		return fmt.Errorf("%w: %s", errFinalSnapshotFailed, rec.FinalSnapshotIdentifier)
+	if completed {
+		s.recordFinalSnapshotCreated(ctx, accountID, rec, snapshotID)
 	}
 	return nil
 }
 
-// Whether the final snapshot's record already names its EC2 snapshot. A read
-// that fails answers false, so the engine stop it guards still runs.
-func (s *Service) finalSnapshotCut(ctx context.Context, kv *kvstore.Bucket, rec *DBInstanceRecord) bool {
-	var record DBSnapshotRecord
-	found, err := getJSON(ctx, kv, DBSnapshotKey(rec.FinalSnapshotIdentifier), &record)
-	return err == nil && found && record.SnapshotID != "" && finalSnapshotIsOurs(&record, rec)
-}
-
-// How long a deletion has spent tearing down, which transitionTimeout bounds:
-// from the later of its start and its final snapshot's creation, so a retaken
-// snapshot starts afresh, less the time that snapshot was pending.
-func (s *Service) deletionElapsed(ctx context.Context, kv *kvstore.Bucket, rec *DBInstanceRecord) (time.Duration, error) {
-	start := transitionStarted(rec)
-	if rec.FinalSnapshotIdentifier == "" {
-		return time.Since(start), nil
-	}
-	var record DBSnapshotRecord
-	found, err := getJSON(ctx, kv, DBSnapshotKey(rec.FinalSnapshotIdentifier), &record)
-	if err != nil {
-		return 0, fmt.Errorf("rds: read the final snapshot %s: %w", rec.FinalSnapshotIdentifier, err)
-	}
-	if !found || !finalSnapshotIsOurs(&record, rec) {
-		return time.Since(start), nil
-	}
-	if record.CreatedAt.After(start) {
-		start = record.CreatedAt
-	}
-	settled := time.Now()
-	if record.SettledAt != nil {
-		settled = *record.SettledAt
-	}
-	return time.Since(start) - max(settled.Sub(record.CreatedAt), 0), nil
-}
-
-// Writes only the snapshot fields under CAS, preserving tags changed while the
-// EC2 snapshot was being cut. It returns the record it wrote, nil when the
-// record already said as much.
+// Completes only the snapshot fields under CAS, preserving tags changed while
+// the EC2 snapshot was being cut.
 func (s *Service) completeFinalSnapshot(ctx context.Context, kv *kvstore.Bucket,
-	source *DBInstanceRecord, snapshotID, state string) (*DBSnapshotRecord, error) {
+	source *DBInstanceRecord, snapshotID string) (bool, error) {
 	key := DBSnapshotKey(source.FinalSnapshotIdentifier)
 	for range tagWriteAttempts {
 		var record DBSnapshotRecord
 		rev, found, err := getJSONRevision(ctx, kv, key, &record)
 		if err != nil {
-			return nil, err
+			return false, err
 		}
 		if !found {
-			return nil, fmt.Errorf("final snapshot reservation %s is missing", source.FinalSnapshotIdentifier)
+			return false, fmt.Errorf("final snapshot reservation %s is missing", source.FinalSnapshotIdentifier)
 		}
 		if !finalSnapshotIsOurs(&record, source) {
-			return nil, awserrors.Errorf(awserrors.ErrorDBSnapshotAlreadyExists,
+			return false, awserrors.Errorf(awserrors.ErrorDBSnapshotAlreadyExists,
 				"DB snapshot %s already exists", source.FinalSnapshotIdentifier)
 		}
-		if record.SnapshotID != "" && record.SnapshotID != snapshotID {
-			return nil, fmt.Errorf("final snapshot %s already points to %s, not %s",
-				source.FinalSnapshotIdentifier, record.SnapshotID, snapshotID)
-		}
-		want, known := rdsSnapshotStatus(state)
-		if !known {
-			return nil, fmt.Errorf("final snapshot %s: EC2 snapshot %s reports state %q",
-				source.FinalSnapshotIdentifier, snapshotID, state)
-		}
-		if record.Status == want && record.SnapshotID == snapshotID {
-			return nil, nil
+		if record.Status == SnapshotStatusAvailable {
+			if record.SnapshotID != snapshotID {
+				return false, fmt.Errorf("final snapshot %s already points to %s, not %s",
+					source.FinalSnapshotIdentifier, record.SnapshotID, snapshotID)
+			}
+			return false, nil
 		}
 		if record.Status != SnapshotStatusCreating {
-			return nil, awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
+			return false, awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
 				"DB snapshot %s is %s; expected %s", source.FinalSnapshotIdentifier,
 				record.Status, SnapshotStatusCreating)
 		}
 
 		record.SnapshotID = snapshotID
-		record.Status = want
-		if want != SnapshotStatusCreating {
-			now := time.Now().UTC()
-			record.SettledAt = &now
-		}
+		record.Status = SnapshotStatusAvailable
 		if err := updateJSON(ctx, kv, key, rev, &record); err == nil {
-			return &record, nil
+			return true, nil
 		} else if !errors.Is(err, kvstore.ErrConflict) {
-			return nil, err
+			return false, err
 		}
 	}
-	return nil, fmt.Errorf("rds: completing final snapshot %s contended after %d attempts",
+	return false, fmt.Errorf("rds: completing final snapshot %s contended after %d attempts",
 		source.FinalSnapshotIdentifier, tagWriteAttempts)
 }
 
+func (s *Service) recordFinalSnapshotCreated(ctx context.Context, accountID string,
+	rec *DBInstanceRecord, snapshotID string) {
+	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.FinalSnapshotIdentifier,
+		"Final DB snapshot created.", EventCategoryBackup, EventCategoryCreation)
+	slog.InfoContext(ctx, "rds: final DB snapshot created",
+		"dbSnapshot", rec.FinalSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier,
+		"snapshotId", snapshotID)
+}
+
 // A viperblock snapshot references its source volume's chunk files rather than
 // copying them, so the volume cannot be deleted while any snapshot survives —
 // including the final one just taken. It is retained instead, recorded with the

spinifex/domains/rds/instance/lifecycle_test.go

@@ -130,15 +130,6 @@ type fakeSnapshots struct {
 	// The tags each created snapshot carries, which is what a tag-filtered
 	// describe resolves a DB snapshot identifier through.
 	tagged map[string]map[string]string
-	// The state a snapshot reports; one absent from it is completed.
-	states map[string]string
-}
-
-func (f *fakeSnapshots) stateOf(id string) string {
-	if state, ok := f.states[id]; ok {
-		return state
-	}
-	return ec2.SnapshotStateCompleted
 }
 
 var _ snapshotProvider = (*fakeSnapshots)(nil)
@@ -165,7 +156,7 @@ func (f *fakeSnapshots) CreateSnapshot(ctx context.Context, input *ec2.CreateSna
 	// A snapshot pins the chunks of the volume it was taken from, so taking one
 	// is also what makes that volume undeletable.
 	f.holding = append(f.holding, id)
-	return &ec2.Snapshot{SnapshotId: aws.String(id), VolumeId: input.VolumeId, State: aws.String(f.stateOf(id))}, nil
+	return &ec2.Snapshot{SnapshotId: aws.String(id), VolumeId: input.VolumeId}, nil
 }
 
 // A tag filter is answered from the recorded tags; anything else reports every
@@ -179,7 +170,7 @@ func (f *fakeSnapshots) DescribeSnapshots(_ context.Context, input *ec2.Describe
 		if !f.matchesFilters(id, input) {
 			continue
 		}
-		out.Snapshots = append(out.Snapshots, &ec2.Snapshot{SnapshotId: aws.String(id), State: aws.String(f.stateOf(id))})
+		out.Snapshots = append(out.Snapshots, &ec2.Snapshot{SnapshotId: aws.String(id)})
 	}
 	if f.afterDescribe != nil {
 		f.afterDescribe()

spinifex/domains/rds/instance/pending_snapshot_test.go

@@ -1,667 +0,0 @@
-package instance
-
-import (
-	"context"
-	"errors"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/rds"
-	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const pendingFinalSnapshotID = "orders-db-final"
-
-// ec2Snapshots is the real EC2 snapshot service over the memory provider, so
-// a snapshot can answer pending and settle later as an OCI backup does. It
-// records every snapshot delete it is asked for, and can fail the volume
-// describe a teardown's release reads.
-type ec2Snapshots struct {
-	*snapshot.SnapshotServiceImpl
-
-	mem *ebsprovider.MemoryProvider
-
-	mu          sync.Mutex
-	deleted     []string
-	describeErr error
-	// strictErr fails the strict describe a reconciler reads a snapshot's state by.
-	strictErr error
-}
-
-var _ snapshotProvider = (*ec2Snapshots)(nil)
-
-func (e *ec2Snapshots) DescribeSnapshots(ctx context.Context, input *ec2.DescribeSnapshotsInput, accountID string) (*ec2.DescribeSnapshotsOutput, error) {
-	e.mu.Lock()
-	err := e.describeErr
-	e.mu.Unlock()
-	if err != nil {
-		return nil, err
-	}
-	return e.SnapshotServiceImpl.DescribeSnapshots(ctx, input, accountID)
-}
-
-func (e *ec2Snapshots) DescribeSnapshotsStrict(ctx context.Context, input *ec2.DescribeSnapshotsInput, accountID string) (*ec2.DescribeSnapshotsOutput, error) {
-	e.mu.Lock()
-	err := e.strictErr
-	e.mu.Unlock()
-	if err != nil {
-		return nil, err
-	}
-	return e.SnapshotServiceImpl.DescribeSnapshotsStrict(ctx, input, accountID)
-}
-
-func (e *ec2Snapshots) DeleteSnapshot(ctx context.Context, input *ec2.DeleteSnapshotInput, accountID string) (*ec2.DeleteSnapshotOutput, error) {
-	e.mu.Lock()
-	e.deleted = append(e.deleted, aws.StringValue(input.SnapshotId))
-	e.mu.Unlock()
-	return e.SnapshotServiceImpl.DeleteSnapshot(ctx, input, accountID)
-}
-
-// withEC2Snapshots replaces the harness's snapshot fake with the real service,
-// holding the instance's data volume as the system account's.
-func withEC2Snapshots(t *testing.T, h *lifecycleHarness) *ec2Snapshots {
-	t.Helper()
-	store := objectstore.NewMemoryObjectStore()
-	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
-	require.NoError(t, ebsmetadata.NewStore(store, cfg.Predastore.Bucket).PutVolume(t.Context(), ebsmetadata.Volume{
-		VolumeID: "vol-rdsdata01", TenantID: awsidentifiers.GlobalAccountID, CapacityGiB: 20, State: "in-use",
-		AvailabilityZone: "us-east-1a", ProviderHandle: "memory://volume/vol-rdsdata01",
-	}))
-	mem := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	_, err := mem.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rdsdata01",
-		CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 20 << 30},
-	})
-	require.NoError(t, err)
-	svc := snapshot.NewSnapshotServiceImplWithStore(cfg, store, nil)
-	svc.SetEBSProvider(mem)
-	e := &ec2Snapshots{SnapshotServiceImpl: svc, mem: mem}
-	h.svc.deps.Snapshots = e
-	return e
-}
-
-// settle finishes the provider's backup and lets the EC2 reaper record it.
-func (e *ec2Snapshots) settle(t *testing.T, snapshotID string, state ebsprovider.SnapshotState) {
-	t.Helper()
-	require.NoError(t, e.mem.SettleSnapshot(snapshotID, state))
-	_, err := e.NewPendingSnapshotReaper().Sweep(t.Context())
-	require.NoError(t, err)
-}
-
-func (e *ec2Snapshots) deletes() []string {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	return append([]string(nil), e.deleted...)
-}
-
-func (h *lifecycleHarness) rewriteSnapshotRecord(t *testing.T, id string, edit func(*DBSnapshotRecord)) {
-	t.Helper()
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	rec, found := h.snapshotRecord(t, id)
-	require.True(t, found)
-	edit(&rec)
-	require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(id), &rec))
-}
-
-func eventsOf(t *testing.T, h *lifecycleHarness, sourceType, id string) []string {
-	t.Helper()
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	var ring eventRing
-	found, err := getJSON(t.Context(), kv, EventRingKey(sourceType, id), &ring)
-	require.NoError(t, err)
-	if !found {
-		return nil
-	}
-	messages := make([]string, 0, len(ring.Events))
-	for _, event := range ring.Events {
-		messages = append(messages, event.Message)
-	}
-	return messages
-}
-
-func countContaining(messages []string, part string) int {
-	n := 0
-	for _, m := range messages {
-		if strings.Contains(m, part) {
-			n++
-		}
-	}
-	return n
-}
-
-// A DB snapshot whose EC2 snapshot is pending stays creating, past the resolve
-// timeout and across a reconciler restart, and settles with the EC2 snapshot:
-// completed makes it available, error makes it failed with an event.
-func TestReconciler_SettlesACreatingDBSnapshotFromItsEC2Snapshot(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name string
-		// adopt seeds the record a dead worker left, naming no EC2 snapshot,
-		// and the EC2 snapshot it cut, instead of a CreateDBSnapshot.
-		adopt      bool
-		ambiguous  bool // the provider's first reply is lost after it accepts
-		settleTo   ebsprovider.SnapshotState
-		wantStatus string
-	}{
-		{name: "completed", settleTo: ebsprovider.SnapshotStateCompleted, wantStatus: SnapshotStatusAvailable},
-		{name: "error", settleTo: ebsprovider.SnapshotStateError, wantStatus: SnapshotStatusFailed},
-		{name: "completed after a lost reply", ambiguous: true, settleTo: ebsprovider.SnapshotStateCompleted,
-			wantStatus: SnapshotStatusAvailable},
-		{name: "adopted, completed", adopt: true, settleTo: ebsprovider.SnapshotStateCompleted, wantStatus: SnapshotStatusAvailable},
-		{name: "adopted, error", adopt: true, settleTo: ebsprovider.SnapshotStateError, wantStatus: SnapshotStatusFailed},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			h := newLifecycleHarness(t, false)
-			e := withEC2Snapshots(t, h)
-			e.mem.AnswerSnapshotsPending(true)
-			if tc.ambiguous {
-				e.mem.FailCreateSnapshotsAfterAccept(1, nil)
-			}
-			rec := availableRecord()
-			seedInstance(t, h.svc, rec)
-
-			if tc.adopt {
-				kv, err := h.svc.bucket(t.Context(), testAccountID)
-				require.NoError(t, err)
-				stale := newDBSnapshotRecord(testAccountID, &rec, &validatedSnapshot{DBSnapshotIdentifier: testSnapshotID})
-				stale.CreatedAt = time.Now().UTC().Add(-2 * snapshotResolveTimeout)
-				require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(testSnapshotID), &stale))
-				_, _, _, err = h.svc.snapshotDataVolume(t.Context(), testAccountID, &rec, StatusStopped, testSnapshotID)
-				require.NoError(t, err)
-			} else {
-				out, err := h.svc.CreateDBSnapshot(t.Context(), &rds.CreateDBSnapshotInput{
-					DBSnapshotIdentifier: aws.String(testSnapshotID), DBInstanceIdentifier: aws.String(testDBID),
-				}, testAccountID)
-				require.NoError(t, err)
-				assert.Equal(t, SnapshotStatusCreating, aws.StringValue(out.DBSnapshot.Status))
-				assert.Equal(t, StatusAvailable, h.record(t).Status, "the instance leaves backing-up once CreateSnapshot returns")
-			}
-
-			require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
-			stored, found := h.snapshotRecord(t, testSnapshotID)
-			require.True(t, found)
-			assert.Equal(t, SnapshotStatusCreating, stored.Status)
-			require.NotEmpty(t, stored.SnapshotID, "the record names its EC2 snapshot")
-			assert.Equal(t, tc.adopt, stored.CrashConsistent)
-			ec2ID := stored.SnapshotID
-
-			// Past the resolve timeout, and on a reconciler that never saw the record.
-			h.rewriteSnapshotRecord(t, testSnapshotID, func(r *DBSnapshotRecord) {
-				r.CreatedAt = time.Now().UTC().Add(-2 * snapshotResolveTimeout)
-			})
-			restarted := testReconciler(t, h.svc, "node-b")
-			revisit, err := restarted.reconcileOnce(t.Context())
-			require.NoError(t, err)
-			assert.LessOrEqual(t, revisit, reconcileInterval, "a pending EC2 snapshot is read again every pass")
-			stored, _ = h.snapshotRecord(t, testSnapshotID)
-			assert.Equal(t, SnapshotStatusCreating, stored.Status, "a pending EC2 snapshot keeps the DB snapshot creating")
-			assert.Equal(t, ec2ID, stored.SnapshotID)
-
-			e.settle(t, ec2ID, tc.settleTo)
-			require.NoError(t, onePass(t, restarted))
-			require.NoError(t, onePass(t, restarted))
-
-			stored, found = h.snapshotRecord(t, testSnapshotID)
-			require.True(t, found, "a settled DB snapshot is never withdrawn")
-			assert.Equal(t, tc.wantStatus, stored.Status)
-			assert.Equal(t, ec2ID, stored.SnapshotID)
-			assert.NotNil(t, stored.SettledAt)
-			events := eventsOf(t, h, EventSourceTypeDBSnapshot, testSnapshotID)
-			if tc.wantStatus == SnapshotStatusFailed {
-				assert.Equal(t, 1, countContaining(events, "ended in error"), "one failure event: %v", events)
-			} else {
-				assert.Equal(t, 1, countContaining(events, "DB snapshot created."), "one creation event: %v", events)
-			}
-			assert.Empty(t, e.deletes())
-		})
-	}
-}
-
-// Two lost replies are an EC2 InternalError, and the DB snapshot record that
-// only described that call is discarded rather than left creating.
-func TestCreateDBSnapshot_DiscardsTheRecordWhenEC2AnswersInternalError(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	e := withEC2Snapshots(t, h)
-	e.mem.AnswerSnapshotsPending(true)
-	e.mem.FailCreateSnapshotsAfterAccept(2, nil)
-	seedInstance(t, h.svc, availableRecord())
-
-	_, err := h.svc.CreateDBSnapshot(t.Context(), &rds.CreateDBSnapshotInput{
-		DBSnapshotIdentifier: aws.String(testSnapshotID), DBInstanceIdentifier: aws.String(testDBID),
-	}, testAccountID)
-	require.Error(t, err)
-
-	_, found := h.snapshotRecord(t, testSnapshotID)
-	assert.False(t, found)
-	assert.Equal(t, StatusAvailable, h.record(t).Status)
-}
-
-// startPendingFinalSnapshot deletes the instance with a final snapshot the
-// provider answers pending, then backdates the delete past its handler's claim
-// and transitionTimeout, as a backup that outlasts both does.
-func startPendingFinalSnapshot(t *testing.T, h *lifecycleHarness, e *ec2Snapshots, ago time.Duration) string {
-	t.Helper()
-	e.mem.AnswerSnapshotsPending(true)
-	seedInstance(t, h.svc, availableRecord())
-
-	out, err := h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
-		DBInstanceIdentifier: aws.String(testDBID), FinalDBSnapshotIdentifier: aws.String(pendingFinalSnapshotID),
-	}, testAccountID)
-	require.NoError(t, err, "a pending final snapshot is not a failed delete")
-	assert.Equal(t, string(StatusDeleting), aws.StringValue(out.DBInstance.DBInstanceStatus))
-
-	rec := h.record(t)
-	started := time.Now().UTC().Add(-ago)
-	rec.TransitionStartedAt = &started
-	seedInstance(t, h.svc, rec)
-	h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(r *DBSnapshotRecord) { r.CreatedAt = started.Add(time.Minute) })
-
-	final, found := h.snapshotRecord(t, pendingFinalSnapshotID)
-	require.True(t, found)
-	assert.Equal(t, SnapshotStatusCreating, final.Status)
-	require.NotEmpty(t, final.SnapshotID)
-	return final.SnapshotID
-}
-
-// A deletion stays in deleting while its final snapshot is pending, however
-// long that takes, and the wait does not count toward transitionTimeout once
-// the snapshot completes; the time after it still does.
-func TestReconciler_HoldsADeletionWhileItsFinalSnapshotIsPending(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	e := withEC2Snapshots(t, h)
-	ec2ID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
-	stopsBefore := len(h.agent.received())
-
-	r := testReconciler(t, h.svc, "node-a")
-	require.NoError(t, onePass(t, r))
-	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-b")))
-
-	assert.Equal(t, StatusDeleting, h.record(t).Status, "transitionTimeout does not run while the final snapshot is pending")
-	assert.Len(t, h.agent.received(), stopsBefore, "a teardown waiting on its cut snapshot does not stop the engine again")
-	assert.Empty(t, h.volumes.deleted)
-
-	// Completes, but the volume release then fails: the three hours waited do
-	// not count, so the deletion is retried rather than failed.
-	e.mu.Lock()
-	e.describeErr = errors.New("snapshot metadata temporarily unavailable")
-	e.mu.Unlock()
-	e.settle(t, ec2ID, ebsprovider.SnapshotStateCompleted)
-	require.NoError(t, onePass(t, r))
-
-	final, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
-	assert.Equal(t, SnapshotStatusAvailable, final.Status)
-	assert.Equal(t, StatusDeleting, h.record(t).Status, "the wait on the final snapshot is excluded from transitionTimeout")
-
-	// Still failing transitionTimeout after it settled, the deletion fails.
-	h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(r *DBSnapshotRecord) {
-		settled := r.SettledAt.Add(-transitionTimeout - time.Minute)
-		r.SettledAt = &settled
-	})
-	require.NoError(t, onePass(t, r))
-	assert.Equal(t, StatusFailed, h.record(t).Status)
-	assert.Empty(t, h.volumes.deleted)
-
-	// Once the release can be read, the retained volume is recorded and the
-	// instance goes, as for a final snapshot that completed at once.
-	e.mu.Lock()
-	e.describeErr = nil
-	e.mu.Unlock()
-	rec := h.record(t)
-	rec.Status = StatusDeleting
-	now := time.Now().UTC().Add(-deleteTimeout - time.Minute)
-	rec.TransitionStartedAt = &now
-	seedInstance(t, h.svc, rec)
-	require.NoError(t, onePass(t, r))
-	assert.False(t, h.recordExists(t, testDBID))
-	retained, found := h.retainedVolume(t, "vol-rdsdata01")
-	require.True(t, found)
-	assert.Equal(t, []string{ec2ID}, retained.Snapshots)
-	assert.Empty(t, h.volumes.deleted)
-	assert.Equal(t, 1, countContaining(eventsOf(t, h, EventSourceTypeDBSnapshot, pendingFinalSnapshotID), "Final DB snapshot created."))
-}
-
-// A pending final snapshot whose state cannot be read for a while does not
-// fail the deletion past transitionTimeout: the time it has been pending is
-// excluded before it settles, and the deletion finishes once it completes.
-func TestReconciler_PendingFinalSnapshotReadFailureDoesNotFailTheDeletion(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	e := withEC2Snapshots(t, h)
-	ec2ID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
-	r := testReconciler(t, h.svc, "node-a")
-
-	e.mu.Lock()
-	e.strictErr = errors.New("snapshot metadata temporarily unavailable")
-	e.mu.Unlock()
-	require.ErrorContains(t, onePass(t, r), "temporarily unavailable", "the creating DB snapshot's settle reports the read")
-	assert.Equal(t, StatusDeleting, h.record(t).Status, "a pending backup's wait is excluded even when its state cannot be read")
-	assert.Empty(t, h.volumes.deleted)
-
-	e.mu.Lock()
-	e.strictErr = nil
-	e.mu.Unlock()
-	require.NoError(t, onePass(t, r))
-	assert.Equal(t, StatusDeleting, h.record(t).Status, "still pending, so still waiting")
-
-	// One pass records the snapshot available, and the next finishes the deletion.
-	e.settle(t, ec2ID, ebsprovider.SnapshotStateCompleted)
-	require.NoError(t, onePass(t, r))
-	require.NoError(t, onePass(t, r))
-	assert.False(t, h.recordExists(t, testDBID), "the deletion recovers once the backup completes")
-	final, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
-	assert.Equal(t, SnapshotStatusAvailable, final.Status)
-}
-
-// A final snapshot whose EC2 snapshot errors or is gone keeps the instance in
-// deleting, with one event naming the failed snapshot, however many passes follow.
-func TestReconciler_KeepsADeletionWhoseFinalSnapshotFailed(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name    string
-		breakIt func(t *testing.T, e *ec2Snapshots, ec2ID string)
-	}{
-		{name: "ended in error", breakIt: func(t *testing.T, e *ec2Snapshots, ec2ID string) {
-			e.settle(t, ec2ID, ebsprovider.SnapshotStateError)
-		}},
-		{name: "gone", breakIt: func(t *testing.T, e *ec2Snapshots, ec2ID string) {
-			require.NoError(t, e.MetadataStore().DeleteSnapshot(t.Context(), awsidentifiers.GlobalAccountID, ec2ID))
-		}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			h := newLifecycleHarness(t, false)
-			e := withEC2Snapshots(t, h)
-			ec2ID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
-
-			tc.breakIt(t, e, ec2ID)
-			r := testReconciler(t, h.svc, "node-a")
-			for range 3 {
-				require.NoError(t, onePass(t, r))
-			}
-
-			assert.Equal(t, StatusDeleting, h.record(t).Status)
-			final, found := h.snapshotRecord(t, pendingFinalSnapshotID)
-			require.True(t, found)
-			assert.Equal(t, SnapshotStatusFailed, final.Status)
-			instanceEvents := eventsOf(t, h, EventSourceTypeDBInstance, testDBID)
-			assert.Equal(t, 1, countContaining(instanceEvents, "The final DB snapshot "+pendingFinalSnapshotID+" failed"), "%v", instanceEvents)
-			assert.Equal(t, 1, countContaining(eventsOf(t, h, EventSourceTypeDBSnapshot, pendingFinalSnapshotID), ec2ID))
-			assert.Empty(t, h.volumes.deleted, "the data volume is the only copy left")
-			_, retained := h.retainedVolume(t, "vol-rdsdata01")
-			assert.False(t, retained, "the teardown stops before it decides the volume's fate")
-
-			// A retried DeleteDBInstance reports the deletion and changes nothing.
-			_, err := h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
-				DBInstanceIdentifier: aws.String(testDBID), FinalDBSnapshotIdentifier: aws.String(pendingFinalSnapshotID),
-			}, testAccountID)
-			require.NoError(t, err)
-			assert.Empty(t, h.volumes.deleted)
-		})
-	}
-}
-
-// Verifies: ADR-0003:S7.
-//
-// TestS7_RDSDeletionNeverDeletesTheDataVolumeWithoutACompletedFinalSnapshot
-// drives a deletion whose final snapshot is pending or errored through every
-// pass past every timeout, and requires no volume or snapshot was deleted.
-func TestS7_RDSDeletionNeverDeletesTheDataVolumeWithoutACompletedFinalSnapshot(t *testing.T) {
-	t.Parallel()
-	for _, settleTo := range []ebsprovider.SnapshotState{"", ebsprovider.SnapshotStateError} {
-		name := "pending"
-		if settleTo != "" {
-			name = string(settleTo)
-		}
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-			h := newLifecycleHarness(t, false)
-			e := withEC2Snapshots(t, h)
-			ec2ID := startPendingFinalSnapshot(t, h, e, 24*time.Hour)
-			if settleTo != "" {
-				e.settle(t, ec2ID, settleTo)
-			}
-			for _, holder := range []string{"node-a", "node-b", "node-a"} {
-				require.NoError(t, onePass(t, testReconciler(t, h.svc, holder)))
-			}
-
-			assert.Empty(t, h.volumes.deleted, "ADR-0003:S7: the data volume was deleted with no completed final snapshot")
-			assert.Empty(t, e.deletes(), "ADR-0003:S7: an EC2 snapshot was deleted")
-			_, err := e.mem.GetVolume(t.Context(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rdsdata01"})
-			assert.NoError(t, err, "ADR-0003:S7: the data volume is gone at the provider")
-			_, held := e.mem.Snapshot(ec2ID)
-			assert.True(t, held, "ADR-0003:S7: the final snapshot is gone at the provider")
-			assert.True(t, h.recordExists(t, testDBID), "ADR-0003:S7: the DB instance record is gone")
-		})
-	}
-}
-
-// A creating record whose EC2 snapshot is gone fails with an event, once: no
-// pass can make that snapshot again, so creating would wait on it forever.
-func TestReconciler_FailsACreatingDBSnapshotWhoseEC2SnapshotIsGone(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	rec := availableRecord()
-	named := newDBSnapshotRecord(testAccountID, &rec, &validatedSnapshot{DBSnapshotIdentifier: testSnapshotID})
-	named.SnapshotID = "snap-named01"
-	require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(testSnapshotID), &named))
-
-	r := testReconciler(t, h.svc, "node-a")
-	for range 3 {
-		require.NoError(t, onePass(t, r))
-	}
-
-	stored, found := h.snapshotRecord(t, testSnapshotID)
-	require.True(t, found, "a failed DB snapshot is kept for the customer to see and delete")
-	assert.Equal(t, SnapshotStatusFailed, stored.Status)
-	assert.NotNil(t, stored.SettledAt)
-	events := eventsOf(t, h, EventSourceTypeDBSnapshot, testSnapshotID)
-	assert.Equal(t, 1, countContaining(events, "snap-named01 no longer exists"), "one failure event: %v", events)
-}
-
-// A creating record naming an EC2 snapshot that reports no state, or cannot
-// be read, is left creating: neither says the data is restorable or lost.
-func TestReconciler_LeavesACreatingDBSnapshotItCannotSettle(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name    string
-		setup   func(f *fakeSnapshots)
-		wantErr bool
-	}{
-		{name: "the EC2 snapshot reports no state", setup: func(f *fakeSnapshots) {
-			f.holding = []string{"snap-named01"}
-			f.states = map[string]string{"snap-named01": ""}
-		}},
-		{name: "the EC2 snapshot cannot be read", setup: func(f *fakeSnapshots) {
-			f.describeErr = errors.New("snapshot metadata temporarily unavailable")
-		}, wantErr: true},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			h := newLifecycleHarness(t, false)
-			if tc.setup != nil {
-				tc.setup(h.snaps)
-			}
-			kv, err := h.svc.bucket(t.Context(), testAccountID)
-			require.NoError(t, err)
-			rec := availableRecord()
-			named := newDBSnapshotRecord(testAccountID, &rec, &validatedSnapshot{DBSnapshotIdentifier: testSnapshotID})
-			named.SnapshotID = "snap-named01"
-			named.CreatedAt = time.Now().UTC().Add(-2 * snapshotResolveTimeout)
-			require.NoError(t, putJSON(t.Context(), kv, DBSnapshotKey(testSnapshotID), &named))
-
-			err = onePass(t, testReconciler(t, h.svc, "node-a"))
-			if tc.wantErr {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-			stored, found := h.snapshotRecord(t, testSnapshotID)
-			require.True(t, found)
-			assert.Equal(t, SnapshotStatusCreating, stored.Status)
-			assert.Equal(t, "snap-named01", stored.SnapshotID)
-		})
-	}
-}
-
-// The state the creating handler's own CreateSnapshot answers decides the
-// final snapshot: error holds the deletion with the volume kept, and a state
-// it cannot read fails the request with the volume kept.
-func TestDeleteDBInstance_FinalSnapshotStateAtCreate(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name       string
-		state      string
-		wantErr    bool
-		wantStatus string
-	}{
-		{name: "error", state: ec2.SnapshotStateError, wantStatus: SnapshotStatusFailed},
-		{name: "no state", state: "", wantErr: true, wantStatus: SnapshotStatusCreating},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			h := newLifecycleHarness(t, false)
-			h.snaps.states = map[string]string{"snap-0001": tc.state}
-			seedInstance(t, h.svc, availableRecord())
-
-			_, err := h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
-				DBInstanceIdentifier: aws.String(testDBID), FinalDBSnapshotIdentifier: aws.String(pendingFinalSnapshotID),
-			}, testAccountID)
-
-			if tc.wantErr {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-			final, found := h.snapshotRecord(t, pendingFinalSnapshotID)
-			require.True(t, found)
-			assert.Equal(t, tc.wantStatus, final.Status)
-			assert.Equal(t, StatusDeleting, h.record(t).Status)
-			assert.Empty(t, h.volumes.deleted)
-
-			if tc.wantStatus == SnapshotStatusCreating {
-				rec := h.record(t)
-				h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(r *DBSnapshotRecord) {
-					r.CreatedAt = time.Now().UTC().Add(-2 * time.Hour)
-				})
-				elapsed, err := h.svc.deletionElapsed(t.Context(), mustBucket(t, h), &rec)
-				require.NoError(t, err)
-				assert.Less(t, elapsed, time.Minute, "an unsettled final snapshot excludes the time since it was reserved")
-			}
-		})
-	}
-}
-
-func mustBucket(t *testing.T, h *lifecycleHarness) *kvstore.Bucket {
-	t.Helper()
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	return kv
-}
-
-// A failed final snapshot can be deleted, as on AWS, and the deletion then
-// takes it again: the data volume stays until a final snapshot is available,
-// and the failed backup's hours do not count against the retake's timeout.
-func TestDeleteDBSnapshot_AFailedFinalSnapshotIsDeletedAndTakenAgain(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	e := withEC2Snapshots(t, h)
-	failedID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
-	del := &rds.DeleteDBSnapshotInput{DBSnapshotIdentifier: aws.String(pendingFinalSnapshotID)}
-	r := testReconciler(t, h.svc, "node-a")
-
-	_, err := h.svc.DeleteDBSnapshot(t.Context(), del, testAccountID)
-	require.True(t, awserrors.IsErrorCode(err, awserrors.ErrorDBSnapshotInvalidState), "a creating DB snapshot is refused: %v", err)
-
-	e.settle(t, failedID, ebsprovider.SnapshotStateError)
-	require.NoError(t, onePass(t, r))
-	failed, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
-	require.Equal(t, SnapshotStatusFailed, failed.Status)
-
-	_, err = h.svc.DeleteDBSnapshot(t.Context(), del, testAccountID)
-	require.NoError(t, err)
-	assert.Contains(t, e.deletes(), failedID)
-
-	require.NoError(t, onePass(t, r))
-	retake, found := h.snapshotRecord(t, pendingFinalSnapshotID)
-	require.True(t, found, "the deletion reserves its final snapshot again")
-	assert.Equal(t, SnapshotStatusCreating, retake.Status)
-	require.NotEmpty(t, retake.SnapshotID)
-	assert.NotEqual(t, failedID, retake.SnapshotID)
-	assert.Empty(t, h.volumes.deleted)
-
-	// Completes, but the volume release then fails: the retake starts its own
-	// timeout, so the deletion is retried rather than failed.
-	e.mu.Lock()
-	e.describeErr = errors.New("snapshot metadata temporarily unavailable")
-	e.mu.Unlock()
-	e.settle(t, retake.SnapshotID, ebsprovider.SnapshotStateCompleted)
-	require.NoError(t, onePass(t, r))
-	require.NoError(t, onePass(t, r))
-	final, _ := h.snapshotRecord(t, pendingFinalSnapshotID)
-	require.Equal(t, SnapshotStatusAvailable, final.Status)
-	assert.Equal(t, StatusDeleting, h.record(t).Status, "the failed backup's wait was counted against the retake")
-	assert.Empty(t, h.volumes.deleted)
-
-	e.mu.Lock()
-	e.describeErr = nil
-	e.mu.Unlock()
-	require.NoError(t, onePass(t, r))
-	assert.False(t, h.recordExists(t, testDBID))
-}
-
-// A delete removes only the DB snapshot it read: the same snapshot rewritten
-// since, by a tag write, is still deleted, and a delayed delete of a failed
-// final snapshot leaves the reservation the deletion has since retaken.
-func TestRemoveDBSnapshot_DeletesOnlyTheSnapshotItRead(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	e := withEC2Snapshots(t, h)
-	failedID := startPendingFinalSnapshot(t, h, e, 3*time.Hour)
-	e.settle(t, failedID, ebsprovider.SnapshotStateError)
-	r := testReconciler(t, h.svc, "node-a")
-	require.NoError(t, onePass(t, r))
-	kv := mustBucket(t, h)
-
-	stale, staleRev, err := h.svc.getDBSnapshot(t.Context(), kv, pendingFinalSnapshotID)
-	require.NoError(t, err)
-	require.Equal(t, SnapshotStatusFailed, stale.Status)
-	h.rewriteSnapshotRecord(t, pendingFinalSnapshotID, func(*DBSnapshotRecord) {})
-	require.NoError(t, h.svc.removeDBSnapshot(t.Context(), kv, testAccountID, stale, staleRev))
-	_, found := h.snapshotRecord(t, pendingFinalSnapshotID)
-	require.False(t, found, "a rewrite of the same snapshot does not stop its delete")
-
-	require.NoError(t, onePass(t, r))
-	retake, found := h.snapshotRecord(t, pendingFinalSnapshotID)
-	require.True(t, found)
-
-	err = h.svc.removeDBSnapshot(t.Context(), kv, testAccountID, stale, staleRev)
-	require.True(t, awserrors.IsErrorCode(err, awserrors.ErrorDBSnapshotInvalidState), "%v", err)
-	kept, found := h.snapshotRecord(t, pendingFinalSnapshotID)
-	require.True(t, found, "a delayed delete removed the retaken reservation")
-	assert.Equal(t, retake.SnapshotID, kept.SnapshotID)
-	assert.Empty(t, h.volumes.deleted)
-}

spinifex/domains/rds/instance/reconciler.go

@@ -597,19 +597,15 @@ func (r *Reconciler) reconcileDeleting(ctx context.Context, kv *kvstore.Bucket,
 		return nil
 	}
 	err := r.svc.teardownDBInstance(ctx, kv, accountID, rec, false)
-	// A failed final snapshot keeps the instance, and its data volume, in
-	// deleting: failing the delete would let a retry skip the only copy.
-	if err == nil || errors.Is(err, errFinalSnapshotInProgress) || errors.Is(err, errFinalSnapshotFailed) ||
-		errors.Is(err, errModifyInProgress) {
+	if err == nil || errors.Is(err, errFinalSnapshotInProgress) || errors.Is(err, errModifyInProgress) {
 		return nil
 	}
-	elapsed, waitErr := r.svc.deletionElapsed(ctx, kv, rec)
-	if waitErr == nil && elapsed > transitionTimeout {
+	if time.Since(transitionStarted(rec)) > transitionTimeout {
 		return r.transition(ctx, kv, rev, rec, StatusFailed,
 			fmt.Sprintf("the DB instance could not be deleted within %s: %v", transitionTimeout, err))
 	}
 	slog.WarnContext(ctx, "rds reconciler: resuming a delete failed; retrying next pass",
-		"dbInstance", rec.DBInstanceIdentifier, "err", errors.Join(err, waitErr))
+		"dbInstance", rec.DBInstanceIdentifier, "err", err)
 	return nil
 }
 
@@ -679,15 +675,6 @@ func (r *Reconciler) reconcileSnapshots(ctx context.Context, kv *kvstore.Bucket,
 		if !found || rec.Status != SnapshotStatusCreating {
 			continue
 		}
-		// A record naming its EC2 snapshot waits on that snapshot settling, which
-		// no KV write announces, so it is read every pass.
-		if rec.SnapshotID != "" {
-			revisit = reconciler.Earliest(revisit, reconcileInterval)
-			if err := r.settleCreatingSnapshot(ctx, kv, rev, accountID, &rec); err != nil {
-				failures = append(failures, fmt.Errorf("settle DB snapshot %s: %w", id, err))
-			}
-			continue
-		}
 		// Not due yet, so ask to be back when it is rather than leaving a dead
 		// worker's record to the resync.
 		if due := time.Until(rec.CreatedAt.Add(snapshotResolveTimeout)); due > 0 {
@@ -706,7 +693,7 @@ func (r *Reconciler) reconcileSnapshots(ctx context.Context, kv *kvstore.Bucket,
 // the name forever while naming nothing a customer can restore.
 func (r *Reconciler) resolveCreatingSnapshot(ctx context.Context, kv *kvstore.Bucket, rev uint64,
 	accountID string, rec *DBSnapshotRecord) error {
-	snapshotID, state, err := r.svc.findEC2SnapshotFor(ctx, accountID, rec.DBSnapshotIdentifier)
+	snapshotID, err := r.svc.findEC2SnapshotFor(ctx, accountID, rec.DBSnapshotIdentifier)
 	if err != nil {
 		return err
 	}
@@ -729,146 +716,30 @@ func (r *Reconciler) resolveCreatingSnapshot(ctx context.Context, kv *kvstore.Bu
 	}
 
 	rec.SnapshotID = snapshotID
+	rec.Status = SnapshotStatusAvailable
 	// The worker died before it could report whether the quiesce held, so the
 	// conservative reading is the one that never overstates the snapshot.
 	rec.CrashConsistent = true
-	slog.InfoContext(ctx, "rds reconciler: adopted the EC2 snapshot of an unfinished DB snapshot",
-		"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", snapshotID, "accountId", accountID, "state", state)
-	return r.svc.applyEC2SnapshotState(ctx, kv, rev, accountID, rec, state, true)
-}
-
-// Settles a creating record that names its EC2 snapshot from that snapshot's
-// state; a pending one is left for the next pass.
-func (r *Reconciler) settleCreatingSnapshot(ctx context.Context, kv *kvstore.Bucket, rev uint64,
-	accountID string, rec *DBSnapshotRecord) error {
-	state, err := r.svc.ec2SnapshotState(ctx, rec.SnapshotID)
-	if err != nil {
-		return err
-	}
-	return r.svc.applyEC2SnapshotState(ctx, kv, rev, accountID, rec, state, false)
-}
-
-// ec2SnapshotGone is the state ec2SnapshotState reports for an EC2 snapshot
-// whose document is gone. Nothing recreates one, so it settles the record.
-const ec2SnapshotGone = "gone"
-
-// rdsSnapshotStatus is the DB snapshot status an EC2 snapshot's state gives:
-// pending is creating, completed available, and error or gone failed. known is
-// false for any other state, which says neither restorable nor lost.
-func rdsSnapshotStatus(ec2State string) (status string, known bool) {
-	switch ec2State {
-	case ec2.SnapshotStatePending:
-		return SnapshotStatusCreating, true
-	case ec2.SnapshotStateCompleted:
-		return SnapshotStatusAvailable, true
-	case ec2.SnapshotStateError, ec2SnapshotGone:
-		return SnapshotStatusFailed, true
-	}
-	return "", false
-}
-
-// Moves a creating record, read at rev, to what its EC2 snapshot's state says,
-// writing a pending one only when adopted says the record has changed. A state
-// rdsSnapshotStatus does not know leaves it.
-func (s *Service) applyEC2SnapshotState(ctx context.Context, kv *kvstore.Bucket, rev uint64,
-	accountID string, rec *DBSnapshotRecord, state string, adopted bool) error {
-	status, known := rdsSnapshotStatus(state)
-	switch {
-	case !known:
-		slog.WarnContext(ctx, "rds reconciler: the EC2 snapshot of a creating DB snapshot reports an unknown state; leaving it creating",
-			"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", rec.SnapshotID, "accountId", accountID, "state", state)
-		return nil
-	case status == SnapshotStatusCreating:
-		if !adopted {
-			return nil
-		}
-	default:
-		now := time.Now().UTC()
-		rec.Status = status
-		rec.SettledAt = &now
-	}
 	if err := updateJSON(ctx, kv, DBSnapshotKey(rec.DBSnapshotIdentifier), rev, rec); err != nil {
 		if errors.Is(err, kvstore.ErrConflict) {
 			return nil
 		}
 		return err
 	}
-	s.recordSnapshotSettled(ctx, accountID, rec, state)
+	r.svc.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.DBSnapshotIdentifier,
+		"DB snapshot created.", EventCategoryBackup, EventCategoryCreation)
+	slog.InfoContext(ctx, "rds reconciler: adopted the EC2 snapshot of an unfinished DB snapshot",
+		"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", snapshotID, "accountId", accountID)
 	return nil
 }
 
-// Reports a DB snapshot record just written available or failed; one still
-// creating reports nothing. Every settlement of a DB snapshot comes through here.
-func (s *Service) recordSnapshotSettled(ctx context.Context, accountID string, rec *DBSnapshotRecord, state string) {
-	switch rec.Status {
-	case SnapshotStatusAvailable:
-		message := "DB snapshot created."
-		if rec.FinalSnapshot {
-			message = "Final DB snapshot created."
-		}
-		s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.DBSnapshotIdentifier,
-			message, EventCategoryBackup, EventCategoryCreation)
-		slog.InfoContext(ctx, "rds: DB snapshot created",
-			"dbSnapshot", rec.DBSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier, "snapshotId", rec.SnapshotID,
-			"snapshotType", rec.SnapshotType, "crashConsistent", rec.CrashConsistent, "finalSnapshot", rec.FinalSnapshot)
-	case SnapshotStatusFailed:
-		s.recordDBSnapshotFailed(ctx, accountID, rec, state)
-	}
-}
-
-// Reports a DB snapshot whose EC2 snapshot settled in error or is gone, and
-// for a final snapshot the DB instance its deletion now holds in deleting.
-func (s *Service) recordDBSnapshotFailed(ctx context.Context, accountID string, rec *DBSnapshotRecord, state string) {
-	why := " ended in error."
-	if state == ec2SnapshotGone {
-		why = " no longer exists."
-	}
-	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, rec.DBSnapshotIdentifier,
-		"The DB snapshot failed: its EC2 snapshot "+rec.SnapshotID+why,
-		EventCategoryFailure, EventCategoryBackup)
-	if rec.FinalSnapshot {
-		s.RecordEvent(ctx, accountID, EventSourceTypeDBInstance, rec.DBInstanceIdentifier,
-			"The final DB snapshot "+rec.DBSnapshotIdentifier+" failed; the DB instance stays in deleting and its data volume is kept.",
-			EventCategoryFailure, EventCategoryDeletion)
-	}
-	slog.WarnContext(ctx, "rds: a DB snapshot's EC2 snapshot ended in error or is gone",
-		"dbSnapshot", rec.DBSnapshotIdentifier, "snapshotId", rec.SnapshotID, "accountId", accountID,
-		"finalSnapshot", rec.FinalSnapshot, "state", state)
-}
-
-// The state of an EC2 snapshot, ec2SnapshotGone when it is gone. The strict
-// describe fails on unreadable metadata, so a partial read is never absence.
-func (s *Service) ec2SnapshotState(ctx context.Context, snapshotID string) (string, error) {
+// The EC2 snapshot an account-scoped DB snapshot identifier owns, empty when
+// none was cut. Strict lookup is required because a partial metadata scan cannot
+// prove absence and must never cause reconciliation to withdraw the RDS record.
+func (s *Service) findEC2SnapshotFor(ctx context.Context, accountID, dbSnapshotIdentifier string) (string, error) {
 	if s.deps.Snapshots == nil {
 		return "", errors.New("rds: no snapshot service configured")
 	}
-	out, err := s.deps.Snapshots.DescribeSnapshotsStrict(ctx, &ec2.DescribeSnapshotsInput{
-		SnapshotIds: aws.StringSlice([]string{snapshotID}),
-	}, awsidentifiers.GlobalAccountID)
-	if err != nil {
-		if awserrors.IsNotFound(err) {
-			return ec2SnapshotGone, nil
-		}
-		return "", fmt.Errorf("rds: read the EC2 snapshot %s: %w", snapshotID, err)
-	}
-	if out == nil {
-		return ec2SnapshotGone, nil
-	}
-	for _, snapshot := range out.Snapshots {
-		if aws.StringValue(snapshot.SnapshotId) == snapshotID {
-			return aws.StringValue(snapshot.State), nil
-		}
-	}
-	return ec2SnapshotGone, nil
-}
-
-// The EC2 snapshot an account-scoped DB snapshot identifier owns and its state,
-// empty when none was cut. Strict lookup is required because a partial metadata
-// scan cannot prove absence and must never cause reconciliation to withdraw the RDS record.
-func (s *Service) findEC2SnapshotFor(ctx context.Context, accountID, dbSnapshotIdentifier string) (snapshotID, state string, err error) {
-	if s.deps.Snapshots == nil {
-		return "", "", errors.New("rds: no snapshot service configured")
-	}
 	out, err := s.deps.Snapshots.DescribeSnapshotsStrict(ctx, &ec2.DescribeSnapshotsInput{
 		Filters: []*ec2.Filter{
 			{
@@ -883,28 +754,28 @@ func (s *Service) findEC2SnapshotFor(ctx context.Context, accountID, dbSnapshotI
 	}, awsidentifiers.GlobalAccountID)
 	if err != nil {
 		if awserrors.IsNotFound(err) {
-			return "", "", nil
+			return "", nil
 		}
-		return "", "", fmt.Errorf("rds: find the EC2 snapshot of %s in account %s: %w",
+		return "", fmt.Errorf("rds: find the EC2 snapshot of %s in account %s: %w",
 			dbSnapshotIdentifier, accountID, err)
 	}
 	if out == nil {
-		return "", "", nil
+		return "", nil
 	}
 
-	var found, foundState string
+	var found string
 	for _, snapshot := range out.Snapshots {
 		id := aws.StringValue(snapshot.SnapshotId)
 		if id == "" {
 			continue
 		}
 		if found != "" {
-			return "", "", fmt.Errorf("rds: DB snapshot %s in account %s has multiple EC2 snapshots (%s and %s)",
+			return "", fmt.Errorf("rds: DB snapshot %s in account %s has multiple EC2 snapshots (%s and %s)",
 				dbSnapshotIdentifier, accountID, found, id)
 		}
-		found, foundState = id, aws.StringValue(snapshot.State)
+		found = id
 	}
-	return found, foundState, nil
+	return found, nil
 }
 
 // When the transition began. A record written by an older control plane carries

spinifex/domains/rds/instance/records.go

@@ -271,8 +271,6 @@ const (
 	// in between leaves a reconcilable trace rather than an orphaned EC2 snapshot.
 	SnapshotStatusCreating  = "creating"
 	SnapshotStatusAvailable = "available"
-	// The EC2 snapshot settled in error, so the record holds no restorable data.
-	SnapshotStatusFailed = "failed"
 )
 
 // SnapshotOperation is the snapshot operation holding a DB instance, written under the same CAS that
@@ -336,9 +334,6 @@ type DBSnapshotRecord struct {
 	Tags map[string]string `json:"tags,omitempty"`
 
 	CreatedAt time.Time `json:"createdAt"`
-	// When the EC2 snapshot settled to available or failed. A deletion waiting
-	// on its final snapshot excludes the wait between this and CreatedAt.
-	SettledAt *time.Time `json:"settledAt,omitempty"`
 }
 
 var _ TaggedRecord = (*DBSnapshotRecord)(nil)

spinifex/domains/rds/instance/snapshot.go

@@ -114,7 +114,7 @@ func (s *Service) snapshotDBInstance(ctx context.Context, kv *kvstore.Bucket, re
 		return nil, err
 	}
 
-	snapshotID, state, crashConsistent, err := s.snapshotDataVolume(ctx, accountID, rec, resume, req.DBSnapshotIdentifier)
+	snapshotID, crashConsistent, err := s.snapshotDataVolume(ctx, accountID, rec, resume, req.DBSnapshotIdentifier)
 	if err != nil {
 		// The record only ever described a snapshot that now does not exist, so it
 		// goes with it rather than being left for the reconciler to puzzle over.
@@ -123,35 +123,26 @@ func (s *Service) snapshotDBInstance(ctx context.Context, kv *kvstore.Bucket, re
 	}
 
 	record.SnapshotID = snapshotID
-	record.CrashConsistent = crashConsistent
-	// A pending EC2 snapshot's contents are already fixed, so the quiesce is
-	// over; the record stays creating until the reconciler sees it settle.
-	if status, _ := rdsSnapshotStatus(state); status == SnapshotStatusCreating {
-		if err := putJSON(ctx, kv, key, &record); err != nil {
-			return nil, err
-		}
-		slog.InfoContext(ctx, "rds: DB snapshot cut; waiting for its EC2 snapshot to complete",
-			"dbSnapshot", req.DBSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier,
-			"snapshotType", record.SnapshotType, "snapshotId", snapshotID, "crashConsistent", crashConsistent)
-		return &record, nil
-	}
-	now := time.Now().UTC()
 	record.Status = SnapshotStatusAvailable
-	record.SettledAt = &now
+	record.CrashConsistent = crashConsistent
 	if err := putJSON(ctx, kv, key, &record); err != nil {
 		return nil, err
 	}
 
-	s.recordSnapshotSettled(ctx, accountID, &record, state)
+	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, req.DBSnapshotIdentifier,
+		"DB snapshot created.", EventCategoryBackup, EventCategoryCreation)
+	slog.InfoContext(ctx, "rds: DB snapshot created",
+		"dbSnapshot", req.DBSnapshotIdentifier, "dbInstance", rec.DBInstanceIdentifier,
+		"snapshotType", record.SnapshotType, "snapshotId", snapshotID, "crashConsistent", crashConsistent)
 
 	return &record, nil
 }
 
 // Quiesces the engine, snapshots the data volume, and releases the engine again
-// however the snapshot went. Reports the EC2 snapshot, its state, and whether
-// the engine was still writing when it was taken.
+// however the snapshot went. Reports the EC2 snapshot and whether the engine was
+// still writing when it was taken.
 func (s *Service) snapshotDataVolume(ctx context.Context, accountID string, rec *DBInstanceRecord,
-	resume Status, dbSnapshotIdentifier string) (snapshotID, state string, crashConsistent bool, err error) {
+	resume Status, dbSnapshotIdentifier string) (snapshotID string, crashConsistent bool, err error) {
 	// A stopped instance has no engine to hold: its datadir was sealed by the
 	// graceful stop, which is the checkpoint a quiesce would be forcing.
 	if resume == StatusAvailable {
@@ -185,12 +176,12 @@ func (s *Service) snapshotDataVolume(ctx context.Context, accountID string, rec
 		}},
 	}, awsidentifiers.GlobalAccountID)
 	if err != nil {
-		return "", "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: %w", rec.DBInstanceIdentifier, err)
+		return "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: %w", rec.DBInstanceIdentifier, err)
 	}
 	if snapshot == nil || aws.StringValue(snapshot.SnapshotId) == "" {
-		return "", "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: empty snapshot id", rec.DBInstanceIdentifier)
+		return "", crashConsistent, fmt.Errorf("rds: snapshot the data volume of %s: empty snapshot id", rec.DBInstanceIdentifier)
 	}
-	return aws.StringValue(snapshot.SnapshotId), aws.StringValue(snapshot.State), crashConsistent, nil
+	return aws.StringValue(snapshot.SnapshotId), crashConsistent, nil
 }
 
 // The half of the warning that holds for every engine. What restoring such a
@@ -318,11 +309,11 @@ func (s *Service) DeleteDBSnapshot(ctx context.Context, input *rds.DeleteDBSnaps
 	if err != nil {
 		return nil, err
 	}
-	rec, rev, err := s.getDBSnapshot(ctx, kv, id)
+	rec, _, err := s.getDBSnapshot(ctx, kv, id)
 	if err != nil {
 		return nil, err
 	}
-	if err := s.removeDBSnapshot(ctx, kv, accountID, rec, rev); err != nil {
+	if err := s.removeDBSnapshot(ctx, kv, accountID, rec); err != nil {
 		return nil, err
 	}
 	// AWS answers with the snapshot as it last stood, the way a delete of a DB
@@ -334,16 +325,16 @@ func (s *Service) DeleteDBSnapshot(ctx context.Context, input *rds.DeleteDBSnaps
 // data volume behind it. Shared with the retention sweep, which cannot go
 // through DeleteDBSnapshot: that rejects the rds: namespace automated snapshots
 // live in, so a customer can never delete one by hand.
-func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, accountID string, rec *DBSnapshotRecord, rev uint64) error {
+func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, accountID string, rec *DBSnapshotRecord) error {
 	if s.deps.Snapshots == nil {
 		return errors.New("rds: no snapshot service configured")
 	}
 	id := rec.DBSnapshotIdentifier
-	// A snapshot still being taken has an in-flight writer that would recreate
-	// the record; the reconciler settles it available or failed, as AWS's are.
-	if rec.Status != SnapshotStatusAvailable && rec.Status != SnapshotStatusFailed {
+	// A snapshot still being taken has no data to remove and an in-flight writer
+	// that would recreate the record; the reconciler resolves it either way.
+	if rec.Status != SnapshotStatusAvailable {
 		return awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
-			"DB snapshot %s is %s; it must be %s or %s to be deleted", id, rec.Status, SnapshotStatusAvailable, SnapshotStatusFailed)
+			"DB snapshot %s is %s; it must be %s to be deleted", id, rec.Status, SnapshotStatusAvailable)
 	}
 
 	if err := s.deleteEC2Snapshot(ctx, kv, accountID, rec); err != nil {
@@ -356,8 +347,8 @@ func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, acco
 	}
 	// Last: while it exists the snapshot is still nameable, and a retry re-runs
 	// the steps above, each of which tolerates work it has already done.
-	if err := deleteDBSnapshotRecord(ctx, kv, rec, rev); err != nil {
-		return err
+	if err := kv.Delete(ctx, DBSnapshotKey(id)); err != nil {
+		return fmt.Errorf("rds: delete the DB snapshot record for %s: %w", id, err)
 	}
 
 	s.RecordEvent(ctx, accountID, EventSourceTypeDBSnapshot, id, "DB snapshot deleted.", EventCategoryDeletion)
@@ -365,35 +356,6 @@ func (s *Service) removeDBSnapshot(ctx context.Context, kv *kvstore.Bucket, acco
 	return nil
 }
 
-// Deletes the record read at rev, or the same snapshot rewritten since, such as
-// by a tag change. A record of the same name for another snapshot, such as a
-// retaken final snapshot's reservation, is left alone.
-func deleteDBSnapshotRecord(ctx context.Context, kv *kvstore.Bucket, rec *DBSnapshotRecord, rev uint64) error {
-	id := rec.DBSnapshotIdentifier
-	for range tagWriteAttempts {
-		err := kv.CompareAndDelete(ctx, DBSnapshotKey(id), rev)
-		if !errors.Is(err, kvstore.ErrConflict) {
-			if err != nil {
-				return fmt.Errorf("rds: delete the DB snapshot record for %s: %w", id, err)
-			}
-			return nil
-		}
-		var current DBSnapshotRecord
-		var found bool
-		if rev, found, err = getJSONRevision(ctx, kv, DBSnapshotKey(id), &current); err != nil {
-			return err
-		}
-		if !found {
-			return nil
-		}
-		if current.SnapshotID != rec.SnapshotID || !current.CreatedAt.Equal(rec.CreatedAt) {
-			return awserrors.Errorf(awserrors.ErrorDBSnapshotInvalidState,
-				"DB snapshot %s was replaced while it was being deleted", id)
-		}
-	}
-	return fmt.Errorf("rds: deleting the DB snapshot record for %s contended after %d attempts", id, tagWriteAttempts)
-}
-
 // A snapshot with a volume still reading through it cannot go, which is exactly
 // the case where an instance restored from it is still running. That is
 // translated into an RDS fault naming the instance rather than surfaced as the

spinifex/ingress/aws/pipeline/lambda_invoke_wire_test.go

@@ -59,6 +59,14 @@ func (activeMeta) UpdateConfiguration(context.Context, string, string, function.
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (activeMeta) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (activeMeta) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 // countingExec records how many invocations reached it.
 type countingExec struct{ calls atomic.Int32 }
 

spinifex/operator/admin/image_import.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
@@ -32,15 +31,8 @@ type ImportOpts struct {
 	// Progress receives the importer's own output (a percentage counter).
 	// Nil discards it.
 	Progress io.Writer
-	// SnapshotTimeout bounds the wait for a snapshot the provider answers
-	// pending to complete. Zero leaves only ctx to bound it.
-	SnapshotTimeout time.Duration
 }
 
-// snapshotPollInterval is how often a pending snapshot is read again; a test
-// shortens it so the wait does not sleep.
-var snapshotPollInterval = 5 * time.Second
-
 // ImportImage writes a local raw disk image into a new provider volume over
 // NBD, then optionally snapshots it. Everything here goes through the provider
 // contract, so an import works against any conforming provider rather than
@@ -99,63 +91,18 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 	}
 
 	if opts.Snapshot {
-		snapshotID := SnapPrefix(opts.VolumeID)
-		created, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+		if _, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
 			Versioned:  ebsprovider.NewVersioned(),
-			SnapshotID: snapshotID,
+			SnapshotID: SnapPrefix(opts.VolumeID),
 			VolumeID:   opts.VolumeID,
-		})
-		if err != nil {
+		}); err != nil {
 			return fmt.Errorf("import: snapshot volume %s: %w", opts.VolumeID, err)
 		}
-		if created != nil && created.State == ebsprovider.SnapshotStatePending {
-			if err := waitForSnapshot(ctx, provider, snapshotID, opts.SnapshotTimeout); err != nil {
-				return fmt.Errorf("import: snapshot volume %s: %w", opts.VolumeID, err)
-			}
-		}
 	}
 
 	return nil
 }
 
-// waitForSnapshot reads a pending snapshot every snapshotPollInterval until it
-// is completed, failing when it errors, is gone, or timeout passes. A read that
-// fails is retried, and reported if the timeout passes first.
-func waitForSnapshot(ctx context.Context, provider ebsprovider.EBSProvider, snapshotID string, timeout time.Duration) error {
-	if timeout > 0 {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, timeout)
-		defer cancel()
-	}
-	ticker := time.NewTicker(snapshotPollInterval)
-	defer ticker.Stop()
-	var lastErr error
-	for {
-		select {
-		case <-ctx.Done():
-			if lastErr != nil {
-				return fmt.Errorf("snapshot %s did not complete: %w (last read failed: %w)", snapshotID, ctx.Err(), lastErr)
-			}
-			return fmt.Errorf("snapshot %s did not complete: %w", snapshotID, ctx.Err())
-		case <-ticker.C:
-		}
-		ref, found, err := ebsprovider.ReadSnapshot(ctx, provider, snapshotID)
-		if err != nil {
-			lastErr = err
-			continue
-		}
-		if !found {
-			return fmt.Errorf("snapshot %s is no longer held by the provider", snapshotID)
-		}
-		switch ref.State {
-		case ebsprovider.SnapshotStateCompleted:
-			return nil
-		case ebsprovider.SnapshotStateError:
-			return fmt.Errorf("snapshot %s ended in error", snapshotID)
-		}
-	}
-}
-
 // writeImage is the bulk copy step, indirected so a test can drive the verb
 // sequence around it without a live NBD export to write to.
 var writeImage = nbd.WriteRawImage

spinifex/operator/admin/image_import_test.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"io"
 	"testing"
-	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/stretchr/testify/assert"
@@ -26,21 +25,6 @@ type recordingProvider struct {
 	// publishExport, when its Kind is set, replaces the nbd export built
 	// from publishURI.
 	publishExport ebsprovider.VolumeExport
-
-	// onList runs before read n (from 1) of the snapshot; a non-nil error
-	// fails that read.
-	onList func(n int) error
-	lists  int
-}
-
-func (p *recordingProvider) ListSnapshots(ctx context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
-	p.lists++
-	if p.onList != nil {
-		if err := p.onList(p.lists); err != nil {
-			return nil, err
-		}
-	}
-	return p.MemoryProvider.ListSnapshots(ctx, req)
 }
 
 func newRecordingProvider() *recordingProvider {
@@ -200,87 +184,3 @@ func TestImportImage_RejectsIncompleteOpts(t *testing.T) {
 		})
 	}
 }
-
-// A snapshot the provider answers pending is read until it completes, and the
-// import fails if it errors, disappears, or outlasts SnapshotTimeout.
-func TestImportImage_WaitsForAPendingSnapshot(t *testing.T) {
-	original := snapshotPollInterval
-	snapshotPollInterval = time.Millisecond
-	t.Cleanup(func() { snapshotPollInterval = original })
-	snapshotID := SnapPrefix("ami-import001")
-
-	cases := []struct {
-		name      string
-		pending   bool
-		onList    func(p *recordingProvider) func(n int) error
-		wantErr   string // empty: the import succeeds
-		wantLists int    // -1: at least one
-	}{
-		{name: "answered completed is not read", wantLists: 0},
-		{name: "completes on the second read", pending: true, wantLists: 2,
-			onList: func(p *recordingProvider) func(int) error {
-				return func(n int) error {
-					if n == 2 {
-						return p.SettleSnapshot(snapshotID, ebsprovider.SnapshotStateCompleted)
-					}
-					return nil
-				}
-			}},
-		{name: "a failed read is retried", pending: true, wantLists: 2,
-			onList: func(p *recordingProvider) func(int) error {
-				return func(n int) error {
-					if n == 1 {
-						return ebsprovider.ErrUnavailable
-					}
-					return p.SettleSnapshot(snapshotID, ebsprovider.SnapshotStateCompleted)
-				}
-			}},
-		{name: "ends in error", pending: true, wantErr: "ended in error", wantLists: 1,
-			onList: func(p *recordingProvider) func(int) error {
-				return func(int) error { return p.SettleSnapshot(snapshotID, ebsprovider.SnapshotStateError) }
-			}},
-		{name: "disappears", pending: true, wantErr: "no longer held", wantLists: 1,
-			onList: func(p *recordingProvider) func(int) error {
-				return func(int) error {
-					return p.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{
-						Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID,
-					})
-				}
-			}},
-		{name: "outlasts the timeout", pending: true, wantErr: context.DeadlineExceeded.Error(), wantLists: -1},
-		{name: "outlasts the timeout while reads fail", pending: true, wantErr: "last read failed", wantLists: -1,
-			onList: func(*recordingProvider) func(int) error {
-				return func(int) error { return ebsprovider.ErrUnavailable }
-			}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			stubWriteImage(t, nil, nil)
-			provider := newRecordingProvider()
-			provider.AnswerSnapshotsPending(tc.pending)
-			if tc.onList != nil {
-				provider.onList = tc.onList(provider)
-			}
-
-			err := ImportImage(context.Background(), provider, ImportOpts{
-				VolumeID: "ami-import001", NodeID: "node-a", SizeBytes: 1 << 30,
-				SourcePath: "/tmp/image.raw", Snapshot: true, SnapshotTimeout: 50 * time.Millisecond,
-			})
-
-			if tc.wantErr == "" {
-				require.NoError(t, err)
-				snap, ok := provider.Snapshot(snapshotID)
-				require.True(t, ok)
-				assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
-			} else {
-				require.Error(t, err)
-				assert.Contains(t, err.Error(), tc.wantErr)
-			}
-			if tc.wantLists < 0 {
-				assert.Positive(t, provider.lists)
-			} else {
-				assert.Equal(t, tc.wantLists, provider.lists)
-			}
-		})
-	}
-}

spinifex/operator/cli/admin.go

@@ -480,9 +480,8 @@ func init() {
 
 const bytesPerGiB = 1024 * 1024 * 1024
 
-// imageImportTimeout bounds each provider call the import makes, and the wait
-// for a snapshot the provider answers pending. Generous because the snapshot
-// at the end runs over the whole imported volume.
+// imageImportTimeout bounds each provider call the import makes. Generous
+// because the snapshot at the end runs over the whole imported volume.
 const imageImportTimeout = 30 * time.Minute
 
 // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes.
@@ -896,7 +895,6 @@ func registerExtractedImage(image imagecatalog.Images, amiNameOverride, imagePat
 		SourcePath:       extractedImagePath,
 		Snapshot:         true,
 		Progress:         os.Stdout,
-		SnapshotTimeout:  imageImportTimeout,
 	})
 	if err != nil {
 		return "", fmt.Errorf("import image: %w", err)

spinifex/operator/cli/admin_ochre.go

@@ -786,7 +786,6 @@ func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvi
 		SourcePath:       imagePath,
 		Snapshot:         true,
 		Progress:         os.Stdout,
-		SnapshotTimeout:  imageImportTimeout,
 	}); err != nil {
 		return "", fmt.Errorf("could not import weights volume %s: %w", volumeId, err)
 	}

spinifex/providers/ebs/conformance/conformance.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/stretchr/testify/assert"
@@ -125,64 +124,6 @@ func drainSnapshotIDs(t *testing.T, provider ebsprovider.EBSProvider, maxResults
 	}
 }
 
-// countOf is how many times id appears in ids.
-func countOf(ids []string, id string) int {
-	n := 0
-	for _, each := range ids {
-		if each == id {
-			n++
-		}
-	}
-	return n
-}
-
-// filterVersion is the version the suite's filter-contract cases are
-// sent at: the first that carries the filter.
-var filterVersion = ebsprovider.Versioned{SchemaVersion: ebsprovider.SnapshotFilterSchemaVersion}
-
-// readSnapshot reads snapshotID through ebsprovider.ReadSnapshot and fails
-// unless the reply holds that snapshot.
-func readSnapshot(t *testing.T, provider ebsprovider.EBSProvider, snapshotID string) ebsprovider.SnapshotRef {
-	t.Helper()
-	ref, found, err := ebsprovider.ReadSnapshot(context.Background(), provider, snapshotID)
-	require.NoError(t, err)
-	require.Truef(t, found, "a snapshot the provider accepted must be read back by its ID, never reported absent")
-	require.Equal(t, snapshotID, ref.ID)
-	return ref
-}
-
-// settleTimeout bounds the wait for a pending snapshot to complete, so a
-// provider that never settles one fails the test instead of hanging it.
-const settleTimeout = 30 * time.Second
-
-// createSnapshot takes snapshotID of volumeID and waits, through the filtered
-// ListSnapshots, until it is completed: CreateSnapshot may answer pending, so
-// nothing about the snapshot is asserted before it settles.
-func createSnapshot(t *testing.T, provider ebsprovider.EBSProvider, snapshotID, volumeID string) *ebsprovider.Snapshot {
-	t.Helper()
-	snap, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
-	require.NoError(t, err)
-	require.NotNil(t, snap)
-	require.Containsf(t, []ebsprovider.SnapshotState{ebsprovider.SnapshotStatePending, ebsprovider.SnapshotStateCompleted}, snap.State,
-		"CreateSnapshot answers pending or completed, never %q", snap.State)
-	require.Equal(t, snapshotID, snap.ID)
-
-	deadline := time.Now().Add(settleTimeout)
-	for {
-		ref := readSnapshot(t, provider, snapshotID)
-		switch ref.State {
-		case ebsprovider.SnapshotStateCompleted:
-			return snap
-		case ebsprovider.SnapshotStatePending:
-			require.Equalf(t, ebsprovider.SnapshotStatePending, snap.State, "snapshot %s answered completed and then read pending", snapshotID)
-		default:
-			require.Failf(t, "snapshot did not complete", "snapshot %s settled to %q", snapshotID, ref.State)
-		}
-		require.Truef(t, time.Now().Before(deadline), "snapshot %s still pending after %s", snapshotID, settleTimeout)
-		time.Sleep(10 * time.Millisecond)
-	}
-}
-
 // SuiteConfig tunes a run for the provider it is pointed at.
 type SuiteConfig struct {
 	// NamePrefix is inserted into every volume and snapshot ID the suite
@@ -374,7 +315,10 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 				CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
 			})
 			require.NoError(t, err)
-			snap := createSnapshot(t, provider, cfg.id("snap-snapsrc"), cfg.id("vol-snapsrc-origin"))
+			snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+				Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-snapsrc"), VolumeID: cfg.id("vol-snapsrc-origin"),
+			})
+			require.NoError(t, err)
 			require.Equal(t, cfg.id("vol-snapsrc-origin"), snap.SourceVolumeID, "a snapshot must report the volume it was taken from")
 
 			restored, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
@@ -397,7 +341,10 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 				CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
 			})
 			require.NoError(t, err)
-			snap := createSnapshot(t, provider, cfg.id("snap-wrongorigin"), cfg.id("vol-wrongorigin-src"))
+			snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+				Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-wrongorigin"), VolumeID: cfg.id("vol-wrongorigin-src"),
+			})
+			require.NoError(t, err)
 
 			_, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
 				Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-wrongorigin-restored"),
@@ -661,55 +608,12 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snap-src"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			snap := createSnapshot(t, provider, cfg.id("snap-ok"), cfg.id("vol-snap-src"))
+			snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-ok"), VolumeID: cfg.id("vol-snap-src")})
+			require.NoError(t, err)
+			require.NotNil(t, snap)
 			assert.Equal(t, cfg.id("snap-ok"), snap.ID)
 			assert.Equal(t, cfg.id("vol-snap-src"), snap.SourceVolumeID)
-		})
-
-		// CSI's idempotency rule for its name field: a repeat is answered with
-		// the snapshot the first call took, so a caller whose answer was lost
-		// can ask again without taking a second snapshot.
-		t.Run("a repeat with the same source returns the first snapshot", func(t *testing.T) {
-			provider := newProvider(t)
-			ctx := context.Background()
-			volumeID, snapshotID := cfg.id("vol-snap-repeat"), cfg.id("snap-repeat")
-			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-			require.NoError(t, err)
-			first := createSnapshot(t, provider, snapshotID, volumeID)
-
-			again, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
-			require.NoError(t, err, "a repeat naming the same source volume is not a conflict")
-			require.NotNil(t, again)
-			assert.Equal(t, first.ID, again.ID)
-			assert.Equal(t, first.SourceVolumeID, again.SourceVolumeID)
-			assert.Equal(t, first.Handle, again.Handle, "the repeat must answer with the snapshot the first call created")
-			assert.Equal(t, ebsprovider.SnapshotStateCompleted, readSnapshot(t, provider, snapshotID).State,
-				"a repeat of a completed snapshot must not reopen it")
-			if capabilities.SnapshotEnumeration {
-				listed := drainSnapshotIDs(t, provider, 0)
-				assert.Equal(t, 1, countOf(listed, snapshotID), "a repeat must not hold a second snapshot under the same ID")
-			}
-		})
-
-		// A caller that lost the first answer may only ask again after the
-		// source volume is gone, so the repeat must not need the volume.
-		t.Run("a repeat after the source volume is deleted returns the first snapshot", func(t *testing.T) {
-			provider := newProvider(t)
-			ctx := context.Background()
-			volumeID, snapshotID := cfg.id("vol-snap-gone"), cfg.id("snap-gone")
-			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-			require.NoError(t, err)
-			first := createSnapshot(t, provider, snapshotID, volumeID)
-			require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}))
-
-			again, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
-			require.NoError(t, err, "a repeat must be answered from the snapshot, not the deleted volume")
-			require.NotNil(t, again)
-			assert.Equal(t, first.ID, again.ID)
-			assert.Equal(t, volumeID, again.SourceVolumeID)
-			assert.Equal(t, first.Handle, again.Handle)
-			assert.False(t, again.CreatedAt.IsZero(), "the repeat must report when the snapshot was taken")
-			assert.Equal(t, ebsprovider.SnapshotStateCompleted, readSnapshot(t, provider, snapshotID).State)
+			assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
 		})
 
 		t.Run("not_found on absent source volume", func(t *testing.T) {
@@ -725,7 +629,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.NoError(t, err)
 			_, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snap-b"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			createSnapshot(t, provider, cfg.id("snap-conflict"), cfg.id("vol-snap-a"))
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-conflict"), VolumeID: cfg.id("vol-snap-a")})
+			require.NoError(t, err)
 			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-conflict"), VolumeID: cfg.id("vol-snap-b")})
 			require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists)
 		})
@@ -764,7 +669,9 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 				}
 			}
 
-			createSnapshot(t, provider, cfg.id("snap-held-owner"), volumeID)
+			snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-held-owner"), VolumeID: volumeID})
+			require.NoError(t, err, "the holder answers for its own volume")
+			assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
 		})
 	})
 
@@ -774,7 +681,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delsnap-src"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			createSnapshot(t, provider, cfg.id("snap-delete-ok"), cfg.id("vol-delsnap-src"))
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-delete-ok"), VolumeID: cfg.id("vol-delsnap-src")})
+			require.NoError(t, err)
 			require.NoError(t, provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-delete-ok")}))
 		})
 
@@ -792,7 +700,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-src"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			createSnapshot(t, provider, cfg.id("snap-copysnap-src"), cfg.id("vol-copysnap-src"))
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-src"), VolumeID: cfg.id("vol-copysnap-src")})
+			require.NoError(t, err)
 
 			copied, err := provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{
 				Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-src"), DestinationSnapshotID: cfg.id("snap-copysnap-dst"), VolumeID: cfg.id("vol-copysnap-src"),
@@ -825,8 +734,10 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-conflict"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			createSnapshot(t, provider, cfg.id("snap-copysnap-a"), cfg.id("vol-copysnap-conflict"))
-			createSnapshot(t, provider, cfg.id("snap-copysnap-b"), cfg.id("vol-copysnap-conflict"))
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-a"), VolumeID: cfg.id("vol-copysnap-conflict")})
+			require.NoError(t, err)
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-b"), VolumeID: cfg.id("vol-copysnap-conflict")})
+			require.NoError(t, err)
 			_, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{
 				Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-a"), DestinationSnapshotID: cfg.id("snap-copysnap-b"), VolumeID: cfg.id("vol-copysnap-conflict"),
 			})
@@ -844,7 +755,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			ctx := context.Background()
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-same"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			createSnapshot(t, provider, cfg.id("snap-copysnap-same"), cfg.id("vol-copysnap-same"))
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-same"), VolumeID: cfg.id("vol-copysnap-same")})
+			require.NoError(t, err)
 			_, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{
 				Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-same"), DestinationSnapshotID: cfg.id("snap-copysnap-same"), VolumeID: cfg.id("vol-copysnap-same"),
 			})
@@ -858,7 +770,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.NoError(t, err)
 			_, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-foreign"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			createSnapshot(t, provider, cfg.id("snap-copysnap-owned"), cfg.id("vol-copysnap-owner"))
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-owned"), VolumeID: cfg.id("vol-copysnap-owner")})
+			require.NoError(t, err)
 			_, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{
 				Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-owned"), DestinationSnapshotID: cfg.id("snap-copysnap-owned-dst"), VolumeID: cfg.id("vol-copysnap-foreign"),
 			})
@@ -883,93 +796,10 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability)
 		})
 
-		t.Run("a filtered read returns the one snapshot", func(t *testing.T) {
-			provider := newProvider(t)
-			ctx := context.Background()
-			volumeID := cfg.id("vol-snapfilter-src")
-			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-			require.NoError(t, err)
-			snap := createSnapshot(t, provider, cfg.id("snap-filter-one"), volumeID)
-			createSnapshot(t, provider, cfg.id("snap-filter-other"), volumeID)
-
-			resp, err := provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: filterVersion, SnapshotID: snap.ID, MaxResults: 1})
-			require.NoError(t, err)
-			require.Len(t, resp.Snapshots, 1, "a filtered read answers with the one snapshot, not a page of them")
-			assert.Equal(t, snap.ID, resp.Snapshots[0].ID)
-			assert.Equal(t, ebsprovider.SnapshotStateCompleted, resp.Snapshots[0].State)
-			assert.Empty(t, resp.NextToken, "a filtered read has no further page")
-			if resp.Snapshots[0].SourceVolumeID != "" {
-				assert.Equal(t, volumeID, resp.Snapshots[0].SourceVolumeID)
-			}
-
-			require.NoError(t, provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snap.ID}))
-			gone, err := provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: filterVersion, SnapshotID: snap.ID})
-			require.NoError(t, err)
-			assert.Empty(t, gone.Snapshots, "a deleted snapshot must read as absent")
-		})
-
-		t.Run("a filtered read of an absent snapshot returns none", func(t *testing.T) {
-			resp, err := newProvider(t).ListSnapshots(context.Background(), ebsprovider.ListSnapshotsRequest{Versioned: filterVersion, SnapshotID: cfg.id("snap-filter-absent")})
-			require.NoError(t, err, "absence is an answer, not an error")
-			require.NotNil(t, resp)
-			assert.Empty(t, resp.Snapshots)
-			assert.Empty(t, resp.NextToken)
-		})
-
-		// A provider at an earlier version would drop the filter and answer with
-		// every snapshot it holds, which a caller would read as the one it asked for.
-		t.Run("the filter below the version that defines it is refused as unsupported", func(t *testing.T) {
-			provider := newProvider(t)
-			for _, version := range contractVersions {
-				if version >= ebsprovider.SnapshotFilterSchemaVersion {
-					continue
-				}
-				_, err := provider.ListSnapshots(context.Background(), ebsprovider.ListSnapshotsRequest{
-					Versioned: ebsprovider.Versioned{SchemaVersion: version}, SnapshotID: cfg.id("snap-filter-old"),
-				})
-				require.ErrorIsf(t, err, ebsprovider.ErrUnsupportedVersion, "v%d", version)
-			}
-		})
-
 		if !capabilities.SnapshotEnumeration {
 			return
 		}
 
-		t.Run("state is enumerated only from the version that defines it", func(t *testing.T) {
-			provider := newProvider(t)
-			ctx := context.Background()
-			volumeID := cfg.id("vol-snaplist-state-src")
-			snapshotID := cfg.id("snap-list-state")
-			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-			require.NoError(t, err)
-			createSnapshot(t, provider, snapshotID, volumeID)
-
-			for _, version := range contractVersions {
-				listed := false
-				var token string
-				for pages := 0; ; pages++ {
-					require.Less(t, pages, maxListPages)
-					resp, err := provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, StartingToken: token})
-					require.NoError(t, err)
-					for _, ref := range resp.Snapshots {
-						if ref.ID != snapshotID {
-							continue
-						}
-						listed = true
-						if version < ebsprovider.SnapshotFilterSchemaVersion {
-							assert.Emptyf(t, ref.State, "v%d does not define state, so its reply must not carry it", version)
-						} else {
-							assert.Equalf(t, ebsprovider.SnapshotStateCompleted, ref.State, "v%d", version)
-						}
-					}
-					if token = resp.NextToken; token == "" {
-						break
-					}
-				}
-				assert.Truef(t, listed, "v%d: snapshot %s was not enumerated", version, snapshotID)
-			}
-		})
-
 		t.Run("a created snapshot is enumerated and a deleted one is not", func(t *testing.T) {
 			provider := newProvider(t)
 			ctx := context.Background()
@@ -977,7 +807,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			snapshotID := cfg.id("snap-list-visible")
 			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
 			require.NoError(t, err)
-			createSnapshot(t, provider, snapshotID, volumeID)
+			_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
+			require.NoError(t, err)
 			assert.Contains(t, drainSnapshotIDs(t, provider, 0), snapshotID,
 				"enumeration must report a snapshot the provider holds; it is the only way to find one whose control-plane document is lost")
 
@@ -994,7 +825,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.NoError(t, err)
 			want := []string{cfg.id("snap-list-page-a"), cfg.id("snap-list-page-b"), cfg.id("snap-list-page-c")}
 			for _, snapshotID := range want {
-				createSnapshot(t, provider, snapshotID, volumeID)
+				_, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
+				require.NoError(t, err)
 			}
 
 			// One snapshot per page forces every token boundary to be exercised.
@@ -1019,7 +851,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.NoError(t, err)
 			want := []string{cfg.id("snap-list-cap-a"), cfg.id("snap-list-cap-b")}
 			for _, snapshotID := range want {
-				createSnapshot(t, provider, snapshotID, volumeID)
+				_, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
+				require.NoError(t, err)
 			}
 
 			resp, err := provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{

spinifex/providers/ebs/conformance/conformance_test.go

@@ -1,7 +1,6 @@
 package conformance_test
 
 import (
-	"context"
 	"testing"
 	"time"
 
@@ -111,50 +110,3 @@ func TestNATSProviderConformance(t *testing.T) {
 		return ebsprovider.NewNATSProvider(conn, 5*time.Second)
 	})
 }
-
-// settlingProvider is a MemoryProvider answering every CreateSnapshot pending.
-// It settles a pending snapshot once a filtered read has seen it pending, so
-// the suite's wait for completion is exercised and still ends.
-type settlingProvider struct {
-	*ebsprovider.MemoryProvider
-}
-
-var _ ebsprovider.EBSProvider = settlingProvider{}
-
-func newSettlingProvider() settlingProvider {
-	provider := ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities)
-	provider.AnswerSnapshotsPending(true)
-	return settlingProvider{MemoryProvider: provider}
-}
-
-func (p settlingProvider) ListSnapshots(ctx context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
-	resp, err := p.MemoryProvider.ListSnapshots(ctx, req)
-	if err == nil && req.SnapshotID != "" && len(resp.Snapshots) == 1 && resp.Snapshots[0].State == ebsprovider.SnapshotStatePending {
-		if err := p.SettleSnapshot(req.SnapshotID, ebsprovider.SnapshotStateCompleted); err != nil {
-			return nil, err
-		}
-	}
-	return resp, err
-}
-
-// TestMemoryProviderPendingSnapshotConformance runs the suite against a
-// provider whose CreateSnapshot answers pending, as one whose backup outlives
-// the request does, in process and over NATS.
-func TestMemoryProviderPendingSnapshotConformance(t *testing.T) {
-	t.Run("in process", func(t *testing.T) {
-		conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider {
-			t.Helper()
-			return newSettlingProvider()
-		})
-	})
-	t.Run("over NATS", func(t *testing.T) {
-		conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider {
-			t.Helper()
-			_, conn := testutil.StartTestNATS(t)
-			stop, err := natsserve.Serve(t.Context(), conn, newSettlingProvider(), natsserve.Options{})
-			require.NoError(t, err)
-			t.Cleanup(stop)
-			return ebsprovider.NewNATSProvider(conn, 5*time.Second)
-		})
-	})
-}

spinifex/providers/ebs/hostile/verbs.go

@@ -63,8 +63,7 @@ func corruptSnapshot(snapshot *ebsprovider.Snapshot, flavour string) *ebsprovide
 	case "handle":
 		corrupted.SourceVolumeID = "vol-hostilewrongsrc"
 	case "state":
-		// Pending is a legitimate answer, so the lie is a failure that never was.
-		corrupted.State = ebsprovider.SnapshotStateError
+		corrupted.State = ebsprovider.SnapshotStatePending
 	}
 	return &corrupted
 }

spinifex/providers/ebs/memory.go

@@ -26,12 +26,6 @@ type MemoryProvider struct {
 	// Zero takes the defaults; tests shorten them.
 	fenceNoticeTimeout time.Duration
 	fenceNoticeEvery   time.Duration
-
-	// Test-set CreateSnapshot behaviour; see AnswerSnapshotsPending and
-	// FailCreateSnapshotsAfterAccept.
-	snapshotsPending   bool
-	failAfterAccept    int
-	failAfterAcceptErr error
 }
 
 type memoryVolume struct {
@@ -253,80 +247,24 @@ func (m *MemoryProvider) CreateSnapshot(_ context.Context, req CreateSnapshotReq
 		if existing.SourceVolumeID != req.VolumeID {
 			return nil, fmt.Errorf("%w: snapshot %s", ErrAlreadyExists, req.SnapshotID)
 		}
-		return m.answerAccepted(existing)
+		return cloneSnapshot(existing), nil
 	}
 	volume := m.volumes[req.VolumeID]
 	if volume == nil || (req.VolumeHandle != "" && req.VolumeHandle != volume.volume.Handle) {
 		return nil, fmt.Errorf("%w: volume %s", ErrNotFound, req.VolumeID)
 	}
-	state := SnapshotStateCompleted
-	if m.snapshotsPending {
-		state = SnapshotStatePending
-	}
 	snapshot := &Snapshot{
 		ID:             req.SnapshotID,
 		SourceVolumeID: req.VolumeID,
 		SizeBytes:      volume.volume.CapacityBytes,
 		CreatedAt:      m.now().UTC(),
-		State:          state,
+		State:          SnapshotStateCompleted,
 		Handle:         "memory://snapshot/" + req.SnapshotID,
 	}
 	m.snapshots[req.SnapshotID] = snapshot
-	return m.answerAccepted(snapshot)
-}
-
-// answerAccepted answers a CreateSnapshot that holds snapshot, or the failure
-// FailCreateSnapshotsAfterAccept owes. The caller holds the provider lock.
-func (m *MemoryProvider) answerAccepted(snapshot *Snapshot) (*Snapshot, error) {
-	if m.failAfterAccept > 0 {
-		m.failAfterAccept--
-		return nil, m.failAfterAcceptErr
-	}
 	return cloneSnapshot(snapshot), nil
 }
 
-// AnswerSnapshotsPending makes CreateSnapshot answer each new snapshot pending,
-// as a provider whose backup outlives the request does, until SettleSnapshot
-// settles it. False restores answering completed.
-func (m *MemoryProvider) AnswerSnapshotsPending(pending bool) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	m.snapshotsPending = pending
-}
-
-// FailCreateSnapshotsAfterAccept makes the next n CreateSnapshot calls that hold
-// their snapshot return err, a reply lost after acceptance. A nil err is
-// context.DeadlineExceeded; pass ErrUnavailable for the provider's own code.
-func (m *MemoryProvider) FailCreateSnapshotsAfterAccept(n int, err error) {
-	if err == nil {
-		err = context.DeadlineExceeded
-	}
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	m.failAfterAccept = n
-	m.failAfterAcceptErr = err
-}
-
-// SettleSnapshot moves a pending snapshot to state, completed or error, as the
-// provider finishing its backup does. It fails if the snapshot is absent or
-// already settled.
-func (m *MemoryProvider) SettleSnapshot(snapshotID string, state SnapshotState) error {
-	if state != SnapshotStateCompleted && state != SnapshotStateError {
-		return fmt.Errorf("%w: a snapshot settles to completed or error, not %q", ErrInvalidArgument, state)
-	}
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	snapshot := m.snapshots[snapshotID]
-	if snapshot == nil {
-		return fmt.Errorf("%w: snapshot %s", ErrNotFound, snapshotID)
-	}
-	if snapshot.State != SnapshotStatePending {
-		return fmt.Errorf("%w: snapshot %s is already %s", ErrInvalidArgument, snapshotID, snapshot.State)
-	}
-	snapshot.State = state
-	return nil
-}
-
 // Snapshot reports what the provider holds under snapshotID, so a
 // control-plane test can assert a snapshot really reached the provider rather
 // than existing only as a control-plane document.
@@ -401,19 +339,9 @@ func (m *MemoryProvider) CopySnapshot(_ context.Context, req CopySnapshotRequest
 // so a token stays meaningful across calls. The token is the ID to resume
 // after, which keeps paging correct when snapshots come and go mid-walk.
 func (m *MemoryProvider) ListSnapshots(_ context.Context, req ListSnapshotsRequest) (*ListSnapshotsResponse, error) {
-	if err := CheckListSnapshotsVersion(req); err != nil {
+	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
-	if req.SnapshotID != "" {
-		m.mu.RLock()
-		defer m.mu.RUnlock()
-		snapshot := m.snapshots[req.SnapshotID]
-		if snapshot == nil {
-			return FilteredSnapshots(req, nil), nil
-		}
-		ref := memorySnapshotRef(req.SchemaVersion, snapshot)
-		return FilteredSnapshots(req, &ref), nil
-	}
 	if !m.capabilities.SnapshotEnumeration {
 		return nil, fmt.Errorf("%w: snapshot enumeration", ErrUnsupportedCapability)
 	}
@@ -425,22 +353,15 @@ func (m *MemoryProvider) ListSnapshots(_ context.Context, req ListSnapshotsReque
 	response := &ListSnapshotsResponse{Versioned: ReplyVersioned(req.SchemaVersion)}
 	response.Snapshots, response.NextToken = Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) SnapshotRef {
-			return memorySnapshotRef(req.SchemaVersion, m.snapshots[id])
+			return SnapshotRef{
+				ID:             id,
+				SourceVolumeID: m.snapshots[id].SourceVolumeID,
+				Handle:         m.snapshots[id].Handle,
+			}
 		})
 	return response, nil
 }
 
-// memorySnapshotRef is snapshot as a ref in a reply at version. The caller holds the
-// provider lock.
-func memorySnapshotRef(version uint16, snapshot *Snapshot) SnapshotRef {
-	return SnapshotRef{
-		ID:             snapshot.ID,
-		SourceVolumeID: snapshot.SourceVolumeID,
-		Handle:         snapshot.Handle,
-		State:          RefState(version, snapshot.State),
-	}
-}
-
 func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeRequest) (*PublishedVolume, error) {
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err

spinifex/providers/ebs/memory_test.go

@@ -430,113 +430,3 @@ func TestValidateSeedData(t *testing.T) {
 	require.NoError(t, ValidateSeedData(make([]byte, MaxSeedBytes)))
 	require.ErrorIs(t, ValidateSeedData(make([]byte, MaxSeedBytes+1)), ErrInvalidArgument)
 }
-
-// memoryWithVolume is a provider holding vol-src, for the snapshot tests below.
-func memoryWithVolume(t *testing.T) *MemoryProvider {
-	t.Helper()
-	provider := NewMemoryProvider(Capabilities{})
-	_, err := provider.CreateVolume(t.Context(), CreateVolumeRequest{
-		Versioned: NewVersioned(), VolumeID: "vol-src", CapacityRange: CapacityRange{RequiredBytes: 1 << 30},
-	})
-	require.NoError(t, err)
-	return provider
-}
-
-func readMemorySnapshot(t *testing.T, provider *MemoryProvider, snapshotID string) []SnapshotRef {
-	t.Helper()
-	resp, err := provider.ListSnapshots(t.Context(), ListSnapshotsRequest{Versioned: NewLatestVersioned(), SnapshotID: snapshotID})
-	require.NoError(t, err)
-	return resp.Snapshots
-}
-
-// TestMemoryProviderPendingSnapshots covers the option later control-plane
-// tests stand a slow backup on: the answer is pending, the filtered read
-// follows it, and only SettleSnapshot moves it on.
-func TestMemoryProviderPendingSnapshots(t *testing.T) {
-	for _, settled := range []SnapshotState{SnapshotStateCompleted, SnapshotStateError} {
-		t.Run(string(settled), func(t *testing.T) {
-			provider := memoryWithVolume(t)
-			provider.AnswerSnapshotsPending(true)
-			create := CreateSnapshotRequest{Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-src"}
-
-			snapshot, err := provider.CreateSnapshot(t.Context(), create)
-			require.NoError(t, err)
-			assert.Equal(t, SnapshotStatePending, snapshot.State)
-			assert.Equal(t, SnapshotStatePending, readMemorySnapshot(t, provider, "snap-1")[0].State)
-			repeated, err := provider.CreateSnapshot(t.Context(), create)
-			require.NoError(t, err)
-			assert.Equal(t, SnapshotStatePending, repeated.State, "a repeat answers with the snapshot as it stands")
-
-			require.NoError(t, provider.SettleSnapshot("snap-1", settled))
-			assert.Equal(t, settled, readMemorySnapshot(t, provider, "snap-1")[0].State)
-			require.ErrorIs(t, provider.SettleSnapshot("snap-1", SnapshotStateCompleted), ErrInvalidArgument, "a snapshot settles once")
-		})
-	}
-
-	t.Run("answering completed again", func(t *testing.T) {
-		provider := memoryWithVolume(t)
-		provider.AnswerSnapshotsPending(true)
-		provider.AnswerSnapshotsPending(false)
-		snapshot, err := provider.CreateSnapshot(t.Context(), CreateSnapshotRequest{Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-src"})
-		require.NoError(t, err)
-		assert.Equal(t, SnapshotStateCompleted, snapshot.State)
-	})
-
-	t.Run("settle refusals", func(t *testing.T) {
-		provider := memoryWithVolume(t)
-		require.ErrorIs(t, provider.SettleSnapshot("snap-absent", SnapshotStateCompleted), ErrNotFound)
-		require.ErrorIs(t, provider.SettleSnapshot("snap-absent", SnapshotStatePending), ErrInvalidArgument)
-	})
-}
-
-// TestMemoryProviderFailsAfterAccept covers the ambiguous failure: the caller
-// sees an error, yet the snapshot is held, and the failures owed run out.
-func TestMemoryProviderFailsAfterAccept(t *testing.T) {
-	tests := []struct {
-		name    string
-		err     error
-		wantErr error
-	}{
-		{name: "unavailable", err: ErrUnavailable, wantErr: ErrUnavailable},
-		{name: "default is a timeout", err: nil, wantErr: context.DeadlineExceeded},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			provider := memoryWithVolume(t)
-			provider.FailCreateSnapshotsAfterAccept(2, tt.err)
-			create := CreateSnapshotRequest{Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-src"}
-
-			for range 2 {
-				_, err := provider.CreateSnapshot(t.Context(), create)
-				require.ErrorIs(t, err, tt.wantErr)
-				assert.Len(t, readMemorySnapshot(t, provider, "snap-1"), 1, "the snapshot was accepted before the failure")
-			}
-			snapshot, err := provider.CreateSnapshot(t.Context(), create)
-			require.NoError(t, err, "only the failures asked for are owed")
-			assert.Equal(t, "snap-1", snapshot.ID)
-		})
-	}
-
-	t.Run("a definite refusal owes no failure", func(t *testing.T) {
-		provider := memoryWithVolume(t)
-		provider.FailCreateSnapshotsAfterAccept(1, ErrUnavailable)
-		_, err := provider.CreateSnapshot(t.Context(), CreateSnapshotRequest{Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-absent"})
-		require.ErrorIs(t, err, ErrNotFound)
-		_, err = provider.CreateSnapshot(t.Context(), CreateSnapshotRequest{Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-src"})
-		require.ErrorIs(t, err, ErrUnavailable)
-	})
-}
-
-// TestMemoryProviderFilteredReadNeedsNoEnumeration covers the filter's reach:
-// a provider that cannot enumerate still answers for one snapshot by its ID.
-func TestMemoryProviderFilteredReadNeedsNoEnumeration(t *testing.T) {
-	provider := memoryWithVolume(t)
-	_, err := provider.CreateSnapshot(t.Context(), CreateSnapshotRequest{Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-src"})
-	require.NoError(t, err)
-
-	assert.Equal(t, []SnapshotRef{{ID: "snap-1", SourceVolumeID: "vol-src", Handle: "memory://snapshot/snap-1", State: SnapshotStateCompleted}},
-		readMemorySnapshot(t, provider, "snap-1"))
-	assert.Empty(t, readMemorySnapshot(t, provider, "snap-absent"))
-	_, err = provider.ListSnapshots(t.Context(), ListSnapshotsRequest{Versioned: NewLatestVersioned()})
-	require.ErrorIs(t, err, ErrUnsupportedCapability, "unfiltered enumeration still needs the capability")
-}

spinifex/providers/ebs/nats.go

@@ -432,10 +432,8 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 	if completed.OperationID != accepted.OperationID {
 		return nil, fmt.Errorf("snapshot operation mismatch: got %q, want %q", completed.OperationID, accepted.OperationID)
 	}
-	// A pending completion is the provider's answer that it fixed the contents
-	// and will settle the snapshot later; a filtered ListSnapshots follows it.
-	if completed.Snapshot == nil || (completed.Snapshot.State != SnapshotStateCompleted && completed.Snapshot.State != SnapshotStatePending) {
-		return nil, fmt.Errorf("snapshot %s completion returned no completed or pending snapshot", req.SnapshotID)
+	if completed.Snapshot == nil || completed.Snapshot.State != SnapshotStateCompleted {
+		return nil, fmt.Errorf("snapshot %s completion returned no completed snapshot", req.SnapshotID)
 	}
 	return completed.Snapshot, nil
 }
@@ -479,7 +477,7 @@ func (p *NATSProvider) CopySnapshot(ctx context.Context, req CopySnapshotRequest
 // ListSnapshots asks the provider what it holds. Like ListVolumes it is not
 // owner-routed: the question is about the whole provider, not one snapshot.
 func (p *NATSProvider) ListSnapshots(ctx context.Context, req ListSnapshotsRequest) (*ListSnapshotsResponse, error) {
-	if err := CheckListSnapshotsVersion(req); err != nil {
+	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response ListSnapshotsResponse
@@ -489,20 +487,9 @@ func (p *NATSProvider) ListSnapshots(ctx context.Context, req ListSnapshotsReque
 	if err := responseError(response.SchemaVersion, response.Error); err != nil {
 		return nil, err
 	}
-	if req.SnapshotID != "" && (len(response.Snapshots) > 1 || (len(response.Snapshots) == 1 && response.Snapshots[0].ID != req.SnapshotID)) {
-		return nil, fmt.Errorf("%s filtered on %s answered with %v: %w", ListSnapshotsSubject, req.SnapshotID, snapshotRefIDs(response.Snapshots), ErrUnreadableReply)
-	}
 	return &response, nil
 }
 
-func snapshotRefIDs(refs []SnapshotRef) []string {
-	ids := make([]string, 0, len(refs))
-	for _, ref := range refs {
-		ids = append(ids, ref.ID)
-	}
-	return ids
-}
-
 func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeRequest) (*PublishedVolume, error) {
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err

spinifex/providers/ebs/nats_test.go

@@ -132,57 +132,37 @@ func TestNATSProviderReturnsTypedProviderError(t *testing.T) {
 	assert.Equal(t, "volume is mounted", err.Error())
 }
 
-// TestNATSProviderSnapshotWaitsForAsyncCompletion covers what the client makes
-// of the completion it waits for: a completed or pending snapshot is the
-// provider's answer, and anything else is refused.
 func TestNATSProviderSnapshotWaitsForAsyncCompletion(t *testing.T) {
-	tests := []struct {
-		name     string
-		snapshot *Snapshot
-		wantErr  string
-	}{
-		{name: "completed", snapshot: &Snapshot{ID: "snap-1", SourceVolumeID: "vol-1", State: SnapshotStateCompleted}},
-		{name: "pending", snapshot: &Snapshot{ID: "snap-1", SourceVolumeID: "vol-1", State: SnapshotStatePending}},
-		{name: "error state", snapshot: &Snapshot{ID: "snap-1", SourceVolumeID: "vol-1", State: SnapshotStateError},
-			wantErr: "snapshot snap-1 completion returned no completed or pending snapshot"},
-		{name: "no snapshot", wantErr: "snapshot snap-1 completion returned no completed or pending snapshot"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			_, conn := testutil.StartTestNATS(t)
-			subject, err := SnapshotSubject("vol-1")
-			require.NoError(t, err)
-			completionSubject, err := SnapshotCompletionSubject("snap-1")
-			require.NoError(t, err)
+	_, conn := testutil.StartTestNATS(t)
+	subject, err := SnapshotSubject("vol-1")
+	require.NoError(t, err)
+	completionSubject, err := SnapshotCompletionSubject("snap-1")
+	require.NoError(t, err)
 
-			sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
-				accepted, marshalErr := json.Marshal(CreateSnapshotResponse{
-					Versioned: NewVersioned(), OperationID: "op-snap-1", CompletionSubject: completionSubject,
-				})
-				require.NoError(t, marshalErr)
-				require.NoError(t, msg.Respond(accepted))
-
-				completed, marshalErr := json.Marshal(CreateSnapshotResponse{
-					Versioned: NewVersioned(), OperationID: "op-snap-1", Snapshot: tt.snapshot,
-				})
-				require.NoError(t, marshalErr)
-				require.NoError(t, conn.Publish(completionSubject, completed))
-			})
-			require.NoError(t, err)
-			t.Cleanup(func() { _ = sub.Unsubscribe() })
-			require.NoError(t, conn.Flush())
-
-			snapshot, err := NewNATSProvider(conn, time.Second).CreateSnapshot(t.Context(), CreateSnapshotRequest{
-				Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-1",
-			})
-			if tt.wantErr != "" {
-				require.EqualError(t, err, tt.wantErr)
-				return
-			}
-			require.NoError(t, err)
-			assert.Equal(t, tt.snapshot, snapshot)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		accepted, marshalErr := json.Marshal(CreateSnapshotResponse{
+			Versioned: NewVersioned(), OperationID: "op-snap-1", CompletionSubject: completionSubject,
 		})
-	}
+		require.NoError(t, marshalErr)
+		require.NoError(t, msg.Respond(accepted))
+
+		completed, marshalErr := json.Marshal(CreateSnapshotResponse{
+			Versioned: NewVersioned(), OperationID: "op-snap-1",
+			Snapshot: &Snapshot{ID: "snap-1", SourceVolumeID: "vol-1", State: SnapshotStateCompleted},
+		})
+		require.NoError(t, marshalErr)
+		require.NoError(t, conn.Publish(completionSubject, completed))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+
+	provider := NewNATSProvider(conn, time.Second)
+	snapshot, err := provider.CreateSnapshot(t.Context(), CreateSnapshotRequest{
+		Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-1",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "snap-1", snapshot.ID)
 }
 
 func TestNATSProviderRejectsUnsafeSubjectTokens(t *testing.T) {
@@ -565,40 +545,6 @@ func TestNATSProviderPropagatesProviderErrors(t *testing.T) {
 		require.ErrorIs(t, err, ErrUnsupportedCapability)
 	})
 
-	t.Run("ListSnapshots filtered on one snapshot", func(t *testing.T) {
-		ref := SnapshotRef{ID: "snap-1", SourceVolumeID: "vol-1", Handle: "vb://snap-1", State: SnapshotStatePending}
-		tests := []struct {
-			name    string
-			refs    []SnapshotRef
-			wantErr bool
-		}{
-			{name: "the one snapshot", refs: []SnapshotRef{ref}},
-			{name: "none", refs: nil},
-			{name: "another snapshot", refs: []SnapshotRef{{ID: "snap-2"}}, wantErr: true},
-			{name: "more than one", refs: []SnapshotRef{ref, {ID: "snap-2"}}, wantErr: true},
-		}
-		for _, tt := range tests {
-			t.Run(tt.name, func(t *testing.T) {
-				_, conn := testutil.StartTestNATS(t)
-				respondWith(t, conn, ListSnapshotsSubject, ListSnapshotsResponse{Versioned: NewLatestVersioned(), Snapshots: tt.refs})
-				page, err := NewNATSProvider(conn, time.Second).ListSnapshots(t.Context(), ListSnapshotsRequest{Versioned: NewLatestVersioned(), SnapshotID: "snap-1"})
-				if tt.wantErr {
-					require.ErrorIs(t, err, ErrUnreadableReply, "a filtered reply naming another snapshot must not pass for the one asked about")
-					return
-				}
-				require.NoError(t, err)
-				assert.Equal(t, tt.refs, page.Snapshots)
-			})
-		}
-	})
-
-	t.Run("ListSnapshots filter below its version is refused before it is sent", func(t *testing.T) {
-		_, err := NewNATSProvider(nil, time.Second).ListSnapshots(t.Context(), ListSnapshotsRequest{
-			Versioned: Versioned{SchemaVersion: NoSealSchemaVersion}, SnapshotID: "snap-1",
-		})
-		require.ErrorIs(t, err, ErrUnsupportedVersion)
-	})
-
 	t.Run("ListSnapshots page", func(t *testing.T) {
 		_, conn := testutil.StartTestNATS(t)
 		respondWith(t, conn, ListSnapshotsSubject, ListSnapshotsResponse{

spinifex/providers/ebs/natsserve/handlers_test.go

@@ -775,45 +775,3 @@ func TestServeStopIsIdempotent(t *testing.T) {
 	_, err = conn.Request(ebsprovider.CapabilitiesSubject, []byte(fmt.Sprintf(`{"schema_version":%d}`, ebsprovider.SchemaVersion)), 200*time.Millisecond)
 	require.ErrorIs(t, err, nats.ErrNoResponders, "stop must unsubscribe every subject Serve registered")
 }
-
-// listCountingProvider counts the ListSnapshots calls that reach it.
-type listCountingProvider struct {
-	*ebsprovider.MemoryProvider
-
-	lists atomic.Int32
-}
-
-func (p *listCountingProvider) ListSnapshots(ctx context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
-	p.lists.Add(1)
-	return p.MemoryProvider.ListSnapshots(ctx, req)
-}
-
-// TestListSnapshotsFilterOnTheWire pins the server half of the snapshot filter:
-// refused below version 4 without reaching the provider, and answered with the
-// one snapshot and its state at 4.
-func TestListSnapshotsFilterOnTheWire(t *testing.T) {
-	provider := &listCountingProvider{MemoryProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
-	conn := serveProvider(t, provider, natsserve.Options{NoQueueGroup: true})
-	ctx := context.Background()
-	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-	require.NoError(t, err)
-	provider.AnswerSnapshotsPending(true)
-	_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-1"})
-	require.NoError(t, err)
-	request := ebsprovider.ListSnapshotsRequest{SnapshotID: "snap-1"}
-
-	msg, err := conn.Request(ebsprovider.ListSnapshotsSubject, withVersion(t, request, ebsprovider.NoSealSchemaVersion), requestTimeout)
-	require.NoError(t, err)
-	envelope := decodeEnvelope(t, msg.Data)
-	require.NotNil(t, envelope.Error)
-	assert.Equal(t, ebsprovider.ErrorCodeUnsupportedVersion, envelope.Error.Code)
-	assert.Equal(t, ebsprovider.NoSealSchemaVersion, envelope.SchemaVersion)
-	assert.Zero(t, provider.lists.Load(), "the refused filter must not have reached the provider")
-
-	msg, err = conn.Request(ebsprovider.ListSnapshotsSubject, withVersion(t, request, ebsprovider.SnapshotFilterSchemaVersion), requestTimeout)
-	require.NoError(t, err)
-	var resp ebsprovider.ListSnapshotsResponse
-	require.NoError(t, json.Unmarshal(msg.Data, &resp))
-	require.Nil(t, resp.Error)
-	assert.Equal(t, []ebsprovider.SnapshotRef{{ID: "snap-1", SourceVolumeID: "vol-1", Handle: "memory://snapshot/snap-1", State: ebsprovider.SnapshotStatePending}}, resp.Snapshots)
-}

spinifex/providers/ebs/natsserve/server.go

@@ -326,10 +326,6 @@ func handleListSnapshots(provider ebsprovider.EBSProvider) msgHandler {
 			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
-		if err := ebsprovider.CheckListSnapshotsVersion(req); err != nil {
-			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: classifyError(err)})
-			return
-		}
 		resp, err := provider.ListSnapshots(ctx, req)
 		if err != nil {
 			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: classifyError(err)})

spinifex/providers/ebs/provider.go

@@ -29,15 +29,10 @@ const (
 	// the request as unsupported rather than dropping the flag and sealing.
 	NoSealSchemaVersion uint16 = 3
 
-	// SnapshotFilterSchemaVersion is the first version that carries
-	// ListSnapshotsRequest.SnapshotID and SnapshotRef.State. A provider that
-	// does not serve it refuses the filter rather than listing everything.
-	SnapshotFilterSchemaVersion uint16 = 4
-
 	// SchemaVersion is the newest wire-contract version understood by this
 	// build. Every request and response carries a version so an unsupported
 	// representation fails at the boundary rather than silently decoding it.
-	SchemaVersion uint16 = SnapshotFilterSchemaVersion
+	SchemaVersion uint16 = NoSealSchemaVersion
 )
 
 // Versioned is embedded in every request and response sent over NATS.
@@ -94,9 +89,6 @@ type EBSProvider interface {
 	ListVolumes(context.Context, ListVolumesRequest) (*ListVolumesResponse, error)
 	ExpandVolume(context.Context, ExpandVolumeRequest) (*Volume, error)
 	DeleteVolume(context.Context, DeleteVolumeRequest) error
-	// CreateSnapshot answers pending or completed, and either way the contents
-	// are fixed at the answer: later writes are not in it. A pending snapshot
-	// settles to completed or error, which a filtered ListSnapshots reports.
 	CreateSnapshot(context.Context, CreateSnapshotRequest) (*Snapshot, error)
 	DeleteSnapshot(context.Context, DeleteSnapshotRequest) error
 	CopySnapshot(context.Context, CopySnapshotRequest) (*Snapshot, error)
@@ -408,9 +400,8 @@ type Snapshot struct {
 	Handle         string        `json:"handle"`
 }
 
-// CreateSnapshotRequest snapshots VolumeID as SnapshotID, owner node first,
-// then SnapshotSubject. A repeat returns the snapshot held under SnapshotID;
-// ErrAlreadyExists only when the provider knows it came from another volume.
+// CreateSnapshotRequest snapshots VolumeID as SnapshotID. It is sent to the
+// volume's owner node first, then to SnapshotSubject.
 type CreateSnapshotRequest struct {
 	Versioned
 
@@ -479,11 +470,6 @@ type SnapshotRef struct {
 	ID             string `json:"id"`
 	SourceVolumeID string `json:"source_volume_id,omitempty"`
 	Handle         string `json:"handle,omitempty"`
-
-	// State is the provider's current answer: pending until it settles, then
-	// completed or error. It is reported from SnapshotFilterSchemaVersion on
-	// (RefState), and is empty at earlier versions.
-	State SnapshotState `json:"state,omitempty"`
 }
 
 // ListSnapshotsRequest pages through the snapshots a provider holds. It is
@@ -493,56 +479,6 @@ type ListSnapshotsRequest struct {
 
 	MaxResults    int32  `json:"max_results,omitempty"`
 	StartingToken string `json:"starting_token,omitempty"`
-
-	// SnapshotID reads that one snapshot, needing no SnapshotEnumeration: the
-	// reply holds it or none, and paging is ignored. Carried only from
-	// SnapshotFilterSchemaVersion on (CheckListSnapshotsVersion).
-	SnapshotID string `json:"snapshot_id,omitempty"`
-}
-
-// CheckListSnapshotsVersion refuses a request this build cannot serve, and a
-// SnapshotID filter sent below SnapshotFilterSchemaVersion as unsupported: a
-// provider at that version would drop the filter and list every snapshot.
-func CheckListSnapshotsVersion(req ListSnapshotsRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
-		return err
-	}
-	if req.SnapshotID != "" && req.SchemaVersion < SnapshotFilterSchemaVersion {
-		return fmt.Errorf("%w: snapshot_id filter requires schema version %d, got %d", ErrUnsupportedVersion, SnapshotFilterSchemaVersion, req.SchemaVersion)
-	}
-	return nil
-}
-
-// ReadSnapshot reads snapshotID from p through the filtered ListSnapshots at
-// the latest contract version. found is false when p holds no such snapshot.
-func ReadSnapshot(ctx context.Context, p EBSProvider, snapshotID string) (ref SnapshotRef, found bool, err error) {
-	resp, err := p.ListSnapshots(ctx, ListSnapshotsRequest{Versioned: NewLatestVersioned(), SnapshotID: snapshotID})
-	if err != nil {
-		return SnapshotRef{}, false, err
-	}
-	if resp == nil || len(resp.Snapshots) == 0 {
-		return SnapshotRef{}, false, nil
-	}
-	return resp.Snapshots[0], true, nil
-}
-
-// FilteredSnapshots is the reply to a filtered ListSnapshots: ref, or none
-// when ref is nil because the provider does not hold the snapshot.
-func FilteredSnapshots(req ListSnapshotsRequest, ref *SnapshotRef) *ListSnapshotsResponse {
-	response := &ListSnapshotsResponse{Versioned: ReplyVersioned(req.SchemaVersion)}
-	if ref != nil {
-		response.Snapshots = []SnapshotRef{*ref}
-	}
-	return response
-}
-
-// RefState is the State a SnapshotRef carries in a reply at version: state
-// from SnapshotFilterSchemaVersion on, and empty before it.
-func RefState(version uint16, state SnapshotState) SnapshotState {
-	if version < SnapshotFilterSchemaVersion {
-		return ""
-	}
-	return state
 }
 
 // ListSnapshotsResponse carries one page. An empty NextToken means the last

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -435,14 +435,6 @@ func handleListSnapshots(ctx context.Context, cfg *Config, msg *nats.Msg) {
 		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
-	if err := ebsprovider.CheckListSnapshotsVersion(req); err != nil {
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()}})
-		return
-	}
-	if req.SnapshotID != "" {
-		respondFilteredSnapshot(ctx, cfg, msg, req)
-		return
-	}
 
 	ids, err := listSnapshotPrefixes(ctx, providerObjectStoreFactory(cfg), cfg.Bucket)
 	if err != nil {
@@ -454,60 +446,11 @@ func handleListSnapshots(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	response := ebsprovider.ListSnapshotsResponse{Versioned: v}
 	response.Snapshots, response.NextToken = ebsprovider.Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.SnapshotRef {
-			return snapshotRef(req.SchemaVersion, id)
+			return ebsprovider.SnapshotRef{ID: id, Handle: snapshotHandle(id)}
 		})
 	respondProvider(ctx, msg, response)
 }
 
-// respondFilteredSnapshot answers a ListSnapshots filtered on one ID. Only a
-// snapshot whose config is written is held: CreateSnapshot writes it last, so
-// a snapshot that failed part way is not reported.
-func respondFilteredSnapshot(ctx context.Context, cfg *Config, msg *nats.Msg, req ebsprovider.ListSnapshotsRequest) {
-	if !validVolumeName(req.SnapshotID) || !strings.HasPrefix(req.SnapshotID, snapshotIDPrefix) {
-		respondProvider(ctx, msg, ebsprovider.FilteredSnapshots(req, nil))
-		return
-	}
-	held, err := snapshotConfigStored(ctx, providerObjectStoreFactory(cfg), cfg.Bucket, req.SnapshotID)
-	if err != nil {
-		slog.Error("ebs.provider.snapshot.list: failed to look up snapshot", "snapshot", req.SnapshotID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion), Error: internalError("look up snapshot: %v", err)})
-		return
-	}
-	if !held {
-		respondProvider(ctx, msg, ebsprovider.FilteredSnapshots(req, nil))
-		return
-	}
-	ref := snapshotRef(req.SchemaVersion, req.SnapshotID)
-	respondProvider(ctx, msg, ebsprovider.FilteredSnapshots(req, &ref))
-}
-
-// snapshotConfigStored reports whether bucket holds snapshotID's config. It
-// lists rather than heads the key, so absence is an empty page, not an error
-// whose code differs between object stores.
-func snapshotConfigStored(ctx context.Context, store objectstore.ObjectStore, bucket, snapshotID string) (bool, error) {
-	key := snapshotID + "/config.json"
-	out, err := store.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
-		Bucket:  awssdk.String(bucket),
-		Prefix:  awssdk.String(key),
-		MaxKeys: awssdk.Int64(1),
-	})
-	if err != nil {
-		return false, fmt.Errorf("list snapshot %s config: %w", snapshotID, err)
-	}
-	return len(out.Contents) > 0 && awssdk.StringValue(out.Contents[0].Key) == key, nil
-}
-
-// snapshotRef is snapshotID as a ref in a reply at version. SourceVolumeID is
-// left empty, and every snapshot is completed, because CreateSnapshot writes
-// it before the completion is published.
-func snapshotRef(version uint16, snapshotID string) ebsprovider.SnapshotRef {
-	return ebsprovider.SnapshotRef{
-		ID:     snapshotID,
-		Handle: snapshotHandle(snapshotID),
-		State:  ebsprovider.RefState(version, ebsprovider.SnapshotStateCompleted),
-	}
-}
-
 // buildProviderVBConfig assembles the viperblock.VB config CreateVolume hands
 // to viperblock.New. Only storage-owned facts go into VolumeMetadata here:
 // TenantID, Tags, VolumeType, IOPS, Throughput and AvailabilityZone are
@@ -1235,25 +1178,21 @@ func snapshotErrorCode(err error) ebsprovider.ErrorCode {
 	}
 }
 
-// checkSnapshotIDFree reports whether snapshotID is already this volume's
-// snapshot, and refuses it as already_exists if it is another's. An identity
-// that cannot be read is refused as internal, since it may yet read as ours.
-func checkSnapshotIDFree(vb *viperblock.VB, volumeID, snapshotID string) (taken bool, err error) {
+// checkSnapshotIDFree refuses a snapshot ID that already exists unless this
+// volume is the one it was taken from. An existing snapshot whose identity
+// cannot be read is refused too: unprovable ownership must not overwrite it.
+func checkSnapshotIDFree(vb *viperblock.VB, volumeID, snapshotID string) error {
 	exists, err := snapshotConfigExists(vb, snapshotID)
 	if err != nil {
-		return false, err
+		return err
 	}
 	if !exists {
-		return false, nil
+		return nil
 	}
-	_, ident, err := vb.LoadSnapshotBlockMap(snapshotID)
-	if err != nil {
-		return false, fmt.Errorf("read snapshot %s identity: %w", snapshotID, err)
+	if _, ident, err := vb.LoadSnapshotBlockMap(snapshotID); err == nil && ident.SourceVolumeName == volumeID {
+		return nil
 	}
-	if ident.SourceVolumeName != volumeID {
-		return false, fmt.Errorf("%w: %s", errSnapshotExistsElsewhere, snapshotID)
-	}
-	return true, nil
+	return fmt.Errorf("%w: %s", errSnapshotExistsElsewhere, snapshotID)
 }
 
 // snapshotVolumeEngine creates snapshotID off volumeID's live checkpoint,
@@ -1282,124 +1221,35 @@ func snapshotVolumeEngine(ctx context.Context, cfg *Config, volumeID, snapshotID
 	}
 	defer releaseWriterLock(ctx, lock, volumeID)
 
-	// A repeat is answered from the stored snapshot before the volume is
-	// opened, so it still answers once the source volume is deleted.
-	stored, storedErr := storedSnapshot(ctx, cfg, volumeID, snapshotID)
-	switch {
-	case storedErr == nil && stored != nil:
-		return stored, nil
-	case errors.Is(storedErr, errSnapshotExistsElsewhere):
-		return nil, storedErr
-	case storedErr != nil:
-		// The engine below repeats the lookup; this failure stands in for
-		// its not_found, which an unread snapshot does not prove.
-		slog.WarnContext(ctx, "ebs.provider.snapshot.create: stored snapshot lookup failed", "volume", volumeID, "snapshot", snapshotID, "err", storedErr)
-	}
-
 	vb, err := openLeasedVolumeVB(ctx, cfg, volumeID, lease)
 	if err != nil {
-		if storedErr != nil && errors.Is(err, viperblock.ErrStateNotFound) {
-			return nil, storedErr
-		}
 		return nil, err
 	}
 	defer vb.Detach()
 	return createSnapshotWithVB(vb, volumeID, snapshotID)
 }
 
-// storedSnapshot answers a repeated CreateSnapshot from the snapshot already
-// stored under snapshotID, without opening or leasing volumeID. It returns nil
-// when no snapshot is stored, and refuses one taken from another volume.
-func storedSnapshot(ctx context.Context, cfg *Config, volumeID, snapshotID string) (*ebsprovider.Snapshot, error) {
-	vbconfig, s3cfg := volumeVBConfig(cfg, volumeID)
-	vb, err := viperblock.NewStateReader(&vbconfig, "s3", s3cfg)
-	if err != nil {
-		return nil, fmt.Errorf("new snapshot reader: %w", err)
-	}
-	if err := vb.InitBackendForRead(ctx); err != nil {
-		return nil, fmt.Errorf("backend init: %w", err)
-	}
-	taken, err := checkSnapshotIDFree(vb, volumeID, snapshotID)
-	if err != nil || !taken {
-		return nil, err
-	}
-	createdAt, err := snapshotCreatedAt(vb, snapshotID)
-	if err != nil {
-		return nil, err
-	}
-	// The snapshot records no size of its own, so it is the source volume's
-	// size while that volume exists, and zero once it is deleted.
-	var sizeBytes int64
-	switch volume, err := describeVolumeEngine(ctx, cfg, volumeID); {
-	case err == nil:
-		sizeBytes = volume.CapacityBytes
-	case !errors.Is(err, viperblock.ErrStateNotFound):
-		return nil, fmt.Errorf("read source volume %s: %w", volumeID, err)
-	}
-	return completedSnapshot(snapshotID, volumeID, sizeBytes, createdAt), nil
-}
-
-// completedSnapshot is the answer for snapshotID of volumeID, which this
-// provider takes whole before it answers.
-func completedSnapshot(snapshotID, volumeID string, sizeBytes int64, createdAt time.Time) *ebsprovider.Snapshot {
-	return &ebsprovider.Snapshot{
-		ID:             snapshotID,
-		SourceVolumeID: volumeID,
-		SizeBytes:      sizeBytes,
-		CreatedAt:      createdAt,
-		State:          ebsprovider.SnapshotStateCompleted,
-		Handle:         snapshotHandle(snapshotID),
-	}
-}
-
-// snapshotCreatedAt reads the creation time stored in snapshotID's config.
-// The caller has authenticated the config through checkSnapshotIDFree; an
-// encrypted config is an envelope whose payload is the plaintext state.
-func snapshotCreatedAt(vb *viperblock.VB, snapshotID string) (time.Time, error) {
-	data, err := vb.Backend.ReadFrom(snapshotID, vbtypes.FileTypeConfig, 0, 0, 0)
-	if err != nil {
-		return time.Time{}, fmt.Errorf("read snapshot %s config: %w", snapshotID, err)
-	}
-	var config struct {
-		Payload   json.RawMessage `json:"payload"`
-		CreatedAt time.Time       `json:"CreatedAt"`
-	}
-	if err := json.Unmarshal(data, &config); err != nil {
-		return time.Time{}, fmt.Errorf("parse snapshot %s config: %w", snapshotID, err)
-	}
-	if len(config.Payload) > 0 {
-		if err := json.Unmarshal(config.Payload, &config); err != nil {
-			return time.Time{}, fmt.Errorf("parse snapshot %s config payload: %w", snapshotID, err)
-		}
-	}
-	return config.CreatedAt.UTC(), nil
-}
-
 // createSnapshotWithVB takes the snapshot against an already-resolved VB,
 // shared by snapshotVolumeEngine's mounted and freshly-opened branches so the
 // two cannot drift on what a snapshot of a volume means.
 func createSnapshotWithVB(vb *viperblock.VB, volumeID, snapshotID string) (*ebsprovider.Snapshot, error) {
-	taken, err := checkSnapshotIDFree(vb, volumeID, snapshotID)
-	if err != nil {
+	if err := checkSnapshotIDFree(vb, volumeID, snapshotID); err != nil {
 		return nil, err
 	}
-	// A repeat that raced the first call past storedSnapshot returns the first
-	// snapshot rather than retaking it over writes made since.
-	if taken {
-		createdAt, err := snapshotCreatedAt(vb, snapshotID)
-		if err != nil {
-			return nil, err
-		}
-		return completedSnapshot(snapshotID, volumeID, safecast.Uint64ToInt64(vb.GetVolumeSize()), createdAt), nil
-	}
 	if err := vb.LoadLiveCheckpoint(); err != nil {
 		return nil, fmt.Errorf("load live checkpoint: %w", err)
 	}
-	state, err := vb.CreateSnapshot(snapshotID)
-	if err != nil {
+	if _, err := vb.CreateSnapshot(snapshotID); err != nil {
 		return nil, fmt.Errorf("create snapshot: %w", err)
 	}
-	return completedSnapshot(snapshotID, volumeID, safecast.Uint64ToInt64(vb.GetVolumeSize()), state.CreatedAt.UTC()), nil
+	return &ebsprovider.Snapshot{
+		ID:             snapshotID,
+		SourceVolumeID: volumeID,
+		SizeBytes:      safecast.Uint64ToInt64(vb.GetVolumeSize()),
+		CreatedAt:      time.Now().UTC(),
+		State:          ebsprovider.SnapshotStateCompleted,
+		Handle:         snapshotHandle(snapshotID),
+	}, nil
 }
 
 // ownerVerbHandlers pairs each owner-routable verb's subject builder with

spinifex/providers/ebs/viperblock/snapshot_filter_test.go

@@ -1,155 +0,0 @@
-package viperblock
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	vbtypes "github.com/mulgadc/viperblock/types"
-	"github.com/mulgadc/viperblock/viperblock"
-	"github.com/mulgadc/viperblock/viperblock/backends/file"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// A repeat of CreateSnapshot must return the snapshot the first call took. A
-// retake would fix it over writes made after the first answer, which a caller
-// that released a quiesce on that answer relies on it not containing.
-func TestCreateSnapshotWithVB_RepeatReturnsTheFirstSnapshot(t *testing.T) {
-	for _, tc := range []struct {
-		name      string
-		encrypted bool
-	}{
-		{name: "plain"},
-		{name: "encrypted", encrypted: true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			const volumeName = "vol-snaprepeat0001"
-			const snapshotID = "snap-repeat0000001"
-			var vb *viperblock.VB
-			if tc.encrypted {
-				vb = createEncryptedTestVB(t, volumeName)
-			} else {
-				vb = createTestVBWithState(t, volumeName)
-			}
-
-			first, err := createSnapshotWithVB(vb, volumeName, snapshotID)
-			require.NoError(t, err)
-			firstConfig, err := vb.Backend.ReadFrom(snapshotID, vbtypes.FileTypeConfig, 0, 0, 0)
-			require.NoError(t, err)
-
-			again, err := createSnapshotWithVB(vb, volumeName, snapshotID)
-			require.NoError(t, err)
-			againConfig, err := vb.Backend.ReadFrom(snapshotID, vbtypes.FileTypeConfig, 0, 0, 0)
-			require.NoError(t, err)
-
-			assert.Equal(t, firstConfig, againConfig, "the repeat rewrote the snapshot instead of returning it")
-			assert.Equal(t, first.ID, again.ID)
-			assert.Equal(t, first.SourceVolumeID, again.SourceVolumeID)
-			assert.Equal(t, first.Handle, again.Handle)
-			assert.Equal(t, ebsprovider.SnapshotStateCompleted, again.State)
-			assert.False(t, again.CreatedAt.IsZero(), "the repeat must report the stored creation time")
-			assert.True(t, first.CreatedAt.Equal(again.CreatedAt), "the repeat reported %s, the first call %s", again.CreatedAt, first.CreatedAt)
-		})
-	}
-}
-
-// createEncryptedTestVB is createTestVBWithState with a master key, so its
-// snapshot configs are sealed envelopes.
-func createEncryptedTestVB(t *testing.T, volumeName string) *viperblock.VB {
-	t.Helper()
-	baseDir := t.TempDir()
-	vb, err := viperblock.New(&viperblock.VB{
-		VolumeName:        volumeName,
-		VolumeSize:        1,
-		BaseDir:           baseDir,
-		WALSyncInterval:   -1,
-		Cache:             viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
-		MasterKey:         testMasterKey(t),
-		EncryptionEnabled: true,
-	}, "file", file.FileConfig{VolumeName: volumeName, BaseDir: baseDir})
-	require.NoError(t, err)
-	t.Cleanup(vb.Detach)
-	require.NoError(t, vb.Backend.Init())
-	require.NoError(t, vb.SaveState())
-	return vb
-}
-
-func TestProviderHandlers_ListSnapshots_FilteredOnOneSnapshot(t *testing.T) {
-	_, natsURL := setupEmbeddedNATS(t)
-	cfg := setupTestConfig(t, natsURL)
-	nc := startProviderSubjects(t, cfg, natsURL)
-
-	store := objectstore.NewMemoryObjectStore()
-	prev := providerObjectStoreFactory
-	providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store }
-	t.Cleanup(func() { providerObjectStoreFactory = prev })
-
-	putTestObject(t, store, cfg.Bucket, "snap-filterheld0001/config.json")
-	putTestObject(t, store, cfg.Bucket, "snap-filterheld0001/checkpoints/blocks.00000001.bin")
-	// A create that failed after its checkpoint and before its config.
-	putTestObject(t, store, cfg.Bucket, "snap-filterpartial01/checkpoints/blocks.00000001.bin")
-	putTestObject(t, store, cfg.Bucket, "snap-filterheld0001x/config.json")
-
-	tests := []struct {
-		name       string
-		version    uint16
-		snapshotID string
-		want       []ebsprovider.SnapshotRef
-		wantCode   ebsprovider.ErrorCode
-	}{
-		{name: "held snapshot is the one ref, completed", version: ebsprovider.SnapshotFilterSchemaVersion, snapshotID: "snap-filterheld0001",
-			want: []ebsprovider.SnapshotRef{{ID: "snap-filterheld0001", Handle: snapshotHandle("snap-filterheld0001"), State: ebsprovider.SnapshotStateCompleted}}},
-		{name: "snapshot without a config is not held", version: ebsprovider.SnapshotFilterSchemaVersion, snapshotID: "snap-filterpartial01"},
-		{name: "absent snapshot answers none", version: ebsprovider.SnapshotFilterSchemaVersion, snapshotID: "snap-filterabsent01"},
-		{name: "a volume ID names no snapshot", version: ebsprovider.SnapshotFilterSchemaVersion, snapshotID: "vol-filternotasnap1"},
-		{name: "an unsafe ID names no snapshot", version: ebsprovider.SnapshotFilterSchemaVersion, snapshotID: "snap-../config"},
-		{name: "filter below its version is unsupported", version: ebsprovider.NoSealSchemaVersion, snapshotID: "snap-filterheld0001",
-			wantCode: ebsprovider.ErrorCodeUnsupportedVersion},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			body := marshalRequest(t, map[string]any{"schema_version": tt.version, "snapshot_id": tt.snapshotID})
-			msg := requestProvider(t, nc, ebsprovider.ListSnapshotsSubject, body)
-
-			var resp ebsprovider.ListSnapshotsResponse
-			require.NoError(t, json.Unmarshal(msg.Data, &resp))
-			if tt.wantCode != "" {
-				require.NotNil(t, resp.Error)
-				assert.Equal(t, tt.wantCode, resp.Error.Code)
-				return
-			}
-			require.Nil(t, resp.Error)
-			assert.Equal(t, tt.want, resp.Snapshots)
-			assert.Empty(t, resp.NextToken)
-		})
-	}
-}
-
-// Enumeration reports a snapshot's state only to a request at the version that
-// defines it, so an older caller's reply keeps the shape it was written for.
-func TestProviderHandlers_ListSnapshots_StateFromItsVersion(t *testing.T) {
-	_, natsURL := setupEmbeddedNATS(t)
-	cfg := setupTestConfig(t, natsURL)
-	nc := startProviderSubjects(t, cfg, natsURL)
-
-	store := objectstore.NewMemoryObjectStore()
-	prev := providerObjectStoreFactory
-	providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store }
-	t.Cleanup(func() { providerObjectStoreFactory = prev })
-	putTestObject(t, store, cfg.Bucket, "snap-liststate00001/config.json")
-
-	for version, want := range map[uint16]ebsprovider.SnapshotState{
-		ebsprovider.NoSealSchemaVersion:         "",
-		ebsprovider.SnapshotFilterSchemaVersion: ebsprovider.SnapshotStateCompleted,
-	} {
-		body := marshalRequest(t, map[string]any{"schema_version": version})
-		msg := requestProvider(t, nc, ebsprovider.ListSnapshotsSubject, body)
-		var resp ebsprovider.ListSnapshotsResponse
-		require.NoError(t, json.Unmarshal(msg.Data, &resp))
-		require.Nil(t, resp.Error)
-		require.Len(t, resp.Snapshots, 1)
-		assert.Equalf(t, want, resp.Snapshots[0].State, "v%d", version)
-	}
-}

spinifex/runtime/compute/vm/uefi_image_invariants_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -26,10 +25,6 @@ func (f *uefiFixture) createImage(t *testing.T) string {
 	svc := ec2image.NewImageServiceImplWithConfig(cfg, f.objects, nil)
 	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
 	svc.SetEBSProvider(provider)
-	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, f.objects, nil)
-	snapshots.SetEBSProvider(provider)
-	svc.SetSnapshotCreator(snapshots)
-	svc.SetImageClaims(snapshots)
 	svc.SetUEFIStores(f.records, "node-a")
 	created, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},

spinifex/runtime/daemon/cluster_wide_reaper.go

@@ -1,23 +0,0 @@
-package daemon
-
-import (
-	"context"
-
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-)
-
-// sweeper is a domain's reaper body, which names its class and sweeps but
-// leaves its scheduling scope to the runtime that registers it.
-type sweeper interface {
-	Class() string
-	Sweep(ctx context.Context) (int, error)
-}
-
-// clusterWideReaper runs a domain sweeper once per cluster, on the leader.
-type clusterWideReaper struct {
-	sweeper
-}
-
-var _ vm.Reaper = clusterWideReaper{}
-
-func (clusterWideReaper) Scope() vm.ReaperScope { return vm.ScopeClusterWide }

spinifex/runtime/daemon/daemon.go

@@ -1363,6 +1363,8 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{lambdafunction.SubjectCodeUpdateBegin, handleNATSRequest(d.node, d.lambdaMetaService.BeginCodeUpdate), "spinifex-workers"},
 			natsSub{lambdafunction.SubjectCodeUpdated, handleNATSRequest(d.node, d.lambdaMetaService.CodeUpdated), "spinifex-workers"},
 			natsSub{lambdafunction.SubjectConfigurationUpdated, handleNATSRequest(d.node, d.lambdaMetaService.UpdateConfiguration), "spinifex-workers"},
+			natsSub{lambdafunction.SubjectList, handleNATSRequest(d.node, d.lambdaMetaService.List), "spinifex-workers"},
+			natsSub{lambdafunction.SubjectDelete, handleNATSRequest(d.node, d.lambdaMetaService.Delete), "spinifex-workers"},
 		)
 	}
 
@@ -2084,9 +2086,6 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("failed to initialize snapshot service: %w", err)
 	}
 	d.snapshotService = snap.svc
-	d.snapshotService.SetClaims(ec2snapshot.NewClaims(d.snapshotClaims(), d.node, d.heartbeatLiveness()))
-	d.imageService.SetSnapshotCreator(d.snapshotService)
-	d.imageService.SetImageClaims(d.snapshotService)
 
 	d.volumeService = ec2volume.NewVolumeServiceImpl(d.config, d.natsConn, snap.kv)
 	if err := d.configureEBSProvider(); err != nil {
@@ -2551,10 +2550,6 @@ func (d *Daemon) startCluster() error {
 		slog.Info("Rebuilt mgmt IP allocator from restored instances", "allocated", d.mgmtIPAllocator.AllocatedCount())
 	}
 
-	// Before any request is served, so no claim a request of this process
-	// takes can be released as one of its predecessor's.
-	d.releaseSnapshotClaims()
-
 	if err := d.subscribeAll(); err != nil {
 		return fmt.Errorf("failed to subscribe to NATS topics: %w", err)
 	}
@@ -2646,11 +2641,6 @@ func (d *Daemon) startCluster() error {
 		if uefiWriteBacks := d.vmMgr.NewUEFIWriteBackReaper(); uefiWriteBacks != nil {
 			reapers = append(reapers, uefiWriteBacks)
 		}
-		if d.snapshotService != nil {
-			// One sweep per cluster reads the provider for each pending document;
-			// the snapshot's claim, not the scope, decides who may repeat a call.
-			reapers = append(reapers, clusterWideReaper{d.snapshotService.NewPendingSnapshotReaper()})
-		}
 		if d.volumeService != nil {
 			reapers = append(reapers, d.volumeService.NewVolumeLeakReaper(d.leakedVolumeInstances))
 			reapers = append(reapers, d.volumeService.NewVolumeReclaimReaper())

spinifex/runtime/daemon/daemon_lambda_listdelete_test.go

@@ -0,0 +1,72 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	lambdafunction "github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A daemon without the list and delete subjects refuses both as retryable
+// transport errors and changes nothing; once served, they decode and commit.
+func TestLambdaListDelete_PreviousDaemonRefusesWithoutMutation(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	svc := lambdafunction.NewMetaService(js)
+	client := lambdafunction.NewNATSClient(nc)
+	ctx := t.Context()
+	const region = "ap-southeast-2"
+
+	serveLambdaSubjects(t, nc, svc, true)
+	_, err := client.Create(ctx, lambdaTestAccount, region, lambdafunction.Record{
+		FunctionID: "f1", FunctionName: "hello", Role: "arn:aws:iam::000000000001:role/r", Runtime: "provided.al2023",
+		Handler: "bootstrap", Timeout: 3, MemorySize: 128, Code: lambdafunction.Code{RevisionID: "rev-1"},
+	})
+	require.NoError(t, err)
+	rec, rev, err := client.Get(ctx, lambdaTestAccount, region, "hello")
+	require.NoError(t, err)
+
+	_, _, err = client.List(ctx, lambdaTestAccount, region, "", 1)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+	err = client.Delete(ctx, lambdaTestAccount, region, rec, rev)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+	_, coded := awserrors.ResolveErrorCode(err)
+	assert.False(t, coded, "the gateway maps an uncoded failure to a retryable ServiceException")
+	same, sameRev, err := client.Get(ctx, lambdaTestAccount, region, "hello")
+	require.NoError(t, err)
+	assert.Equal(t, rev, sameRev)
+	assert.Equal(t, rec, same)
+	kv, err := js.KeyValue(ctx, lambdafunction.KVAccountBucket(lambdaTestAccount))
+	require.NoError(t, err)
+	_, err = kv.Get(ctx, lambdafunction.KVDeletedKey(region, "f1"))
+	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "no obligation was recorded")
+
+	for subject, h := range map[string]nats.MsgHandler{
+		lambdafunction.SubjectList:   asMsgHandler(handleNATSRequest("node-1", svc.List)),
+		lambdafunction.SubjectDelete: asMsgHandler(handleNATSRequest("node-1", svc.Delete)),
+	} {
+		sub, err := nc.QueueSubscribe(subject, "spinifex-workers", h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+	listed, next, err := client.List(ctx, lambdaTestAccount, region, "", 1)
+	require.NoError(t, err)
+	require.Len(t, listed, 1)
+	assert.Equal(t, rec, listed[0].Record)
+	assert.Equal(t, rev, listed[0].Revision)
+	assert.Empty(t, next)
+
+	_, _, err = client.List(ctx, lambdaTestAccount, region, "bogus", 1)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	require.NoError(t, client.Delete(ctx, lambdaTestAccount, region, rec, rev))
+	_, _, err = client.Get(ctx, lambdaTestAccount, region, "hello")
+	require.ErrorIs(t, err, lambdafunction.ErrNotFound)
+	err = client.Delete(ctx, lambdaTestAccount, region, rec, rev)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}

spinifex/runtime/daemon/locks.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/dns"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
@@ -31,7 +30,6 @@ type daemonLocks struct {
 	elbv2          *kvsync.Lease
 	eks            *kvsync.LeaseGroup
 	uefi           *kvsync.ClaimGroup
-	snapshot       *kvsync.ClaimGroup
 }
 
 func newDaemonLocks(store kvsync.KVStore, node string) *daemonLocks {
@@ -45,7 +43,6 @@ func newDaemonLocks(store kvsync.KVStore, node string) *daemonLocks {
 		elbv2:          elbv2domain.NewDataPathLease(store, node),
 		eks:            ekscluster.NewClusterLeases(store, node),
 		uefi:           uefistore.NewClaims(store, node),
-		snapshot:       ec2snapshot.NewClaimGroup(store, node),
 	}
 }
 
@@ -83,12 +80,3 @@ func (d *Daemon) uefiClaims() *kvsync.ClaimGroup {
 	}
 	return d.locks.uefi
 }
-
-// snapshotClaims is the pending-snapshot claim group, or nil before
-// startCluster has built the daemon's locks.
-func (d *Daemon) snapshotClaims() *kvsync.ClaimGroup {
-	if d.locks == nil {
-		return nil
-	}
-	return d.locks.snapshot
-}

spinifex/runtime/daemon/locks_test.go

@@ -25,7 +25,6 @@ func TestDaemonLocks_EveryOwnerHasItsOwnKey(t *testing.T) {
 		locks.eks.Get("123456789012/alpha")
 		locks.eks.Get("123456789012/alpha/teardown")
 		locks.uefi.Get("i-0123456789abcdef0")
-		locks.snapshot.Get("snap-0123456789abcdef0")
 	})
 }
 

spinifex/runtime/daemon/snapshot_claims.go

@@ -1,55 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"log/slog"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-)
-
-// snapshotClaimReleaseTimeout bounds the start-up release, which lists every
-// snapshot document before this daemon serves a request.
-const snapshotClaimReleaseTimeout = 30 * time.Second
-
-// releaseSnapshotClaims frees the claims this node's previous process held on
-// pending snapshots, so the reaper is not held off them by a live heartbeat.
-// A failed pass is retried in the background, where it skips this process's own.
-func (d *Daemon) releaseSnapshotClaims() {
-	if d.snapshotService == nil || d.snapshotClaims() == nil {
-		return
-	}
-	if d.releaseSnapshotClaimsOnce(d.ctx) {
-		return
-	}
-	d.shutdownWg.Go(func() {
-		ticker := time.NewTicker(vm.GCInterval)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-d.ctx.Done():
-				return
-			case <-ticker.C:
-				if d.releaseSnapshotClaimsOnce(d.ctx) {
-					return
-				}
-			}
-		}
-	})
-}
-
-// releaseSnapshotClaimsOnce runs one release pass and reports whether it read
-// and released everything it had to.
-func (d *Daemon) releaseSnapshotClaimsOnce(ctx context.Context) bool {
-	ctx, cancel := context.WithTimeout(ctx, snapshotClaimReleaseTimeout)
-	defer cancel()
-	released, err := d.snapshotService.ReleaseNodeClaims(ctx)
-	if released > 0 {
-		slog.InfoContext(ctx, "Released this node's claims on pending snapshots from its previous process", "count", released)
-	}
-	if err != nil {
-		slog.WarnContext(ctx, "Failed to release this node's claims on pending snapshots, retrying in the background", "err", err)
-		return false
-	}
-	return true
-}

spinifex/runtime/daemon/uefi_image_store_reaper_test.go

@@ -16,7 +16,6 @@ import (
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -96,10 +95,6 @@ func (f *uefiReaperFixture) createImage(t *testing.T, instanceID string) ebsmeta
 	svc := ec2image.NewImageServiceImplWithConfig(cfg, f.objects, nil)
 	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
 	svc.SetEBSProvider(provider)
-	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, f.objects, nil)
-	snapshots.SetEBSProvider(provider)
-	svc.SetSnapshotCreator(snapshots)
-	svc.SetImageClaims(snapshots)
 	svc.SetUEFIStores(f.m.UEFIStores(), "node-a")
 	created, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},

spinifex/runtime/daemon/unpublish_obligations_test.go

@@ -357,7 +357,7 @@ func TestVolumeMounterHold_UnpublishSealsOrDropsAsAsked(t *testing.T) {
 		assert.Equal(t, "vol-settle", call.VolumeID)
 		assert.Equal(t, "handle-1", call.Handle, "the obligation's handle names the publish it undoes")
 		assert.Equal(t, "node-hold", call.NodeID)
-		assert.GreaterOrEqual(t, call.SchemaVersion, ebsprovider.NoSealSchemaVersion, "no_seal is carried only from the version that defines it")
+		assert.Equal(t, ebsprovider.NoSealSchemaVersion, call.SchemaVersion)
 	}
 	assert.False(t, calls[0].NoSeal, "a surviving volume is sealed so its writes are kept")
 	assert.True(t, calls[1].NoSeal)

spinifex/runtime/daemon/vm_adapters_contract_test.go

@@ -697,7 +697,7 @@ func TestVolumeMounterAdapter_AbandonUnpublishesWithoutSealing(t *testing.T) {
 	require.Len(t, calls, 4, "every volume is attempted")
 	for _, call := range calls {
 		assert.True(t, call.noSeal, "%s must be dropped unsealed", call.volume)
-		assert.GreaterOrEqual(t, call.version, ebsprovider.NoSealSchemaVersion, "%s must be sent at a version carrying NoSeal", call.volume)
+		assert.Equal(t, ebsprovider.NoSealSchemaVersion, call.version, "%s must be sent at the version carrying NoSeal", call.volume)
 	}
 	assert.Contains(t, err.Error(), "abandon vol-refused (superseded)", "the failure must say why the volume was abandoned")
 	const gaveUp = "Gave up a volume export"

spinifex/runtime/roles/awsgw/lambda_testlink_off_test.go

@@ -52,6 +52,14 @@ func (*spyFunctions) UpdateConfiguration(context.Context, string, string, functi
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (*spyFunctions) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (*spyFunctions) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 // spyExec counts invocations that would start an attempt.
 type spyExec struct{ calls atomic.Int32 }
 

spinifex/runtime/roles/awsgw/lambda_testlink_reply_test.go

@@ -60,6 +60,14 @@ func (storedFunction) UpdateConfiguration(context.Context, string, string, funct
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (storedFunction) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (storedFunction) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 // invokeThrough sends one Invoke through the Lambda handler to the real
 // test-link executor; fakeControl stands in for the daemon and answers with rep.
 func invokeThrough(t *testing.T, rep testlink.Reply) (*httptest.ResponseRecorder, error) {

spinifex/runtime/roles/awsgw/lambda_testlink_seam_test.go

@@ -85,6 +85,14 @@ func (seamFunction) UpdateConfiguration(context.Context, string, string, functio
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (seamFunction) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (seamFunction) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 type urlPresigner struct {
 	url   string
 	calls *atomic.Int32

spinifex/runtime/roles/qemunbdd/provider.go

@@ -146,13 +146,6 @@ func (p *Provider) snapshotPath(snapshotID string) string {
 	return filepath.Join(p.baseDir, "snapshots", snapshotID+".qcow2")
 }
 
-// partialSnapshotPath is where a snapshot's file is written before it is
-// renamed to snapshotPath. It does not end in .qcow2, so no listing, lookup or
-// create ever counts it as a snapshot.
-func (p *Provider) partialSnapshotPath(snapshotID string) string {
-	return filepath.Join(p.baseDir, "snapshots", "."+snapshotID+".qcow2.partial")
-}
-
 func (p *Provider) socketPath(volumeID string) string {
 	return filepath.Join(p.baseDir, "sockets", volumeID+".sock")
 }
@@ -520,10 +513,11 @@ func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSna
 		}
 		return p.buildSnapshot(ctx, req.SnapshotID, meta.sourceVolumeID, snapPath)
 	}
-	// The provider restarted since it took this snapshot, so its source is
-	// unknown; the control plane mints each ID once, so this is a repeat.
+	// A file on disk with no matching snapshotMeta entry means the provider
+	// restarted since it was created: the source-volume link lived only in
+	// memory and cannot be recovered, so the ID is refused rather than silently overwritten or accepted without provenance.
 	if _, err := os.Stat(snapPath); err == nil {
-		return p.buildSnapshot(ctx, req.SnapshotID, "", snapPath)
+		return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrAlreadyExists, req.SnapshotID)
 	} else if !errors.Is(err, os.ErrNotExist) {
 		return nil, err
 	}
@@ -544,10 +538,7 @@ func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSna
 	if pub, ok := p.published[req.VolumeID]; ok && !pub.dead() {
 		return nil, fmt.Errorf("%w: volume %s is published, so it cannot be snapshotted until it is unpublished", ebsprovider.ErrVolumeInUse, req.VolumeID)
 	}
-	if err := p.writeSnapshotFile(req.SnapshotID, func(partial string) error {
-		_, err := p.run.Run(ctx, "qemu-img", "convert", "-O", "qcow2", volPath, partial)
-		return err
-	}); err != nil {
+	if _, err := p.run.Run(ctx, "qemu-img", "convert", "-O", "qcow2", volPath, snapPath); err != nil {
 		return nil, fmt.Errorf("create snapshot %s: %w", req.SnapshotID, err)
 	}
 	p.snapshotMeta[req.SnapshotID] = snapshotMeta{sourceVolumeID: req.VolumeID}
@@ -558,12 +549,9 @@ func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSna
 // snapshot that outlived the process still appears. Its SourceVolumeID comes
 // back empty in that case: the link was only ever in memory.
 func (p *Provider) ListSnapshots(_ context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
-	if err := ebsprovider.CheckListSnapshotsVersion(req); err != nil {
+	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
-	if req.SnapshotID != "" {
-		return p.filteredSnapshot(req)
-	}
 	entries, err := os.ReadDir(filepath.Join(p.baseDir, "snapshots"))
 	if err != nil {
 		return nil, fmt.Errorf("qemunbdd: read snapshots directory: %w", err)
@@ -575,41 +563,15 @@ func (p *Provider) ListSnapshots(_ context.Context, req ebsprovider.ListSnapshot
 	response := &ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion)}
 	response.Snapshots, response.NextToken = ebsprovider.Page(qcow2IDs(entries), req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.SnapshotRef {
-			return p.snapshotRef(req.SchemaVersion, id)
+			return ebsprovider.SnapshotRef{
+				ID:             id,
+				SourceVolumeID: p.snapshotMeta[id].sourceVolumeID,
+				Handle:         p.snapshotPath(id),
+			}
 		})
 	return response, nil
 }
 
-// filteredSnapshot answers a ListSnapshots filtered on one ID from its file. An
-// ID that is not a single path element names no file this provider wrote.
-func (p *Provider) filteredSnapshot(req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) {
-	if filepath.Base(req.SnapshotID) != req.SnapshotID {
-		return ebsprovider.FilteredSnapshots(req, nil), nil
-	}
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	if _, err := os.Stat(p.snapshotPath(req.SnapshotID)); err != nil {
-		if errors.Is(err, os.ErrNotExist) {
-			return ebsprovider.FilteredSnapshots(req, nil), nil
-		}
-		return nil, fmt.Errorf("qemunbdd: stat snapshot %s: %w", req.SnapshotID, err)
-	}
-	ref := p.snapshotRef(req.SchemaVersion, req.SnapshotID)
-	return ebsprovider.FilteredSnapshots(req, &ref), nil
-}
-
-// snapshotRef is snapshotID as a ref in a reply at version. Every snapshot is
-// completed, because CreateSnapshot copies it before answering. The caller
-// holds p.mu.
-func (p *Provider) snapshotRef(version uint16, snapshotID string) ebsprovider.SnapshotRef {
-	return ebsprovider.SnapshotRef{
-		ID:             snapshotID,
-		SourceVolumeID: p.snapshotMeta[snapshotID].sourceVolumeID,
-		Handle:         p.snapshotPath(snapshotID),
-		State:          ebsprovider.RefState(version, ebsprovider.SnapshotStateCompleted),
-	}
-}
-
 // snapshotInUse reports whether any volume file still points at snapPath as
 // its qcow2 backing file, the same relationship CreateVolume establishes for
 // a SourceSnapshotID clone.
@@ -707,61 +669,13 @@ func (p *Provider) CopySnapshot(ctx context.Context, req ebsprovider.CopySnapsho
 	}
 
 	srcPath := p.snapshotPath(req.SourceSnapshotID)
-	if err := p.writeSnapshotFile(req.DestinationSnapshotID, func(partial string) error {
-		return copyFile(srcPath, partial)
-	}); err != nil {
+	if err := copyFile(srcPath, dstPath); err != nil {
 		return nil, fmt.Errorf("copy snapshot %s to %s: %w", req.SourceSnapshotID, req.DestinationSnapshotID, err)
 	}
 	p.snapshotMeta[req.DestinationSnapshotID] = snapshotMeta{sourceVolumeID: req.VolumeID}
 	return p.buildSnapshot(ctx, req.DestinationSnapshotID, req.VolumeID, dstPath)
 }
 
-// writeSnapshotFile has write fill the snapshot's partial file, then syncs it
-// and renames it to the snapshot's name, so an interrupted write never leaves a
-// file there. A partial file an earlier interrupted write left is replaced.
-func (p *Provider) writeSnapshotFile(snapshotID string, write func(partial string) error) error {
-	partial := p.partialSnapshotPath(snapshotID)
-	if err := removeIfExists(partial); err != nil {
-		return fmt.Errorf("remove partial snapshot file: %w", err)
-	}
-	if err := write(partial); err != nil {
-		return errors.Join(err, removeIfExists(partial))
-	}
-	if err := syncPath(partial); err != nil {
-		return errors.Join(fmt.Errorf("sync partial snapshot file: %w", err), removeIfExists(partial))
-	}
-	final := p.snapshotPath(snapshotID)
-	if err := os.Rename(partial, final); err != nil {
-		return errors.Join(fmt.Errorf("publish snapshot file: %w", err), removeIfExists(partial))
-	}
-	// Without the directory's sync the rename may not survive a crash, so the
-	// snapshot is not reported until it has; a failure unpublishes it.
-	if err := syncPath(filepath.Dir(final)); err != nil {
-		return errors.Join(fmt.Errorf("sync snapshots directory: %w", err), removeIfExists(final))
-	}
-	return nil
-}
-
-// syncPath flushes a file's or a directory's contents to stable storage.
-func syncPath(path string) error {
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	if err := f.Sync(); err != nil {
-		_ = f.Close()
-		return err
-	}
-	return f.Close()
-}
-
-func removeIfExists(path string) error {
-	if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-	return nil
-}
-
 // copyFile duplicates a snapshot file directly: CopySnapshot's job is to
 // produce a second, independent file, not a CoW clone, so this is plain
 // filesystem I/O and never touches the runner.

spinifex/runtime/roles/qemunbdd/provider_test.go

@@ -2,7 +2,6 @@ package qemunbdd
 
 import (
 	"context"
-	"errors"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -411,78 +410,8 @@ func TestCreateSnapshot_Argv(t *testing.T) {
 	calls := fr.callsSnapshot()
 	snapPath := p.snapshotPath("snap-1")
 	require.Len(t, calls, 2)
-	assert.Equal(t, []string{"convert", "-O", "qcow2", volPath, p.partialSnapshotPath("snap-1")}, calls[0].args)
+	assert.Equal(t, []string{"convert", "-O", "qcow2", volPath, snapPath}, calls[0].args)
 	assert.Equal(t, []string{"info", "--output=json", "--force-share", snapPath}, calls[1].args)
-	assert.FileExists(t, snapPath)
-	assert.NoFileExists(t, p.partialSnapshotPath("snap-1"))
-}
-
-// partialConvert is a qemu-img convert that dies part way: it leaves some of
-// the copy at its target and fails.
-type partialConvert struct {
-	*fakeRunner
-}
-
-func (r partialConvert) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
-	if name == "qemu-img" && len(args) > 0 && args[0] == "convert" {
-		_ = os.WriteFile(args[len(args)-1], []byte("half a qcow2"), 0o640)
-		return nil, errors.New("qemu-img convert: signal: killed")
-	}
-	return r.fakeRunner.Run(ctx, name, args...)
-}
-
-// assertNoSnapshot requires that no read of p reports snapshotID.
-func assertNoSnapshot(t *testing.T, p *Provider, snapshotID string) {
-	t.Helper()
-	filtered, err := p.ListSnapshots(t.Context(), ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: snapshotID})
-	require.NoError(t, err)
-	assert.Empty(t, filtered.Snapshots, "a partial copy must never read as a snapshot")
-	all, err := p.ListSnapshots(t.Context(), ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewLatestVersioned()})
-	require.NoError(t, err)
-	assert.Empty(t, all.Snapshots, "a partial copy must never be listed")
-	assert.NoFileExists(t, p.snapshotPath(snapshotID))
-}
-
-// A copy that fails part way leaves no snapshot and no partial file, and the
-// same create then succeeds.
-func TestCreateSnapshot_FailedCopyIsNeverASnapshot(t *testing.T) {
-	p, fr := newTestProvider(t)
-	require.NoError(t, os.WriteFile(p.volumePath("vol-snap-src"), nil, 0o640))
-	p.run = partialConvert{fakeRunner: fr}
-	req := ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-cut", VolumeID: "vol-snap-src"}
-
-	_, err := p.CreateSnapshot(t.Context(), req)
-	require.Error(t, err)
-	assertNoSnapshot(t, p, "snap-cut")
-	assert.NoFileExists(t, p.partialSnapshotPath("snap-cut"))
-
-	p.run = fr
-	snap, err := p.CreateSnapshot(t.Context(), req)
-	require.NoError(t, err, "a failed copy must not leave the ID taken")
-	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
-	assert.FileExists(t, p.snapshotPath("snap-cut"))
-}
-
-// A provider killed mid-copy leaves its partial file. After the restart it is
-// no snapshot, and the reaper's repeat of the create replaces it.
-func TestCreateSnapshot_CopyKilledMidwayIsNeverASnapshot(t *testing.T) {
-	dir := t.TempDir()
-	killed, err := newProvider(dir, newFakeRunner())
-	require.NoError(t, err)
-	require.NoError(t, os.WriteFile(killed.volumePath("vol-snap-src"), nil, 0o640))
-	require.NoError(t, os.WriteFile(killed.partialSnapshotPath("snap-cut"), []byte("half a qcow2"), 0o640))
-
-	restarted, err := newProvider(dir, newFakeRunner())
-	require.NoError(t, err)
-	assertNoSnapshot(t, restarted, "snap-cut")
-
-	snap, err := restarted.CreateSnapshot(t.Context(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-cut", VolumeID: "vol-snap-src"})
-	require.NoError(t, err)
-	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
-	written, err := os.ReadFile(restarted.snapshotPath("snap-cut"))
-	require.NoError(t, err)
-	assert.Empty(t, written, "the snapshot is the new copy, not the partial one")
-	assert.NoFileExists(t, restarted.partialSnapshotPath("snap-cut"))
 }
 
 // A published volume is refused before qemu-img runs, and an unpublished one
@@ -527,27 +456,6 @@ func TestCreateSnapshot_IdempotentSameVolume(t *testing.T) {
 	assert.Equal(t, first.SourceVolumeID, second.SourceVolumeID)
 }
 
-func TestCreateSnapshot_RepeatAfterARestartAnswersTheSnapshot(t *testing.T) {
-	dir := t.TempDir()
-	first, err := newProvider(dir, newFakeRunner())
-	require.NoError(t, err)
-	require.NoError(t, os.WriteFile(first.volumePath("vol-snap-restart"), nil, 0o640))
-	req := ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-restart", VolumeID: "vol-snap-restart"}
-	_, err = first.CreateSnapshot(context.Background(), req)
-	require.NoError(t, err)
-
-	fr := newFakeRunner()
-	restarted, err := newProvider(dir, fr)
-	require.NoError(t, err)
-	got, err := restarted.CreateSnapshot(context.Background(), req)
-	require.NoError(t, err)
-	assert.Equal(t, "snap-restart", got.ID)
-	assert.Equal(t, ebsprovider.SnapshotStateCompleted, got.State)
-	for _, c := range fr.callsSnapshot() {
-		assert.NotContains(t, c.args, "convert", "a repeat must not copy the volume again")
-	}
-}
-
 func TestCreateSnapshot_ConflictingSourceVolumeIsAlreadyExists(t *testing.T) {
 	p, _ := newTestProvider(t)
 	volA := p.volumePath("vol-snap-a")

spinifex/runtime/roles/qemunbdd/snapshot_filter_test.go

@@ -1,37 +0,0 @@
-package qemunbdd_test
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qemunbdd"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// A filtered read is answered from the snapshot's file, so an ID that is not a
-// single path element must name nothing rather than a file elsewhere on disk.
-func TestListSnapshotsFilterOnlyNamesSnapshotFiles(t *testing.T) {
-	baseDir := shortTempDir(t)
-	provider, err := qemunbdd.NewProvider(baseDir)
-	require.NoError(t, err)
-	require.NoError(t, os.WriteFile(filepath.Join(baseDir, "volumes", "vol-1.qcow2"), nil, 0o600))
-	require.NoError(t, os.WriteFile(filepath.Join(baseDir, "snapshots", "snap-1.qcow2"), nil, 0o600))
-
-	for _, tt := range []struct {
-		snapshotID string
-		want       []ebsprovider.SnapshotRef
-	}{
-		{snapshotID: "snap-1", want: []ebsprovider.SnapshotRef{{ID: "snap-1", Handle: filepath.Join(baseDir, "snapshots", "snap-1.qcow2"), State: ebsprovider.SnapshotStateCompleted}}},
-		{snapshotID: "snap-absent"},
-		{snapshotID: "../volumes/vol-1"},
-	} {
-		t.Run(tt.snapshotID, func(t *testing.T) {
-			resp, err := provider.ListSnapshots(t.Context(), ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: tt.snapshotID})
-			require.NoError(t, err)
-			assert.Equal(t, tt.want, resp.Snapshots)
-		})
-	}
-}

tests/integration/image_daemon_lite.go

@@ -5,9 +5,7 @@ package integration
 import (
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 )
 
@@ -31,11 +29,6 @@ func StartImageDaemonLite(t *testing.T, gw *Gateway) (*ec2image.ImageServiceImpl
 
 	store := objectstore.NewMemoryObjectStore()
 	svc := ec2image.NewImageServiceImplWithStore(store, testImageBucket)
-	// As the daemon does, CreateImage snapshots through a snapshot service.
-	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(
-		&config.Config{Predastore: config.PredastoreConfig{Bucket: testImageBucket}}, store, nil)
-	svc.SetSnapshotCreator(snapshots)
-	svc.SetImageClaims(snapshots)
 
 	subscribeServiceMethods(t, gw.NATSConn, "ec2", svc)