fix: release only system-tagged addresses in the terminate backstop

An internet-facing ALB's address is now tagged managed-by at allocation, and
ReleaseAddressByInstanceID leaves any untagged record alone with a warning, so
a wrong instance ID can no longer release a customer's own address.

.github/workflows/dev-release.yml

@@ -30,10 +30,6 @@ concurrency:
 jobs:
   prerelease:
     runs-on: ubuntu-26.04
-    env:
-      # Dev prereleases, newest published first: GitHub lists releases by tag name
-      # as text, so list order is not build order. The install Worker uses this rule.
-      DEV_RELEASES_JQ: '[.[] | select(.draft == false and .prerelease and .published_at != null and (.tag_name | test("-dev(\\.|$)")))] | sort_by(.published_at) | reverse'
     steps:
       # Full history and tags, because the tag name is git describe and a shallow
       # clone has no tag to describe against.
@@ -168,12 +164,12 @@ jobs:
               exit 1
           fi
           echo "latest is still $latest"
-          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq "$DEV_RELEASES_JQ | .[0].tag_name")"
+          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
           [ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
 
-      # Kept rather than unbounded: the Worker reads one page of releases to resolve
-      # the channel, so enough prereleases would eventually push the newest off it.
-      # Five leaves a few recent builds to roll back to.
+      # Kept rather than unbounded: the Worker reads the first page of releases to
+      # resolve the channel, so enough prereleases would eventually push the real
+      # ones off it. Five leaves a few recent builds to roll back to.
       - name: Prune old dev prereleases
         if: steps.version.outputs.exists != 'true'
         env:
@@ -181,7 +177,7 @@ jobs:
         run: |
           set -euo pipefail
           gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
-              --jq "$DEV_RELEASES_JQ | .[5:] | .[].tag_name" \
+              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \
               > stale.txt
           while read -r tag; do
               [ -n "$tag" ] || continue

.github/workflows/spinifex.yml

@@ -20,12 +20,6 @@ on:
 permissions:
   contents: read
 
-# A newer push to the same pull request supersedes its run. Pushes and dispatches
-# get a group of their own, so every commit on dev and main keeps its result.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
-  cancel-in-progress: true
-
 jobs:
   build_and_test:
     name: Build and Test

.github/workflows/ui.yml

@@ -30,12 +30,6 @@ on:
 permissions:
   contents: read
 
-# A newer push to the same pull request supersedes its run. Pushes and dispatches
-# get a group of their own, so every commit on dev and main keeps its result.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
-  cancel-in-progress: true
-
 jobs:
   ui_build_and_test:
     name: UI Build and Test

.gitignore

@@ -27,7 +27,6 @@ build/microvm/sqs-worker/.stage.*/
 build/microvm/lambda-runtime/vmlinuz
 build/microvm/lambda-runtime/initramfs.cpio.gz
 build/microvm/lambda-runtime/.stage.*/
-build/microvm/lambda-testlink-runtime/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -82,35 +82,6 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
-# Controlled-test only: builds the tagged daemon and its hard-kill initializer
-# together, pinning the initializer's digest into that daemon build. The daemon
-# keeps its symbols so the pin can be read back from the binary.
-# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
-# build-lambda-testlink-image prints; it is read raw and validated before use.
-LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
-LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
-LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
-LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
-build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
-build-lambda-testlink:
-	@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
-	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
-	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
-	@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
-	@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
-	@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
-	cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
-	scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
-	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
-	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
-	echo "initializer sha256 $$digest"; \
-	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
-	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
-	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
-	@mkdir -p $(LAMBDA_TESTLINK_DIR)
-	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
-	@rmdir $(LAMBDA_TESTLINK_DIR).tmp
-
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
 	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -226,24 +197,6 @@ bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
 
 build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
-	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
-
-# Controlled-test only: a separately built guest whose agent has the
-# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
-# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
-LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
-build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
-	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
-	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
-	$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
-	@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
-	GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
-	./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
-	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
-	install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
-	@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
-	[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
-	echo "LAMBDA_IMAGE_DIGEST=$$digest"
 
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -299,10 +252,6 @@ test-build-scripts:
 	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
 		$(_RUNSH); \
 	done
-	@$(_HDR) "\n....Running scripts/terraform/latitude-spx/*_test.sh...."
-	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find scripts/terraform/latitude-spx -maxdepth 1 -name '*_test.sh' | sort); do \
-		$(_RUNSH); \
-	done
 	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
 	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
 		$(_RUNSH); \
@@ -380,10 +329,7 @@ test-cover:
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
 	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -594,7 +540,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

cmd/lambda-agent/connect.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 )
 
@@ -30,20 +29,20 @@ func defaultStartup() startup {
 
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
-func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {
+func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
 	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
 	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
-		return nil, "", exitRefused, errors.Join(loadErr, err)
+		return nil, exitRefused, errors.Join(loadErr, err)
 	}
 	if loadErr != nil {
-		return nil, "", exitRefused, loadErr
+		return nil, exitRefused, loadErr
 	}
 	conn, err := hostlink.Dial(ctx, id)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
 		}
-		return nil, "", exitLink, err
+		return nil, exitLink, err
 	}
-	return conn, id.Scope, 0, nil
+	return conn, 0, nil
 }

cmd/lambda-agent/grant_testlink.go

@@ -1,66 +0,0 @@
-//go:build lambdatestlink
-
-package main
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
-)
-
-const entryGrant = "test-grant"
-
-// testGrant is the grant the host launched this VM under.
-type testGrant struct {
-	grantID, attemptID, environmentID string
-	generation                        uint64
-}
-
-// readGrant reads the test-grant fw_cfg entry: grant ID, attempt ID,
-// environment ID and generation, one per line.
-func readGrant(fwCfgDir string) (testGrant, error) {
-	b, err := os.ReadFile(filepath.Join(fwCfgDir, entryGrant, "raw"))
-	if err != nil {
-		return testGrant{}, fmt.Errorf("read %s: %w", entryGrant, err)
-	}
-	lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n")
-	if len(lines) != 4 {
-		return testGrant{}, errors.New(entryGrant + " entry malformed")
-	}
-	g := testGrant{grantID: lines[0], attemptID: lines[1], environmentID: lines[2]}
-	g.generation, err = strconv.ParseUint(lines[3], 10, 64)
-	if err != nil || g.generation == 0 || g.grantID == "" || g.attemptID == "" || g.environmentID == "" {
-		return testGrant{}, errors.New(entryGrant + " entry malformed")
-	}
-	return g, nil
-}
-
-// checkBinding refuses a run unless the grant names the authenticated environment
-// generation, the request carries that grant and its deadline is still ahead. The
-// clock check is a local guard only: it cannot detect skew or a superseded grant.
-func checkBinding(scope lambdav1.Scope, g testGrant, req hostlink.Request, now time.Time) error {
-	parts := strings.Split(string(scope), "/")
-	if len(parts) != 5 {
-		return errors.New("authenticated scope malformed")
-	}
-	if parts[3] != g.environmentID || parts[4] != strconv.FormatUint(g.generation, 10) {
-		return errors.New("grant is not for the authenticated environment generation")
-	}
-	if req.GrantID != g.grantID || req.AttemptID != g.attemptID {
-		return errors.New("request does not carry this VM's grant")
-	}
-	if req.DeadlineMs <= 0 {
-		return errors.New("request has no deadline")
-	}
-	if !now.Before(time.UnixMilli(req.DeadlineMs)) {
-		return errors.New("request deadline already passed")
-	}
-	return nil
-}

cmd/lambda-agent/helpers_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/1"
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
 
 // guest is a fake fw_cfg tree; unload removes it as unloading the module would.
 type guest struct {

cmd/lambda-agent/package_testlink.go

@@ -18,7 +18,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-	"unicode"
 
 	"golang.org/x/sys/unix"
 )
@@ -29,7 +28,6 @@ import (
 const (
 	entryPackageSize   = "test-package-size"
 	entryPackageSHA256 = "test-package-sha256"
-	entryHandler       = "test-handler"
 
 	devMaxZipBytes    = 16 << 20
 	devMaxUnzipBytes  = 64 << 20
@@ -39,9 +37,8 @@ const (
 )
 
 type packageSpec struct {
-	size    int64
-	sha256  [sha256.Size]byte
-	handler string
+	size   int64
+	sha256 [sha256.Size]byte
 }
 
 type unpackLimits struct {
@@ -79,13 +76,6 @@ func readPackageSpec(fwCfgDir string) (packageSpec, error) {
 		return spec, errors.New("package sha256 is not 64 hex digits")
 	}
 	copy(spec.sha256[:], b)
-	if spec.handler, err = read(entryHandler); err != nil {
-		return spec, err
-	}
-	if spec.handler == "" || len(spec.handler) > 128 || strings.IndexFunc(spec.handler, unicode.IsSpace) >= 0 ||
-		strings.ContainsRune(spec.handler, 0) {
-		return spec, errors.New("handler must be 1..128 characters with no whitespace")
-	}
 	return spec, nil
 }
 

cmd/lambda-agent/package_testlink_test.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 	"syscall"
 	"testing"
 
@@ -166,23 +165,19 @@ func TestUnpackChecksSizeAndHash(t *testing.T) {
 func TestReadPackageSpecBounds(t *testing.T) {
 	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 	cases := map[string]struct {
-		size, hash, handler string
-		want                string
+		size, hash string
+		want       string
 	}{
-		"valid":            {"10", sum, "bootstrap.h", ""},
-		"zero":             {"0", sum, "bootstrap.h", "outside"},
-		"too large":        {"16777217", sum, "bootstrap.h", "outside"},
-		"not hex":          {"10", "zz", "bootstrap.h", "64 hex digits"},
-		"short":            {"10", sum[:62], "bootstrap.h", "64 hex digits"},
-		"empty handler":    {"10", sum, "", "handler"},
-		"handler space":    {"10", sum, "a b", "handler"},
-		"128-char handler": {"10", sum, strings.Repeat("h", 128), ""},
-		"129-char handler": {"10", sum, strings.Repeat("h", 129), "handler"},
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			dir := t.TempDir()
-			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash, entryHandler: tc.handler} {
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
 				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
 			}
@@ -190,7 +185,6 @@ func TestReadPackageSpecBounds(t *testing.T) {
 			if tc.want == "" {
 				require.NoError(t, err)
 				assert.Equal(t, int64(10), spec.size)
-				assert.Equal(t, tc.handler, spec.handler)
 				return
 			}
 			assert.ErrorContains(t, err, tc.want)

cmd/lambda-agent/start_tcp.go

@@ -25,7 +25,7 @@ func start(ctx context.Context) int {
 
 // run authenticates to the host and holds the connection until it closes.
 func (s startup) run(ctx context.Context) (int, error) {
-	conn, _, code, err := s.connect(ctx)
+	conn, code, err := s.connect(ctx)
 	if err != nil {
 		return code, err
 	}

cmd/lambda-agent/start_testlink.go

@@ -5,8 +5,8 @@ package main
 import (
 	"context"
 	"errors"
+	"flag"
 	"fmt"
-	"io"
 	"log/slog"
 	"net"
 	"net/http"
@@ -24,6 +24,8 @@ const (
 	taskRoot    = "/var/task"
 )
 
+var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
+
 // testlink relays one invocation over the test carrier on the authenticated
 // connection; the carrier is not the guest protocol.
 type testlink struct {
@@ -52,14 +54,13 @@ func start(ctx context.Context) int {
 func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
-	grant, grantErr := readGrant(t.fwCfgDir)
-	conn, scope, code, err := t.connect(parent)
+	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, errors.Join(err, specErr, grantErr)
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
-	if err := errors.Join(specErr, grantErr); err != nil {
-		return exitRefused, err
+	if specErr != nil {
+		return exitRefused, specErr
 	}
 	link := hostlink.NewTestStream(conn)
 
@@ -75,10 +76,6 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
-	// Nothing of the tenant's is installed or run for an unbound request.
-	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
-		return exitRefused, err
-	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -96,7 +93,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	defer cancel(nil)
 	cmd, err := t.startBootstrap(ctx, []string{
 		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
-		"_HANDLER=" + spec.handler,
+		"_HANDLER=" + *handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
 		"PATH=/bin:/sbin:/usr/bin",
 	})
@@ -107,11 +104,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	go func() {
 		defer close(exited)
 		err := cmd.Wait()
-		// Once a response is being received the result stands, so only an
-		// exit before that cancels the invocation.
-		if srv.BootstrapExited() {
-			cancel(fmt.Errorf("bootstrap exited: %w", err))
-		}
+		cancel(fmt.Errorf("bootstrap exited: %w", err))
 	}()
 
 	err = agent.Run(ctx, &oneInvocation{Link: link, first: &first}, srv)
@@ -123,18 +116,11 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// shutdownGrace is how long past the invocation deadline the agent waits for
-// the host's shutdown frame once its result is written.
-const shutdownGrace = 2 * time.Second
-
 // oneInvocation serves the host's first request and then stops the agent:
-// each test VM runs a single invocation. After the result is written it waits
-// for the host's shutdown frame or EOF, so the result is not cut off by the
-// poweroff that follows the agent's exit.
+// each test VM runs a single invocation.
 type oneInvocation struct {
 	hostlink.Link
-	first      *hostlink.Request
-	deadlineMs int64
+	first *hostlink.Request
 }
 
 var _ hostlink.Link = (*oneInvocation)(nil)
@@ -143,36 +129,10 @@ func (o *oneInvocation) Next(ctx context.Context) (hostlink.Request, error) {
 	if err := ctx.Err(); err != nil {
 		return hostlink.Request{}, err
 	}
-	if o.first != nil {
-		req := *o.first
-		o.first = nil
-		o.deadlineMs = req.DeadlineMs
-		return req, nil
+	if o.first == nil {
+		return hostlink.Request{}, hostlink.ErrShutdown
 	}
-	return hostlink.Request{}, o.awaitShutdown(ctx)
-}
-
-// awaitShutdown reads the host's next frame and always ends the agent: with
-// ErrShutdown for a shutdown frame, EOF, a stray frame or an expired wait.
-func (o *oneInvocation) awaitShutdown(ctx context.Context) error {
-	end := time.UnixMilli(o.deadlineMs)
-	if now := time.Now(); end.Before(now) {
-		end = now
-	}
-	wctx, cancel := context.WithDeadline(ctx, end.Add(shutdownGrace))
-	defer cancel()
-	_, err := o.Link.Next(wctx)
-	switch {
-	case errors.Is(err, hostlink.ErrShutdown), errors.Is(err, io.EOF):
-		return hostlink.ErrShutdown
-	case err == nil:
-		slog.Warn("lambda-agent: frame after the invocation was not dispatched")
-		return hostlink.ErrShutdown
-	case ctx.Err() != nil:
-		return err
-	case wctx.Err() != nil:
-		slog.Warn("lambda-agent: no shutdown from the host; exiting")
-		return hostlink.ErrShutdown
-	}
-	return err
+	req := *o.first
+	o.first = nil
+	return req, nil
 }

cmd/lambda-agent/start_testlink_test.go

@@ -25,18 +25,12 @@ import (
 
 const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"
 
-// helperHandlerOut names the file the helper bootstrap writes its _HANDLER to.
-const helperHandlerOut = "LAMBDA_AGENT_TEST_HANDLER_OUT"
-
 // TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
 // echoes one invocation through the Runtime API, then waits for the next.
 func TestHelperBootstrap(t *testing.T) {
 	if os.Getenv(helperEnv) != "1" {
 		return
 	}
-	if out := os.Getenv(helperHandlerOut); out != "" {
-		require.NoError(t, os.WriteFile(out, []byte(os.Getenv("_HANDLER")), 0o600))
-	}
 	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
 	resp, err := http.Get(base + "next")
 	require.NoError(t, err)
@@ -47,28 +41,14 @@ func TestHelperBootstrap(t *testing.T) {
 	post, err := http.Post(base+id+"/response", "application/json", bytes.NewReader(body))
 	require.NoError(t, err)
 	post.Body.Close()
-	if os.Getenv(helperExitAfterPost) == "1" {
-		return
-	}
 	_, _ = http.Get(base + "next")
 }
 
-// helperExitAfterPost makes the helper bootstrap exit once its response is posted.
-const helperExitAfterPost = "LAMBDA_AGENT_TEST_EXIT_AFTER_POST"
-
-// testGrantEntry grants g-1 for attempt a-1 on the fixture scope's env-1/1.
-const testGrantEntry = "g-1\na-1\nenv-1\n1\n"
-
-// testHandler is the handler the fixture's host configures.
-const testHandler = "bootstrap.fixture"
-
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
 	task    string
 	sealed  bool
-	// handlerOut receives the _HANDLER the bootstrap process itself saw.
-	handlerOut string
 }
 
 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -81,16 +61,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	sum := sha256.Sum256(zipBytes)
 	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
 	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
-	if _, ok := entries[entryHandler]; !ok {
-		entries[entryHandler] = []byte(testHandler)
-	}
-	if _, ok := entries[entryGrant]; !ok {
-		entries[entryGrant] = []byte(testGrantEntry)
-	}
 	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
-	b.handlerOut = filepath.Join(t.TempDir(), "handler")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -104,7 +77,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			b.n++
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
-			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
+			cmd.Env = append(env, helperEnv+"=1")
 			return cmd, cmd.Start()
 		},
 	}
@@ -159,105 +132,10 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
-	seen, err := os.ReadFile(b.handlerOut)
-	require.NoError(t, err, "bootstrap process did not report its handler")
-	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
 
-// A bootstrap that posts its response and exits at once does not cost the
-// host the result; the agent then waits for the host's shutdown.
-func TestTestlinkBootstrapExitingAfterResponseStillDelivers(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	h := newHost(t)
-	h.accept(ctx)
-	var b bootstrapStarts
-	tl := b.testlink(t, h.entries())
-	start := tl.startBootstrap
-	tl.startBootstrap = func(ctx context.Context, env []string) (*exec.Cmd, error) {
-		return start(ctx, append(env, helperExitAfterPost+"=1"))
-	}
-	done := runTestlinkAsync(ctx, tl)
-
-	conn := h.guest(t)
-	defer conn.Close()
-	hs := hostlink.NewTestStream(conn)
-	payload := bytes.Repeat([]byte("0123456789"), 300_000)
-	req := hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: payload}
-	require.NoError(t, hs.HostSend(req))
-	got, err := hs.HostResult(ctx)
-	require.NoError(t, err)
-	assert.Equal(t, hostlink.Response{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", Payload: payload}, got)
-
-	require.NoError(t, hs.HostShutdown())
-	res := wait(t, done, "agent to stop after the host's shutdown")
-	assert.Equal(t, exitClosed, res.code)
-	assert.NoError(t, res.err)
-}
-
-// The agent stays up for the host's shutdown rather than exiting as soon as
-// its result is written, and never dispatches a stray frame.
-func TestOneInvocationAfterResult(t *testing.T) {
-	first := hostlink.Request{RequestID: "r-1", DeadlineMs: time.Now().Add(time.Minute).UnixMilli()}
-	cases := map[string]struct {
-		next func(context.Context) (hostlink.Request, error)
-		want error
-	}{
-		"shutdown frame": {func(context.Context) (hostlink.Request, error) { return hostlink.Request{}, hostlink.ErrShutdown }, hostlink.ErrShutdown},
-		"eof":            {func(context.Context) (hostlink.Request, error) { return hostlink.Request{}, io.EOF }, hostlink.ErrShutdown},
-		"a second request is not dispatched": {func(context.Context) (hostlink.Request, error) {
-			return hostlink.Request{RequestID: "r-2"}, nil
-		}, hostlink.ErrShutdown},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			o := &oneInvocation{Link: stubLink{tc.next}, first: &first}
-			_, err := o.Next(context.Background())
-			require.NoError(t, err)
-			req, err := o.Next(context.Background())
-			require.ErrorIs(t, err, tc.want)
-			assert.Empty(t, req.RequestID)
-		})
-	}
-	t.Run("host never sends shutdown", func(t *testing.T) {
-		past := first
-		past.DeadlineMs = time.Now().Add(-time.Second).UnixMilli()
-		o := &oneInvocation{Link: stubLink{func(ctx context.Context) (hostlink.Request, error) {
-			<-ctx.Done()
-			return hostlink.Request{}, ctx.Err()
-		}}, first: &past}
-		_, err := o.Next(context.Background())
-		require.NoError(t, err)
-		start := time.Now()
-		_, err = o.Next(context.Background())
-		require.ErrorIs(t, err, hostlink.ErrShutdown)
-		assert.GreaterOrEqual(t, time.Since(start), shutdownGrace-100*time.Millisecond, "waited the grace")
-	})
-	t.Run("parent cancel is not a shutdown", func(t *testing.T) {
-		o := &oneInvocation{Link: stubLink{func(ctx context.Context) (hostlink.Request, error) {
-			<-ctx.Done()
-			return hostlink.Request{}, ctx.Err()
-		}}, first: &first}
-		_, _ = o.Next(context.Background())
-		ctx, cancel := context.WithCancel(context.Background())
-		cancel()
-		_, err := o.Next(ctx)
-		require.ErrorIs(t, err, context.Canceled)
-	})
-}
-
-type stubLink struct {
-	next func(context.Context) (hostlink.Request, error)
-}
-
-var _ hostlink.Link = stubLink{}
-
-func (l stubLink) Next(ctx context.Context) (hostlink.Request, error) { return l.next(ctx) }
-
-func (stubLink) Complete(context.Context, hostlink.Response) error { return nil }
-
 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -340,8 +218,7 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
 
 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
-		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
 	assert.ErrorContains(t, res.err, "install package")
@@ -367,87 +244,3 @@ func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
 	_, err := os.Stat(tl.fwCfgRoot)
 	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
 }
-
-// A request that does not carry this VM's grant, or a grant for another
-// environment generation than the one authenticated, installs and runs
-// nothing of the tenant's.
-func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
-	cases := map[string]struct {
-		grant            string
-		grantID, attempt string
-		deadlineMs       int64
-		want             string
-	}{
-		"frame with no deadline":      {testGrantEntry, "g-1", "a-1", -1, "no deadline"},
-		"frame with expired deadline": {testGrantEntry, "g-1", "a-1", time.Now().Add(-time.Second).UnixMilli(), "deadline already passed"},
-		"frame for another grant":     {testGrantEntry, "g-2", "a-1", 0, "does not carry this VM's grant"},
-		"frame for another attempt":   {testGrantEntry, "g-1", "a-2", 0, "does not carry this VM's grant"},
-		"frame with no grant":         {testGrantEntry, "", "", 0, "does not carry this VM's grant"},
-		"grant for another env":       {"g-1\na-1\nenv-2\n1\n", "g-1", "a-1", 0, "not for the authenticated environment"},
-		"grant for another gen":       {"g-1\na-1\nenv-1\n2\n", "g-1", "a-1", 0, "not for the authenticated environment"},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			entries := h.entries()
-			entries[entryGrant] = []byte(tc.grant)
-			done := runTestlinkAsync(ctx, b.testlink(t, entries))
-			conn := h.guest(t)
-			defer conn.Close()
-			deadline := time.Now().Add(5 * time.Second).UnixMilli()
-			switch {
-			case tc.deadlineMs < 0:
-				deadline = 0
-			case tc.deadlineMs > 0:
-				deadline = tc.deadlineMs
-			}
-			req := hostlink.Request{GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
-				DeadlineMs: deadline, Payload: []byte(`{}`)}
-			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
-
-			res := wait(t, done, "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, tc.want)
-			assert.Zero(t, b.n)
-			assert.False(t, b.sealed)
-			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
-		})
-	}
-}
-
-// A missing or malformed grant entry refuses before any frame is awaited.
-func TestTestlinkBadGrantEntryRefuses(t *testing.T) {
-	cases := map[string]*string{
-		"missing":       nil,
-		"too few lines": ptr("g-1\na-1\nenv-1\n"),
-		"generation 0":  ptr("g-1\na-1\nenv-1\n0\n"),
-		"empty grant":   ptr("\na-1\nenv-1\n1\n"),
-	}
-	for name, grant := range cases {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			entries := h.entries()
-			if grant != nil {
-				entries[entryGrant] = []byte(*grant)
-			}
-			tl := b.testlink(t, entries)
-			if grant == nil {
-				require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryGrant)))
-			}
-			res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, entryGrant)
-			assert.Zero(t, b.n)
-		})
-	}
-}
-
-func ptr(s string) *string { return &s }

cmd/lambda-image-agent-tag/main.go

@@ -1,221 +0,0 @@
-// Command lambda-image-agent-tag checks whether the lambda-agent archived in a
-// built Lambda guest initramfs carries the lambdatestlink build tag. Build
-// information is recorded by the build; it is not proof of how the agent behaves.
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"compress/gzip"
-	"debug/buildinfo"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"slices"
-	"strconv"
-	"strings"
-)
-
-const (
-	agentPath    = "usr/local/bin/lambda-agent"
-	maxArchive   = 1 << 30
-	maxAgent     = 256 << 20
-	maxName      = 4096
-	newcHeader   = 110
-	trailerName  = "TRAILER!!!"
-	modeTypeMask = 0o170000
-	modeDir      = 0o040000
-	modeRegular  = 0o100000
-)
-
-var agentParents = []string{"usr", "usr/local", "usr/local/bin"}
-
-func main() {
-	if len(os.Args) != 3 || (os.Args[2] != "present" && os.Args[2] != "absent") {
-		fmt.Fprintln(os.Stderr, "usage: lambda-image-agent-tag <initramfs.cpio.gz> present|absent")
-		os.Exit(2)
-	}
-	if err := run(os.Args[1], os.Args[2] == "present"); err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[1], err)
-		os.Exit(1)
-	}
-}
-
-func run(path string, want bool) error {
-	fi, err := os.Lstat(path)
-	if err != nil {
-		return err
-	}
-	if !fi.Mode().IsRegular() {
-		return errors.New("not a regular file")
-	}
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	agent, err := archivedAgent(f)
-	if err != nil {
-		return err
-	}
-	return checkTag(agent, want)
-}
-
-// archivedAgent returns the single agent entry after checking listed names,
-// entry types, the agent's link count and zero-only padding after the trailer.
-// It is not a full kernel-equivalent parser; 070702 CRCs, for one, are unchecked.
-func archivedAgent(r io.Reader) ([]byte, error) {
-	zr, err := gzip.NewReader(bufio.NewReader(r))
-	if err != nil {
-		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
-	}
-	raw, err := io.ReadAll(io.LimitReader(zr, maxArchive+1))
-	if err != nil {
-		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
-	}
-	if len(raw) > maxArchive {
-		return nil, fmt.Errorf("archive larger than %d bytes", maxArchive)
-	}
-
-	seen := map[string]bool{}
-	var agent []byte
-	off := 0
-	for {
-		e, next, err := parseEntry(raw, off)
-		if err != nil {
-			return nil, fmt.Errorf("entry at offset %d: %w", off, err)
-		}
-		off = next
-		if e.name == trailerName {
-			break
-		}
-		if !canonical(e.name) {
-			return nil, fmt.Errorf("entry name %q is not canonical", e.name)
-		}
-		if seen[e.name] {
-			return nil, fmt.Errorf("entry %q appears more than once", e.name)
-		}
-		seen[e.name] = true
-		for _, p := range agentParents {
-			if e.name == p && e.mode&modeTypeMask != modeDir {
-				return nil, fmt.Errorf("%s is not a directory", p)
-			}
-		}
-		if e.name == agentPath {
-			for _, p := range agentParents {
-				if !seen[p] {
-					return nil, fmt.Errorf("%s has no %s directory entry before it", agentPath, p)
-				}
-			}
-			if e.mode&modeTypeMask != modeRegular {
-				return nil, fmt.Errorf("%s is not a regular file", agentPath)
-			}
-			if e.nlink != 1 {
-				return nil, fmt.Errorf("%s has %d links, want 1", agentPath, e.nlink)
-			}
-			if len(e.data) > maxAgent {
-				return nil, fmt.Errorf("%s is larger than %d bytes", agentPath, maxAgent)
-			}
-			agent = e.data
-		}
-	}
-	if i := bytes.IndexFunc(raw[off:], func(r rune) bool { return r != 0 }); i >= 0 {
-		return nil, fmt.Errorf("data after the archive trailer at offset %d", off+i)
-	}
-	if agent == nil {
-		return nil, fmt.Errorf("no %s entry", agentPath)
-	}
-	return agent, nil
-}
-
-type entry struct {
-	name  string
-	mode  uint64
-	nlink uint64
-	data  []byte
-}
-
-func parseEntry(raw []byte, off int) (entry, int, error) {
-	if len(raw)-off < newcHeader {
-		return entry{}, 0, errors.New("truncated header")
-	}
-	h := raw[off : off+newcHeader]
-	if m := string(h[:6]); m != "070701" && m != "070702" {
-		return entry{}, 0, fmt.Errorf("magic %q is not newc", m)
-	}
-	field := func(i int) (uint64, error) {
-		return strconv.ParseUint(string(h[6+8*i:14+8*i]), 16, 32)
-	}
-	mode, err1 := field(1)
-	nlink, err2 := field(4)
-	size, err3 := field(6)
-	nameSize, err4 := field(11)
-	if err := errors.Join(err1, err2, err3, err4); err != nil {
-		return entry{}, 0, fmt.Errorf("bad header field: %w", err)
-	}
-	if nameSize < 2 || nameSize > maxName {
-		return entry{}, 0, fmt.Errorf("name size %d out of range", nameSize)
-	}
-	nameEnd := off + newcHeader + int(nameSize)
-	dataOff := align4(nameEnd)
-	dataEnd := dataOff + int(size)
-	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
-		return entry{}, 0, errors.New("truncated entry")
-	}
-	name := raw[off+newcHeader : nameEnd]
-	if name[len(name)-1] != 0 || bytes.IndexByte(name[:len(name)-1], 0) >= 0 {
-		return entry{}, 0, errors.New("name is not one NUL-terminated string")
-	}
-	return entry{name: string(name[:len(name)-1]), mode: mode, nlink: nlink, data: raw[dataOff:dataEnd]}, align4(dataEnd), nil
-}
-
-func align4(n int) int { return (n + 3) &^ 3 }
-
-// canonical accepts "." and relative names with no empty, "." or ".." component.
-func canonical(name string) bool {
-	if name == "." {
-		return true
-	}
-	if name == "" || strings.HasPrefix(name, "/") {
-		return false
-	}
-	for c := range strings.SplitSeq(name, "/") {
-		if c == "" || c == "." || c == ".." {
-			return false
-		}
-	}
-	return true
-}
-
-func checkTag(agent []byte, want bool) error {
-	info, err := buildinfo.Read(bytes.NewReader(agent))
-	if err != nil {
-		return fmt.Errorf("no Go build information in %s: %w", agentPath, err)
-	}
-	return decide(info, want)
-}
-
-// decide refuses build information with no recorded settings, since their
-// absence does not show the tag was absent.
-func decide(info *buildinfo.BuildInfo, want bool) error {
-	if len(info.Settings) == 0 {
-		return fmt.Errorf("%s records no build settings", agentPath)
-	}
-	if hasTag(info, "lambdatestlink") != want {
-		return fmt.Errorf("%s: lambdatestlink build tag present=%v, want %v", agentPath, !want, want)
-	}
-	return nil
-}
-
-func hasTag(info *buildinfo.BuildInfo, tag string) bool {
-	for _, s := range info.Settings {
-		if s.Key != "-tags" {
-			continue
-		}
-		if slices.Contains(strings.Split(s.Value, ","), tag) {
-			return true
-		}
-	}
-	return false
-}

cmd/lambda-image-agent-tag/main_test.go

@@ -1,154 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"debug/buildinfo"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime/debug"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-type fileEntry struct {
-	name  string
-	mode  uint32
-	nlink uint32
-	data  string
-}
-
-func dir(name string) fileEntry { return fileEntry{name: name, mode: modeDir | 0o755, nlink: 2} }
-func file(name, data string) fileEntry {
-	return fileEntry{name: name, mode: modeRegular | 0o755, nlink: 1, data: data}
-}
-func symlink(name, target string) fileEntry {
-	return fileEntry{name: name, mode: 0o120777, nlink: 1, data: target}
-}
-
-// newc writes entries and a trailer in the format the kernel unpacks.
-func newc(entries ...fileEntry) []byte {
-	var b bytes.Buffer
-	for i, e := range append(entries, fileEntry{name: trailerName, nlink: 1}) {
-		fmt.Fprintf(&b, "070701%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X",
-			i+1, e.mode, 0, 0, e.nlink, 0, len(e.data), 0, 0, 0, 0, len(e.name)+1, 0)
-		b.WriteString(e.name)
-		b.WriteByte(0)
-		pad(&b)
-		b.WriteString(e.data)
-		pad(&b)
-	}
-	return b.Bytes()
-}
-
-func pad(b *bytes.Buffer) {
-	for b.Len()%4 != 0 {
-		b.WriteByte(0)
-	}
-}
-
-func gz(parts ...[]byte) []byte {
-	var b bytes.Buffer
-	for _, p := range parts {
-		w := gzip.NewWriter(&b)
-		_, _ = w.Write(p)
-		_ = w.Close()
-	}
-	return b.Bytes()
-}
-
-func withAgent(agent fileEntry) []byte {
-	return newc(dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent, file("init", "#!/bin/sh"))
-}
-
-func TestArchivedAgent_ReturnsTheOneArchivedAgent(t *testing.T) {
-	got, err := archivedAgent(bytes.NewReader(gz(withAgent(file(agentPath, "agent-bytes")))))
-	require.NoError(t, err)
-	require.Equal(t, "agent-bytes", string(got))
-
-	padded := append(withAgent(file(agentPath, "agent-bytes")), make([]byte, 1024)...)
-	got, err = archivedAgent(bytes.NewReader(gz(padded)))
-	require.NoError(t, err, "zero padding after the trailer")
-	require.Equal(t, "agent-bytes", string(got))
-}
-
-func TestArchivedAgent_Refuses(t *testing.T) {
-	parents := []fileEntry{dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin")}
-	agent := file(agentPath, "untagged")
-	small := newc(append(parents, agent)...)
-	second := newc(append(parents[1:], file(agentPath, "tagged"))...)
-	// The second archive starts inside what would be the first one's final block.
-	sameBlock := append(append([]byte{}, small...), second...)
-	for len(sameBlock)%512 != 0 {
-		sameBlock = append(sameBlock, 0)
-	}
-
-	for name, archive := range map[string][]byte{
-		"aliased duplicate":            gz(newc(append(parents, agent, file("././"+agentPath, "tagged"))...)),
-		"leading slash alias":          gz(newc(append(parents, agent, file("/"+agentPath, "tagged"))...)),
-		"dot-dot component":            gz(newc(append(parents, agent, file("usr/local/../local/bin/lambda-agent", "x"))...)),
-		"empty component":              gz(newc(append(parents, agent, file("usr//local/bin/lambda-agent", "x"))...)),
-		"exact duplicate":              gz(newc(append(parents, agent, file(agentPath, "tagged"))...)),
-		"duplicate directory":          gz(newc(dir("usr"), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent)),
-		"missing directory entry":      gz(newc(dir("usr"), dir("usr/local/bin"), agent)),
-		"directory after the agent":    gz(newc(dir("usr"), dir("usr/local"), agent, dir("usr/local/bin"))),
-		"parent is a symlink":          gz(newc(dir("usr"), dir("usr/local"), symlink("usr/local/bin", "elsewhere"), agent)),
-		"agent is a symlink":           gz(newc(append(parents, symlink(agentPath, "/bin/sh"))...)),
-		"hard-linked agent":            gz(newc(append(parents, fileEntry{name: agentPath, mode: modeRegular | 0o755, nlink: 2, data: "x"})...)),
-		"no agent":                     gz(newc(parents...)),
-		"second archive appended":      gz(small, second),
-		"second archive in same block": gz(sameBlock),
-		"non-zero byte after trailer":  gz(append(append([]byte{}, small...), 0, 0, 1)),
-		"truncated archive":            gz(small[:len(small)-20]),
-		"not newc":                     gz([]byte("not cpio at all, but long enough to fill one newc header: " + string(make([]byte, 120)))),
-		"not gzip":                     small,
-	} {
-		t.Run(name, func(t *testing.T) {
-			_, err := archivedAgent(bytes.NewReader(archive))
-			require.Error(t, err)
-		})
-	}
-}
-
-func TestRun_RefusesASymlinkedImagePath(t *testing.T) {
-	d := t.TempDir()
-	img := filepath.Join(d, "initramfs.cpio.gz")
-	require.NoError(t, os.WriteFile(img, gz(withAgent(file(agentPath, "x"))), 0o644))
-	require.NoError(t, os.Symlink(img, filepath.Join(d, "link")))
-	require.ErrorContains(t, run(filepath.Join(d, "link"), false), "not a regular file")
-}
-
-func TestHasTag_MatchesWholeTagsOnly(t *testing.T) {
-	info := func(tags string) *buildinfo.BuildInfo {
-		return &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}, {Key: "-tags", Value: tags}}}
-	}
-	require.True(t, hasTag(info("lambdatestlink,fips140v1.0"), "lambdatestlink"))
-	require.False(t, hasTag(info("fips140v1.0"), "lambdatestlink"))
-	require.False(t, hasTag(info("lambdatestlinkx,xlambdatestlink"), "lambdatestlink"))
-	require.False(t, hasTag(&buildinfo.BuildInfo{}, "lambdatestlink"))
-}
-
-// The test binary carries real build information, so checkTag reads it and
-// refuses the opposite answer; bytes with no build information are refused.
-func TestCheckTag_ReadsRealBuildInformation(t *testing.T) {
-	exe, err := os.Executable()
-	require.NoError(t, err)
-	raw, err := os.ReadFile(exe)
-	require.NoError(t, err)
-	info, err := buildinfo.Read(bytes.NewReader(raw))
-	require.NoError(t, err)
-	tagged := hasTag(info, "lambdatestlink")
-	require.NoError(t, checkTag(raw, tagged))
-	require.Error(t, checkTag(raw, !tagged))
-	require.Error(t, checkTag([]byte("not a Go binary"), false))
-}
-
-func TestDecide_RefusesMissingBuildSettings(t *testing.T) {
-	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, false), "no build settings")
-	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, true), "no build settings")
-	untagged := &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}}}
-	require.NoError(t, decide(untagged, false))
-	require.Error(t, decide(untagged, true))
-}

cmd/lambda-image-digest/main.go

@@ -1,26 +0,0 @@
-//go:build lambdatestlink
-
-// Command lambda-image-digest prints the test-link runtime image digest of one
-// directory, selecting the boot files the same way the daemon does. The digest
-// identifies the bytes; it does not authenticate where they came from.
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
-)
-
-func main() {
-	if len(os.Args) != 2 {
-		fmt.Fprintln(os.Stderr, "usage: lambda-image-digest <image-dir>")
-		os.Exit(2)
-	}
-	digest, err := testlink.DirDigest(os.Args[1])
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-	fmt.Println(digest)
-}

contracts/cluster/v1/facts.go

@@ -56,8 +56,6 @@ type NodeFacts struct {
 	Memory      MemoryFacts       `json:"memory"`
 	NICs        []NICFacts        `json:"nics"`
 	Planes      []PlaneFacts      `json:"planes"`
-	Disks       []DiskFacts       `json:"disks,omitempty"`
-	Filesystems []FilesystemFacts `json:"filesystems,omitempty"`
 	Sources     map[string]string `json:"sources"`
 }
 
@@ -136,42 +134,6 @@ type MemoryGroup struct {
 	Serials       []string `json:"serials,omitempty"`
 }
 
-// DiskFacts is one physical drive as sysfs reports it. Transport is nvme, sata,
-// usb or virtio, and empty when the device path names none, as a RAID
-// controller's virtual disk does.
-type DiskFacts struct {
-	Name       string `json:"name"`
-	Path       string `json:"path"`
-	SizeBytes  uint64 `json:"size_bytes"`
-	Model      string `json:"model,omitempty"`
-	Transport  string `json:"transport,omitempty"`
-	Rotational bool   `json:"rotational,omitempty"`
-	Removable  bool   `json:"removable,omitempty"`
-}
-
-// FilesystemFacts is one mounted block-device or ZFS filesystem. BindOf names
-// the directory a bind mount shows; Disk names the drives under it. Error says
-// why Usage is missing or stale.
-type FilesystemFacts struct {
-	Mount  string           `json:"mount"`
-	Source string           `json:"source"`
-	Disk   string           `json:"disk,omitempty"`
-	Type   string           `json:"type"`
-	BindOf string           `json:"bind_of,omitempty"`
-	Usage  *FilesystemUsage `json:"usage,omitempty"`
-	Error  string           `json:"error,omitempty"`
-	// ID identifies the mount on its node, so a filesystem replaced at the
-	// same path is read afresh. Empty for a mount the daemon cannot see.
-	ID string `json:"-"`
-}
-
-// FilesystemUsage is a filesystem's last statfs reading, as df counts it.
-type FilesystemUsage struct {
-	SizeBytes  uint64 `json:"size_bytes"`
-	UsedBytes  uint64 `json:"used_bytes"`
-	AvailBytes uint64 `json:"avail_bytes"`
-}
-
 // NICFacts is one physical network interface and, where LLDP reports it, the
 // switch port it is cabled to.
 type NICFacts struct {

scripts/build-lambda-runtime-microvm-image.sh

@@ -3,28 +3,24 @@
 # + initramfs: a guest running the Lambda guest agent, with virtio-net for its
 # host control connection. It is not shipped.
 #
-# Usage: build-lambda-runtime-microvm-image.sh [agent-binary [output-dir]]
-#
 # Inputs:
 #   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
-#   agent binary, default $REPO_ROOT/bin/lambda-agent (make bin/lambda-agent)
+#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
 #
-# Outputs, in output-dir (default $REPO_ROOT/build/microvm/lambda-runtime):
-#   vmlinuz
-#   initramfs.cpio.gz
+# Outputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
+#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
 #
 # Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
 # shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-INPUT_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
-INIT_SH="$INPUT_DIR/init.sh"
-INITTAB="$INPUT_DIR/inittab"
-RUN_AGENT="$INPUT_DIR/run-agent"
-LAMBDA_AGENT_BIN="${1:-$REPO_ROOT/bin/lambda-agent}"
-# The library stages beside BUILD_DIR, so it names the output, not the inputs.
-BUILD_DIR="${2:-$INPUT_DIR}"
+BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+RUN_AGENT="$BUILD_DIR/run-agent"
+LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
 TAG="[build-lambda-runtime-microvm-image]"
 
 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
@@ -42,8 +38,7 @@ MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
 source "$REPO_ROOT/scripts/lib/microvm-image.sh"
 
 echo "$TAG repo root: $REPO_ROOT"
-echo "$TAG input dir: $INPUT_DIR"
-echo "$TAG output dir: $BUILD_DIR"
+echo "$TAG build dir: $BUILD_DIR"
 
 microvm_check_tools
 
@@ -62,9 +57,6 @@ if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
     exit 1
 fi
 
-# Absolute, because packing the initramfs runs from inside the chroot.
-mkdir -p "$BUILD_DIR"
-BUILD_DIR=$(cd "$BUILD_DIR" && pwd)
 microvm_init_dirs
 microvm_make_rootfs
 microvm_empty_dev

scripts/build-lambda-runtime-microvm-image_test.sh

@@ -1,81 +0,0 @@
-#!/bin/sh
-# build-lambda-runtime-microvm-image_test.sh — a build into a chosen output
-# directory stages and publishes there, and leaves the default input directory
-# untouched.
-#
-# The script runs in a fixture repo against the real library, with only the
-# rootfs, kernel and packing steps replaced so no image is downloaded.
-set -eu
-
-SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-echo "-- build-lambda-runtime-microvm-image.sh output directory"
-
-REPO="$TMP/repo"
-IN="$REPO/build/microvm/lambda-runtime"
-mkdir -p "$REPO/scripts/lib" "$IN" "$REPO/bin" "$TMP/bin"
-cp "$SCRIPTS_DIR/build-lambda-runtime-microvm-image.sh" "$REPO/scripts/"
-for f in init.sh inittab run-agent; do printf '%s\n' "$f" > "$IN/$f"; done
-printf 'default image\n' > "$IN/vmlinuz"
-printf '#!/bin/sh\n' > "$REPO/bin/agent-testlink"
-chmod +x "$REPO/bin/agent-testlink"
-printf '#!/bin/sh\necho "ELF 64-bit, statically linked"\n' > "$TMP/bin/file"
-chmod +x "$TMP/bin/file"
-BEFORE=$(cd "$IN" && ls -A | cksum)
-
-# The real library, with the steps that need a rootfs or kernel replaced.
-# Packing runs from inside the chroot as the real step does, and records where
-# the stage is while it still exists.
-cat > "$REPO/scripts/lib/microvm-image.sh" <<EOF
-. "$SCRIPTS_DIR/lib/microvm-image.sh"
-microvm_check_tools() { :; }
-microvm_make_rootfs() { mkdir -p "\$CHROOT_DIR/etc"; }
-microvm_empty_dev() { :; }
-microvm_locate_kernel() { :; }
-microvm_strip_modules() { :; }
-microvm_strip_rootfs() { :; }
-microvm_stage_kernel() { printf 'kernel\n' > "\$STAGE_DIR/vmlinuz"; }
-microvm_pack_initramfs() {
-    (cd "\$CHROOT_DIR" && printf 'initramfs\n' | gzip > "\$STAGE_DIR/initramfs.cpio.gz")
-    dirname "\$STAGE_DIR" > "$TMP/stage-parent"
-    ls -A "$IN" > "$TMP/input-during"
-}
-EOF
-
-# The output is given relative to the repo root, as make passes it.
-OUT="$REPO/build/microvm/lambda-testlink-runtime"
-if ! (cd "$REPO" && PATH="$TMP/bin:$PATH" bash scripts/build-lambda-runtime-microvm-image.sh bin/agent-testlink build/microvm/lambda-testlink-runtime) > "$TMP/log" 2>&1; then
-    fail "build into output dir: $(tr '\n' ' ' < "$TMP/log")"
-else
-    if [ "$(cat "$TMP/stage-parent")" = "$OUT" ]; then
-        pass "stage created in the output directory"
-    else
-        fail "stage created in $(cat "$TMP/stage-parent"), want $OUT"
-    fi
-    if grep -q '^\.stage\.' "$TMP/input-during"; then
-        fail "stage present in the default input directory during the build"
-    else
-        pass "no stage in the default input directory during the build"
-    fi
-    if [ "$(cd "$IN" && ls -A | cksum)" = "$BEFORE" ] && grep -qx 'default image' "$IN/vmlinuz"; then
-        pass "default input directory unchanged"
-    else
-        fail "default input directory changed: $(cd "$IN" && ls -A | tr '\n' ' ')"
-    fi
-    if grep -qx kernel "$OUT/vmlinuz" && gzip -t "$OUT/initramfs.cpio.gz" && [ "$(ls -A "$OUT" | wc -l)" -eq 2 ]; then
-        pass "output directory holds exactly the published pair"
-    else
-        fail "output directory: $(ls -A "$OUT" | tr '\n' ' ')"
-    fi
-fi
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi

scripts/install-node.sh

@@ -260,14 +260,6 @@ out() {
     ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@" 2>/dev/null || true
 }
 
-# remote_join_cmd prints the command that joins one node. The token is not in it: it arrives on
-# stdin and is read after sudo has logged its command, so sudo's log never holds it. The other
-# values are positional arguments of a literal script, never spliced into it.
-remote_join_cmd() {
-    local script='exec spx admin join --force --node "$1" --bind "$2" --cluster-bind "$2" --advertise "$3" --host "$4" --token "$(cat)" --region "$5" --az "$6"'
-    printf 'sudo bash -c %q _ %q %q %q %q %q %q' "$script" "$@"
-}
-
 SETUP_OVN="/usr/local/share/spinifex/setup-ovn.sh"
 
 # --- Preflight -------------------------------------------------------------
@@ -660,18 +652,14 @@ fi
 [ -n "$EXT_IFACE" ] && init_args+=(--external-iface="$EXT_IFACE")
 [ -n "$EMAIL" ] && init_args+=(--email="$EMAIL")
 
-# init prints the join token, so its log is root-only and outside the world-writable /tmp.
-INIT_LOG=/var/log/spx-init.log
-
-# redact_tokens hides join tokens in text about to be shown to the operator.
-redact_tokens() { sed -E 's/spx_join_[A-Za-z0-9_-]+/spx_join_[redacted]/g'; }
+INIT_LOG=/tmp/spx-init.log
 
 log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s) to join)"
 
 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid sh -c 'spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
     exit 0" || fail "${HOSTS[0]}: could not start spx admin init"
 
 TOKEN=""
@@ -687,7 +675,7 @@ else
         elapsed=$((elapsed + 5))
     done
     if [ -z "$TOKEN" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: spx admin init produced no join token within ${TOKEN_TIMEOUT}s"
     fi
     log "  join token issued"
@@ -703,9 +691,11 @@ fi
 # waiting on a shell that node 2 is still holding.
 JOIN_PIDS=()
 for i in $(seq 1 $((N - 1))); do
-    # The token goes to the node on stdin, so it is in no ssh argument and not in sudo's log.
-    # spx admin join takes it only as --token, so it is still in that process's arguments.
-    on "${HOSTS[$i]}" "$(remote_join_cmd "${NODE_NAMES[$i]}" "${LAN_IPS[$i]}" "${WAN_IPS[$i]}" "${LAN_IPS[0]}:$PORT" "$REGION" "$AZ")" <<<"$TOKEN" &
+    on "${HOSTS[$i]}" "sudo spx admin join --force \
+        --node ${NODE_NAMES[$i]} \
+        --bind ${LAN_IPS[$i]} --cluster-bind ${LAN_IPS[$i]} --advertise ${WAN_IPS[$i]} \
+        --host ${LAN_IPS[0]}:$PORT --token $TOKEN \
+        --region $REGION --az $AZ" &
     JOIN_PIDS+=("$!")
 done
 
@@ -734,7 +724,7 @@ if ! $DRY_RUN; then
         elapsed=$((elapsed + 5))
     done
     if [ "$elapsed" -ge "$FORMATION_TIMEOUT" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: formation did not complete within ${FORMATION_TIMEOUT}s"
     fi
     log "  formation complete"

scripts/install-node_test.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
-# sudo's log (which holds the command it was given) never contains it, and the other values must
-# arrive as plain arguments even when they hold shell syntax.
-# preflight runs this with sh
-[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
-set -euo pipefail
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-mkdir "$work/bin"
-cd "$work"
-
-cat > "$work/bin/sudo" <<'SH'
-#!/bin/sh
-echo "$*" >> "$FAKE_SUDO_LOG"
-exec "$@"
-SH
-cat > "$work/bin/spx" <<'SH'
-#!/bin/sh
-for a in "$@"; do printf '<%s>' "$a"; done > "$FAKE_SPX_ARGV"
-SH
-chmod +x "$work/bin/sudo" "$work/bin/spx"
-export FAKE_SUDO_LOG="$work/sudo.log" FAKE_SPX_ARGV="$work/spx.argv" PATH="$work/bin:$PATH"
-
-eval "$(sed -n '/^remote_join_cmd()/,/^}/p' "$here/install-node.sh")"
-
-fails=0
-check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-
-TOKEN=spx_join_FAKETOKEN-abc_1
-cmd="$(remote_join_cmd node2 10.0.0.2 203.0.113.2 '10.0.0.1:4432;touch PWNED' us-east-1 'az $(touch PWNED2)')"
-bash -c "$cmd" <<<"$TOKEN"
-
-check "the token is not in the command" "$(printf '%s' "$cmd" | grep -c FAKETOKEN || true)" 0
-check "the token is not in sudo's log" "$(grep -c FAKETOKEN "$FAKE_SUDO_LOG" || true)" 0
-check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--token><$TOKEN>"
-check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
-check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
-
-[ "$fails" -eq 0 ]

scripts/lib/lambda-testlink-build_test.sh

@@ -1,111 +0,0 @@
-#!/bin/sh
-# lambda-testlink-build_test.sh — the test-link make targets refuse a malformed
-# image digest and an unsupported host or GOARCH before touching any output.
-#
-# Every case is refused on the recipe's first lines, so nothing is compiled.
-set -eu
-
-LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-REPO_ROOT=$(dirname -- "$(dirname -- "$LIB_DIR")")
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-echo "-- make build-lambda-testlink / build-lambda-testlink-image guards"
-
-PWN="$TMP/pwn"
-OUT="$TMP/out"
-HEX=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-mkdir -p "$OUT"
-printf 'old daemon\n' > "$OUT/spx"
-printf 'old initializer\n' > "$OUT/hardkill.so"
-PAIR=$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)
-
-# unchanged <name> — the existing pair is intact, no scratch was made and no
-# injected command ran.
-unchanged() {
-    if [ -e "$PWN" ]; then
-        fail "$1: injected command ran"
-        rm -f "$PWN"
-    elif [ -e "$OUT.tmp" ]; then
-        fail "$1: scratch directory created"
-    elif [ "$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)" != "$PAIR" ]; then
-        fail "$1: existing pair replaced"
-    else
-        pass "$1"
-    fi
-}
-
-# refused <name> <want> <log> <status> — make failed with the expected message.
-refused() {
-    if [ "$4" -eq 0 ]; then
-        fail "$1: make succeeded"
-    elif ! grep -q "$2" "$3"; then
-        fail "$1: wrong refusal: $(tr '\n' ' ' < "$3")"
-    else
-        unchanged "$1"
-    fi
-}
-
-testlink() {
-    make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" "$@" > "$TMP/log" 2>&1
-}
-
-DIGEST_MSG="LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"
-NL='
-'
-for v in '' "\$(shell touch $PWN)" "\$\$(touch $PWN)" "\`touch $PWN\`" "$HEX; touch $PWN" \
-    "$HEX\" -X main.x=\"y" "$HEX -X main.x=y" "$HEX${NL}touch $PWN" \
-    0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF "${HEX%?}" "${HEX}0"; do
-    # GNU make itself expands a command-line $(shell ...) while parsing, for
-    # any target, so that one literal is tested only from the environment.
-    case "$v" in
-    "\$(shell"*) ;;
-    *)
-        status=0
-        testlink LAMBDA_IMAGE_DIGEST="$v" || status=$?
-        refused "command-line digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
-        ;;
-    esac
-    status=0
-    LAMBDA_IMAGE_DIGEST="$v" testlink || status=$?
-    refused "environment digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
-done
-
-status=0
-env -u LAMBDA_IMAGE_DIGEST make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" > "$TMP/log" 2>&1 || status=$?
-refused "digest unset" "$DIGEST_MSG" "$TMP/log" "$status"
-
-ARCH_MSG="supports only an x86_64 host building linux/amd64"
-SHIM="$TMP/shim"
-mkdir -p "$SHIM"
-printf '#!/bin/sh\necho aarch64\n' > "$SHIM/uname"
-chmod +x "$SHIM/uname"
-
-status=0
-GOARCH=arm64 testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
-refused "GOARCH=arm64" "$ARCH_MSG" "$TMP/log" "$status"
-status=0
-PATH="$SHIM:$PATH" testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
-refused "aarch64 host" "$ARCH_MSG" "$TMP/log" "$status"
-
-# The image target refuses before building qboot or any agent binary.
-for c in "GOARCH=arm64" "PATH=$SHIM:$PATH"; do
-    status=0
-    env "$c" make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink-image > "$TMP/log" 2>&1 || status=$?
-    if [ "$status" -eq 0 ]; then
-        fail "image target [${c%%=*}]: make succeeded"
-    elif ! grep -q "$ARCH_MSG" "$TMP/log" || grep -q "qboot\|go build" "$TMP/log"; then
-        fail "image target [${c%%=*}]: did not refuse first: $(tr '\n' ' ' < "$TMP/log")"
-    else
-        pass "image target [${c%%=*}] refused first"
-    fi
-done
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi

scripts/terraform/latitude-spx/.gitignore

@@ -1,10 +0,0 @@
-.terraform/
-terraform.tfstate
-terraform.tfstate.*
-*.tfplan
-*.auto.tfvars
-crash.log
-terraform.tfvars
-*.pem
-*.log
-.terraform.lock.hcl

scripts/terraform/latitude-spx/README.md

@@ -1,137 +0,0 @@
-# latitude-spx
-
-Builds a Spinifex deployment on latitude.sh bare metal from your laptop with `deploy-latitude.sh`.
-The commands, flags, step markers and state layout follow `vultr-spx`.
-
-**Status: not yet run end to end.**
-The script was written from the plan doc findings (`docs/development/feature/latitude-bm.md`, L1 to L26) and from the provider schema.
-No latitude.sh API call or host run was made while building it, so treat the first `up` as the test.
-One node has been the only shape run live, and only in part (see Unverified). The three-node flow, including the floating pool, has not been run: Ben takes that over.
-
-## What it creates
-
-| Resource | Notes |
-|---|---|
-| Project | `spinifex-<name>`, unless `--project` names an existing one (then it is never deleted) |
-| SSH key | `spinifex-<name>`, account scoped |
-| User data | `spinifex-<name>`, host preparation only (no `{{ }}`, latitude.sh substitutes them) |
-| Servers | `m4-metal-medium`, site `DAL`, Ubuntu 26.04, hourly billing, hostnames `spx-<name>-nodeN` |
-| Elastic IP | one by default (`elastic_ip_count` in `variables.tf`, 1 to 5; set `TF_VAR_elastic_ip_count` to change it). With one node it is a static pool address and the count must stay 1, because a second pool is never tried when the first is full; with three nodes the addresses form the `latitude` floating pool, which Spinifex moves between servers. `nginx-webserver` needs one address and runs on one node; `nginx-alb` needs two public addresses and is a three-node workbook |
-| Virtual network | three nodes only: `spinifex-<name>`, one VLAN assignment per server |
-
-## Requirements
-
-OpenTofu (or Terraform) 1.5 or later, `aws` CLI, `python3`, `curl`, `ssh`, an SSH key pair.
-
-| Variable | When | Meaning |
-|---|---|---|
-| `LATITUDESH_AUTH_TOKEN` | always | the account API token; used by the provider and this script only, never sent to a node |
-| `LATITUDESH_NODE_TOKEN` | `--nodes 3` | a second, different, writable API key, copied to each node (`/etc/spinifex/latitude/api-token`, owned by `spinifex-daemon` with mode 0400, in a root-owned 0711 directory; the other services share the `spinifex` group, so no group may read it; ownership and mode are set on every `up`; a symlink or any other owner there is refused) so the daemon can move Elastic IPs |
-
-The script refuses a node key it can see is read-only (it lists `/auth/api_keys` and matches the key's last five characters). If the key is not among the account's keys, write authority is first proven by the first Elastic IP move.
-
-Exposure: a latitude.sh API key is team-wide. It is not scoped to Elastic IPs, so anyone who reads it from a node can do whatever the key's owner can in the team (the API exposes `read_only` and `allowed_ips` on a key, and no narrower scope). Create the node key with `allowed_ips` set to the nodes' public addresses, and revoke it after `destroy`.
-
-Nodes are reached as `ubuntu` with the deployment's SSH key, and remote commands run through passwordless `sudo`: latitude.sh puts the key on the image's login user and root login is not enabled.
-
-## Usage
-
-```
-export LATITUDESH_AUTH_TOKEN=...        # from the latitude.sh dashboard
-./deploy-latitude.sh up --name demo     # one node
-
-export LATITUDESH_NODE_TOKEN=...        # a second, writable key
-./deploy-latitude.sh up --name demo3 --nodes 3
-./deploy-latitude.sh status --name demo
-./deploy-latitude.sh ssh --name demo node1 'systemctl status spinifex.target'
-./deploy-latitude.sh credentials --name demo
-./deploy-latitude.sh stop node1 --name demo
-./deploy-latitude.sh start node1 --name demo
-./deploy-latitude.sh destroy --name demo
-```
-
-Options: `--name`, `--site` (alias `--region`), `--plan`, `--os`, `--project ID|SLUG`, `--distro TARBALL`, `--channel`, `--spx-region`, `--spx-az`, `--ssh-key`, `--image`, `--data-disk-id`, `--vpc-subnet` (the VLAN's /24, three nodes), `--nodes 1|3`, `--yes`.
-`./deploy-latitude.sh --help` lists them with defaults.
-
-Three nodes need `--distro` with a tarball built from this branch (`make distro-amd64`) until a release carries the latitude provider.
-A released spx rejects `source="latitude"` and its daemon restarts in a loop, so `up --nodes 3` without `--distro` stops in preflight.
-Before the pool is written, each node's `/usr/local/bin/spx` is checked for the `latitude_elastic_ip_ids` key name; spx has no validate subcommand, so this is a string search of the binary, not a run of its config check.
-
-## What `up` does
-
-`terraform` then, per node: `host` (wait for SSH as `ubuntu` and cloud-init, `/dev/kvm`), `disk`, `vlan` (three nodes), `install`, `form`, `start`; then `pool` (one node) or `cluster` (three nodes: `install-node.sh` forms the cluster); three nodes then install the node token on every `up` and add the `latitude` pool (`pool`); then `restart` (three nodes wait for the current daemon to log "floating pool ready" for the pool), `public`, `image` and the AWS profile.
-Each step writes `step.<name>[.nodeN]` in the state directory, and a rerun resumes at the first step without one.
-
-- **Data disk.** `host-data-disk.sh` picks the blank NVMe by content and by-id EUI, never by `nvmeXn1` (L1, L11).
-A candidate is at least 100 GB with no signature, partitions, holders or mounts, and not in the root filesystem's ancestry; with none it stops and lists the disks.
-With several (LON2 has two 1.7 TB and one spare 447 GB blank NVMe next to the root disk) it takes the largest, the lowest by-id path among equals, prints what it chose and the blank disks it left, and never touches the others (`--data-disk-id` chooses another).
-It mounts by UUID with `nofail` and the script records the UUID in `data_uuid.<node>`.
-- **Mount check.** Before anything writes under `/var/lib/spinifex`, and on every run, the mounted filesystem UUID must equal the recorded one.
-The units that write there also get a `RequiresMountsFor=/var/lib/spinifex` drop-in.
-The mount is by UUID in fstab, so renamed disks do not change it, and there is no check at service start.
-- **Firewall.** `setup.sh --firewall on` runs before Spinifex first starts.
-- **One node.** The Elastic IP is a static pool (`gateway_ip = "192.0.2.1"`, the node's next hop as `gateway`, no `prefix_len`) with the IMDS addresses moved off the ones latitude.sh routes.
-- **Three nodes.** The `vlan` step makes `eno2.<vid>` with `ip link` and persists it as systemd-networkd files for that link plus a drop-in on the `eno2` unit.
-It never runs `netplan apply`, refuses a subnet that overlaps the SSH peer or a route, and removes the VLAN again if the public interface's address, default route or the route to the SSH peer differ afterwards.
-`form_cluster` then puts the cluster and Geneve planes on that VLAN with `install-node.sh` (`--user ubuntu`); its output is not kept, as it can hold the join token.
-- **Floating pool (three nodes).** The `latitude` pool names the Elastic IP IDs and the project ID from the Terraform outputs, and each node's `provider_host_id` is its server ID. `spinifex.toml` is edited idempotently and read back. Failover moves an address with one `PATCH /elastic_ips/{id}` and waits for it to be active; latitude.sh has no unassign, so a node that loses an address is not told to release it.
-- **State** is in `~/.spinifex/latitude/<name>/`: Terraform state, `ids` (every `latitudesh_*` object in the state, merged on each apply and before each destroy), `step.*` markers, logs and the data disk UUIDs.
-- **AWS profile** `spinifex-latitude-<name>` is replaced on each `up`, with the CA at `~/.spinifex/latitude-<name>-ca.pem`.
-The secret key is written to `~/.aws/credentials` (0600, other profiles kept, the old file saved beside it) and never appears in a process argument.
-
-## Token handling
-
-`LATITUDESH_AUTH_TOKEN` is read from the environment by the provider and by the script's API calls.
-It is not a Terraform variable, is on no command line (curl reads it from stdin), is not sent to a node, and is written to no file.
-`LATITUDESH_NODE_TOKEN` is checked with curl reading it from stdin and reaches each node on stdin of the install script, so it is in no argument on either side.
-`TF_LOG` is unset by the script.
-The join token that `install-node.sh` creates is shared with Vultr and OCI: its init log is root-only, and `spx admin join` still takes it as `--token` on the joining node (tracked separately).
-
-## Recovery
-
-If SSH is lost once the firewall is armed, use the serial console.
-`up` enables the server's out-of-band connection with the deployment's SSH key and prints the `ssh` command it returns before it arms the firewall, falling back to the Out-of-Band tab in the dashboard.
-Logging in there needs an OS user password, which you set yourself while SSH still works.
-A fresh deployment that cannot be recovered is cheaper to `destroy` and `up` again.
-The VLAN step prints the console command first.
-It refuses a subnet that holds the SSH peer or overlaps an existing route, and removes the VLAN again if the route to the SSH peer or the public interface changes.
-
-## destroy
-
-`destroy` asks you to type the deployment name (`--yes` skips only that), then records every ID in the Terraform state and stops without destroying anything if it cannot.
-It runs `terraform destroy`, fetches every recorded ID by itself (2xx present, 404 gone, anything else fails and keeps the records), and deletes what remains of those.
-Servers, keys, user data, virtual networks and projects whose names match the deployment but that are not recorded are listed by ID and deleted only after you type `delete-untracked` for exactly those IDs, which `--yes` never supplies; an ID that shows up later needs its own confirmation, and without a terminal they are kept and `destroy` exits 1.
-Elastic IPs are touched only by recorded ID.
-Elastic IP release is asynchronous and only works once the address is active or in error, so the sweep retries.
-Because the Elastic IP list ignores pagination, `destroy` ends by asking you to check the dashboard for leftovers and type `checked`.
-An `up` that failed while creating can leave an object nothing recorded, such as an Elastic IP; that dashboard check is the only thing that finds it.
-
-## Unverified
-
-Nothing in the three-node flow has run against real servers. For one node, only the parts the person ran have been exercised, and the script has changed since.
-
-- The OS slug `ubuntu_26_04_x64_lts` (the `latitudesh_operating_system` data source fails the plan if it is wrong).
-- That the deploy key is on `ubuntu` and `sudo -n` works for it, and that `sudo` resets nothing the script needs (the SSH peer address is read before `sudo` for that reason).
-- The power action request (`POST /servers/{id}/actions` with `power_off` and `power_on`).
-- Resolving `--project` as a slug through `GET /projects/{slug}`, and the `in_stock` shape of `GET /plans` used by the stock check, which only warns when it cannot read it.
-- The `GET` and `DELETE` paths for `/user_data/{id}` and `/virtual_networks/{id}`.
-- That the Spinifex API certificate names the node's public address, which the profile check needs.
-- That the private NIC is `eno2`, its tagging and MTU (the script uses 1500), and that the networkd files persist the VLAN across a reboot.
-- `GET /auth/api_keys` and its `token_last_slice` and `read_only` fields, which the node key check relies on; and that a key with `allowed_ips` set to the nodes' addresses still works from them.
-- The Elastic IP move itself: `PATCH /elastic_ips/{id}` with `server_id`, its 202 then `active`, the error codes, and how long a move takes; the client waits up to its move timeout and makes one write per call.
-- That the daemon reads the node token and logs "floating pool ready", then fails over when a node stops (`stop node1`) and the address lands on another node.
-- `install-node.sh` as `--user ubuntu` on latitude.sh servers.
-
-## Not exercised
-
-Any real API call from the three-node flow, `destroy`, `host-data-disk.sh` on a host and the VLAN step on a server.
-Guest launch, public reachability of the Elastic IP, the Terraform workbooks and failover are not part of `up`; prove them by hand.
-`e2e-latitude.sh` and removing the old `install-node.sh` in this directory are not done.
-
-## Tests
-
-`sh node-scripts_test.sh` checks the node token install (install, unchanged, rotation, modes, an older install converging, symlinks and owners refused; run it as root too, with `sudo sh node-scripts_test.sh`, to check that only the daemon's user can read the file) and the readiness check (old success then current failure) with stand-in tools.
-`bash deploy-argv_test.sh` checks that neither API token, nor the five characters used to find the node key, is an argument of any external command in the node token paths, and that the SSH peer address comes from the same connection after sudo's environment reset.
-`scripts/install-node_test.sh` (part of `make preflight`) checks that the join token reaches `spx admin join` only on stdin, so sudo's log never holds it.
-The first two are not part of `make preflight`.
-The Go side: `cd spinifex && GOWORK=off go test ./providers/cloud/latitude/ ./domains/network/external/floating/ ./bootstrap/config/ ./runtime/daemon/`.

scripts/terraform/latitude-spx/cloud-init/user-data.yaml

@@ -1,23 +0,0 @@
-#cloud-config
-# Host preparation only. Spinifex is installed and formed over SSH by deploy-latitude.sh.
-# latitude.sh substitutes double-brace variables in user data, so this file holds none.
-write_files:
-  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
-    content: |
-      network: {config: disabled}
-  # The image's netplan lists eno2 with no address and without optional, so boot waits two minutes
-  # for it. --any returns once one link, eno1, is online; it changes the wait, not the network.
-  # UNVERIFIED: the binary path is the Ubuntu one and the drop-in first applies on the next boot.
-  - path: /etc/systemd/system/systemd-networkd-wait-online.service.d/10-spinifex-any.conf
-    content: |
-      [Service]
-      ExecStart=
-      ExecStart=/usr/lib/systemd/systemd-networkd-wait-online --any
-# runcmd carries on past a failing command, so each line stands alone.
-runcmd:
-  - [sh, -c, "command -v ufw >/dev/null && ufw --force disable || true"]
-  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
-  # The image ships empty apt lists and no git, which the workbooks need.
-  - [sh, -c, "apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git || true"]
-  - [sh, -c, "systemctl daemon-reload || true"]
-  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]

scripts/terraform/latitude-spx/compute.tf

@@ -1,70 +0,0 @@
-locals {
-  hostname_prefix = "spx-${var.deployment_name}-node"
-  project         = var.project != "" ? var.project : latitudesh_project.deploy[0].slug
-  project_id      = var.project != "" ? var.project_id : latitudesh_project.deploy[0].id
-}
-
-# Both lookups fail the plan when a slug does not exist, which is the only check of the OS slug.
-data "latitudesh_operating_system" "os" {
-  slug = var.operating_system
-}
-
-data "latitudesh_plan" "plan" {
-  slug = var.plan
-}
-
-resource "latitudesh_project" "deploy" {
-  count             = var.project == "" ? 1 : 0
-  name              = "spinifex-${var.deployment_name}"
-  description       = "Spinifex deployment ${var.deployment_name}"
-  environment       = "Development"
-  provisioning_type = "on_demand"
-}
-
-# The key is team scoped. The server login user is ubuntu, not root.
-resource "latitudesh_ssh_key" "deploy" {
-  name       = "spinifex-${var.deployment_name}"
-  public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
-}
-
-# Host preparation only. Latitude substitutes double-brace variables inside user data, so the file
-# must hold none. Spinifex is installed over SSH by deploy-latitude.sh.
-resource "latitudesh_user_data" "host" {
-  description = "spinifex-${var.deployment_name}"
-  content     = filebase64("${path.module}/cloud-init/user-data.yaml")
-}
-
-# billing must be hourly: the default is monthly, which bills upfront with no refund.
-resource "latitudesh_server" "node" {
-  count            = var.node_count
-  project          = local.project
-  hostname         = "${local.hostname_prefix}${count.index + 1}"
-  site             = var.site
-  plan             = data.latitudesh_plan.plan.slug
-  operating_system = data.latitudesh_operating_system.os.slug
-  billing          = "hourly"
-  ssh_keys         = [latitudesh_ssh_key.deploy.id]
-  user_data        = latitudesh_user_data.host.id
-
-  timeouts = {
-    create = "60m"
-  }
-}
-
-# Elastic IPs are bought here and never created or deleted by Spinifex. Each starts on a server, spread
-# over the nodes, and server_id is ignored afterwards so a move by Spinifex is not undone by a later apply.
-resource "latitudesh_elastic_ip" "pool" {
-  count = var.elastic_ip_count
-  # The provider documents ID or slug but posts the value as project_id unresolved: a slug gave 404.
-  project   = local.project_id
-  server_id = latitudesh_server.node[count.index % var.node_count].id
-
-  lifecycle {
-    ignore_changes = [server_id]
-  }
-}
-
-moved {
-  from = latitudesh_elastic_ip.node1
-  to   = latitudesh_elastic_ip.pool[0]
-}

scripts/terraform/latitude-spx/deploy-argv_test.sh

@@ -1,228 +0,0 @@
-#!/usr/bin/env bash
-# Checks that neither API token is ever an argument of an external command in the node token
-# paths of deploy-latitude.sh. External commands are stand-ins that record their arguments and
-# their standard input; the test fails if a token is in the first and proves it is in the second.
-# preflight runs this with sh
-[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
-set -euo pipefail
-exec </dev/null
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-mkdir "$work/bin" "$work/home"
-real_python="$(command -v python3)"
-
-AUTH=FAKEAUTHTOKEN9
-NODE=FAKENODETOKEN7
-
-# Every command the script runs outside bash records "name args" and, for the first line of
-# standard input, "name stdin: ...".
-for c in curl ssh scp aws tofu ssh-keygen; do
-    cat > "$work/bin/$c" <<'SH'
-#!/bin/sh
-echo "$(basename "$0") $*" >> "$FAKE_ARGV"
-if [ ! -t 0 ]; then
-    in="$(cat)"
-    [ -z "$in" ] || echo "$(basename "$0") stdin: $in" >> "$FAKE_STDIN"
-fi
-case "$(basename "$0")" in
-    curl) printf '{"data":[{"attributes":{"token_last_slice":"%s","read_only":false}}]}\n200' "$(printf '%s' "$FAKE_NODE" | tail -c 5)" ;;
-    ssh)
-        if [ -n "${FAKE_SSH_EVAL:-}" ]; then
-            # Behave like a login on a node: skip the options and host, run the rest with the
-            # connection variable of this connection, and let sudo reset the environment.
-            while [ "$#" -gt 0 ]; do case "$1" in ubuntu@*) shift; break ;; *) shift ;; esac; done
-            SSH_CONNECTION="$FAKE_SSH_CONNECTION" bash -c "$*"
-        else
-            echo installed
-        fi ;;
-esac
-SH
-    chmod +x "$work/bin/$c"
-done
-cat > "$work/bin/sudo" <<'SH'
-#!/bin/sh
-while [ "$#" -gt 0 ]; do case "$1" in -*) shift ;; *) break ;; esac; done
-exec env -i PATH="$PATH" "$@"
-SH
-chmod +x "$work/bin/sudo"
-cat > "$work/bin/python3" <<SH
-#!/bin/sh
-echo "python3 \$*" >> "\$FAKE_ARGV"
-exec "$real_python" "\$@"
-SH
-chmod +x "$work/bin/python3"
-
-export FAKE_ARGV="$work/argv" FAKE_STDIN="$work/stdin" FAKE_NODE="$NODE" PATH="$work/bin:$PATH" HOME="$work/home"
-: > "$FAKE_ARGV"
-: > "$FAKE_STDIN"
-export LATITUDESH_AUTH_TOKEN="$AUTH" LATITUDESH_NODE_TOKEN="$NODE"
-
-# shellcheck source=deploy-latitude.sh
-source "$here/deploy-latitude.sh"
-
-# shellcheck disable=SC2034 # read by the sourced functions
-NODES=3 IP=192.0.2.10 NODE_NAME=node1
-check_node_token >/dev/null
-install_key >/dev/null 2>&1
-
-fails=0
-for tok in "$AUTH" "$NODE" "${NODE: -5}"; do
-    if grep -q "$tok" "$FAKE_ARGV"; then
-        echo "FAIL token $tok is in a command's arguments:"
-        grep "$tok" "$FAKE_ARGV"
-        fails=$((fails + 1))
-    else
-        echo "ok   $tok is in no argument"
-    fi
-done
-for what in "curl stdin:.*Bearer $NODE" "ssh stdin: $NODE"; do
-    if grep -q "$what" "$FAKE_STDIN"; then
-        echo "ok   the check can see the token where it is sent ($what)"
-    else
-        echo "FAIL the token never reached stdin ($what): the test would pass without checking anything"
-        fails=$((fails + 1))
-    fi
-done
-
-# The peer address comes from the same connection that runs the command, whichever source
-# address that connection has, and survives sudo's environment reset.
-export FAKE_SSH_EVAL=1
-for src in 198.51.100.7 203.0.113.99; do
-    export FAKE_SSH_CONNECTION="$src 51000 192.0.2.10 22"
-    got="$(on 'printf %s "$SSH_PEER"')"
-    if [ "$got" = "$src" ]; then echo "ok   peer $src reaches the command"; else echo "FAIL peer for $src was '$got'"; fails=$((fails + 1)); fi
-done
-
-# A three-node run needs a build that knows the latitude source: a release would be refused by its
-# own config check. The guard needs --distro, and the node is checked before it gets the pool.
-check_result() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-check_result "three nodes without --distro stop" "$( (NODES=3 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-check_result "three nodes with --distro go on" "$( (NODES=3 DISTRO=/tmp/x.tar.gz require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node without --distro goes on" "$( (NODES=1 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with one address goes on" "$( (NODES=1 TF_VAR_elastic_ip_count=1 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with no setting goes on" "$( (NODES=1 TF_VAR_elastic_ip_count="" require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with two addresses stops" "$( (NODES=1 TF_VAR_elastic_ip_count=2 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-check_result "three nodes with three addresses go on" "$( (NODES=3 TF_VAR_elastic_ip_count=3 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-printf 'binary with latitude_elastic_ip_ids inside\n' > "$work/spx-new"
-printf 'binary of a release\n' > "$work/spx-old"
-SPX_BIN="$work/spx-new"
-check_result "a build that knows latitude is accepted" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-# shellcheck disable=SC2034 # read by the sourced functions
-SPX_BIN="$work/spx-old"
-check_result "a release is refused" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-# The profile spx writes under sudo is in the ubuntu user's home, so node-side reads name that
-# directory instead of relying on root's HOME.
-mkdir "$work/awsdir"
-printf '[other]\naws_access_key_id = X\n[spinifex]\naws_access_key_id = AK1\naws_secret_access_key = SK1\n[z]\nq = 1\n' > "$work/awsdir/credentials"
-node_ip() { printf 192.0.2.10; }
-# shellcheck disable=SC2034 # read by the sourced functions
-NODE_AWS_DIR="$work/awsdir"
-check_result "credentials come from the node's AWS dir" "$(cmd_credentials | tr '\n' ' ')" "[spinifex] aws_access_key_id = AK1 aws_secret_access_key = SK1 "
-
-# A failed API call logs its status and a documented code, never the response text, which can
-# echo the request's headers.
-mkdir "$work/bin2"
-cat > "$work/bin2/curl" <<'SH'
-#!/bin/sh
-cat > /dev/null
-printf '{"errors":[{"code":"%s","detail":"Authorization: Bearer %s"}]}\n401' "$FAKE_CODE" "$FAKE_LEAK"
-SH
-chmod +x "$work/bin2/curl"
-logged="$(FAKE_CODE=UNAUTHORIZED FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
-check_result "a failed call names its status and code" "$(grep -c 'returned 401 (UNAUTHORIZED)' <<<"$logged" || true)" 1
-check_result "a failed call never logs the token" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
-logged="$(FAKE_CODE="$AUTH" FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
-check_result "an undocumented code is not logged" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
-
-# A recorded ID that now names another deployment's object is not ours to delete.
-# shellcheck disable=SC2034 # read by the sourced functions
-NAME=demo
-latitude() { printf '%s' "$FAKE_LAT"; }
-orig_expected_project_id="$(declare -f expected_project_id)"
-expected_project_id() { echo proj_1; }
-srv() {
-    local frag=',"project":{"id":"proj_1"}'
-    [ "$#" -lt 2 ] || frag="$2"
-    printf '{"data":{"id":"sv_1","attributes":{"hostname":"%s"%s}}}' "$1" "$frag"
-}
-FAKE_LAT="$(srv spx-demo-node2)"
-check_result "a recorded server with our hostname is ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" ours
-FAKE_LAT="$(srv spx-other-node1)"
-check_result "a recorded server with another hostname is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo2-node1)"
-check_result "a hostname that only starts like ours is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
-check_result "our hostname in another project is not ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo-node1 '')"
-check_result "a server with no project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{}')"
-check_result "a project with no id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-FAKE_LAT="$(srv spx-demo-node1 ',"project":null')"
-check_result "a null project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-expected_project_id() { return 1; }
-FAKE_LAT="$(srv spx-demo-node1)"
-check_result "no known project id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-expected_project_id() { echo proj_1; }
-FAKE_LAT='not json'
-check_result "an unreadable object is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-KNOWN_IDS="$work/ids"
-printf 'server sv_1\n' > "$KNOWN_IDS"
-lat_exists() { return 0; }
-FAKE_LAT="$(srv spx-other-node1)"
-check_result "destroy stops before Terraform when a recorded ID was reused" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
-check_result "destroy stops when the same hostname is in another project" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1 '')"
-check_result "destroy stops when the project cannot be read" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1)"
-check_result "destroy goes on when the recorded server is ours" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" went
-leftovers_out="$( (lat_match() { return 0; }; FAKE_LAT="$(srv spx-other-node1)" leftovers) 2>/dev/null || true)"
-check_result "a reused ID is not listed as a leftover to delete" "$(grep -c sv_1 <<<"$leftovers_out" || true)" 0
-
-# After terraform destroy the state's project output is gone; the remembered ID still lets the
-# sweep delete a matching leftover and keep a foreign one.
-eval "$orig_expected_project_id"
-TF_BIN="$work/tofu"
-printf '#!/bin/sh\nif [ -f "%s/gone" ]; then exit 1; fi\nprintf proj_1\n' "$work" > "$TF_BIN"
-chmod +x "$TF_BIN"
-EXPECTED_PROJECT=""
-remember_project_id
-check_result "the project ID is remembered while the state holds it" "$EXPECTED_PROJECT" proj_1
-: > "$work/gone"
-check_result "the state no longer answers after destroy" "$("$TF_BIN" output >/dev/null 2>&1 && echo up || echo gone)" gone
-printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
-lat_match() { return 1; }
-latitude() { if [ "$2" = /servers/sv_1 ]; then srv spx-demo-node1; else srv spx-other-node1 ',"project":{"id":"proj_other"}'; fi; }
-swept="$(leftovers 2>/dev/null || true)"
-check_result "after destroy a matching leftover is still listed" "$(grep -c 'server sv_1 ' <<<"$swept" || true)" 1
-check_result "after destroy a foreign one is kept out" "$(grep -c 'server sv_2 ' <<<"$swept" || true)" 0
-EXPECTED_PROJECT=""
-swept="$(leftovers 2>/dev/null || true)"
-check_result "without the remembered ID a leftover is unknown, not matched" "$(grep -c 'server sv_1 sv_1 (unknown)' <<<"$swept" || true)" 1
-expected_project_id() { echo proj_1; }
-
-# Cleanup never deletes an object whose ownership could not be read, recorded or not.
-deleted="$work/deleted"
-: > "$deleted"
-latitude() { printf '%s %s\n' "$1" "$2" >> "$deleted"; }
-printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
-delete_left $'server sv_1 sv_1 (unknown)\nserver sv_2 spx-demo-node2\n' "" >/dev/null 2>&1
-check_result "an unreadable recorded object is not deleted" "$(grep -c sv_1 "$deleted" || true)" 0
-check_result "a verified recorded object is deleted" "$(grep -c 'DELETE .*sv_2' "$deleted" || true)" 1
-
-# Both deployment sizes write the metadata remap.
-cfg="$work/spinifex.toml"
-extract() { sed -n "/^$1()/,/^}/p" "$here/deploy-latitude.sh" | sed -n "/<<'PY'/,/^PY\$/p" | sed '1d;$d' | sed "s|/etc/spinifex/spinifex.toml|$cfg|"; }
-printf '[network]\nexternal_mode = "nat"\n\n[bootstrap]\ny = 1\n' > "$cfg"
-extract configure_pool | python3 -I - 203.0.113.1 203.0.113.254
-check_result "one node gets the metadata remap" "$(grep -c 'imds_host_meta_ip = "169.254.42.254"' "$cfg")" 1
-printf '[network]\nexternal_mode = "nat"\n\n[nodes.node1]\nx = 1\n\n[nodes.node2]\nx = 1\n\n[nodes.node3]\nx = 1\n\n[bootstrap]\ny = 1\n' > "$cfg"
-extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
-check_result "three nodes get the metadata remap" "$(grep -c 'imds_host_dns_ip  = "169.254.42.253"' "$cfg")" 1
-check_result "three nodes get the latitude pool" "$(grep -c 'source                  = "latitude"' "$cfg")" 1
-extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
-check_result "writing the config again changes nothing" "$(grep -c 'imds_host_meta_ip' "$cfg")" 1
-unset FAKE_SSH_EVAL
-
-[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/host-data-disk.sh

@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-# host-data-disk.sh — format and mount the blank data disk at /var/lib/spinifex. Run as root ON the host.
-#
-#   host-data-disk.sh [/dev/disk/by-id/nvme-eui.<id>]
-#
-# The disk is chosen by content, never by its nvmeXn1 name, which swaps between boots. A candidate is a
-# whole-disk by-id EUI link whose device is not in the root filesystem's ancestry, has no partitions,
-# holders, swap or mounts, no filesystem signature and no partition table. With none the script stops
-# and lists the disks. With several it takes the largest, and the lowest by-id path among equals, and
-# prints what it chose and the blank disks it left alone; pass the by-id path to choose another.
-# It never reformats: a rerun on a host already done prints the UUID and exits. The last line of stdout
-# is DATA_UUID=<uuid>, which the later install phases need.
-
-set -euo pipefail
-
-MP=/var/lib/spinifex
-LABEL=spinifex-data
-FSTAB_BAK=/etc/fstab.pre-spinifex
-MIN_BYTES="${MIN_DATA_BYTES:-100000000000}"
-WANT="${1:-}"
-
-die() { echo "STOP: $*" >&2; exit 1; }
-[ "$(id -u)" = 0 ] || die "run as root"
-
-# Resume: the mount is already there and is ours.
-if mp_uuid="$(findmnt --mountpoint "$MP" -no UUID 2>/dev/null)" && [ -n "$mp_uuid" ]; then
-    src="$(findmnt --mountpoint "$MP" -no SOURCE)"
-    [ "$(blkid -s LABEL -o value "$src")" = "$LABEL" ] || die "$MP is mounted from $src, which is not labelled $LABEL; not touching it"
-    grep -q "UUID=${mp_uuid}[[:space:]]" /etc/fstab || die "$MP is mounted but /etc/fstab has no UUID=$mp_uuid entry"
-    echo "already done: $MP is mounted from $src (UUID $mp_uuid)"
-    echo "DATA_UUID=$mp_uuid"
-    exit 0
-fi
-if grep -q "$MP" /etc/fstab; then die "/etc/fstab mentions $MP but nothing is mounted there; fix that by hand first"; fi
-[ ! -e "$MP" ] || [ -z "$(ls -A "$MP")" ] || die "$MP exists and is not empty"
-
-root_src="$(findmnt -no SOURCE /)" || die "cannot find the root source"
-root_anc="$(lsblk -s -nro NAME "$root_src")" || die "cannot read the root ancestry"
-[ -n "$root_anc" ] || die "empty root ancestry"
-
-# blank_reason ID prints why the disk is not a candidate and returns 1, or returns 0.
-blank_reason() {
-    local id="$1" dev name size kids hold sw mnt rc w
-    dev="$(readlink -e "$id")" || { echo "link does not resolve"; return 1; }
-    [ -b "$dev" ] || { echo "not a block device"; return 1; }
-    name="$(basename "$dev")"
-    if grep -qx "$name" <<<"$root_anc"; then echo "in the root ancestry"; return 1; fi
-    size="$(blockdev --getsize64 "$id")" || { echo "size unreadable"; return 1; }
-    [ "$size" -ge "$MIN_BYTES" ] || { echo "only $size bytes"; return 1; }
-    kids="$(lsblk -nro NAME "$id")" || { echo "children unreadable"; return 1; }
-    [ "$kids" = "$name" ] || { echo "has children"; return 1; }
-    hold="$(ls -A "/sys/block/$name/holders/")" || { echo "holders unreadable"; return 1; }
-    [ -z "$hold" ] || { echo "has holders"; return 1; }
-    sw="$(swapon --show=NAME --noheadings)" || { echo "swapon failed"; return 1; }
-    if grep -q "$name" <<<"$sw"; then echo "is swap"; return 1; fi
-    mnt="$(lsblk -nro MOUNTPOINTS "$id")" || { echo "mounts unreadable"; return 1; }
-    [ -z "${mnt//[[:space:]]/}" ] || { echo "is mounted"; return 1; }
-    set +e; blkid -p "$id" >/dev/null 2>&1; rc=$?; set -e
-    [ "$rc" -eq 2 ] || { echo "blkid -p says it has a signature (rc $rc)"; return 1; }
-    w="$(wipefs -n "$id")" || { echo "wipefs failed"; return 1; }
-    [ -z "$w" ] || { echo "wipefs finds signatures"; return 1; }
-    return 0
-}
-
-# choose_data_disk ID... prints the largest disk's ID, the lowest path among equal sizes, and says on
-# stderr which it left. The choice depends only on size and path, so every node of one plan agrees.
-choose_data_disk() {
-    local chosen rest
-    chosen="$(for id in "$@"; do printf '%s %s\n' "$(blockdev --getsize64 "$id")" "$id"; done |
-        sort -k1,1nr -k2,2 | head -n1)"
-    chosen="${chosen#* }"
-    if [ "$#" -gt 1 ]; then
-        rest="$(printf '%s\n' "$@" | grep -vxF "$chosen" | sort | tr '\n' ' ')"
-        echo "chose $chosen ($(blockdev --getsize64 "$chosen") bytes), the largest of $# blank disks; left untouched: ${rest% }" >&2
-    fi
-    printf '%s\n' "$chosen"
-}
-
-candidates=()
-shopt -s nullglob
-for id in /dev/disk/by-id/nvme-eui.*; do
-    case "$id" in *-part*) continue ;; esac
-    if [ -n "$WANT" ] && [ "$id" != "$WANT" ]; then continue; fi
-    if why="$(blank_reason "$id")"; then
-        candidates+=("$id")
-    else
-        echo "skip $id: $why"
-    fi
-done
-shopt -u nullglob
-
-if [ "${#candidates[@]}" -eq 0 ]; then
-    lsblk -o NAME,SIZE,TYPE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS >&2 || true
-    die "no blank data disk among the by-id EUI disks"
-fi
-ID="$(choose_data_disk "${candidates[@]}")"
-DEV="$(readlink -e "$ID")"
-echo "CHECKS PASSED: $ID -> $DEV"
-
-mkdir -p "$MP"
-mkfs.ext4 -q -L "$LABEL" "$ID"
-U="$(blkid -p -o value -s UUID "$ID")" || die "blkid UUID failed"
-[ -n "$U" ] || die "no UUID after mkfs"
-[ "$(blkid -p -o value -s TYPE "$ID")" = ext4 ] || die "not ext4 after mkfs"
-
-NEW="$(mktemp /etc/fstab.spinifex.XXXXXX)"
-cp /etc/fstab "$NEW"
-echo "UUID=$U $MP ext4 defaults,nofail,x-systemd.device-timeout=30s 0 2" >> "$NEW"
-findmnt --verify --tab-file "$NEW" || { rm -f "$NEW"; die "staged fstab failed verify, /etc/fstab untouched"; }
-BAK="$FSTAB_BAK.$(date +%s)"
-cp -a /etc/fstab "$BAK"
-trap 'cp -a "$BAK" /etc/fstab; systemctl daemon-reload; echo "ROLLED BACK /etc/fstab from $BAK" >&2' EXIT
-chmod 644 "$NEW"
-mv "$NEW" /etc/fstab
-systemctl daemon-reload
-mount "$MP"
-[ "$(findmnt --mountpoint "$MP" -no UUID)" = "$U" ] || die "wrong UUID at $MP"
-[ "$(findmnt --mountpoint "$MP" -no FSTYPE)" = ext4 ] || die "wrong fstype at $MP"
-[ "$(findmnt --mountpoint "$MP" -no MAJ:MIN | tr -d ' ')" = "$(lsblk -dno MAJ:MIN "$ID" | tr -d ' ')" ] || die "wrong device at $MP"
-trap - EXIT
-findmnt --mountpoint "$MP" -o TARGET,SOURCE,MAJ:MIN,UUID,FSTYPE,OPTIONS
-echo "STORAGE OK"
-echo "DATA_UUID=$U"

scripts/terraform/latitude-spx/network.tf

@@ -1,14 +0,0 @@
-# A private network needs two or more servers, so a single node has none. The VLAN carries the cluster
-# and Geneve traffic; its address is set over SSH, since the VID is only known after apply.
-resource "latitudesh_virtual_network" "lan" {
-  count       = var.node_count > 1 ? 1 : 0
-  project     = local.project_id
-  site        = var.site
-  description = "spinifex-${var.deployment_name}"
-}
-
-resource "latitudesh_vlan_assignment" "node" {
-  count              = var.node_count > 1 ? var.node_count : 0
-  server_id          = latitudesh_server.node[count.index].id
-  virtual_network_id = latitudesh_virtual_network.lan[0].id
-}

scripts/terraform/latitude-spx/node-install-api-token.sh

@@ -1,48 +0,0 @@
-#!/bin/sh
-# Runs as root on a node. Reads the API token on stdin and installs it for the daemon.
-# The directory belongs to root and is not writable by anyone else, so no service can create or
-# swap an entry in it. The file belongs to the daemon's user alone (mode 0400): the other
-# services share the spinifex group, so group access would hand them the team-wide token. The
-# temporary file has an unpredictable name and is created exclusively. A symlink, or an owner
-# other than root or the daemon's user, where the directory or the file should be is refused.
-# Ownership and mode are set on every run, so an older install converges. The file's content is
-# replaced only when the token differs. Prints installed or unchanged.
-set -eu
-dir="${API_TOKEN_DIR:-/etc/spinifex/latitude}"
-root="${API_TOKEN_ROOT-root}"
-rootgroup="${API_TOKEN_ROOT_GROUP-root}"
-user="${API_TOKEN_USER-spinifex-daemon}"
-umask 077
-
-fail() { echo "$*" >&2; exit 1; }
-owner_of() { stat -c %U "$1"; }
-
-[ ! -L "$dir" ] || fail "$dir is a symlink"
-if [ ! -e "$dir" ]; then mkdir -p "$dir"; fi
-[ -d "$dir" ] || fail "$dir is not a directory"
-[ "$(owner_of "$dir")" = "$root" ] || fail "$dir is not owned by $root"
-chown "$root:$rootgroup" "$dir"
-chmod 0711 "$dir"
-
-target="$dir/api-token"
-[ ! -L "$target" ] || fail "$target is a symlink"
-if [ -e "$target" ]; then
-    [ -f "$target" ] || fail "$target is not a regular file"
-    holder="$(owner_of "$target")"
-    [ "$holder" = "$root" ] || [ "$holder" = "$user" ] || fail "$target is owned by $holder"
-fi
-
-tmp="$(mktemp "$dir/.api-token.XXXXXX")"
-trap 'rm -f "$tmp"' EXIT
-cat > "$tmp"
-[ -s "$tmp" ] || fail "empty token on stdin"
-if [ -f "$target" ] && cmp -s "$tmp" "$target"; then
-    chown "$user" "$target"
-    chmod 0400 "$target"
-    echo unchanged
-    exit 0
-fi
-chown "$user" "$tmp"
-chmod 0400 "$tmp"
-mv -f "$tmp" "$target"
-echo installed

scripts/terraform/latitude-spx/node-pool-state.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-# Runs as root on a node with the pool name as $1. Reads only the journal of the daemon's
-# current invocation, so a success logged before a restart cannot hide a failure after it.
-# Prints ready, refused or waiting.
-set -eu
-pool="${1:?pool name}"
-id="$(systemctl show -p InvocationID --value spinifex-daemon)"
-[ -n "$id" ] || { echo waiting; exit 0; }
-log="$(journalctl "_SYSTEMD_INVOCATION_ID=$id" -u spinifex-daemon --no-pager -o cat 2>/dev/null || true)"
-named() { grep -F -e "pool=$pool" -e "\"pool\":\"$pool\""; }
-if printf '%s\n' "$log" | grep -F "refused the pool's first call" | named >/dev/null; then
-    echo refused
-elif printf '%s\n' "$log" | grep -F "floating pool ready" | named >/dev/null; then
-    echo ready
-else
-    echo waiting
-fi

scripts/terraform/latitude-spx/node-scripts_test.sh

@@ -1,119 +0,0 @@
-#!/bin/sh
-# Tests node-install-api-token.sh and node-pool-state.sh with stand-in tools.
-set -eu
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-fails=0
-check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-
-# token install: A, A again, then B across a rotation; the file ends as B, owned by the daemon
-# user alone (0400) in a 0711 directory that only root may change
-me="$(id -un)"
-export API_TOKEN_DIR="$work/tok" API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
-install_token() { sh "$here/node-install-api-token.sh"; }
-check "first install" "$(printf 'TOKENA' | install_token)" installed
-check "same token" "$(printf 'TOKENA' | install_token)" unchanged
-check "rotation" "$(printf 'TOKENB' | install_token)" installed
-check "file holds B" "$(cat "$work/tok/api-token")" TOKENB
-check "file mode and owner" "$(stat -c '%U %a' "$work/tok/api-token")" "$me 400"
-check "dir mode and owner" "$(stat -c '%U %a' "$work/tok")" "$me 711"
-check "no temp left after install" "$(ls -A "$work/tok")" api-token
-check "empty refused" "$(printf '' | install_token 2>/dev/null || echo failed)" failed
-check "file kept after refusal" "$(cat "$work/tok/api-token")" TOKENB
-
-# an older install converges on the unchanged path too
-chmod 0640 "$work/tok/api-token"
-chmod 0755 "$work/tok"
-check "unchanged path repairs" "$(printf 'TOKENB' | install_token)" unchanged
-check "mode repaired on unchanged" "$(stat -c %a "$work/tok/api-token")" 400
-check "dir mode repaired on unchanged" "$(stat -c %a "$work/tok")" 711
-
-# who can read it: only the daemon's identity. This needs two real identities, so it runs as root only.
-if [ "$(id -u)" = 0 ]; then
-    chmod 0711 "$work"
-    export API_TOKEN_ROOT=root API_TOKEN_ROOT_GROUP=root API_TOKEN_USER=daemon
-    for mode in installed unchanged; do
-        [ "$mode" = installed ] && rm -f "$work/tok/api-token"
-        check "$mode path" "$(printf 'TOKENB' | install_token)" "$mode"
-        check "daemon reads ($mode)" "$(setpriv --reuid=daemon --regid=daemon --clear-groups cat "$work/tok/api-token")" TOKENB
-        check "another identity in the shared group cannot read ($mode)" \
-            "$(setpriv --reuid=nobody --regid=daemon --clear-groups cat "$work/tok/api-token" 2>/dev/null || echo denied)" denied
-    done
-    export API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
-else
-    echo "skip reading as the daemon and as another service: needs root"
-fi
-
-# a link planted where the file or the directory belongs is refused, and nothing it points at is touched
-echo SENTINEL > "$work/sentinel"
-rm -f "$work/tok/api-token"
-ln -s "$work/sentinel" "$work/tok/api-token"
-check "symlinked file refused" "$(printf 'TOKENC' | sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
-check "symlink target untouched" "$(cat "$work/sentinel")" SENTINEL
-ln -s "$work/tok" "$work/tok-link"
-check "symlinked dir refused" "$(printf 'TOKENC' | API_TOKEN_DIR="$work/tok-link" sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
-check "dir owned by someone else refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=nobody-such install_token 2>/dev/null || echo refused)" refused
-rm -f "$work/tok/api-token"
-printf 'TOKENB' | install_token >/dev/null
-check "file owned by a stranger refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=$me API_TOKEN_USER=nobody-such install_token 2>/dev/null || echo refused)" refused
-check "no temp left" "$(ls -A "$work/tok")" api-token
-
-# data disk choice: the largest blank disk, the lowest path among equals, the rest left alone
-choose_dir="$work/choose"
-mkdir -p "$choose_dir/bin"
-cat > "$choose_dir/bin/blockdev" <<'SH'
-#!/bin/sh
-# blockdev --getsize64 <id>: the fixture's size for that disk
-cat "$CHOOSE_DIR/size.$(basename "$2")"
-SH
-chmod +x "$choose_dir/bin/blockdev"
-eval "$(sed -n '/^choose_data_disk()/,/^}/p' "$here/host-data-disk.sh")"
-size_of() { printf '%s' "$2" > "$choose_dir/size.$1"; }
-pick() { CHOOSE_DIR="$choose_dir" PATH="$choose_dir/bin:$PATH" choose_data_disk "$@" 2>"$choose_dir/err"; }
-size_of nvme-eui.0b 1700000000000
-size_of nvme-eui.0a 1700000000000
-size_of nvme-eui.09 447000000000
-size_of nvme-eui.0c 1900000000000
-check "largest of three wins" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0c)" /dev/disk/by-id/nvme-eui.0c
-check "equal sizes: the lowest path" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a)" /dev/disk/by-id/nvme-eui.0a
-check "order of the list does not matter" "$(pick /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.0a
-check "what was left is said" "$(grep -c 'left untouched: /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b' "$choose_dir/err")" 1
-check "one disk is just chosen" "$(pick /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.09
-check "one disk says nothing" "$(wc -c < "$choose_dir/err" | tr -d ' ')" 0
-
-# pool state: the stand-in journal serves only the lines of the invocation asked for
-mkdir "$work/bin"
-cat > "$work/bin/systemctl" <<'SH'
-#!/bin/sh
-cat "$FAKE_DIR/invocation"
-SH
-cat > "$work/bin/journalctl" <<'SH'
-#!/bin/sh
-for a in "$@"; do
-    case "$a" in _SYSTEMD_INVOCATION_ID=*) id="${a#_SYSTEMD_INVOCATION_ID=}" ;; esac
-done
-cat "$FAKE_DIR/journal.$id" 2>/dev/null
-SH
-chmod +x "$work/bin/systemctl" "$work/bin/journalctl"
-export FAKE_DIR="$work" PATH="$work/bin:$PATH"
-state() { sh "$here/node-pool-state.sh" latitude; }
-
-echo old > "$work/invocation"
-echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.old"
-check "current success" "$(state)" ready
-
-echo new > "$work/invocation"
-echo 'ERROR floating: the provider refused the pool'"'"'s first call; public addresses will fail until fixed pool=latitude' > "$work/journal.new"
-check "old success then current failure" "$(state)" refused
-
-echo 'INFO starting' > "$work/journal.new"
-check "current invocation not yet ready" "$(state)" waiting
-
-echo 'INFO floating pool ready pool=other source=latitude' > "$work/journal.new"
-check "another pool does not count" "$(state)" waiting
-
-echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.new"
-check "current success" "$(state)" ready
-
-[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/outputs.tf

@@ -1,56 +0,0 @@
-output "deployment_name" {
-  value = var.deployment_name
-}
-
-output "site" {
-  value = var.site
-}
-
-output "project" {
-  value = local.project
-}
-
-# provider_host_id is the server ID. public_ip is the server's own /31 address, which SSH and the
-# Spinifex API use; the Elastic IP is separate.
-output "nodes" {
-  value = [for n in latitudesh_server.node : {
-    name             = trimprefix(n.hostname, "spx-${var.deployment_name}-")
-    provider_host_id = n.id
-    public_ip        = n.primary_ipv4
-  }]
-}
-
-# The first address, for a single node's static pool.
-output "elastic_ip" {
-  value = latitudesh_elastic_ip.pool[0].address
-}
-
-# IDs as well as addresses: destroy deletes only these IDs, and the floating pool lists them.
-output "elastic_ips" {
-  value = [for e in latitudesh_elastic_ip.pool : { id = e.id, address = e.address }]
-}
-
-# The VLAN ID of the cluster network; null for one node, which has no private network.
-output "vid" {
-  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].vid
-}
-
-output "ssh_key_id" {
-  value = latitudesh_ssh_key.deploy.id
-}
-
-output "user_data_id" {
-  value = latitudesh_user_data.host.id
-}
-
-output "project_id" {
-  value = local.project_id
-}
-
-output "project_created" {
-  value = length(latitudesh_project.deploy) > 0
-}
-
-output "virtual_network_id" {
-  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].id
-}

scripts/terraform/latitude-spx/provider.tf

@@ -1,4 +0,0 @@
-# The token comes from LATITUDESH_AUTH_TOKEN in the environment, which the provider reads
-# itself (its auth_token argument documents that variable). It is never a variable here, so it
-# is not written to the state or to a tfvars file. The default project is set per resource.
-provider "latitudesh" {}

scripts/terraform/latitude-spx/terraform.tfvars.example

@@ -1,5 +0,0 @@
-deployment_name     = "l1"
-site                = "DAL"
-plan                = "m4-metal-medium"
-node_count          = 1
-ssh_public_key_path = "~/.ssh/id_ed25519.pub"

scripts/terraform/latitude-spx/variables.tf

@@ -1,63 +0,0 @@
-variable "deployment_name" {
-  type        = string
-  description = "Names every resource and the server hostnames, so two deployments in one account never collide."
-  validation {
-    # A hostname is at most 32 characters: "spx-" + name + "-node1".
-    condition     = can(regex("^[a-z0-9][a-z0-9-]{0,15}$", var.deployment_name))
-    error_message = "deployment_name must be 1-16 lowercase letters, digits or hyphens."
-  }
-}
-
-variable "project" {
-  type        = string
-  default     = ""
-  description = "ID or slug of an existing latitude.sh project to deploy into. Empty creates a project spinifex-<deployment_name> and removes it on destroy."
-}
-
-variable "project_id" {
-  type        = string
-  default     = ""
-  description = "ID of the project named by project. The Elastic IP create sends it as project_id without resolving a slug (provider 4.8.0), so it must be an ID. Unused when project is empty."
-}
-
-variable "site" {
-  type        = string
-  default     = "DAL"
-  description = "latitude.sh site slug. Pick one with stock for the plan and Elastic IP support; not SAO."
-}
-
-variable "plan" {
-  type        = string
-  default     = "m4-metal-medium"
-  description = "Server plan slug (the pricing page calls it m4.metal.medium)."
-}
-
-variable "operating_system" {
-  type        = string
-  default     = "ubuntu_26_04_x64_lts"
-  description = "OS slug. UNVERIFIED: the 26.04 slug follows the 24.04 pattern (ubuntu_24_04_x64_lts) and is checked by a data source at plan time."
-}
-
-variable "node_count" {
-  type    = number
-  default = 1
-  validation {
-    condition     = contains([1, 3], var.node_count)
-    error_message = "node_count must be 1 or 3."
-  }
-}
-
-variable "elastic_ip_count" {
-  type        = number
-  default     = 1
-  description = "Elastic IPs to buy, each 4 USD a month. Every one starts on a server and keeps it unless Spinifex moves it. latitude.sh allows 5 per team."
-  validation {
-    condition     = var.elastic_ip_count >= 1 && var.elastic_ip_count <= 5 && floor(var.elastic_ip_count) == var.elastic_ip_count
-    error_message = "elastic_ip_count must be a whole number from 1 to 5."
-  }
-}
-
-variable "ssh_public_key_path" {
-  type    = string
-  default = "~/.ssh/id_ed25519.pub"
-}

scripts/terraform/latitude-spx/versions.tf

@@ -1,7 +0,0 @@
-terraform {
-  required_version = ">= 1.5.0"
-  required_providers {
-    # Pinned exactly: 4.8.0 is the version whose schema these files were written against.
-    latitudesh = { source = "latitudesh/latitudesh", version = "= 4.8.0" }
-  }
-}

spinifex/agents/lambda/agent/agent.go

@@ -12,10 +12,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
 )
 
-// resultWriteGrace is how long past the invocation deadline a result may take
-// to reach the host.
-const resultWriteGrace = 2 * time.Second
-
 const (
 	errTypeTimedOut = "Sandbox.Timedout"
 	errTypeUnknown  = "Runtime.Unknown"
@@ -57,11 +53,6 @@ func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error
 			Payload:     req.Payload,
 		})
 		timedOut := errors.Is(err, runtimeapi.ErrDeadlineExceeded)
-		if errors.Is(err, runtimeapi.ErrResponseBodyFailed) {
-			// The admitted body was lost: report nothing, so the host's
-			// outcome stays unknown.
-			return err
-		}
 		if err != nil {
 			if ctx.Err() != nil {
 				return cause(ctx, err)
@@ -73,15 +64,9 @@ func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error
 			AttemptID:     req.AttemptID,
 			RequestID:     req.RequestID,
 			Payload:       res.Payload,
-			Segments:      res.Segments,
 			FunctionError: res.FunctionError,
 		}
-		// A timed-out invocation writes its result after the deadline, so the
-		// write is bounded from whichever is later, plus the grace.
-		wctx, wcancel := context.WithDeadline(ctx, laterOf(time.UnixMilli(req.DeadlineMs), time.Now()).Add(resultWriteGrace))
-		err = link.Complete(wctx, resp)
-		wcancel()
-		if err != nil {
+		if err := link.Complete(ctx, resp); err != nil {
 			return cause(ctx, err)
 		}
 		if timedOut {
@@ -90,13 +75,6 @@ func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error
 	}
 }
 
-func laterOf(a, b time.Time) time.Time {
-	if a.After(b) {
-		return a
-	}
-	return b
-}
-
 func cause(ctx context.Context, err error) error {
 	if ctx.Err() != nil {
 		return context.Cause(ctx)

spinifex/agents/lambda/agent/agent_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -24,7 +23,6 @@ const api = "/2018-06-01/runtime/"
 type fakeLink struct {
 	reqs []hostlink.Request
 	got  chan hostlink.Response
-	ctxs chan context.Context // when set, the context of each Complete
 }
 
 var _ hostlink.Link = (*fakeLink)(nil)
@@ -39,9 +37,6 @@ func (l *fakeLink) Next(ctx context.Context) (hostlink.Request, error) {
 }
 
 func (l *fakeLink) Complete(ctx context.Context, r hostlink.Response) error {
-	if l.ctxs != nil {
-		l.ctxs <- ctx
-	}
 	l.got <- r
 	return nil
 }
@@ -145,58 +140,3 @@ func TestRunReturnsLinkError(t *testing.T) {
 	err := Run(context.Background(), &errLink{}, runtimeapi.New())
 	require.EqualError(t, err, "link down")
 }
-
-func TestRunBoundsTheResultWriteByTheDeadlinePlusGrace(t *testing.T) {
-	t.Run("on time", func(t *testing.T) {
-		srv := runtimeapi.New()
-		ts := httptest.NewServer(srv.Handler())
-		defer ts.Close()
-		runtime(t, ts.URL, func(string, []byte) (string, string) { return "/response", "ok" })
-		dl := deadline(time.Minute)
-		link := &fakeLink{got: make(chan hostlink.Response, 1), ctxs: make(chan context.Context, 1),
-			reqs: []hostlink.Request{{RequestID: "r", DeadlineMs: dl}}}
-		require.NoError(t, Run(context.Background(), link, srv))
-		d, ok := (<-link.ctxs).Deadline()
-		require.True(t, ok)
-		assert.Equal(t, time.UnixMilli(dl).Add(resultWriteGrace), d)
-	})
-	t.Run("a timed-out result is written after the deadline", func(t *testing.T) {
-		srv := runtimeapi.New() // nothing answers: the invocation expires
-		link := &fakeLink{got: make(chan hostlink.Response, 1), ctxs: make(chan context.Context, 1),
-			reqs: []hostlink.Request{{RequestID: "r", DeadlineMs: deadline(30 * time.Millisecond)}}}
-		require.ErrorIs(t, Run(context.Background(), link, srv), ErrInvocationTimedOut)
-		wctx := <-link.ctxs
-		d, ok := wctx.Deadline()
-		require.True(t, ok)
-		assert.Greater(t, time.Until(d), -resultWriteGrace, "the write deadline is not already past")
-		assert.Equal(t, "Sandbox.Timedout", (<-link.got).FunctionError)
-	})
-}
-
-// A body lost after admission sends the host nothing, so it reports the
-// outcome unknown, whether or not the bootstrap is still alive.
-func TestRunSendsNoResultWhenTheBodyIsLost(t *testing.T) {
-	srv := runtimeapi.New()
-	ts := httptest.NewServer(srv.Handler())
-	defer ts.Close()
-	link := &fakeLink{got: make(chan hostlink.Response, 1), reqs: []hostlink.Request{{RequestID: "r", DeadlineMs: deadline(time.Minute)}}}
-	done := make(chan error, 1)
-	go func() { done <- Run(context.Background(), link, srv) }()
-
-	resp, err := http.Get(ts.URL + api + "invocation/next")
-	require.NoError(t, err)
-	resp.Body.Close()
-	conn, err := net.Dial("tcp", ts.Listener.Addr().String())
-	require.NoError(t, err)
-	_, err = conn.Write([]byte("POST " + api + "invocation/r/response HTTP/1.1\r\nHost: x\r\nContent-Length: 100\r\n\r\npartial"))
-	require.NoError(t, err)
-	conn.Close()
-
-	select {
-	case err := <-done:
-		require.ErrorIs(t, err, runtimeapi.ErrResponseBodyFailed)
-	case <-time.After(10 * time.Second):
-		t.Fatal("Run waited for the invocation deadline")
-	}
-	assert.Empty(t, link.got, "no result may be reported")
-}

spinifex/agents/lambda/agent/agent_testlink_test.go

@@ -1,61 +0,0 @@
-//go:build lambdatestlink
-
-package agent_test
-
-import (
-	"context"
-	"net"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
-)
-
-// An expired invocation's timeout result is written over a real connection
-// after the deadline, within the grace, and the host decodes it.
-func TestRunWritesTheTimeoutFrameAfterTheDeadline(t *testing.T) {
-	guestConn, hostConn := net.Pipe()
-	t.Cleanup(func() {
-		_ = guestConn.Close()
-		_ = hostConn.Close()
-	})
-	guest := hostlink.NewTestStream(guestConn)
-	host := hostlink.NewTestStream(hostConn)
-
-	dl := time.Now().Add(50 * time.Millisecond)
-	req := hostlink.Request{GrantID: "g", AttemptID: "a", RequestID: "r", DeadlineMs: dl.UnixMilli()}
-	sent := make(chan error, 1)
-	go func() { sent <- host.HostSend(req) }()
-
-	done := make(chan error, 1)
-	go func() { done <- agent.Run(t.Context(), guest, runtimeapi.New()) }()
-	select {
-	case err := <-sent:
-		require.NoError(t, err)
-	case <-time.After(5 * time.Second):
-		t.Fatal("the request was not delivered")
-	}
-
-	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
-	defer cancel()
-	resp, err := host.HostResult(ctx)
-	got := time.Now()
-	require.NoError(t, err)
-	assert.Equal(t, "Sandbox.Timedout", resp.FunctionError)
-	assert.Equal(t, "r", resp.RequestID)
-	assert.Equal(t, "g", resp.GrantID)
-	assert.Equal(t, "a", resp.AttemptID)
-	assert.False(t, got.Before(time.UnixMilli(req.DeadlineMs)), "the result was read before the deadline")
-
-	select {
-	case err := <-done:
-		require.ErrorIs(t, err, agent.ErrInvocationTimedOut)
-	case <-time.After(5 * time.Second):
-		t.Fatal("Run did not return after writing the timeout result")
-	}
-}

spinifex/agents/lambda/hostlink/link.go

@@ -5,7 +5,6 @@ package hostlink
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 )
 
@@ -24,31 +23,13 @@ type Request struct {
 	Payload     []byte `json:"payload"`
 }
 
-// Response is the outcome of a Request. FunctionError is empty on success. A
-// non-nil Segments is the payload and takes precedence over Payload; only a
-// streaming carrier can send it, and encoding/json refuses it.
+// Response is the outcome of a Request. FunctionError is empty on success.
 type Response struct {
-	GrantID       string   `json:"grant_id"`
-	AttemptID     string   `json:"attempt_id"`
-	RequestID     string   `json:"request_id"`
-	Payload       []byte   `json:"payload"`
-	Segments      [][]byte `json:"-"`
-	FunctionError string   `json:"function_error,omitempty"`
-}
-
-// ErrSegmentedNotMarshalable is returned when a segmented payload reaches
-// encoding/json, which would otherwise drop it.
-var ErrSegmentedNotMarshalable = errors.New("hostlink: a segmented payload can only be streamed by a carrier")
-
-type responseAlias Response
-
-// MarshalJSON encodes r as the plain struct, and fails for a segmented
-// payload rather than emit a frame without it.
-func (r Response) MarshalJSON() ([]byte, error) {
-	if r.Segments != nil {
-		return nil, ErrSegmentedNotMarshalable
-	}
-	return json.Marshal(responseAlias(r))
+	GrantID       string `json:"grant_id"`
+	AttemptID     string `json:"attempt_id"`
+	RequestID     string `json:"request_id"`
+	Payload       []byte `json:"payload"`
+	FunctionError string `json:"function_error,omitempty"`
 }
 
 // Link carries invocations from the host to the guest and results back.

spinifex/agents/lambda/hostlink/test_stream.go

@@ -3,17 +3,13 @@
 package hostlink
 
 import (
-	"bufio"
 	"context"
-	"encoding/base64"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"net"
-	"sync/atomic"
-	"time"
+	"sync"
 )
 
 // maxFrame bounds one frame: a 6 MiB payload as base64 plus headroom.
@@ -37,14 +33,9 @@ type frame struct {
 // TestStream is a test-only carrier over an io.ReadWriter, framing JSON with
 // a 4-byte big-endian length. It is not the guest protocol.
 type TestStream struct {
-	rw    io.ReadWriter
-	conn  net.Conn // rw when it is a net.Conn; results are written only then
-	in    chan frame
-	wslot chan struct{}
-
-	// closed is set, under the write slot, once the connection was closed
-	// with a result possibly half sent.
-	closed atomic.Bool
+	rw  io.ReadWriter
+	in  chan frame
+	wmu sync.Mutex
 
 	err error // set before in is closed
 }
@@ -53,8 +44,7 @@ var _ Link = (*TestStream)(nil)
 
 // NewTestStream starts reading frames from rw.
 func NewTestStream(rw io.ReadWriter) *TestStream {
-	s := &TestStream{rw: rw, in: make(chan frame), wslot: make(chan struct{}, 1)}
-	s.conn, _ = rw.(net.Conn)
+	s := &TestStream{rw: rw, in: make(chan frame)}
 	go s.readLoop()
 	return s
 }
@@ -102,218 +92,12 @@ func (s *TestStream) Next(ctx context.Context) (Request, error) {
 	}
 }
 
-// ErrNoDeadlineTransport is returned by Complete when the stream's transport
-// is not a net.Conn, since a blocked write could then never be interrupted.
-var ErrNoDeadlineTransport = errors.New("hostlink: result needs a net.Conn transport")
-
-// ErrStreamClosed is returned by Complete after a failed write closed the
-// connection.
-var ErrStreamClosed = errors.New("hostlink: stream closed after a failed result write")
-
-// pastDeadline is a write deadline that is always already over.
-var pastDeadline = time.Unix(1, 0)
-
-// acquire takes the write slot or gives up when ctx ends.
-func (s *TestStream) acquire(ctx context.Context) error {
-	select {
-	case s.wslot <- struct{}{}:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}
-
-func (s *TestStream) release() { <-s.wslot }
-
-// Complete implements Link. The result is streamed as the same JSON frame
-// json.Marshal makes for it, without holding a copy of the payload.
-func (s *TestStream) Complete(ctx context.Context, resp Response) (err error) {
-	if s.conn == nil {
-		return ErrNoDeadlineTransport
-	}
-	if s.closed.Load() {
-		return ErrStreamClosed
-	}
+// Complete implements Link.
+func (s *TestStream) Complete(ctx context.Context, resp Response) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
-	rf, err := newResultFrame(resp)
-	if err != nil {
-		return err
-	}
-	if rf.size > maxFrame {
-		return ErrFrameTooLarge
-	}
-	if err := s.acquire(ctx); err != nil {
-		return err
-	}
-	// The slot is released last, after the deadline has been cleared.
-	defer s.release()
-	if s.closed.Load() {
-		return ErrStreamClosed
-	}
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-	// Once any frame byte may be out, the connection is closed before the
-	// slot is released so no later writer can append to a partial frame.
-	closeConn := func() {
-		s.closed.Store(true)
-		_ = s.conn.Close()
-	}
-
-	dl, _ := ctx.Deadline()
-	if err := s.conn.SetWriteDeadline(dl); err != nil {
-		closeConn()
-		return fmt.Errorf("hostlink: set write deadline: %w", err)
-	}
-	stop, watched, watchErr := make(chan struct{}), make(chan struct{}), make(chan error, 1)
-	go func() {
-		defer close(watched)
-		select {
-		case <-ctx.Done():
-			if err := s.conn.SetWriteDeadline(pastDeadline); err != nil {
-				// Closing is what unblocks the pending write.
-				s.closed.Store(true)
-				_ = s.conn.Close()
-				watchErr <- err
-			}
-		case <-stop:
-		}
-	}()
-	cw := &countingWriter{w: s.conn}
-	werr := rf.writeTo(cw)
-	close(stop)
-	<-watched
-	select {
-	case e := <-watchErr:
-		return fmt.Errorf("hostlink: interrupt blocked write: %w", errors.Join(e, werr))
-	default:
-	}
-	if werr == nil && cw.n != int64(4+rf.size) {
-		werr = fmt.Errorf("hostlink: frame wrote %d bytes, expected %d", cw.n, 4+rf.size)
-	}
-	if werr != nil {
-		closeConn()
-		return werr
-	}
-	if cerr := s.conn.SetWriteDeadline(time.Time{}); cerr != nil {
-		closeConn()
-		return fmt.Errorf("hostlink: clear write deadline: %w", errors.Join(cerr, werr))
-	}
-	return nil
-}
-
-// countingWriter writes everything to w, in full, and counts the bytes.
-type countingWriter struct {
-	w io.Writer
-	n int64
-}
-
-func (c *countingWriter) Write(p []byte) (int, error) {
-	done := 0
-	for done < len(p) {
-		n, err := c.w.Write(p[done:])
-		done += n
-		c.n += int64(n)
-		if err != nil {
-			return done, err
-		}
-		if n == 0 {
-			return done, io.ErrShortWrite
-		}
-	}
-	return done, nil
-}
-
-// resultFrame is the result frame in the field order json.Marshal gives
-// frame{Type, Response}, split around the payload value.
-type resultFrame struct {
-	head, tail []byte
-	payload    [][]byte
-	null       bool
-	size       int // JSON length, excluding the 4-byte prefix
-}
-
-func newResultFrame(r Response) (*resultFrame, error) {
-	q := func(v string) ([]byte, error) { return json.Marshal(v) }
-	var parts [3][]byte
-	for i, v := range []string{r.GrantID, r.AttemptID, r.RequestID} {
-		b, err := q(v)
-		if err != nil {
-			return nil, err
-		}
-		parts[i] = b
-	}
-	rf := &resultFrame{}
-	rf.head = append(rf.head, `{"type":"result","response":{"grant_id":`...)
-	rf.head = append(rf.head, parts[0]...)
-	rf.head = append(rf.head, `,"attempt_id":`...)
-	rf.head = append(rf.head, parts[1]...)
-	rf.head = append(rf.head, `,"request_id":`...)
-	rf.head = append(rf.head, parts[2]...)
-	rf.head = append(rf.head, `,"payload":`...)
-	if r.FunctionError != "" {
-		b, err := q(r.FunctionError)
-		if err != nil {
-			return nil, err
-		}
-		rf.tail = append(rf.tail, `,"function_error":`...)
-		rf.tail = append(rf.tail, b...)
-	}
-	rf.tail = append(rf.tail, `}}`...)
-
-	total := 0
-	switch {
-	case r.Segments != nil:
-		rf.payload = r.Segments
-	case r.Payload != nil:
-		rf.payload = [][]byte{r.Payload}
-	default:
-		rf.null = true
-	}
-	for _, seg := range rf.payload {
-		total += len(seg)
-	}
-	rf.size = len(rf.head) + len(rf.tail)
-	if rf.null {
-		rf.size += len("null")
-	} else {
-		// A payload too large for maxFrame is refused before its length matters.
-		if total > maxFrame {
-			rf.size = maxFrame + 1
-			return rf, nil
-		}
-		rf.size += 2 + base64.StdEncoding.EncodedLen(total)
-	}
-	return rf, nil
-}
-
-// writeTo writes the length prefix and the frame, in order, through a small
-// buffer so the payload is never held a second time.
-func (rf *resultFrame) writeTo(w io.Writer) error {
-	bw := bufio.NewWriterSize(w, 32<<10)
-	var hdr [4]byte
-	binary.BigEndian.PutUint32(hdr[:], uint32(rf.size))
-	_, _ = bw.Write(hdr[:])
-	_, _ = bw.Write(rf.head)
-	if rf.null {
-		_, _ = bw.WriteString("null")
-	} else {
-		_ = bw.WriteByte('"')
-		enc := base64.NewEncoder(base64.StdEncoding, bw)
-		for _, seg := range rf.payload {
-			if _, err := enc.Write(seg); err != nil {
-				return err
-			}
-		}
-		if err := enc.Close(); err != nil {
-			return err
-		}
-		_ = bw.WriteByte('"')
-	}
-	_, _ = bw.Write(rf.tail)
-	return bw.Flush()
+	return s.write(frame{Type: typeResult, Response: &resp})
 }
 
 // HostSend writes an invoke frame, as the host side of the carrier.
@@ -349,8 +133,8 @@ func (s *TestStream) write(f frame) error {
 	buf := make([]byte, 4+len(b))
 	binary.BigEndian.PutUint32(buf, uint32(len(b)))
 	copy(buf[4:], b)
-	_ = s.acquire(context.Background())
-	defer s.release()
+	s.wmu.Lock()
+	defer s.wmu.Unlock()
 	_, err = s.rw.Write(buf)
 	return err
 }

spinifex/agents/lambda/hostlink/test_stream_complete_test.go

@@ -1,612 +0,0 @@
-//go:build lambdatestlink
-
-package hostlink
-
-//test:in-package reaches the unexported write slot, frame types and readFrame to force orderings
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"encoding/json"
-	"errors"
-	"io"
-	"net"
-	"os"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
-)
-
-// fakeConn is a net.Conn that records what is written and the deadlines set,
-// and can fail or block on demand.
-type fakeConn struct {
-	mu        sync.Mutex
-	buf       bytes.Buffer
-	deadlines []time.Time
-	maxWrite  int
-
-	failDeadline func(n int, t time.Time) error
-	blockWrites  bool
-	started      chan struct{}
-	unblock      chan struct{}
-	closedCh     chan struct{}
-	startOnce    sync.Once
-	unblockOnce  sync.Once
-	closeOnce    sync.Once
-	closes       atomic.Int32
-}
-
-var _ net.Conn = (*fakeConn)(nil)
-
-func newFakeConn() *fakeConn {
-	return &fakeConn{started: make(chan struct{}), unblock: make(chan struct{}), closedCh: make(chan struct{})}
-}
-
-func (c *fakeConn) Read([]byte) (int, error) { <-c.closedCh; return 0, io.EOF }
-
-func (c *fakeConn) Write(p []byte) (int, error) {
-	if c.blockWrites {
-		c.startOnce.Do(func() { close(c.started) })
-		select {
-		case <-c.unblock:
-			return 0, os.ErrDeadlineExceeded
-		case <-c.closedCh:
-			return 0, net.ErrClosed
-		}
-	}
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.maxWrite = max(c.maxWrite, len(p))
-	return c.buf.Write(p)
-}
-
-func (c *fakeConn) Close() error {
-	c.closes.Add(1)
-	c.closeOnce.Do(func() { close(c.closedCh) })
-	return nil
-}
-
-func (c *fakeConn) SetWriteDeadline(t time.Time) error {
-	c.mu.Lock()
-	c.deadlines = append(c.deadlines, t)
-	n := len(c.deadlines)
-	c.mu.Unlock()
-	if c.failDeadline != nil {
-		if err := c.failDeadline(n, t); err != nil {
-			return err
-		}
-	}
-	if !t.IsZero() && !t.After(time.Now()) {
-		c.unblockOnce.Do(func() { close(c.unblock) })
-	}
-	return nil
-}
-
-func (c *fakeConn) LocalAddr() net.Addr             { return nil }
-func (c *fakeConn) RemoteAddr() net.Addr            { return nil }
-func (c *fakeConn) SetDeadline(time.Time) error     { return nil }
-func (c *fakeConn) SetReadDeadline(time.Time) error { return nil }
-
-func (c *fakeConn) written() []byte {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return bytes.Clone(c.buf.Bytes())
-}
-
-func (c *fakeConn) deadlineLog() []time.Time {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return append([]time.Time(nil), c.deadlines...)
-}
-
-// await bounds a receive so a regression fails promptly instead of hanging.
-func await[T any](t *testing.T, ch <-chan T) T {
-	t.Helper()
-	select {
-	case v := <-ch:
-		return v
-	case <-time.After(15 * time.Second):
-		t.Fatal("timed out waiting on a channel")
-		panic("unreachable")
-	}
-}
-
-// pattern returns n deterministic bytes.
-func pattern(n int) []byte {
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = byte(i*7 + i/251)
-	}
-	return b
-}
-
-func split(b []byte, seg int) [][]byte {
-	out := [][]byte{}
-	for len(b) > 0 {
-		n := min(seg, len(b))
-		out = append(out, b[:n])
-		b = b[n:]
-	}
-	return out
-}
-
-func segmented(b []byte, n int) [][]byte { return split(b, n) }
-
-func joined(r Response) []byte {
-	if r.Segments != nil {
-		return append([]byte{}, bytes.Join(r.Segments, nil)...)
-	}
-	return r.Payload
-}
-
-// assertTodaysFrame checks the written bytes are exactly the length prefix
-// and the json.Marshal of the frame, and that readFrame decodes it.
-func assertTodaysFrame(t *testing.T, got []byte, r Response) {
-	t.Helper()
-	plain := r
-	plain.Segments = nil
-	plain.Payload = joined(r)
-	want, err := json.Marshal(frame{Type: typeResult, Response: &plain})
-	require.NoError(t, err)
-	require.GreaterOrEqual(t, len(got), 4)
-	assert.Equal(t, uint32(len(got)-4), binary.BigEndian.Uint32(got[:4]), "length prefix must match the bytes after it")
-	if !assert.True(t, bytes.Equal(want, got[4:]), "frame differs from json.Marshal (%d vs %d bytes)", len(want), len(got)-4) {
-		return
-	}
-	f, err := readFrame(bytes.NewReader(got))
-	require.NoError(t, err)
-	var wf frame
-	require.NoError(t, json.Unmarshal(want, &wf))
-	assert.Equal(t, *wf.Response, *f.Response)
-}
-
-func complete(t *testing.T, r Response) []byte {
-	t.Helper()
-	c := newFakeConn()
-	defer c.Close()
-	require.NoError(t, NewTestStream(c).Complete(context.Background(), r))
-	return c.written()
-}
-
-func TestCompleteMatchesJSONMarshal(t *testing.T) {
-	meta := Response{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1"}
-	t.Run("6 MiB from one buffer and from many segments", func(t *testing.T) {
-		p := pattern(6 << 20)
-		one := meta
-		one.Payload = p
-		assertTodaysFrame(t, complete(t, one), one)
-		many := meta
-		many.Segments = segmented(p, 256<<10)
-		assertTodaysFrame(t, complete(t, many), many)
-		odd := meta
-		odd.Segments = segmented(p, 256<<10+1)
-		odd.FunctionError = "Unhandled"
-		assertTodaysFrame(t, complete(t, odd), odd)
-	})
-	t.Run("segment lengths against totals of each length mod 3", func(t *testing.T) {
-		for _, seg := range []int{1, 2, 3, 4, 5, 256<<10 + 1} {
-			totals := []int{0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66}
-			if seg > 5 {
-				totals = append(totals, seg-1, seg, seg+1, 2*seg+1, 2*seg+2)
-			}
-			for _, total := range totals {
-				r := meta
-				r.Segments = segmented(pattern(total), seg)
-				assertTodaysFrame(t, complete(t, r), r)
-			}
-		}
-	})
-	t.Run("nil and empty payloads", func(t *testing.T) {
-		nilp := meta
-		got := complete(t, nilp)
-		assert.Contains(t, string(got[4:]), `"payload":null`)
-		assertTodaysFrame(t, got, nilp)
-
-		empty := meta
-		empty.Payload = []byte{}
-		got = complete(t, empty)
-		assert.Contains(t, string(got[4:]), `"payload":""`)
-		assertTodaysFrame(t, got, empty)
-
-		noSegs := meta
-		noSegs.Segments = [][]byte{}
-		assertTodaysFrame(t, complete(t, noSegs), noSegs)
-	})
-	t.Run("metadata is escaped as json.Marshal escapes it", func(t *testing.T) {
-		for _, s := range []string{
-			`quo"te`, `back\slash`, "ctl\x00\x01\x1f\n\t\r", "naïve 日本語 😀", "<script>&amp;</script>",
-			`","payload":"x"`, `{"type":"invoke"}`, `"payload":`, "bad\xffutf8", ``,
-		} {
-			r := Response{GrantID: s, AttemptID: s + "a", RequestID: "r" + s, Payload: []byte("p"), FunctionError: s}
-			assertTodaysFrame(t, complete(t, r), r)
-			r.FunctionError = ""
-			r.Segments = [][]byte{[]byte("a"), []byte("bc")}
-			assertTodaysFrame(t, complete(t, r), r)
-		}
-	})
-	t.Run("a payload landing exactly on maxFrame", func(t *testing.T) {
-		for pad := range 4 {
-			r := Response{GrantID: "g", AttemptID: "a", RequestID: strings.Repeat("r", 1+pad)}
-			rf, err := newResultFrame(r)
-			require.NoError(t, err)
-			room := maxFrame - rf.size + 2
-			if room%4 != 0 {
-				continue
-			}
-			r.Payload = pattern(room / 4 * 3)
-			rf, err = newResultFrame(r)
-			require.NoError(t, err)
-			require.Equal(t, maxFrame, rf.size)
-			assertTodaysFrame(t, complete(t, r), r)
-			return
-		}
-		t.Fatal("no request id length lands on maxFrame")
-	})
-}
-
-func TestCompleteRefusesOverMaxFrameBeforeWriting(t *testing.T) {
-	c := newFakeConn()
-	defer c.Close()
-	s := NewTestStream(c)
-	err := s.Complete(context.Background(), Response{Segments: segmented(make([]byte, maxFrame), 1<<20)})
-	require.ErrorIs(t, err, ErrFrameTooLarge)
-	assert.Empty(t, c.written())
-	assert.Empty(t, c.deadlineLog(), "nothing is touched before the size check")
-}
-
-// plainRW has no deadline support at all.
-type plainRW struct {
-	io.Reader
-	wrote atomic.Int32
-}
-
-func (p *plainRW) Write(b []byte) (int, error) { p.wrote.Add(1); return len(b), nil }
-
-// deadlineRW has SetWriteDeadline but is not a net.Conn.
-type deadlineRW struct{ plainRW }
-
-func (*deadlineRW) SetWriteDeadline(time.Time) error { return nil }
-
-func TestCompleteRefusesNonNetConn(t *testing.T) {
-	p := &plainRW{Reader: strings.NewReader("")}
-	d := &deadlineRW{plainRW{Reader: strings.NewReader("")}}
-	for name, tc := range map[string]struct {
-		rw    io.ReadWriter
-		wrote *atomic.Int32
-	}{
-		"plain":                 {p, &p.wrote},
-		"with SetWriteDeadline": {d, &d.wrote},
-	} {
-		t.Run(name, func(t *testing.T) {
-			s := NewTestStream(tc.rw)
-			err := s.Complete(context.Background(), Response{Payload: []byte{1}})
-			require.ErrorIs(t, err, ErrNoDeadlineTransport)
-			assert.Zero(t, tc.wrote.Load(), "no byte may be written")
-		})
-	}
-}
-
-// sigCtx reports the first time something selects on its Done channel.
-type sigCtx struct {
-	context.Context
-	waiting chan struct{}
-	once    sync.Once
-}
-
-func (c *sigCtx) Done() <-chan struct{} {
-	c.once.Do(func() { close(c.waiting) })
-	return c.Context.Done()
-}
-
-func TestCompleteCancelledWhileAnotherWriteHoldsTheSlot(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	s := NewTestStream(c)
-	require.NoError(t, s.acquire(context.Background())) // another writer holds the slot
-
-	base, cancel := context.WithCancel(context.Background())
-	ctx := &sigCtx{Context: base, waiting: make(chan struct{})}
-	done := make(chan error, 1)
-	go func() { done <- s.Complete(ctx, Response{Payload: []byte("x")}) }()
-	await(t, ctx.waiting)
-	cancel()
-	require.ErrorIs(t, await(t, done), context.Canceled)
-	assert.Empty(t, c.written(), "the peer receives no bytes")
-	assert.Empty(t, c.deadlineLog())
-
-	s.release()
-	require.NoError(t, s.Complete(context.Background(), Response{Payload: []byte("x")}), "the slot is still usable")
-	c.Close()
-}
-
-func TestCompletePreCancelledReleasesSlot(t *testing.T) {
-	c := newFakeConn()
-	defer c.Close()
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	require.ErrorIs(t, s.Complete(ctx, Response{Payload: []byte("x")}), context.Canceled)
-	assert.Empty(t, c.written())
-	assert.Zero(t, len(s.wslot), "slot released")
-}
-
-func TestCompleteInitialDeadlineFailureClosesAndWritesNothing(t *testing.T) {
-	c := newFakeConn()
-	c.failDeadline = func(int, time.Time) error { return errors.New("no deadlines") }
-	s := NewTestStream(c)
-	err := s.Complete(context.Background(), Response{Payload: []byte("x")})
-	require.ErrorContains(t, err, "set write deadline")
-	assert.Equal(t, int32(1), c.closes.Load())
-	assert.Empty(t, c.written())
-	assert.Zero(t, len(s.wslot))
-}
-
-func TestCompleteInterruptFailureClosesAndReportsLinkError(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	c.blockWrites = true
-	c.failDeadline = func(_ int, t time.Time) error {
-		if !t.IsZero() && t.Before(time.Now()) {
-			return errors.New("cannot interrupt")
-		}
-		return nil
-	}
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	done := make(chan error, 1)
-	go func() { done <- s.Complete(ctx, Response{Payload: []byte("x")}) }()
-	await(t, c.started)
-	cancel()
-	var err error
-	select {
-	case err = <-done:
-	case <-time.After(5 * time.Second):
-		c.Close()
-		t.Fatal("Complete stayed blocked: the watcher did not close the connection")
-	}
-	require.ErrorContains(t, err, "interrupt blocked write")
-	assert.Equal(t, int32(1), c.closes.Load())
-	assert.Zero(t, len(s.wslot))
-}
-
-func TestCompleteClearFailureAfterWriteClosesButFrameWasSent(t *testing.T) {
-	c := newFakeConn()
-	c.failDeadline = func(n int, _ time.Time) error {
-		if n == 2 {
-			return errors.New("cannot clear")
-		}
-		return nil
-	}
-	s := NewTestStream(c)
-	r := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Payload: []byte("done")}
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-	defer cancel()
-	err := s.Complete(ctx, r)
-	require.ErrorContains(t, err, "clear write deadline")
-	assert.Equal(t, int32(1), c.closes.Load())
-	// The host already holds a complete valid frame and may accept it.
-	assertTodaysFrame(t, c.written(), r)
-}
-
-func TestCompleteReleasesSlotOnlyAfterDeadlineCleared(t *testing.T) {
-	c := newFakeConn()
-	clearing, proceed := make(chan struct{}), make(chan struct{})
-	c.failDeadline = func(n int, t time.Time) error {
-		if n == 2 { // writer A clearing its deadline
-			close(clearing)
-			<-proceed
-		}
-		return nil
-	}
-	s := NewTestStream(c)
-	ctxA, cancelA := context.WithTimeout(context.Background(), time.Minute)
-	defer cancelA()
-	aDone := make(chan error, 1)
-	go func() { aDone <- s.Complete(ctxA, Response{Payload: []byte("a")}) }()
-	await(t, clearing)
-
-	base, cancelB := context.WithTimeout(context.Background(), 2*time.Minute)
-	defer cancelB()
-	ctxB := &sigCtx{Context: base, waiting: make(chan struct{})}
-	bDone := make(chan error, 1)
-	go func() { bDone <- s.Complete(ctxB, Response{Payload: []byte("b")}) }()
-	await(t, ctxB.waiting)
-	assert.Len(t, c.deadlineLog(), 2, "B has set no deadline while A is still cleaning up")
-
-	close(proceed)
-	require.NoError(t, await(t, aDone))
-	require.NoError(t, await(t, bDone))
-	log := c.deadlineLog()
-	require.Len(t, log, 4)
-	assert.False(t, log[0].IsZero())
-	assert.True(t, log[1].IsZero(), "A clears first")
-	assert.False(t, log[2].IsZero())
-	assert.True(t, log[3].IsZero())
-	c.Close()
-}
-
-func TestCompleteStopsAtDeadlineWhenPeerNeverReads(t *testing.T) {
-	g, h := net.Pipe()
-	defer h.Close()
-	s := NewTestStream(g)
-	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
-	defer cancel()
-	err := s.Complete(ctx, Response{GrantID: "g", Payload: []byte("x")})
-	require.Error(t, err)
-	assert.ErrorIs(t, err, os.ErrDeadlineExceeded)
-	assert.Zero(t, len(s.wslot))
-
-	// The host never received a frame, so it has nothing to accept.
-	host := NewTestStream(h)
-	g.Close()
-	_, err = host.HostResult(context.Background())
-	require.Error(t, err)
-}
-
-func TestCompleteCancelInterruptsBlockedWrite(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	c.blockWrites = true
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	done := make(chan error, 1)
-	go func() { done <- s.Complete(ctx, Response{Payload: []byte("x")}) }()
-	await(t, c.started)
-	cancel()
-	require.ErrorIs(t, await(t, done), os.ErrDeadlineExceeded)
-	assert.Equal(t, int32(1), c.closes.Load(), "a frame may be half sent, so the connection is closed")
-	assert.ErrorIs(t, s.Complete(context.Background(), Response{Payload: []byte("x")}), ErrStreamClosed)
-}
-
-func TestCompleteCancelAfterWriteDoesNotTouchConn(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	require.NoError(t, s.Complete(ctx, Response{Payload: []byte("x")}))
-	before := len(c.deadlineLog())
-	cancel()
-	assert.Len(t, c.deadlineLog(), before, "no watcher is left to move the deadline")
-	assert.Zero(t, c.closes.Load())
-	c.Close()
-}
-
-func TestCompleteWritesNeverInterleave(t *testing.T) {
-	c := newFakeConn()
-	defer c.Close()
-	s := NewTestStream(c)
-	const n = 16
-	var wg sync.WaitGroup
-	for i := range n {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if i%2 == 0 {
-				assert.NoError(t, s.Complete(context.Background(), Response{GrantID: "g", Segments: segmented(pattern(100000+i), 7001)}))
-			} else {
-				assert.NoError(t, s.HostSend(Request{GrantID: "g", Payload: pattern(100000 + i)}))
-			}
-		}()
-	}
-	wg.Wait()
-	r := bytes.NewReader(c.written())
-	var results, invokes int
-	for r.Len() > 0 {
-		f, err := readFrame(r)
-		require.NoError(t, err)
-		switch f.Type {
-		case typeResult:
-			results++
-		case typeInvoke:
-			invokes++
-		}
-	}
-	assert.Equal(t, n/2, results)
-	assert.Equal(t, n/2, invokes)
-}
-
-// discardConn drops writes so a benchmark does not hold the frame.
-type discardConn struct{ *fakeConn }
-
-func (discardConn) Write(p []byte) (int, error) { return len(p), nil }
-
-func TestCompleteMakesNoPayloadSizedAllocation(t *testing.T) {
-	r := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Segments: segmented(pattern(6<<20), 256<<10)}
-	c := newFakeConn()
-	defer c.Close()
-	require.NoError(t, NewTestStream(c).Complete(context.Background(), r))
-	assert.Less(t, c.maxWrite, 64<<10, "no single write is payload sized")
-
-	d := discardConn{newFakeConn()}
-	defer d.Close()
-	bs := NewTestStream(d)
-	br := testing.Benchmark(func(b *testing.B) {
-		b.ReportAllocs()
-		for range b.N {
-			if err := bs.Complete(context.Background(), r); err != nil {
-				b.Fatal(err)
-			}
-		}
-	})
-	assert.Less(t, br.AllocedBytesPerOp(), int64(1<<20), "writing a 6 MiB result")
-}
-
-func TestResponseJSONCompatibility(t *testing.T) {
-	plain := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Payload: []byte("p<&>"), FunctionError: "Unhandled"}
-
-	t.Run("nil segments encode as the plain struct", func(t *testing.T) {
-		got, err := json.Marshal(plain)
-		require.NoError(t, err)
-		want := `{"grant_id":"g","attempt_id":"a","request_id":"r","payload":"cDwmPg==","function_error":"Unhandled"}`
-		assert.Equal(t, want, string(got))
-		inFrame, err := json.Marshal(frame{Type: typeResult, Response: &plain})
-		require.NoError(t, err)
-		assert.Equal(t, `{"type":"result","response":`+want+`}`, string(inFrame))
-		nilPayload, err := json.Marshal(Response{GrantID: "g"})
-		require.NoError(t, err)
-		assert.Equal(t, `{"grant_id":"g","attempt_id":"","request_id":"","payload":null}`, string(nilPayload))
-	})
-	t.Run("a legacy frame decodes as before", func(t *testing.T) {
-		legacy, err := json.Marshal(frame{Type: typeResult, Response: &plain})
-		require.NoError(t, err)
-		var f frame
-		require.NoError(t, json.Unmarshal(legacy, &f))
-		assert.Equal(t, plain, *f.Response)
-	})
-	t.Run("unknown fields including segments are ignored", func(t *testing.T) {
-		var f frame
-		in := `{"type":"result","extra":1,"response":{"grant_id":"g","segments":{"x":1},"unknown":[1],"payload":"cA=="}}`
-		require.NoError(t, json.Unmarshal([]byte(in), &f))
-		assert.Equal(t, Response{GrantID: "g", Payload: []byte("p")}, *f.Response)
-	})
-	t.Run("a segmented payload cannot be marshalled", func(t *testing.T) {
-		r := Response{GrantID: "g", Segments: segmented([]byte("data"), 2)}
-		_, err := json.Marshal(r)
-		require.ErrorIs(t, err, ErrSegmentedNotMarshalable)
-		_, err = json.Marshal(frame{Type: typeResult, Response: &r})
-		require.ErrorIs(t, err, ErrSegmentedNotMarshalable)
-		_, err = json.Marshal(Response{GrantID: "g", Segments: [][]byte{}})
-		require.ErrorIs(t, err, ErrSegmentedNotMarshalable)
-	})
-}
-
-// partialConn accepts n bytes of a write and then fails.
-type partialConn struct {
-	*fakeConn
-	budget int
-}
-
-func (p *partialConn) Write(b []byte) (int, error) {
-	if p.budget < len(b) {
-		n := p.budget
-		p.budget = 0
-		p.fakeConn.Write(b[:n])
-		return n, errors.New("link broke")
-	}
-	p.budget -= len(b)
-	return p.fakeConn.Write(b)
-}
-
-func TestCompleteAfterPartialWriteSendsNothingMore(t *testing.T) {
-	pc := &partialConn{fakeConn: newFakeConn(), budget: 20}
-	s := NewTestStream(pc)
-	err := s.Complete(context.Background(), Response{GrantID: "g", Payload: pattern(100)})
-	require.ErrorContains(t, err, "link broke")
-	assert.Equal(t, int32(1), pc.closes.Load(), "closed while the slot was held")
-	assert.Zero(t, len(s.wslot))
-	partial := len(pc.written())
-	assert.Less(t, partial, 4+100, "the host holds an incomplete frame")
-
-	pc.budget = 1 << 20
-	require.ErrorIs(t, s.Complete(context.Background(), Response{GrantID: "g2", Payload: []byte("again")}), ErrStreamClosed)
-	assert.Len(t, pc.written(), partial, "the peer receives no further bytes")
-	_, err = readFrame(bytes.NewReader(pc.written()))
-	assert.Error(t, err, "the host has no valid frame to accept")
-}

spinifex/agents/lambda/runtimeapi/server.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"strconv"
@@ -43,9 +42,6 @@ var (
 	ErrDeadlineExceeded = errors.New("runtimeapi: invocation deadline exceeded")
 	// ErrInvalidInvocation is returned by Invoke for an invocation with no request ID.
 	ErrInvalidInvocation = errors.New("runtimeapi: invocation has no request id")
-	// ErrResponseBodyFailed is returned by Invoke when a response or error
-	// body that was admitted could not be read in full. No result exists.
-	ErrResponseBodyFailed = errors.New("runtimeapi: admitted response body was not received")
 )
 
 // Invocation is one event handed to the runtime.
@@ -58,11 +54,9 @@ type Invocation struct {
 }
 
 // Result is what the runtime answered. FunctionError is empty on success and
-// otherwise the error type. A non-nil Segments is the payload, in order, and
-// takes precedence over Payload.
+// otherwise the error type.
 type Result struct {
 	Payload       []byte
-	Segments      [][]byte
 	FunctionError string
 }
 
@@ -72,25 +66,10 @@ type InitError struct {
 	Body []byte
 }
 
-type flightState int
-
-const (
-	stWaiting flightState = iota
-	stRunning
-	stReceiving
-	stCompleted
-	stExited
-)
-
-type outcome struct {
-	res Result
-	err error
-}
-
 type flight struct {
 	inv   Invocation
-	state flightState
-	done  chan outcome
+	taken bool
+	done  chan Result
 }
 
 // Server implements the Runtime API for a single in-flight invocation.
@@ -99,11 +78,6 @@ type Server struct {
 	cur  *flight
 	wake chan struct{}
 
-	// exited records that the bootstrap process has ended; settled that the
-	// last flight has its result, so a later exit no longer matters.
-	exited  bool
-	settled bool
-
 	initOnce sync.Once
 	initDone chan struct{}
 	initErr  InitError
@@ -133,16 +107,12 @@ func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
 	if inv.RequestID == "" {
 		return Result{}, ErrInvalidInvocation
 	}
-	f := &flight{inv: inv, done: make(chan outcome, 1)}
+	f := &flight{inv: inv, done: make(chan Result, 1)}
 	s.mu.Lock()
 	if s.cur != nil {
 		s.mu.Unlock()
 		return Result{}, ErrBusy
 	}
-	if s.exited {
-		f.state = stExited
-	}
-	s.settled = false
 	s.cur = f
 	s.mu.Unlock()
 	select {
@@ -154,8 +124,8 @@ func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
 	defer timer.Stop()
 	var cause error
 	select {
-	case out := <-f.done:
-		return out.res, out.err
+	case res := <-f.done:
+		return res, nil
 	case <-timer.C:
 		cause = ErrDeadlineExceeded
 	case <-ctx.Done():
@@ -170,27 +140,7 @@ func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
 	}
 	s.mu.Unlock()
 	// Completed while we were giving up; the result is already queued.
-	out := <-f.done
-	return out.res, out.err
-}
-
-// BootstrapExited records that the runtime process has ended and reports
-// whether the caller should cancel the invocation. Before the response is
-// admitted it should; afterwards the result stands.
-func (s *Server) BootstrapExited() bool {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.exited = true
-	f := s.cur
-	switch {
-	case f == nil:
-		return !s.settled
-	case f.state < stReceiving:
-		f.state = stExited
-		return true
-	default:
-		return false
-	}
+	return <-f.done, nil
 }
 
 // Handler returns the Runtime API routes.
@@ -207,8 +157,8 @@ func (s *Server) next(w http.ResponseWriter, r *http.Request) {
 	for {
 		s.mu.Lock()
 		f := s.cur
-		if f != nil && f.state == stWaiting {
-			f.state = stRunning
+		if f != nil && !f.taken {
+			f.taken = true
 			s.mu.Unlock()
 			h := w.Header()
 			h.Set(headerRequestID, f.inv.RequestID)
@@ -229,149 +179,58 @@ func (s *Server) next(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// admit claims the single response slot of the invocation named id, so only
-// one POST per invocation is ever read. It returns nil if id is not the
-// delivered, unclaimed invocation of a runtime that is still running.
-func (s *Server) admit(id string) *flight {
+// claimable reports whether id names the delivered, uncompleted invocation.
+func (s *Server) claimable(id string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.cur != nil && s.cur.taken && s.cur.inv.RequestID == id
+}
+
+// complete finishes the invocation named id, reporting whether it was live.
+func (s *Server) complete(id string, res Result) bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	f := s.cur
-	if f == nil || f.state != stRunning || f.inv.RequestID != id {
-		return nil
-	}
-	f.state = stReceiving
-	return f
-}
-
-// complete records res for an admitted flight, reporting whether the
-// invocation was still waiting for it.
-func (s *Server) complete(f *flight, res Result) bool {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.cur != f {
+	if f == nil || !f.taken || f.inv.RequestID != id {
 		return false
 	}
 	s.cur = nil
-	s.settled = true
-	f.state = stCompleted
-	f.done <- outcome{res: res}
+	f.done <- res
 	return true
 }
 
-// readFailed ends an admitted flight whose body was lost. The outcome is
-// published before the flight is detached, so Invoke never waits on it.
-func (s *Server) readFailed(f *flight, err error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.cur != f {
-		return
-	}
-	s.cur = nil
-	s.settled = true
-	f.state = stCompleted
-	f.done <- outcome{err: fmt.Errorf("%w: %w", ErrResponseBodyFailed, err)}
-}
-
 func (s *Server) response(w http.ResponseWriter, r *http.Request) {
 	id := r.PathValue("id")
-	f := s.admit(id)
-	if f == nil {
+	if !s.claimable(id) {
 		invalidID(w, id)
 		return
 	}
-	s.finishResponse(w, r, f)
-}
-
-// segmentBytes is the largest piece an undeclared-length body is read into.
-const segmentBytes = 256 << 10
-
-// finishResponse reads the admitted response body and completes f.
-func (s *Server) finishResponse(w http.ResponseWriter, r *http.Request, f *flight) {
-	var res Result
-	var oversize bool
-	var err error
-	switch cl := r.ContentLength; {
-	case cl > MaxResponseBytes:
-		oversize, err = discardToCap(r.Body)
-	case cl >= 0:
-		res.Payload = make([]byte, cl)
-		_, err = io.ReadFull(r.Body, res.Payload)
-	default:
-		res.Segments, oversize, err = readSegments(r.Body)
-	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes+1))
 	if err != nil {
-		s.readFailed(f, err)
 		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
 		return
 	}
-	if oversize {
+	if len(body) > MaxResponseBytes {
 		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
-		s.complete(f, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeUnhandled})
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
 		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
 		return
 	}
-	if !s.complete(f, res) {
-		invalidID(w, f.inv.RequestID)
+	if !s.complete(id, Result{Payload: body}) {
+		invalidID(w, id)
 		return
 	}
 	writeStatus(w)
 }
 
-// discardToCap counts body through a small buffer. It reports oversize once
-// byte MaxResponseBytes+1 arrives and an error if the body ends first.
-func discardToCap(body io.Reader) (bool, error) {
-	n, err := io.CopyN(io.Discard, body, MaxResponseBytes+1)
-	if n == MaxResponseBytes+1 {
-		return true, nil
-	}
-	if err == nil || errors.Is(err, io.EOF) {
-		err = io.ErrUnexpectedEOF
-	}
-	return false, err
-}
-
-// readSegments reads body into segments of at most segmentBytes, never
-// concatenating them. It reports oversize and drops them at byte
-// MaxResponseBytes+1.
-func readSegments(body io.Reader) ([][]byte, bool, error) {
-	segs := [][]byte{}
-	total := 0
-	for {
-		n := min(segmentBytes, MaxResponseBytes+1-total)
-		seg := make([]byte, n)
-		got := 0
-		var err error
-		for got < n && err == nil {
-			var c int
-			c, err = body.Read(seg[got:])
-			got += c
-		}
-		total += got
-		if total > MaxResponseBytes {
-			return nil, true, nil
-		}
-		if got > 0 {
-			segs = append(segs, seg[:got])
-		}
-		if errors.Is(err, io.EOF) {
-			return segs, false, nil
-		}
-		if err != nil {
-			return nil, false, err
-		}
-	}
-}
-
 func (s *Server) invocationError(w http.ResponseWriter, r *http.Request) {
 	id := r.PathValue("id")
-	f := s.admit(id)
-	if f == nil {
+	if !s.claimable(id) {
 		invalidID(w, id)
 		return
 	}
 	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes))
 	if err != nil {
-		s.readFailed(f, err)
 		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
 		return
 	}
@@ -379,7 +238,7 @@ func (s *Server) invocationError(w http.ResponseWriter, r *http.Request) {
 	if errType == "" {
 		errType = ErrTypeUnhandled
 	}
-	if !s.complete(f, Result{Payload: body, FunctionError: errType}) {
+	if !s.complete(id, Result{Payload: body, FunctionError: errType}) {
 		invalidID(w, id)
 		return
 	}

spinifex/agents/lambda/runtimeapi/server_admission_test.go

@@ -1,488 +0,0 @@
-package runtimeapi
-
-//test:in-package drives the unexported admit and exit transitions to force both orders
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
-)
-
-// await bounds a receive so a regression fails promptly instead of hanging.
-func await[T any](t *testing.T, ch <-chan T) T {
-	t.Helper()
-	select {
-	case v := <-ch:
-		return v
-	case <-time.After(15 * time.Second):
-		t.Fatal("timed out waiting on a channel")
-		panic("unreachable")
-	}
-}
-
-// arm puts a delivered, unclaimed flight in place without any HTTP traffic.
-func arm(s *Server, id string) *flight {
-	f := &flight{inv: inv(id, time.Minute), state: stRunning, done: make(chan outcome, 1)}
-	s.mu.Lock()
-	s.cur = f
-	s.mu.Unlock()
-	return f
-}
-
-func postTo(s *Server, kind, id string, body io.Reader, length int64) *httptest.ResponseRecorder {
-	r := httptest.NewRequest(http.MethodPost, base+"invocation/"+id+"/"+kind, body)
-	r.ContentLength = length
-	r.SetPathValue("id", id)
-	rec := httptest.NewRecorder()
-	if kind == "response" {
-		s.response(rec, r)
-	} else {
-		s.invocationError(rec, r)
-	}
-	return rec
-}
-
-// patterned yields n bytes of a repeating pattern without holding them.
-type patterned struct{ n, off int }
-
-func (p *patterned) Read(b []byte) (int, error) {
-	if p.off >= p.n {
-		return 0, io.EOF
-	}
-	c := min(len(b), p.n-p.off)
-	for i := range c {
-		b[i] = byte((p.off + i) % 251)
-	}
-	p.off += c
-	return c, nil
-}
-
-func pattern(n int) []byte {
-	b := make([]byte, n)
-	_, _ = io.ReadFull(&patterned{n: n}, b)
-	return b
-}
-
-func concat(r Result) []byte {
-	if r.Segments == nil {
-		return r.Payload
-	}
-	return bytes.Join(r.Segments, nil)
-}
-
-// counting records whether the handler read the body at all.
-type counting struct{ reads atomic.Int32 }
-
-func (c *counting) Read([]byte) (int, error) { c.reads.Add(1); return 0, io.EOF }
-
-// gated blocks its first Read until released and reports that it started.
-type gated struct {
-	started chan struct{}
-	release chan struct{}
-	rest    io.Reader
-	once    atomic.Bool
-}
-
-func (g *gated) Read(b []byte) (int, error) {
-	if g.once.CompareAndSwap(false, true) {
-		close(g.started)
-		<-g.release
-	}
-	return g.rest.Read(b)
-}
-
-func TestOnlyOnePostPerInvocationIsRead(t *testing.T) {
-	for _, first := range []string{"response", "error"} {
-		for _, second := range []string{"response", "error"} {
-			t.Run(first+"_then_"+second, func(t *testing.T) {
-				s := New()
-				f := arm(s, "req-1")
-				g := &gated{started: make(chan struct{}), release: make(chan struct{}), rest: bytes.NewReader([]byte("body"))}
-				firstDone := make(chan *httptest.ResponseRecorder, 1)
-				go func() { firstDone <- postTo(s, first, "req-1", g, 4) }()
-				await(t, g.started)
-
-				c := &counting{}
-				rec := postTo(s, second, "req-1", c, 4)
-				assert.Equal(t, http.StatusBadRequest, rec.Code)
-				assert.Equal(t, "InvalidRequestID", errType(t, rec.Body.Bytes()))
-				assert.Zero(t, c.reads.Load(), "the refused POST's body was read")
-				assert.Empty(t, f.done, "nothing completes while the admitted body is still arriving")
-
-				close(g.release)
-				assert.Equal(t, http.StatusAccepted, (await(t, firstDone)).Code)
-				assert.Equal(t, "body", string(concat((await(t, f.done)).res)))
-			})
-		}
-	}
-}
-
-func TestDeclaredOversizeCompletesWithoutPayloadBuffer(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	n := MaxResponseBytes + 10
-	rec := postTo(s, "response", "req-1", &patterned{n: n}, int64(n))
-	assert.Equal(t, http.StatusRequestEntityTooLarge, rec.Code)
-	res := (await(t, f.done)).res
-	assert.Equal(t, ErrTypeUnhandled, res.FunctionError)
-	assert.Equal(t, ErrTypeResponseTooLarge, errType(t, res.Payload))
-
-	br := testing.Benchmark(func(b *testing.B) {
-		b.ReportAllocs()
-		for range b.N {
-			s := New()
-			f := arm(s, "req-1")
-			postTo(s, "response", "req-1", &patterned{n: n}, int64(n))
-			await(t, f.done)
-		}
-	})
-	assert.Less(t, br.AllocedBytesPerOp(), int64(256<<10), "an oversize body must not be buffered")
-}
-
-func TestDeclaredOversizeCutShortIsUnknown(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes}, MaxResponseBytes+10)
-	assert.Equal(t, http.StatusBadRequest, rec.Code)
-	assert.ErrorIs(t, (await(t, f.done)).err, ErrResponseBodyFailed, "no result is made from a partial body")
-
-	rec = postTo(s, "response", "req-1", &patterned{n: 1}, 1)
-	assert.Equal(t, "InvalidRequestID", errType(t, rec.Body.Bytes()), "the slot stays consumed")
-}
-
-func TestKnownLengthWithinCapAllocatesOnce(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes}, MaxResponseBytes)
-	assert.Equal(t, http.StatusAccepted, rec.Code)
-	res := (await(t, f.done)).res
-	assert.Nil(t, res.Segments)
-	assert.Equal(t, pattern(MaxResponseBytes), res.Payload)
-
-	s = New()
-	f = arm(s, "req-1")
-	rec = postTo(s, "response", "req-1", &patterned{n: 100}, 200)
-	assert.Equal(t, http.StatusBadRequest, rec.Code, "a body ending short of its length is truncated")
-	out := await(t, f.done)
-	assert.ErrorIs(t, out.err, ErrResponseBodyFailed)
-	assert.Empty(t, out.res.Payload)
-}
-
-func TestChunkedBodyCap(t *testing.T) {
-	t.Run("exactly the cap is accepted in order", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes}, -1)
-		assert.Equal(t, http.StatusAccepted, rec.Code)
-		res := (await(t, f.done)).res
-		assert.Empty(t, res.FunctionError)
-		require.NotNil(t, res.Segments)
-		for _, seg := range res.Segments {
-			assert.LessOrEqual(t, len(seg), segmentBytes)
-		}
-		assert.Equal(t, pattern(MaxResponseBytes), concat(res))
-	})
-	t.Run("one byte over completes as too large", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes + 1}, -1)
-		assert.Equal(t, http.StatusRequestEntityTooLarge, rec.Code)
-		res := (await(t, f.done)).res
-		assert.Equal(t, ErrTypeResponseTooLarge, errType(t, res.Payload))
-	})
-	t.Run("empty body is an empty non-nil payload", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		postTo(s, "response", "req-1", &patterned{}, -1)
-		res := (await(t, f.done)).res
-		require.NotNil(t, res.Segments)
-		assert.Empty(t, concat(res))
-	})
-	t.Run("peak allocation stays within the cap plus one segment", func(t *testing.T) {
-		for _, n := range []int{MaxResponseBytes, MaxResponseBytes + 1} {
-			br := testing.Benchmark(func(b *testing.B) {
-				b.ReportAllocs()
-				for range b.N {
-					s := New()
-					f := arm(s, "req-1")
-					postTo(s, "response", "req-1", &patterned{n: n}, -1)
-					await(t, f.done)
-				}
-			})
-			assert.Less(t, br.AllocedBytesPerOp(), int64(MaxResponseBytes+segmentBytes+64<<10), "body of %d bytes", n)
-		}
-	})
-}
-
-func TestErrorPostOverCapKeepsTruncatedPayload(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	n := MaxResponseBytes + 100
-	rec := postTo(s, "error", "req-1", &patterned{n: n}, int64(n))
-	assert.Equal(t, http.StatusAccepted, rec.Code)
-	res := (await(t, f.done)).res
-	assert.Equal(t, ErrTypeUnhandled, res.FunctionError)
-	assert.Equal(t, pattern(MaxResponseBytes), res.Payload)
-}
-
-func TestBootstrapExitBeforeAdmissionCancels(t *testing.T) {
-	t.Run("before the flight exists", func(t *testing.T) {
-		s := New()
-		assert.True(t, s.BootstrapExited())
-		ctx := t.Context()
-		go func() { _, _ = s.Invoke(ctx, inv("req-1", time.Minute)) }()
-		require.Eventually(t, func() bool {
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			return s.cur != nil && s.cur.state == stExited
-		}, time.Second, time.Millisecond, "the flight is created already exited")
-	})
-	t.Run("before next", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		f.state = stWaiting
-		assert.True(t, s.BootstrapExited())
-		assert.Equal(t, stExited, f.state)
-	})
-	t.Run("after next", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		assert.True(t, s.BootstrapExited())
-		assert.Equal(t, stExited, f.state)
-	})
-}
-
-func TestExitWinsTheRace(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	require.True(t, s.BootstrapExited())
-
-	c := &counting{}
-	rec := postTo(s, "response", "req-1", c, 4)
-	assert.Equal(t, http.StatusBadRequest, rec.Code)
-	assert.Equal(t, "InvalidRequestID", errType(t, rec.Body.Bytes()))
-	assert.Zero(t, c.reads.Load())
-	assert.Empty(t, f.done)
-}
-
-func TestAdmissionWinsTheRace(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	adm := s.admit("req-1")
-	require.NotNil(t, adm)
-	assert.False(t, s.BootstrapExited(), "an admitted response is not cancelled by the exit")
-
-	r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader([]byte("late body")))
-	rec := httptest.NewRecorder()
-	s.finishResponse(rec, r, adm)
-	assert.Equal(t, http.StatusAccepted, rec.Code)
-	assert.Equal(t, "late body", string((await(t, f.done)).res.Payload))
-	assert.False(t, s.BootstrapExited(), "a recorded result stands")
-}
-
-func TestExitedAfterResultIsIgnored(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	postTo(s, "response", "req-1", bytes.NewReader([]byte("ok")), 2)
-	await(t, f.done)
-	assert.False(t, s.BootstrapExited())
-}
-
-// failing yields some bytes and then a read error, as a reset connection does.
-type failing struct{ left int }
-
-func (f *failing) Read(b []byte) (int, error) {
-	if f.left == 0 {
-		return 0, io.ErrClosedPipe
-	}
-	n := min(len(b), f.left)
-	f.left -= n
-	return n, nil
-}
-
-// invokeReady runs Invoke on a real flight that the runtime has taken.
-func invokeReady(t *testing.T, s *Server, ctx context.Context) <-chan invokeOut {
-	t.Helper()
-	out := make(chan invokeOut, 1)
-	go func() {
-		res, err := s.Invoke(ctx, inv("req-1", time.Minute))
-		out <- invokeOut{res, err}
-	}()
-	require.Eventually(t, func() bool {
-		s.mu.Lock()
-		defer s.mu.Unlock()
-		if s.cur != nil && s.cur.state == stWaiting {
-			s.cur.state = stRunning
-		}
-		return s.cur != nil && s.cur.state == stRunning
-	}, time.Second, time.Millisecond)
-	return out
-}
-
-func TestLiveBootstrapTruncatedBodyEndsInvokeAtOnce(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	s := New()
-	out := invokeReady(t, s, context.Background()) // a minute-long deadline
-	adm := s.admit("req-1")
-	require.NotNil(t, adm)
-	s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 3}), adm)
-	select {
-	case got := <-out:
-		require.ErrorIs(t, got.err, ErrResponseBodyFailed)
-		assert.Empty(t, got.res.Payload)
-		assert.Empty(t, got.res.FunctionError)
-	case <-time.After(5 * time.Second):
-		t.Fatal("Invoke waited for its deadline")
-	}
-}
-
-func TestIncompleteBodyNeverStrandsInvoke(t *testing.T) {
-	// A body that fails after admission loses the result. The flight must end
-	// with an explicit outcome whichever of the read failure, the bootstrap
-	// exit and the invocation ending comes first.
-	steps := map[string]func(s *Server, adm *flight, cancel context.CancelFunc){
-		"read fails": func(s *Server, adm *flight, _ context.CancelFunc) {
-			s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 3}), adm)
-		},
-		"bootstrap exits":      func(s *Server, _ *flight, _ context.CancelFunc) { s.BootstrapExited() },
-		"invocation cancelled": func(_ *Server, _ *flight, cancel context.CancelFunc) { cancel() },
-	}
-	orders := [][]string{
-		{"read fails", "bootstrap exits", "invocation cancelled"},
-		{"bootstrap exits", "read fails", "invocation cancelled"},
-		{"invocation cancelled", "read fails", "bootstrap exits"},
-		{"invocation cancelled", "bootstrap exits", "read fails"},
-		{"read fails", "invocation cancelled", "bootstrap exits"},
-		{"bootstrap exits", "invocation cancelled", "read fails"},
-	}
-	for _, order := range orders {
-		t.Run(order[0]+"_"+order[1]+"_"+order[2], func(t *testing.T) {
-			defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-			s := New()
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-			out := invokeReady(t, s, ctx)
-			adm := s.admit("req-1")
-			require.NotNil(t, adm)
-			// Whichever of the read failure and the cancellation comes first
-			// decides the outcome; the exit changes nothing once admitted.
-			want := ErrResponseBodyFailed
-			for _, step := range order {
-				if step == "invocation cancelled" {
-					want = context.Canceled
-				}
-				if step != "bootstrap exits" {
-					break
-				}
-			}
-			var got invokeOut
-			returned := false
-			for _, step := range order {
-				steps[step](s, adm, cancel)
-				if step == "invocation cancelled" {
-					// Settle the cancellation before the next transition.
-					select {
-					case got = <-out:
-						returned = true
-					case <-time.After(5 * time.Second):
-						t.Fatal("Invoke did not return after cancel")
-					}
-				}
-			}
-			if !returned {
-				select {
-				case got = <-out:
-				case <-time.After(5 * time.Second):
-					t.Fatal("Invoke did not return")
-				}
-			}
-			require.ErrorIs(t, got.err, want)
-			assert.Empty(t, got.res.Payload)
-			assert.Empty(t, got.res.FunctionError)
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			assert.Nil(t, s.cur, "the flight must be detached")
-		})
-	}
-}
-
-func TestIncompleteBodyAndDeadlineInBothOrders(t *testing.T) {
-	t.Run("deadline first", func(t *testing.T) {
-		defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-		s := New()
-		out := make(chan invokeOut, 1)
-		go func() {
-			res, err := s.Invoke(context.Background(), inv("req-1", 20*time.Millisecond))
-			out <- invokeOut{res, err}
-		}()
-		require.Eventually(t, func() bool {
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			if s.cur != nil && s.cur.state == stWaiting {
-				s.cur.state = stRunning
-			}
-			return s.cur != nil
-		}, time.Second, time.Millisecond)
-		adm := s.admit("req-1")
-		require.NotNil(t, adm)
-		select {
-		case got := <-out:
-			require.ErrorIs(t, got.err, ErrDeadlineExceeded)
-		case <-time.After(5 * time.Second):
-			t.Fatal("Invoke did not time out")
-		}
-		s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 1}), adm)
-		assert.Empty(t, adm.done, "a late failure publishes nothing")
-	})
-	t.Run("body failure first", func(t *testing.T) {
-		defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-		s := New()
-		out := invokeReady(t, s, context.Background())
-		adm := s.admit("req-1")
-		require.NotNil(t, adm)
-		s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 1}), adm)
-		require.ErrorIs(t, (await(t, out)).err, ErrResponseBodyFailed)
-	})
-}
-
-// The reply a bootstrap sees to an oversize body is recorded here, not
-// claimed to match AWS.
-func TestOversizeBootstrapVisibleReply(t *testing.T) {
-	cases := map[string]struct {
-		length int64
-		size   int
-	}{
-		"declared length": {MaxResponseBytes + 2, MaxResponseBytes + 2},
-		"chunked":         {-1, MaxResponseBytes + 2},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			s, ts := start(t)
-			out := invokeAsync(s, inv("req-1", 10*time.Second))
-			get(t, ts.URL+base+"invocation/next")
-			req, err := http.NewRequest(http.MethodPost, ts.URL+base+"invocation/req-1/response", io.NopCloser(&patterned{n: tc.size}))
-			require.NoError(t, err)
-			req.ContentLength = tc.length
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				t.Logf("bootstrap-visible: transport error: %v", err)
-			} else {
-				defer resp.Body.Close()
-				t.Logf("bootstrap-visible: status %d", resp.StatusCode)
-				assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
-			}
-			got := await(t, out)
-			require.NoError(t, got.err)
-			assert.Equal(t, ErrTypeResponseTooLarge, errType(t, got.res.Payload))
-		})
-	}
-}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -170,7 +170,7 @@ func TestOversizeResponse(t *testing.T) {
 
 	got := <-out
 	require.NoError(t, got.err)
-	assert.Equal(t, "Unhandled", got.res.FunctionError)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
 	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
 }
 

spinifex/agents/rds/agent/parameters.go

@@ -4,13 +4,11 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"math"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
-	"github.com/mulgadc/bluebottle/pkg/safecast"
 	rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
 )
 
@@ -108,29 +106,12 @@ func (s parameterStore) chownToEngine(path string) error {
 	if err != nil {
 		return err
 	}
-	uid, err := ownerID(credential.Uid)
-	if err != nil {
-		return fmt.Errorf("uid of %s: %w", s.osUser, err)
-	}
-	gid, err := ownerID(credential.Gid)
-	if err != nil {
-		return fmt.Errorf("gid of %s: %w", s.osUser, err)
-	}
-	if err := os.Chown(path, uid, gid); err != nil {
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
 		return fmt.Errorf("hand %s to %s: %w", path, s.osUser, err)
 	}
 	return nil
 }
 
-// Rejects rather than saturates: a clamped ID would hand the file to another
-// account while the engine still runs as the real one.
-func ownerID(id uint32) (int, error) {
-	if uint64(id) > math.MaxInt {
-		return 0, fmt.Errorf("%d does not fit an int", id)
-	}
-	return safecast.Uint64ToInt(uint64(id)), nil
-}
-
 // Snapshots the installed set as the rollback target, but only once the engine
 // is running all of it. A set still pending a restart has not been shown to
 // serve, so promoting it would point the rollback at a config about to fail.

spinifex/bootstrap/config/config.go

@@ -111,7 +111,7 @@ type AWSConfig struct {
 // ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
 type ExternalPool struct {
 	Name       string   `mapstructure:"name"`        // Pool identifier (e.g., "wan", "dc1-primary")
-	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale", "vultr" or "latitude"
+	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale" or "vultr"
 	BindBridge string   `mapstructure:"bind_bridge"` // Linux bridge for DHCP DORA (source=dhcp only)
 	DHCPMAC    string   `mapstructure:"dhcp_mac"`    // DHCP client MAC strategy: "derived" (default) or "interface" (source=dhcp only)
 	RangeStart string   `mapstructure:"range_start"` // First IP in range (static source only)
@@ -155,17 +155,6 @@ type ExternalPool struct {
 	VultrRegion  string `mapstructure:"vultr_region"`   // Vultr region ID, e.g. "ord"; never derived from the Spinifex region
 	VultrKeyFile string `mapstructure:"vultr_key_file"` // API key file (optional; see vultr.DefaultKeyFile)
 	LabelPrefix  string `mapstructure:"label_prefix"`   // Marks this deployment's addresses, e.g. "spinifex-prod"
-
-	// Latitude names the project, token file and fixed set of Elastic IPs
-	// (source=latitude only). Each node's server ID is provider_host_id on the node.
-	LatitudeProjectID    string   `mapstructure:"latitude_project_id"`     // latitude.sh project ID the Elastic IPs belong to
-	LatitudeKeyFile      string   `mapstructure:"latitude_key_file"`       // API token file (optional; see latitude.DefaultKeyFile)
-	LatitudeElasticIPIDs []string `mapstructure:"latitude_elastic_ip_ids"` // Elastic IP IDs the pool hands out
-}
-
-// hasLatitudeKeys reports whether any latitude_* key is set.
-func (p ExternalPool) hasLatitudeKeys() bool {
-	return p.LatitudeProjectID != "" || p.LatitudeKeyFile != "" || len(p.LatitudeElasticIPIDs) > 0
 }
 
 // hasVultrKeys reports whether any vultr_* key, or the label prefix only a
@@ -262,7 +251,7 @@ type Config struct {
 
 	// ProviderHostID is this node's instance ID at its cloud provider, which a
 	// floating public address is attached to. Required on every node when a
-	// source="vultr" or source="latitude" pool exists.
+	// source="vultr" pool exists.
 	ProviderHostID string `json:"ProviderHostID,omitempty" mapstructure:"provider_host_id"`
 
 	Daemon      DaemonConfig      `json:"Daemon" mapstructure:"daemon"`
@@ -818,53 +807,6 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
-// MaxLatitudeElasticIPs bounds a latitude pool: latitude.sh allows five Elastic IPs per team, and
-// every allocation reads each configured address from the API.
-const MaxLatitudeElasticIPs = 5
-
-// validateLatitudePool checks a source="latitude" pool. The Elastic IPs are
-// bought ahead of time and named here, the API picks nothing, so the range,
-// gateway and label keys mean nothing, and every node must name its server.
-func validateLatitudePool(cc *ClusterConfig, p ExternalPool) error {
-	if p.LatitudeProjectID == "" || len(p.LatitudeElasticIPIDs) == 0 {
-		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires latitude_project_id and latitude_elastic_ip_ids", p.Name)
-	}
-	if len(p.LatitudeElasticIPIDs) > MaxLatitudeElasticIPs {
-		return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids lists %d addresses, at most %d are allowed", p.Name, len(p.LatitudeElasticIPIDs), MaxLatitudeElasticIPs)
-	}
-	seen := map[string]struct{}{}
-	for _, id := range p.LatitudeElasticIPIDs {
-		if id == "" {
-			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids has an empty entry", p.Name)
-		}
-		if _, dup := seen[id]; dup {
-			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids lists %q twice", p.Name, id)
-		}
-		seen[id] = struct{}{}
-	}
-	if p.LatitudeKeyFile != "" && !filepath.IsAbs(p.LatitudeKeyFile) {
-		return fmt.Errorf("config: [[network.external_pools]] %q: latitude_key_file must be an absolute path, got %q", p.Name, p.LatitudeKeyFile)
-	}
-	if p.BindBridge != "" || p.DHCPMAC != "" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)
-	}
-	if p.RangeStart != "" || p.RangeEnd != "" || p.Gateway != "" || p.GatewayIP != "" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: range_start/range_end/gateway/gateway_ip not allowed with source=\"latitude\" (the Elastic IPs are listed by ID)", p.Name)
-	}
-	if p.GwLrpRangeStart != "" || p.GwLrpRangeEnd != "" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: gw_lrp_range_start/gw_lrp_range_end not allowed with source=\"latitude\" (a gateway address per VPC would use up the Elastic IP limit)", p.Name)
-	}
-	if cc.Network.ExternalMode != "nat" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires [network] external_mode = \"nat\" (routed); a routed Elastic IP is delivered to the server's own MAC", p.Name)
-	}
-	for name, n := range cc.Nodes {
-		if n.ProviderHostID == "" {
-			return fmt.Errorf("config: [nodes.%s] provider_host_id is required with a source=\"latitude\" pool (%q): it is the latitude.sh server ID the node's Elastic IPs move to", name, p.Name)
-		}
-	}
-	return nil
-}
-
 // validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
 // providers differ. A peer stanza without one is unknown, not the default: the
 // generated config omits it, and the cluster's KV lock is the authoritative check.
@@ -937,9 +879,6 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		end   netip.Addr
 	}
 	var ranges []poolRange
-	// latitudeIDs maps each Elastic IP ID to the pool naming it. Every pool keeps its own
-	// record of what it handed out, so an ID in two pools would be given to two guests.
-	latitudeIDs := map[string]string{}
 	for _, p := range cc.Network.ExternalPools {
 		if p.Source != "oci" && (p.OCICompartmentID != "" || p.OCIVNICID != "" || p.OCIVNICIface != "" || p.OCISubnetID != "" || p.OCIPublicIPPool != "" || p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
 			return fmt.Errorf("config: [[network.external_pools]] %q: oci_* keys are only valid with source=\"oci\"", p.Name)
@@ -947,9 +886,6 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if p.Source != "exoscale" && p.hasExoscaleKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: exoscale_* keys are only valid with source=\"exoscale\"", p.Name)
 		}
-		if p.Source != "latitude" && p.hasLatitudeKeys() {
-			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_* keys are only valid with source=\"latitude\"", p.Name)
-		}
 		if p.Source != "vultr" && p.hasVultrKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: vultr_* and label_prefix keys are only valid with source=\"vultr\"", p.Name)
 		}
@@ -1049,19 +985,8 @@ func validateClusterConfig(cc *ClusterConfig) error {
 				return err
 			}
 			continue
-		case "latitude":
-			if err := validateLatitudePool(cc, p); err != nil {
-				return err
-			}
-			for _, id := range p.LatitudeElasticIPIDs {
-				if other, dup := latitudeIDs[id]; dup {
-					return fmt.Errorf("config: [[network.external_pools]] %q: Elastic IP %q is also named by pool %q; each Elastic IP may belong to one pool", p.Name, id, other)
-				}
-				latitudeIDs[id] = p.Name
-			}
-			continue
 		default:
-			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\", \"vultr\" or \"latitude\"", p.Name, p.Source)
+			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\" or \"vultr\"", p.Name, p.Source)
 		}
 		if p.RangeStart == "" || p.RangeEnd == "" {
 			continue

spinifex/bootstrap/config/config_test.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/spf13/viper"
@@ -1551,83 +1550,6 @@ func TestLoadConfig_NetworkPoolVultrRejections(t *testing.T) {
 	}
 }
 
-const latitudePoolBase = `
-source = "latitude"
-latitude_project_id = "proj_1"
-latitude_elastic_ip_ids = ["eip_1", "eip_2"]
-`
-
-func TestLoadConfig_NetworkPoolLatitudeSourceAccepted(t *testing.T) {
-	cfg, err := loadVultrPool(t, "nat", latitudePoolBase+"latitude_key_file = \"/etc/spinifex/latitude/api-token\"\n",
-		[2]string{"sv_a", "sv_b"})
-	require.NoError(t, err)
-	p := cfg.Network.ExternalPools[0]
-	assert.Equal(t, "latitude", p.Source)
-	assert.Equal(t, "proj_1", p.LatitudeProjectID)
-	assert.Equal(t, []string{"eip_1", "eip_2"}, p.LatitudeElasticIPIDs)
-	assert.Equal(t, "/etc/spinifex/latitude/api-token", p.LatitudeKeyFile)
-	assert.Equal(t, "sv_a", cfg.Nodes["n1"].ProviderHostID)
-}
-
-func latitudeIDs(n int) string {
-	ids := make([]string, n)
-	for i := range ids {
-		ids[i] = fmt.Sprintf("%q", fmt.Sprintf("eip_%d", i))
-	}
-	return "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [" + strings.Join(ids, ", ") + "]\n"
-}
-
-func TestLoadConfig_NetworkPoolLatitudeSetIsCapped(t *testing.T) {
-	cfg, err := loadVultrPool(t, "nat", latitudeIDs(MaxLatitudeElasticIPs), [2]string{"a", "b"})
-	require.NoError(t, err)
-	assert.Len(t, cfg.Network.ExternalPools[0].LatitudeElasticIPIDs, MaxLatitudeElasticIPs)
-
-	_, err = loadVultrPool(t, "nat", latitudeIDs(MaxLatitudeElasticIPs+1), [2]string{"a", "b"})
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "at most 5 are allowed")
-}
-
-func TestLoadConfig_NetworkPoolLatitudeRejections(t *testing.T) {
-	both := [2]string{"a", "b"}
-	cases := []struct {
-		name, mode, body string
-		nodes            [2]string
-		want             string
-	}{
-		{"pool mode", "pool", latitudePoolBase, both, `requires [network] external_mode = "nat"`},
-		{"no project", "nat", "source = \"latitude\"\nlatitude_elastic_ip_ids = [\"e\"]\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
-		{"no ids", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
-		{"duplicate id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e\", \"e\"]\n", both, "twice"},
-		{"empty id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"\"]\n", both, "empty entry"},
-		{"relative key file", "nat", latitudePoolBase + "latitude_key_file = \"token\"\n", both, "absolute path"},
-		{"label prefix", "nat", latitudePoolBase + "label_prefix = \"x\"\n", both, "only valid with source=\"vultr\""},
-		{"range", "nat", latitudePoolBase + "range_start = \"1.2.3.4\"\nrange_end = \"1.2.3.4\"\n", both, "listed by ID"},
-		{"a node without its server id", "nat", latitudePoolBase, [2]string{"a", ""}, "[nodes.n2] provider_host_id is required"},
-		{"latitude keys on static", "nat", "latitude_project_id = \"p\"\n", both, `only valid with source="latitude"`},
-		{"vultr keys on latitude", "nat", latitudePoolBase + "vultr_region = \"ord\"\n", both, `only valid with source="vultr"`},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := loadVultrPool(t, tc.mode, tc.body, tc.nodes)
-			require.Error(t, err)
-			assert.Contains(t, err.Error(), tc.want)
-		})
-	}
-}
-
-func TestLoadConfig_NetworkPoolLatitudeIDInTwoPoolsIsRejected(t *testing.T) {
-	resetViper(t)
-	body := "node = \"n1\"\n\n[network]\nexternal_mode = \"nat\"\n\n" +
-		"[[network.external_pools]]\nname = \"a\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e1\", \"e2\"]\n\n" +
-		"[[network.external_pools]]\nname = \"b\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e3\", \"e2\"]\n\n" +
-		"[nodes.n1]\nregion = \"us-east-1\"\nprovider_host_id = \"sv_1\"\n"
-	path := filepath.Join(t.TempDir(), "spinifex.toml")
-	require.NoError(t, os.WriteFile(path, []byte(body), 0600))
-	_, err := LoadConfig(path)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), `"e2" is also named by pool "a"`)
-}
-
 func TestLoadConfig_NorthstarNameservers(t *testing.T) {
 	load := func(t *testing.T, block string) (*ClusterConfig, error) {
 		t.Helper()

spinifex/domains/ec2/accountcleanup/compute.go

@@ -1,349 +0,0 @@
-package accountcleanup
-
-import (
-	"context"
-	"log/slog"
-	"slices"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-)
-
-// Compute kinds and the extra dispositions and reasons the compute run reports.
-const (
-	KindInstance    = "instance"
-	KindSpotRequest = "spot-request"
-
-	// DispositionCancelled: a spot request this run moved to cancelled.
-	DispositionCancelled = "cancelled"
-	// DispositionTerminal: a spot request that was already closed, cancelled or failed.
-	DispositionTerminal = "terminal"
-	// DispositionPending: a terminate was acknowledged but the teardown read cannot
-	// confirm completion, because terminated records expire.
-	DispositionPending = "pending"
-
-	// ReasonTerminating: a terminate is in progress or its teardown is unfinished.
-	ReasonTerminating = "terminating"
-	// ReasonTerminationUnproven: the terminate was not acknowledged, so the
-	// instance may still be running on a node that did not answer.
-	ReasonTerminationUnproven = "termination-unproven"
-	// ReasonManagedServiceOwner: a managed service still owns the instance.
-	ReasonManagedServiceOwner = "managed-service-owner"
-	// ReasonSpotCancelFailed: a spot request that can still launch could not be cancelled.
-	ReasonSpotCancelFailed = "spot-cancel-failed"
-	// ReasonSpotStateUnrecognised: a spot request in a state outside the EC2 model.
-	ReasonSpotStateUnrecognised = "spot-state-unrecognised"
-	// ReasonTeardownUnconfirmed: no unfinished teardown was observed, which is
-	// not proof the teardown finished.
-	ReasonTeardownUnconfirmed = "teardown-unconfirmed"
-
-	// Incomplete reasons: the run stopped or could not prove a part of its result.
-	ReasonSpotInventoryIncomplete     = "spot-inventory-incomplete"
-	ReasonInstanceInventoryIncomplete = "instance-inventory-incomplete"
-	ReasonManagedOwnershipUnknown     = "managed-ownership-unknown"
-	ReasonTeardownUnread              = "teardown-unread"
-)
-
-// Compute run statuses.
-const (
-	StatusClean      = "clean"
-	StatusIncomplete = "incomplete"
-)
-
-// ComputeRequest names the account whose compute is cleaned. Force is accepted
-// for callers that carry one and bypasses nothing: an unproven terminate, an
-// incomplete inventory or a managed owner is never overridden.
-type ComputeRequest struct {
-	AccountID      string `json:"account_id"`
-	CorrelationRef string `json:"correlation_ref,omitempty"`
-	Force          bool   `json:"force,omitempty"`
-}
-
-// Outcome is what one compute or storage run found and did. Clean means nothing
-// was blocked or pending and every read completed; it is a snapshot, not
-// evidence of absence.
-type Outcome struct {
-	AccountID         string   `json:"account_id"`
-	CorrelationRef    string   `json:"correlation_ref,omitempty"`
-	Status            string   `json:"status"`
-	IncompleteReasons []string `json:"incomplete_reasons,omitempty"`
-	Items             []Item   `json:"items,omitempty"`
-}
-
-// ComputeResult and StorageResult are the outcomes of the two runs.
-type (
-	ComputeResult = Outcome
-	StorageResult = Outcome
-)
-
-// ComputeCleaner terminates an account's instances and cancels its spot
-// requests. It is destructive; the caller must hold an accepted account deletion.
-type ComputeCleaner interface {
-	CleanupCompute(ctx context.Context, req *ComputeRequest) (*ComputeResult, error)
-}
-
-// SpotRequestOwner is the spot-request owner's account scan and cancel.
-type SpotRequestOwner interface {
-	ScanAccount(ctx context.Context, accountID string) (*ec2spotinstance.AccountScan, error)
-	CancelSpotInstanceRequests(ctx context.Context, input *ec2.CancelSpotInstanceRequestsInput, accountID string) (*ec2.CancelSpotInstanceRequestsOutput, error)
-}
-
-// InstanceInventory lists every non-terminated instance the account owns, or
-// fails. It is DescribeAccountInstancesStrict; a partial list is never returned.
-type InstanceInventory interface {
-	AccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error)
-}
-
-// InstanceTerminator sends the existing EC2 terminate for one instance and
-// returns once the owning node acknowledged it.
-type InstanceTerminator interface {
-	TerminateInstance(ctx context.Context, accountID, instanceID string) error
-}
-
-// TeardownReader reports instances whose terminate is not fully torn down.
-type TeardownReader interface {
-	AccountTeardownOutstanding(accountID string) ([]ec2instance.OutstandingInstance, error)
-}
-
-// ManagedInstanceReader reads the complete set of instance IDs a managed
-// service, such as an EKS nodegroup, still owns in the account, or an error.
-// The owner of that record answers it; EC2 never reads those records.
-type ManagedInstanceReader interface {
-	AccountManagedInstances(ctx context.Context, accountID string) (map[string]bool, error)
-}
-
-var (
-	_ SpotRequestOwner = (*ec2spotinstance.SpotInstanceServiceImpl)(nil)
-	_ TeardownReader   = (*ec2instance.InstanceServiceImpl)(nil)
-	_ ComputeCleaner   = (*ComputeCleanup)(nil)
-)
-
-// ComputeCleanup composes the compute owners.
-type ComputeCleanup struct {
-	spot      SpotRequestOwner
-	instances InstanceInventory
-	terminate InstanceTerminator
-	teardown  TeardownReader
-	managed   ManagedInstanceReader
-}
-
-// NewCompute builds a compute cleanup. A nil dependency makes every run fail
-// before it acts: a missing managed-instance reader is a wiring error, not "none owned".
-func NewCompute(spot SpotRequestOwner, instances InstanceInventory, terminate InstanceTerminator, teardown TeardownReader, managed ManagedInstanceReader) *ComputeCleanup {
-	return &ComputeCleanup{spot: spot, instances: instances, terminate: terminate, teardown: teardown, managed: managed}
-}
-
-// CleanupCompute cancels the account's open and active spot requests, then
-// terminates every instance that is not already terminating and no managed
-// service owns. It never waits for teardown: outstanding work is reported and a
-// rerun converges it. A spot, inventory or ownership read that does not
-// complete terminates nothing.
-func (c *ComputeCleanup) CleanupCompute(ctx context.Context, req *ComputeRequest) (*ComputeResult, error) {
-	if req == nil || req.AccountID == "" {
-		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
-	}
-	if !awsidentifiers.IsAccountID(req.AccountID) {
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The account ID must be 12 digits.")
-	}
-	if c == nil || c.spot == nil || c.instances == nil || c.terminate == nil || c.teardown == nil || c.managed == nil {
-		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Compute cleanup is not fully wired.")
-	}
-	res := &ComputeResult{AccountID: req.AccountID, CorrelationRef: req.CorrelationRef}
-	defer res.finish()
-
-	if !c.cancelSpot(ctx, req.AccountID, res) {
-		return res, nil
-	}
-
-	inventory, err := c.instances.AccountInstances(ctx, req.AccountID)
-	if err != nil {
-		slog.WarnContext(ctx, "CleanupCompute: instance inventory incomplete, nothing terminated", "account_id", req.AccountID, "err", err)
-		res.incomplete(ReasonInstanceInventoryIncomplete)
-		return res, nil
-	}
-	managed, err := c.managed.AccountManagedInstances(ctx, req.AccountID)
-	if err != nil {
-		slog.WarnContext(ctx, "CleanupCompute: managed ownership unknown, nothing terminated", "account_id", req.AccountID, "err", err)
-		res.incomplete(ReasonManagedOwnershipUnknown)
-		return res, nil
-	}
-
-	if !inventoryUsable(inventory) {
-		slog.WarnContext(ctx, "CleanupCompute: inventory holds an entry with no identity, nothing terminated", "account_id", req.AccountID)
-		res.incomplete(ReasonInstanceInventoryIncomplete)
-		return res, nil
-	}
-
-	asked := c.terminateAll(ctx, req.AccountID, inventory, managed, res)
-	c.reportTeardown(req.AccountID, asked, res)
-	return res, nil
-}
-
-// cancelSpot cancels open and active requests and rescans. It reports false
-// when the spot state is not proven, which stops the run before any terminate.
-func (c *ComputeCleanup) cancelSpot(ctx context.Context, accountID string, res *ComputeResult) bool {
-	scan, err := c.spot.ScanAccount(ctx, accountID)
-	if err != nil {
-		slog.WarnContext(ctx, "CleanupCompute: spot scan failed, nothing terminated", "account_id", accountID, "err", err)
-		res.incomplete(ReasonSpotInventoryIncomplete)
-		return false
-	}
-	var open []*string
-	cancelled := map[string]bool{}
-	for _, r := range scan.Active {
-		if !terminalSpotState(r.State) {
-			open = append(open, aws.String(r.RequestID))
-			cancelled[r.RequestID] = true
-		}
-	}
-	if len(open) > 0 {
-		if _, err := c.spot.CancelSpotInstanceRequests(ctx, &ec2.CancelSpotInstanceRequestsInput{SpotInstanceRequestIds: open}, accountID); err != nil {
-			slog.WarnContext(ctx, "CleanupCompute: spot cancel failed, nothing terminated", "account_id", accountID, "err", err)
-			for _, id := range open {
-				res.Items = append(res.Items, Item{ResourceID: aws.StringValue(id), Kind: KindSpotRequest, Disposition: DispositionBlocked, Reason: ReasonSpotCancelFailed})
-			}
-			res.incomplete(ReasonSpotInventoryIncomplete)
-			return false
-		}
-		scan, err = c.spot.ScanAccount(ctx, accountID)
-		if err != nil {
-			slog.WarnContext(ctx, "CleanupCompute: spot rescan failed, nothing terminated", "account_id", accountID, "err", err)
-			res.incomplete(ReasonSpotInventoryIncomplete)
-			return false
-		}
-	}
-	proven := true
-	for _, r := range scan.Active {
-		if !terminalSpotState(r.State) {
-			proven = false
-			res.Items = append(res.Items, Item{ResourceID: r.RequestID, Kind: KindSpotRequest, State: r.State, Disposition: DispositionBlocked, Reason: ReasonSpotCancelFailed})
-		}
-	}
-	for _, r := range scan.Terminal {
-		if !terminalSpotState(r.State) {
-			proven = false
-			res.Items = append(res.Items, Item{ResourceID: r.RequestID, Kind: KindSpotRequest, State: r.State, Disposition: DispositionBlocked, Reason: ReasonSpotStateUnrecognised})
-			continue
-		}
-		disposition := DispositionTerminal
-		if cancelled[r.RequestID] {
-			disposition = DispositionCancelled
-		}
-		res.Items = append(res.Items, Item{ResourceID: r.RequestID, Kind: KindSpotRequest, State: r.State, Disposition: disposition})
-	}
-	if !proven {
-		res.incomplete(ReasonSpotInventoryIncomplete)
-	}
-	return proven
-}
-
-// terminalSpotState: closed, cancelled and failed requests launch nothing more.
-// Every other state, disabled and unknown ones included, must be cancelled.
-func terminalSpotState(state string) bool {
-	return state == ec2.SpotInstanceStateClosed || state == ec2.SpotInstanceStateCancelled || state == ec2.SpotInstanceStateFailed
-}
-
-// inventoryUsable reports whether every entry names an instance. An entry that
-// does not could be a live instance, so the run terminates nothing.
-func inventoryUsable(out *ec2.DescribeInstancesOutput) bool {
-	if out == nil {
-		return false
-	}
-	for _, reservation := range out.Reservations {
-		if reservation == nil {
-			return false
-		}
-		for _, in := range reservation.Instances {
-			if in == nil || aws.StringValue(in.InstanceId) == "" {
-				return false
-			}
-		}
-	}
-	return true
-}
-
-// terminateAll terminates every listed instance that is not terminating and
-// not managed, and returns the IDs whose terminate is in hand.
-func (c *ComputeCleanup) terminateAll(ctx context.Context, accountID string, inventory *ec2.DescribeInstancesOutput, managed map[string]bool, res *ComputeResult) map[string]bool {
-	asked := map[string]bool{}
-	for _, reservation := range inventory.Reservations {
-		for _, in := range reservation.Instances {
-			id := aws.StringValue(in.InstanceId)
-			state := ""
-			if in.State != nil {
-				state = aws.StringValue(in.State.Name)
-			}
-			switch {
-			case managed[id]:
-				res.Items = append(res.Items, Item{ResourceID: id, Kind: KindInstance, State: state, Disposition: DispositionBlocked, Reason: ReasonManagedServiceOwner})
-			case state == ec2.InstanceStateNameTerminated, state == ec2.InstanceStateNameShuttingDown:
-				// Already terminating: the teardown read below reports it.
-				asked[id] = true
-			default:
-				if err := c.terminate.TerminateInstance(ctx, accountID, id); err != nil {
-					slog.WarnContext(ctx, "CleanupCompute: terminate not acknowledged", "account_id", accountID, "instance_id", id, "err", err)
-					res.Items = append(res.Items, Item{ResourceID: id, Kind: KindInstance, State: state, Disposition: DispositionBlocked, Reason: ReasonTerminationUnproven})
-					continue
-				}
-				asked[id] = true
-			}
-		}
-	}
-	return asked
-}
-
-// reportTeardown reads the unfinished teardowns once, without waiting, and
-// reports each instance still in progress as blocked.
-func (c *ComputeCleanup) reportTeardown(accountID string, asked map[string]bool, res *ComputeResult) {
-	outstanding, err := c.teardown.AccountTeardownOutstanding(accountID)
-	if err != nil {
-		res.incomplete(ReasonTeardownUnread)
-		for id := range asked {
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindInstance, Disposition: DispositionBlocked, Reason: ReasonTerminationUnproven})
-		}
-		return
-	}
-	open := map[string]bool{}
-	for _, o := range outstanding {
-		open[o.InstanceID] = true
-		if !res.hasInstance(o.InstanceID) {
-			res.Items = append(res.Items, Item{ResourceID: o.InstanceID, Kind: KindInstance, State: o.State, Disposition: DispositionBlocked, Reason: ReasonTerminating})
-		}
-	}
-	for id := range asked {
-		if !open[id] {
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindInstance, Disposition: DispositionPending, Reason: ReasonTeardownUnconfirmed})
-		}
-	}
-}
-
-func (r *ComputeResult) hasInstance(id string) bool {
-	return slices.ContainsFunc(r.Items, func(it Item) bool { return it.Kind == KindInstance && it.ResourceID == id })
-}
-
-func (r *ComputeResult) incomplete(reason string) {
-	if !slices.Contains(r.IncompleteReasons, reason) {
-		r.IncompleteReasons = append(r.IncompleteReasons, reason)
-	}
-}
-
-func (r *ComputeResult) finish() {
-	r.Status = StatusClean
-	if len(r.IncompleteReasons) > 0 {
-		r.Status = StatusIncomplete
-	}
-	for _, it := range r.Items {
-		switch it.Disposition {
-		case DispositionBlocked, DispositionPending, DispositionFailed, DispositionResidue:
-			r.Status = StatusIncomplete
-		}
-	}
-	slices.SortFunc(r.Items, func(a, b Item) int {
-		return strings.Compare(a.Kind+a.ResourceID, b.Kind+b.ResourceID)
-	})
-}

spinifex/domains/ec2/accountcleanup/compute_test.go

@@ -1,396 +0,0 @@
-package accountcleanup
-
-//test:in-package — the compute owners are fakes so each test controls the
-// read or terminate that fails and can assert no terminate happened.
-
-import (
-	"context"
-	"errors"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	runningID  = "i-0aaaaaaaaaaaaaaa1"
-	stoppedID  = "i-0aaaaaaaaaaaaaaa2"
-	managedID  = "i-0aaaaaaaaaaaaaaa3"
-	stoppingID = "i-0aaaaaaaaaaaaaaa4"
-)
-
-type fakeSpot struct {
-	scans      []*ec2spotinstance.AccountScan
-	scanErr    error
-	cancelErr  error
-	scanCalls  int
-	cancelled  []string
-	cancelCall int
-}
-
-func (f *fakeSpot) ScanAccount(context.Context, string) (*ec2spotinstance.AccountScan, error) {
-	f.scanCalls++
-	if f.scanErr != nil {
-		return nil, f.scanErr
-	}
-	i := min(f.scanCalls-1, len(f.scans)-1)
-	return f.scans[i], nil
-}
-
-func (f *fakeSpot) CancelSpotInstanceRequests(_ context.Context, in *ec2.CancelSpotInstanceRequestsInput, _ string) (*ec2.CancelSpotInstanceRequestsOutput, error) {
-	f.cancelCall++
-	if f.cancelErr != nil {
-		return nil, f.cancelErr
-	}
-	for _, id := range in.SpotInstanceRequestIds {
-		f.cancelled = append(f.cancelled, aws.StringValue(id))
-	}
-	return &ec2.CancelSpotInstanceRequestsOutput{}, nil
-}
-
-type fakeInventory struct {
-	out *ec2.DescribeInstancesOutput
-	err error
-}
-
-func (f *fakeInventory) AccountInstances(context.Context, string) (*ec2.DescribeInstancesOutput, error) {
-	return f.out, f.err
-}
-
-type fakeTerminator struct {
-	failFor    map[string]error
-	terminated []string
-}
-
-func (f *fakeTerminator) TerminateInstance(_ context.Context, _, id string) error {
-	if err := f.failFor[id]; err != nil {
-		return err
-	}
-	f.terminated = append(f.terminated, id)
-	return nil
-}
-
-type fakeTeardown struct {
-	outstanding []ec2instance.OutstandingInstance
-	err         error
-	reads       int
-}
-
-func (f *fakeTeardown) AccountTeardownOutstanding(string) ([]ec2instance.OutstandingInstance, error) {
-	f.reads++
-	return f.outstanding, f.err
-}
-
-type fakeManaged struct {
-	ids map[string]bool
-	err error
-}
-
-func (f *fakeManaged) AccountManagedInstances(context.Context, string) (map[string]bool, error) {
-	return f.ids, f.err
-}
-
-func listing(states map[string]string) *ec2.DescribeInstancesOutput {
-	res := &ec2.Reservation{}
-	for id, state := range states {
-		res.Instances = append(res.Instances, &ec2.Instance{InstanceId: aws.String(id), State: &ec2.InstanceState{Name: aws.String(state)}})
-	}
-	return &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{res}}
-}
-
-type computeRig struct {
-	spot       *fakeSpot
-	inventory  *fakeInventory
-	terminator *fakeTerminator
-	teardown   *fakeTeardown
-	managed    *fakeManaged
-}
-
-func newComputeRig() *computeRig {
-	return &computeRig{
-		spot:       &fakeSpot{scans: []*ec2spotinstance.AccountScan{{}}},
-		inventory:  &fakeInventory{out: listing(map[string]string{runningID: ec2.InstanceStateNameRunning, stoppedID: ec2.InstanceStateNameStopped})},
-		terminator: &fakeTerminator{},
-		teardown:   &fakeTeardown{},
-		managed:    &fakeManaged{ids: map[string]bool{}},
-	}
-}
-
-func (r *computeRig) cleanup() *ComputeCleanup {
-	return NewCompute(r.spot, r.inventory, r.terminator, r.teardown, r.managed)
-}
-
-func (r *computeRig) run(t *testing.T, force bool) *ComputeResult {
-	t.Helper()
-	res, err := r.cleanup().CleanupCompute(t.Context(), &ComputeRequest{AccountID: testAccount, Force: force})
-	require.NoError(t, err)
-	return res
-}
-
-func itemFor(res *ComputeResult, id string) *Item {
-	for i := range res.Items {
-		if res.Items[i].ResourceID == id {
-			return &res.Items[i]
-		}
-	}
-	return nil
-}
-
-func TestCleanupCompute_TerminatesEveryLiveInstance(t *testing.T) {
-	rig := newComputeRig()
-	res := rig.run(t, false)
-
-	assert.ElementsMatch(t, []string{runningID, stoppedID}, rig.terminator.terminated)
-	assert.Equal(t, StatusIncomplete, res.Status, "an absent teardown record is not proof of completion")
-	assert.Equal(t, DispositionPending, itemFor(res, runningID).Disposition)
-	assert.Equal(t, ReasonTeardownUnconfirmed, itemFor(res, runningID).Reason)
-}
-
-func TestCleanupCompute_SkipsInstanceAlreadyTerminating(t *testing.T) {
-	rig := newComputeRig()
-	rig.inventory.out = listing(map[string]string{stoppingID: ec2.InstanceStateNameShuttingDown})
-	rig.teardown.outstanding = []ec2instance.OutstandingInstance{{InstanceID: stoppingID, State: "shutting-down"}}
-
-	res := rig.run(t, false)
-
-	assert.Empty(t, rig.terminator.terminated, "an instance already terminating is not terminated again")
-	item := itemFor(res, stoppingID)
-	require.NotNil(t, item)
-	assert.Equal(t, DispositionBlocked, item.Disposition)
-	assert.Equal(t, ReasonTerminating, item.Reason)
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-func TestCleanupCompute_UnacknowledgedTerminateIsUnproven(t *testing.T) {
-	rig := newComputeRig()
-	rig.terminator.failFor = map[string]error{runningID: errors.New("no responders")}
-
-	res := rig.run(t, false)
-
-	item := itemFor(res, runningID)
-	require.NotNil(t, item)
-	assert.Equal(t, DispositionBlocked, item.Disposition)
-	assert.Equal(t, ReasonTerminationUnproven, item.Reason)
-	assert.Equal(t, DispositionPending, itemFor(res, stoppedID).Disposition, "one unproven instance does not stop the others")
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-func TestCleanupCompute_TeardownOutstandingIsBlockedTerminating(t *testing.T) {
-	rig := newComputeRig()
-	rig.teardown.outstanding = []ec2instance.OutstandingInstance{
-		{InstanceID: runningID, State: "shutting-down", Teardown: map[string]string{"qemu": "pending"}},
-	}
-
-	res := rig.run(t, false)
-
-	item := itemFor(res, runningID)
-	assert.Equal(t, DispositionBlocked, item.Disposition)
-	assert.Equal(t, ReasonTerminating, item.Reason)
-	assert.Equal(t, DispositionPending, itemFor(res, stoppedID).Disposition)
-}
-
-func TestCleanupCompute_UnreadableTeardownLeavesTerminatesUnproven(t *testing.T) {
-	rig := newComputeRig()
-	rig.teardown.err = errors.New("store down")
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, StatusIncomplete, res.Status)
-	assert.Contains(t, res.IncompleteReasons, ReasonTeardownUnread)
-	for _, id := range []string{runningID, stoppedID} {
-		assert.Equal(t, ReasonTerminationUnproven, itemFor(res, id).Reason)
-	}
-}
-
-func TestCleanupCompute_CancelsOpenAndActiveSpotRequests(t *testing.T) {
-	rig := newComputeRig()
-	rig.spot.scans = []*ec2spotinstance.AccountScan{
-		{Active: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-open", State: ec2.SpotInstanceStateOpen}, {RequestID: "sir-active", State: ec2.SpotInstanceStateActive}}},
-		{Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-open", State: ec2.SpotInstanceStateCancelled}, {RequestID: "sir-active", State: ec2.SpotInstanceStateCancelled}, {RequestID: "sir-old", State: ec2.SpotInstanceStateClosed}}},
-	}
-
-	res := rig.run(t, false)
-
-	assert.ElementsMatch(t, []string{"sir-open", "sir-active"}, rig.spot.cancelled)
-	assert.Equal(t, DispositionCancelled, itemFor(res, "sir-open").Disposition)
-	assert.Equal(t, DispositionTerminal, itemFor(res, "sir-old").Disposition)
-	assert.NotContains(t, res.IncompleteReasons, ReasonSpotInventoryIncomplete)
-}
-
-// A failed request is terminal, like closed and cancelled: it is neither
-// cancelled again nor does it keep the run incomplete.
-func TestCleanupCompute_FailedSpotRequestIsTerminal(t *testing.T) {
-	rig := newComputeRig()
-	rig.spot.scans = []*ec2spotinstance.AccountScan{{
-		Active:   []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-failed-active", State: ec2.SpotInstanceStateFailed}},
-		Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-failed", State: ec2.SpotInstanceStateFailed}},
-	}}
-
-	res := rig.run(t, false)
-
-	assert.Zero(t, rig.spot.cancelCall, "a failed request is not cancelled")
-	assert.NotContains(t, res.IncompleteReasons, ReasonSpotInventoryIncomplete)
-	assert.Equal(t, DispositionTerminal, itemFor(res, "sir-failed").Disposition)
-	assert.ElementsMatch(t, []string{runningID, stoppedID}, rig.terminator.terminated, "a failed spot request does not hold up the terminate")
-}
-
-func TestCleanupCompute_SpotStillOpenAfterCancelIsBlocked(t *testing.T) {
-	rig := newComputeRig()
-	open := &ec2spotinstance.AccountScan{Active: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-open", State: ec2.SpotInstanceStateOpen}}}
-	rig.spot.scans = []*ec2spotinstance.AccountScan{open, open}
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, ReasonSpotCancelFailed, itemFor(res, "sir-open").Reason)
-	assert.Empty(t, rig.terminator.terminated, "an instance could be relaunched while a request is open")
-}
-
-// ADR-0003:S7. A read the run cannot complete terminates nothing.
-func TestS7_CleanupComputeUnprovenReadTerminatesNothing(t *testing.T) {
-	cases := map[string]func(*computeRig){
-		"instance inventory error": func(r *computeRig) { r.inventory.err = errors.New("incomplete listing") },
-		"spot scan error":          func(r *computeRig) { r.spot.scanErr = errors.New("scan failed") },
-		"spot cancel error": func(r *computeRig) {
-			r.spot.scans = []*ec2spotinstance.AccountScan{{Active: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-open", State: ec2.SpotInstanceStateOpen}}}}
-			r.spot.cancelErr = errors.New("cancel failed")
-		},
-		"managed reader error": func(r *computeRig) { r.managed.err = errors.New("nodegroup store down") },
-	}
-	for name, arrange := range cases {
-		for _, force := range []bool{false, true} {
-			t.Run(name, func(t *testing.T) {
-				rig := newComputeRig()
-				arrange(rig)
-
-				res := rig.run(t, force)
-
-				assert.Empty(t, rig.terminator.terminated, "ADR-0003:S7: an unproven read must terminate nothing (force=%v)", force)
-				assert.Equal(t, StatusIncomplete, res.Status)
-				assert.NotEmpty(t, res.IncompleteReasons)
-			})
-		}
-	}
-}
-
-// ADR-0003:S7. An instance a managed service owns is never terminated.
-func TestS7_CleanupComputeManagedInstanceIsNeverTerminated(t *testing.T) {
-	for _, force := range []bool{false, true} {
-		rig := newComputeRig()
-		rig.inventory.out = listing(map[string]string{managedID: ec2.InstanceStateNameRunning, runningID: ec2.InstanceStateNameRunning})
-		rig.managed.ids = map[string]bool{managedID: true}
-
-		res := rig.run(t, force)
-
-		assert.Equal(t, []string{runningID}, rig.terminator.terminated, "ADR-0003:S7: a managed-owned instance must not be terminated (force=%v)", force)
-		item := itemFor(res, managedID)
-		assert.Equal(t, DispositionBlocked, item.Disposition)
-		assert.Equal(t, ReasonManagedServiceOwner, item.Reason)
-		assert.Equal(t, StatusIncomplete, res.Status)
-	}
-}
-
-// ADR-0003:S7. A missing managed-instance reader is a wiring error and not
-// the same as "none owned".
-func TestS7_CleanupComputeNilManagedReaderTerminatesNothing(t *testing.T) {
-	rig := newComputeRig()
-
-	_, err := NewCompute(rig.spot, rig.inventory, rig.terminator, rig.teardown, nil).
-		CleanupCompute(t.Context(), &ComputeRequest{AccountID: testAccount})
-
-	require.Error(t, err)
-	assert.Empty(t, rig.terminator.terminated, "ADR-0003:S7: no terminate without a managed-ownership answer")
-	assert.Zero(t, rig.spot.cancelCall+rig.spot.scanCalls, "nothing is touched before the wiring is proven")
-}
-
-// ADR-0003:S8. The call returns with outcomes while teardown is outstanding;
-// it neither polls the teardown nor blocks on it.
-func TestS8_CleanupComputeReturnsWhileTeardownOutstanding(t *testing.T) {
-	rig := newComputeRig()
-	rig.teardown.outstanding = []ec2instance.OutstandingInstance{
-		{InstanceID: runningID, State: "shutting-down"}, {InstanceID: stoppedID, State: "shutting-down"},
-	}
-	ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
-	defer cancel()
-
-	start := time.Now()
-	res, err := rig.cleanup().CleanupCompute(ctx, &ComputeRequest{AccountID: testAccount})
-
-	require.NoError(t, err)
-	assert.Less(t, time.Since(start), time.Second)
-	assert.Equal(t, StatusIncomplete, res.Status)
-	assert.Equal(t, ReasonTerminating, itemFor(res, runningID).Reason)
-	assert.Equal(t, 1, rig.teardown.reads, "teardown is read once, not polled")
-}
-
-func TestCleanupCompute_RejectsBadRequest(t *testing.T) {
-	c := newComputeRig().cleanup()
-	for _, req := range []*ComputeRequest{nil, {}, {AccountID: "123"}} {
-		_, err := c.CleanupCompute(t.Context(), req)
-		assert.Error(t, err)
-	}
-}
-
-// Terminated records expire, so a missing one is never read as a finished teardown.
-func TestCleanupCompute_NoOutstandingRecordIsNotProofOfCompletion(t *testing.T) {
-	rig := newComputeRig()
-	res := rig.run(t, false)
-
-	for _, id := range []string{runningID, stoppedID} {
-		assert.Equal(t, DispositionPending, itemFor(res, id).Disposition)
-	}
-	assert.Equal(t, StatusIncomplete, res.Status)
-
-	rig.inventory.out = listing(nil)
-	assert.Equal(t, StatusClean, rig.run(t, false).Status, "a rerun that finds no instance converges")
-}
-
-// A disabled request can resume, so it is cancelled; an unknown state is never drained.
-func TestS7_CleanupComputeSpotStateIsFailClosed(t *testing.T) {
-	rig := newComputeRig()
-	rig.spot.scans = []*ec2spotinstance.AccountScan{
-		{Active: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-disabled", State: ec2.SpotInstanceStateDisabled}}},
-		{Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-disabled", State: ec2.SpotInstanceStateCancelled}}},
-	}
-	rig.run(t, false)
-	assert.Equal(t, []string{"sir-disabled"}, rig.spot.cancelled)
-
-	unknown := newComputeRig()
-	unknown.spot.scans = []*ec2spotinstance.AccountScan{{Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-odd", State: "mystery"}}}}
-	res := unknown.run(t, false)
-	assert.Equal(t, ReasonSpotStateUnrecognised, itemFor(res, "sir-odd").Reason)
-	assert.Empty(t, unknown.terminator.terminated, "ADR-0003:S7: an unrecognised spot state could still launch, so nothing is terminated")
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-// ADR-0003:S7. An inventory entry with no identity could be a live instance,
-// so the run terminates nothing and is not clean.
-func TestS7_CleanupComputeUnidentifiedInventoryEntryTerminatesNothing(t *testing.T) {
-	bad := map[string]*ec2.DescribeInstancesOutput{
-		"nil output":      nil,
-		"nil reservation": {Reservations: []*ec2.Reservation{nil}},
-		"nil instance":    {Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{nil}}}},
-		"empty id":        {Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{{InstanceId: aws.String("")}}}}},
-	}
-	for name, out := range bad {
-		for _, force := range []bool{false, true} {
-			t.Run(name, func(t *testing.T) {
-				rig := newComputeRig()
-				if out != nil {
-					out.Reservations = append(out.Reservations, listing(map[string]string{runningID: ec2.InstanceStateNameRunning}).Reservations...)
-				}
-				rig.inventory.out = out
-
-				res := rig.run(t, force)
-
-				assert.Empty(t, rig.terminator.terminated, "ADR-0003:S7: nothing is terminated from a list with an unidentified entry (force=%v)", force)
-				assert.Equal(t, StatusIncomplete, res.Status)
-				assert.Contains(t, res.IncompleteReasons, ReasonInstanceInventoryIncomplete)
-			})
-		}
-	}
-}

spinifex/domains/ec2/accountcleanup/storage.go

@@ -1,331 +0,0 @@
-package accountcleanup
-
-import (
-	"context"
-	"log/slog"
-	"slices"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-)
-
-// Storage kinds and the extra reasons the storage run reports.
-const (
-	KindVolume = "volume"
-	KindTag    = "tag"
-
-	// ReasonHolderPresent: the volume record shows an attachment, or a live
-	// instance still maps the volume.
-	ReasonHolderPresent = "holder-present"
-	// ReasonVolumeStateUnproven: the volume is not in a state that proves it is
-	// free, so it is kept.
-	ReasonVolumeStateUnproven = "volume-state-unproven"
-	// ReasonVolumeDeleting: the owner is reclaiming the volume; the record is
-	// not gone yet.
-	ReasonVolumeDeleting = "volume-deleting"
-
-	// Incomplete reasons.
-	ReasonVolumeInventoryIncomplete = "volume-inventory-incomplete"
-	ReasonHolderProofIncomplete     = "holder-proof-incomplete"
-	ReasonTagIndexUnread            = "tag-index-unread"
-	ReasonTagOwnerUnresolved        = "tag-owner-unresolved"
-)
-
-// StorageRequest names the account whose volumes and tag residue are cleaned.
-// Force is accepted and bypasses nothing: a holder, an unproven terminate or an
-// incomplete inventory always keeps the volume.
-type StorageRequest struct {
-	AccountID      string `json:"account_id"`
-	CorrelationRef string `json:"correlation_ref,omitempty"`
-	Force          bool   `json:"force,omitempty"`
-}
-
-// StorageCleaner deletes an account's free volumes and the tag residue of
-// instances, volumes and spot requests that no longer exist. It is
-// destructive; the caller must hold an accepted account deletion.
-type StorageCleaner interface {
-	CleanupStorage(ctx context.Context, req *StorageRequest) (*StorageResult, error)
-}
-
-// VolumeLister is the complete-or-error volume inventory.
-type VolumeLister interface {
-	VolumeInventory(ctx context.Context, accountID string) ([]accountinventory.VolumeItem, error)
-}
-
-// VolumeDeleter sends the existing EC2 volume delete for one volume. It
-// returns once the owner accepted the delete, not when the backing is reclaimed.
-type VolumeDeleter interface {
-	DeleteVolume(ctx context.Context, accountID, volumeID string) error
-}
-
-// SpotScanner is the spot-request owner's account scan.
-type SpotScanner interface {
-	ScanAccount(ctx context.Context, accountID string) (*ec2spotinstance.AccountScan, error)
-}
-
-var (
-	_ VolumeLister   = (*accountinventory.Inventory)(nil)
-	_ StorageCleaner = (*StorageCleanup)(nil)
-)
-
-// Resource-ID prefixes whose tag residue the storage run removes.
-var storageTagPrefixes = []string{"i-", "vol-", "sir-"}
-
-// StorageCleanup composes the storage owners.
-type StorageCleanup struct {
-	volumes   VolumeLister
-	deleter   VolumeDeleter
-	instances InstanceInventory
-	teardown  TeardownReader
-	spot      SpotScanner
-	tags      TagIndexOwner
-}
-
-// NewStorage builds a storage cleanup. A nil dependency makes every run fail
-// before it acts.
-func NewStorage(volumes VolumeLister, deleter VolumeDeleter, instances InstanceInventory, teardown TeardownReader, spot SpotScanner, tags TagIndexOwner) *StorageCleanup {
-	return &StorageCleanup{volumes: volumes, deleter: deleter, instances: instances, teardown: teardown, spot: spot, tags: tags}
-}
-
-// CleanupStorage deletes each volume that is available, shows no attachment and
-// that no live instance maps, then removes tag-index entries whose owner record
-// is gone. It never waits for reclaim: a volume the owner is deleting is
-// reported pending and a rerun converges it. A read that does not complete
-// deletes nothing.
-func (c *StorageCleanup) CleanupStorage(ctx context.Context, req *StorageRequest) (*StorageResult, error) {
-	if req == nil || req.AccountID == "" {
-		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
-	}
-	if !awsidentifiers.IsAccountID(req.AccountID) {
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The account ID must be 12 digits.")
-	}
-	if c == nil || c.volumes == nil || c.deleter == nil || c.instances == nil || c.teardown == nil || c.spot == nil || c.tags == nil {
-		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Storage cleanup is not fully wired.")
-	}
-	res := &StorageResult{AccountID: req.AccountID, CorrelationRef: req.CorrelationRef}
-	defer res.finish()
-
-	volumes, err := c.volumes.VolumeInventory(ctx, req.AccountID)
-	if err != nil {
-		slog.WarnContext(ctx, "CleanupStorage: volume inventory incomplete, nothing deleted", "account_id", req.AccountID, "err", err)
-		res.incomplete(ReasonVolumeInventoryIncomplete)
-		return res, nil
-	}
-	live, err := c.instances.AccountInstances(ctx, req.AccountID)
-	if err != nil || !inventoryUsable(live) {
-		slog.WarnContext(ctx, "CleanupStorage: no complete instance view, nothing deleted", "account_id", req.AccountID, "err", err)
-		res.incomplete(ReasonHolderProofIncomplete)
-		return res, nil
-	}
-
-	outstanding, err := c.teardown.AccountTeardownOutstanding(req.AccountID)
-	if err != nil {
-		slog.WarnContext(ctx, "CleanupStorage: teardown unread, nothing deleted", "account_id", req.AccountID, "err", err)
-		res.incomplete(ReasonHolderProofIncomplete)
-		return res, nil
-	}
-
-	c.deleteFreeVolumes(ctx, req.AccountID, volumes, live, outstanding, res)
-	c.clearTagResidue(ctx, req.AccountID, live, outstanding, res)
-	return res, nil
-}
-
-// deleteFreeVolumes deletes the free volumes, re-reads the inventory and
-// reports each volume from that re-read.
-func (c *StorageCleanup) deleteFreeVolumes(ctx context.Context, accountID string, volumes []accountinventory.VolumeItem, live *ec2.DescribeInstancesOutput, outstanding []ec2instance.OutstandingInstance, res *StorageResult) {
-	issued := map[string]bool{}
-	blocked := map[string]string{}
-	unproven := terminatedTeardownOutstanding(outstanding)
-	for _, v := range volumes {
-		switch {
-		case v.State == ec2.VolumeStateDeleting:
-			// The owner is already reclaiming it; the re-read reports it.
-		case v.AttachedInstance != "" || volumeMappedByLiveInstance(live, v.VolumeID):
-			blocked[v.VolumeID] = ReasonHolderPresent
-		case unproven:
-			// A terminated record whose teardown is unfinished may still have a
-			// QEMU holding a volume this record no longer names.
-			blocked[v.VolumeID] = ReasonTerminationUnproven
-		case v.State != ec2.VolumeStateAvailable:
-			blocked[v.VolumeID] = ReasonVolumeStateUnproven
-		default:
-			if err := c.deleter.DeleteVolume(ctx, accountID, v.VolumeID); err != nil {
-				slog.WarnContext(ctx, "CleanupStorage: volume delete refused", "account_id", accountID, "volume_id", v.VolumeID, "err", err)
-				if code, ok := awserrors.ResolveErrorCode(err); ok && code == awserrors.ErrorVolumeInUse {
-					blocked[v.VolumeID] = ReasonHolderPresent
-					continue
-				}
-				res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: v.State, Disposition: DispositionFailed, Reason: ReasonOwnerDeleteFailed})
-				continue
-			}
-			issued[v.VolumeID] = true
-		}
-	}
-
-	after, err := c.volumes.VolumeInventory(ctx, accountID)
-	if err != nil {
-		slog.WarnContext(ctx, "CleanupStorage: volume re-read failed", "account_id", accountID, "err", err)
-		res.incomplete(ReasonVolumeInventoryIncomplete)
-		for _, v := range volumes {
-			if issued[v.VolumeID] || v.State == ec2.VolumeStateDeleting {
-				res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: v.State, Disposition: DispositionPending, Reason: ReasonAbsenceUnproven})
-			}
-		}
-	} else {
-		present := map[string]accountinventory.VolumeItem{}
-		for _, v := range after {
-			present[v.VolumeID] = v
-		}
-		initial := map[string]bool{}
-		for _, v := range volumes {
-			initial[v.VolumeID] = true
-		}
-		for _, v := range after {
-			if !initial[v.VolumeID] {
-				res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: v.State, Disposition: DispositionPending, Reason: ReasonAppearedDuringCleanup})
-			}
-		}
-		for _, v := range volumes {
-			now, still := present[v.VolumeID]
-			switch {
-			case !issued[v.VolumeID] && v.State != ec2.VolumeStateDeleting:
-				continue
-			case !still:
-				res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, Disposition: DispositionDeleted})
-			case now.State == ec2.VolumeStateDeleting:
-				res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: now.State, Disposition: DispositionPending, Reason: ReasonVolumeDeleting})
-			default:
-				res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: now.State, Disposition: DispositionFailed, Reason: ReasonStillPresent})
-			}
-		}
-	}
-	for id, reason := range blocked {
-		res.Items = append(res.Items, Item{ResourceID: id, Kind: KindVolume, Disposition: DispositionBlocked, Reason: reason})
-	}
-}
-
-// terminatedTeardownOutstanding reports whether an instance whose record says
-// terminated still has unfinished teardown. Its mappings are gone from the
-// live listing, so no volume can be shown free of it.
-func terminatedTeardownOutstanding(outstanding []ec2instance.OutstandingInstance) bool {
-	return slices.ContainsFunc(outstanding, func(o ec2instance.OutstandingInstance) bool {
-		return o.State == ec2.InstanceStateNameTerminated
-	})
-}
-
-// volumeMappedByLiveInstance reports whether a live instance's block device
-// mappings name the volume. A volume record's own attachment fields can drift,
-// so this is checked as well; terminated instances are not in the listing.
-func volumeMappedByLiveInstance(live *ec2.DescribeInstancesOutput, volumeID string) bool {
-	for _, reservation := range live.Reservations {
-		for _, in := range reservation.Instances {
-			if in.State != nil && aws.StringValue(in.State.Name) == ec2.InstanceStateNameTerminated {
-				continue
-			}
-			for _, bdm := range in.BlockDeviceMappings {
-				if bdm != nil && bdm.Ebs != nil && aws.StringValue(bdm.Ebs.VolumeId) == volumeID {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}
-
-// clearTagResidue removes the tag entries of instances, volumes and spot
-// requests whose owner re-read resolved and holds no record. An owner that did
-// not resolve keeps its entries and the run incomplete.
-func (c *StorageCleanup) clearTagResidue(ctx context.Context, accountID string, live *ec2.DescribeInstancesOutput, outstanding []ec2instance.OutstandingInstance, res *StorageResult) {
-	tagged, err := c.tags.ScanAccountResourceIDs(ctx, accountID, storageTagPrefixes...)
-	if err != nil {
-		slog.WarnContext(ctx, "CleanupStorage: tag index read failed", "account_id", accountID, "err", err)
-		res.incomplete(ReasonTagIndexUnread)
-		return
-	}
-	owned, resolved := c.ownedResourceIDs(ctx, accountID, live, outstanding)
-	var cleared []string
-	for _, id := range tagged {
-		prefix := tagPrefix(id)
-		switch {
-		case !resolved[prefix]:
-			res.incomplete(ReasonTagOwnerUnresolved)
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionResidue, Reason: ReasonOwnerRecordsUnresolved})
-		case owned[id]:
-			// The resource still exists; its tags are not residue.
-		default:
-			if err := c.tags.DeleteAllTags(ctx, accountID, id); err != nil {
-				slog.WarnContext(ctx, "CleanupStorage: tag clear failed", "account_id", accountID, "resource_id", id, "err", err)
-				res.incomplete(ReasonOwnerDeleteFailed)
-				res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionFailed, Reason: ReasonOwnerDeleteFailed})
-				continue
-			}
-			cleared = append(cleared, id)
-		}
-	}
-	if len(cleared) == 0 {
-		return
-	}
-	after, err := c.tags.ScanAccountResourceIDs(ctx, accountID, storageTagPrefixes...)
-	if err != nil {
-		res.incomplete(ReasonTagIndexUnread)
-	}
-	for _, id := range cleared {
-		switch {
-		case err != nil:
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionPending, Reason: ReasonAbsenceUnproven})
-		case slices.Contains(after, id):
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionFailed, Reason: ReasonStillPresent})
-		default:
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionDeleted})
-		}
-	}
-}
-
-// ownedResourceIDs reads, per prefix, the IDs whose owner still holds a record,
-// and whether that read completed. Instances are owned while listed live or
-// while their teardown is unfinished; a missing teardown record is not proof
-// of completion, but tags of an instance with no live record are residue.
-func (c *StorageCleanup) ownedResourceIDs(ctx context.Context, accountID string, live *ec2.DescribeInstancesOutput, outstanding []ec2instance.OutstandingInstance) (map[string]bool, map[string]bool) {
-	owned := map[string]bool{}
-	resolved := map[string]bool{}
-
-	volumes, err := c.volumes.VolumeInventory(ctx, accountID)
-	if err == nil {
-		resolved["vol-"] = true
-		for _, v := range volumes {
-			owned[v.VolumeID] = true
-		}
-	}
-	scan, err := c.spot.ScanAccount(ctx, accountID)
-	if err == nil {
-		resolved["sir-"] = true
-		for _, r := range slices.Concat(scan.Active, scan.Terminal) {
-			owned[r.RequestID] = true
-		}
-	}
-	resolved["i-"] = true
-	for _, o := range outstanding {
-		owned[o.InstanceID] = true
-	}
-	for _, reservation := range live.Reservations {
-		for _, in := range reservation.Instances {
-			owned[aws.StringValue(in.InstanceId)] = true
-		}
-	}
-	return owned, resolved
-}
-
-func tagPrefix(id string) string {
-	for _, prefix := range storageTagPrefixes {
-		if strings.HasPrefix(id, prefix) {
-			return prefix
-		}
-	}
-	return ""
-}

spinifex/domains/ec2/accountcleanup/storage_test.go

@@ -1,436 +0,0 @@
-package accountcleanup
-
-//test:in-package — the storage owners are fakes so each test controls the
-// read or delete that fails and can assert no delete happened.
-
-import (
-	"context"
-	"errors"
-	"slices"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	freeVol     = "vol-0aaaaaaaaaaaaaaa1"
-	attachedVol = "vol-0aaaaaaaaaaaaaaa2"
-	mappedVol   = "vol-0aaaaaaaaaaaaaaa3"
-	deletingVol = "vol-0aaaaaaaaaaaaaaa4"
-	driftedVol  = "vol-0aaaaaaaaaaaaaaa5"
-)
-
-type fakeVolumes struct {
-	vols []accountinventory.VolumeItem
-	// failAt makes the nth read (1-based) fail; 0 means never.
-	failAt int
-	reads  int
-}
-
-func (f *fakeVolumes) VolumeInventory(context.Context, string) ([]accountinventory.VolumeItem, error) {
-	f.reads++
-	if f.failAt == f.reads || f.failAt < 0 {
-		return nil, errors.New("volume store unreadable")
-	}
-	return slices.Clone(f.vols), nil
-}
-
-// fakeDeleter acts on the volume fake: reclaim "gone" removes the record,
-// "deleting" marks it, anything else leaves it.
-type fakeDeleter struct {
-	store    *fakeVolumes
-	reclaim  string
-	errFor   map[string]error
-	deleted  []string
-	onDelete func()
-}
-
-func (f *fakeDeleter) DeleteVolume(_ context.Context, _, id string) error {
-	if err := f.errFor[id]; err != nil {
-		return err
-	}
-	f.deleted = append(f.deleted, id)
-	if f.onDelete != nil {
-		f.onDelete()
-	}
-	f.store.vols = slices.DeleteFunc(f.store.vols, func(v accountinventory.VolumeItem) bool {
-		if v.VolumeID != id {
-			return false
-		}
-		return f.reclaim == "gone"
-	})
-	if f.reclaim == "deleting" {
-		for i := range f.store.vols {
-			if f.store.vols[i].VolumeID == id {
-				f.store.vols[i].State = ec2.VolumeStateDeleting
-			}
-		}
-	}
-	return nil
-}
-
-type fakeSpotScan struct {
-	scan *ec2spotinstance.AccountScan
-	err  error
-}
-
-func (f *fakeSpotScan) ScanAccount(context.Context, string) (*ec2spotinstance.AccountScan, error) {
-	return f.scan, f.err
-}
-
-type fakeTags struct {
-	ids       []string
-	scanErr   error
-	rescanErr error
-	deleteErr map[string]error
-	scans     int
-	deleted   []string
-	keep      map[string]bool // ids a delete does not actually remove
-}
-
-func (f *fakeTags) ScanAccountResourceIDs(context.Context, string, ...string) ([]string, error) {
-	f.scans++
-	if f.scanErr != nil {
-		return nil, f.scanErr
-	}
-	if f.scans > 1 && f.rescanErr != nil {
-		return nil, f.rescanErr
-	}
-	return slices.Clone(f.ids), nil
-}
-
-func (f *fakeTags) DeleteAllTags(_ context.Context, _, id string) error {
-	if err := f.deleteErr[id]; err != nil {
-		return err
-	}
-	f.deleted = append(f.deleted, id)
-	if !f.keep[id] {
-		f.ids = slices.DeleteFunc(f.ids, func(s string) bool { return s == id })
-	}
-	return nil
-}
-
-func volume(id, state, attached string) accountinventory.VolumeItem {
-	return accountinventory.VolumeItem{VolumeID: id, State: state, AttachedInstance: attached}
-}
-
-type storageRig struct {
-	volumes   *fakeVolumes
-	deleter   *fakeDeleter
-	instances *fakeInventory
-	teardown  *fakeTeardown
-	spot      *fakeSpotScan
-	tags      *fakeTags
-}
-
-func newStorageRig() *storageRig {
-	store := &fakeVolumes{vols: []accountinventory.VolumeItem{volume(freeVol, ec2.VolumeStateAvailable, "")}}
-	return &storageRig{
-		volumes:   store,
-		deleter:   &fakeDeleter{store: store, reclaim: "gone"},
-		instances: &fakeInventory{out: listing(nil)},
-		teardown:  &fakeTeardown{},
-		spot:      &fakeSpotScan{scan: &ec2spotinstance.AccountScan{}},
-		tags:      &fakeTags{},
-	}
-}
-
-func (r *storageRig) cleanup() *StorageCleanup {
-	return NewStorage(r.volumes, r.deleter, r.instances, r.teardown, r.spot, r.tags)
-}
-
-func (r *storageRig) run(t *testing.T, force bool) *StorageResult {
-	t.Helper()
-	res, err := r.cleanup().CleanupStorage(t.Context(), &StorageRequest{AccountID: testAccount, Force: force})
-	require.NoError(t, err)
-	return res
-}
-
-func mapping(volumeID string) *ec2.Instance {
-	return &ec2.Instance{
-		InstanceId:          aws.String(runningID),
-		State:               &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)},
-		BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(volumeID)}}},
-	}
-}
-
-func TestCleanupStorage_DeletesFreeVolume(t *testing.T) {
-	rig := newStorageRig()
-	res := rig.run(t, false)
-
-	assert.Equal(t, []string{freeVol}, rig.deleter.deleted)
-	assert.Equal(t, DispositionDeleted, itemFor(res, freeVol).Disposition)
-	assert.Equal(t, StatusClean, res.Status)
-}
-
-// ADR-0003:S7. A read the run cannot complete deletes nothing.
-func TestS7_CleanupStorageUnprovenReadDeletesNothing(t *testing.T) {
-	cases := map[string]func(*storageRig){
-		"volume inventory error": func(r *storageRig) { r.volumes.failAt = -1 },
-		"instance inventory error": func(r *storageRig) {
-			r.instances.err = errors.New("incomplete listing")
-		},
-		"unidentified instance entry": func(r *storageRig) {
-			r.instances.out = &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{nil}}}}
-		},
-	}
-	for name, arrange := range cases {
-		for _, force := range []bool{false, true} {
-			t.Run(name, func(t *testing.T) {
-				rig := newStorageRig()
-				arrange(rig)
-
-				res := rig.run(t, force)
-
-				assert.Empty(t, rig.deleter.deleted, "ADR-0003:S7: an unproven read must delete nothing (force=%v)", force)
-				assert.Equal(t, StatusIncomplete, res.Status)
-				assert.NotEmpty(t, res.IncompleteReasons)
-			})
-		}
-	}
-}
-
-// ADR-0003:S7. A volume with an attachment, a live mapping or an unproven
-// state is never deleted, whatever Force says. The drifted record has no state
-// at all, as a running root volume's can.
-func TestS7_CleanupStorageHeldVolumeIsNeverDeleted(t *testing.T) {
-	for _, force := range []bool{false, true} {
-		rig := newStorageRig()
-		rig.volumes.vols = []accountinventory.VolumeItem{
-			volume(attachedVol, ec2.VolumeStateInUse, runningID),
-			volume(mappedVol, ec2.VolumeStateAvailable, ""),
-			volume(driftedVol, "", ""),
-		}
-		rig.instances.out = &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{mapping(mappedVol)}}}}
-
-		res := rig.run(t, force)
-
-		assert.Empty(t, rig.deleter.deleted, "ADR-0003:S7: a held volume must not be deleted (force=%v)", force)
-		for _, id := range []string{attachedVol, mappedVol} {
-			assert.Equal(t, ReasonHolderPresent, itemFor(res, id).Reason, id)
-		}
-		assert.Equal(t, ReasonVolumeStateUnproven, itemFor(res, driftedVol).Reason)
-		assert.Equal(t, StatusIncomplete, res.Status)
-	}
-}
-
-// A mapping on a terminated instance is stale; it does not hold the volume.
-func TestCleanupStorage_TerminatedMappingDoesNotHold(t *testing.T) {
-	rig := newStorageRig()
-	stale := mapping(freeVol)
-	stale.State.Name = aws.String(ec2.InstanceStateNameTerminated)
-	rig.instances.out = &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{stale}}}}
-
-	rig.run(t, false)
-
-	assert.Equal(t, []string{freeVol}, rig.deleter.deleted)
-}
-
-// The delete path re-checks the holder; its refusal is a held volume, not a failure.
-func TestCleanupStorage_OwnerRefusalIsHolderPresent(t *testing.T) {
-	rig := newStorageRig()
-	rig.deleter.errFor = map[string]error{freeVol: errors.New(awserrors.ErrorVolumeInUse)}
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, ReasonHolderPresent, itemFor(res, freeVol).Reason)
-	assert.Equal(t, DispositionBlocked, itemFor(res, freeVol).Disposition)
-
-	rig = newStorageRig()
-	rig.deleter.errFor = map[string]error{freeVol: errors.New("store down")}
-	res = rig.run(t, false)
-	assert.Equal(t, DispositionFailed, itemFor(res, freeVol).Disposition)
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-func TestCleanupStorage_DeletingVolumeIsPendingAndNotRedeleted(t *testing.T) {
-	rig := newStorageRig()
-	rig.volumes.vols = []accountinventory.VolumeItem{volume(deletingVol, ec2.VolumeStateDeleting, "")}
-
-	res := rig.run(t, false)
-
-	assert.Empty(t, rig.deleter.deleted)
-	assert.Equal(t, DispositionPending, itemFor(res, deletingVol).Disposition)
-	assert.Equal(t, ReasonVolumeDeleting, itemFor(res, deletingVol).Reason)
-	assert.Len(t, res.Items, 1, "one outcome for the volume, not a second blocked entry")
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-// ADR-0003:S8. The call returns with a pending outcome while the owner
-// reclaims; it re-reads once and does not poll for the record to go.
-func TestS8_CleanupStorageReturnsWhileReclaimPending(t *testing.T) {
-	rig := newStorageRig()
-	rig.deleter.reclaim = "deleting"
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, DispositionPending, itemFor(res, freeVol).Disposition)
-	assert.Equal(t, StatusIncomplete, res.Status)
-	assert.Equal(t, []string{freeVol}, rig.deleter.deleted, "deleted once, not retried inside the call")
-	assert.Equal(t, 3, rig.volumes.reads, "before, after the delete, and for tag ownership; no polling")
-}
-
-func TestCleanupStorage_RereadFailureKeepsDeletePending(t *testing.T) {
-	rig := newStorageRig()
-	rig.volumes.failAt = 2
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, DispositionPending, itemFor(res, freeVol).Disposition)
-	assert.Equal(t, ReasonAbsenceUnproven, itemFor(res, freeVol).Reason)
-	assert.Contains(t, res.IncompleteReasons, ReasonVolumeInventoryIncomplete)
-}
-
-func TestCleanupStorage_ClearsTagResidueOfMissingOwners(t *testing.T) {
-	rig := newStorageRig()
-	rig.volumes.vols = nil
-	rig.spot.scan = &ec2spotinstance.AccountScan{Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-closed", State: ec2.SpotInstanceStateClosed}}}
-	rig.tags.ids = []string{"i-0gone", "vol-0gone", "sir-0gone", "sir-closed"}
-
-	res := rig.run(t, false)
-
-	assert.ElementsMatch(t, []string{"i-0gone", "vol-0gone", "sir-0gone"}, rig.tags.deleted)
-	for _, id := range rig.tags.deleted {
-		assert.Equal(t, DispositionDeleted, itemFor(res, id).Disposition)
-	}
-	assert.Nil(t, itemFor(res, "sir-closed"), "a request that still has a record keeps its tags")
-	assert.Equal(t, StatusClean, res.Status)
-}
-
-func TestCleanupStorage_KeepsTagsOfResourcesThatStillExist(t *testing.T) {
-	rig := newStorageRig()
-	rig.volumes.vols = []accountinventory.VolumeItem{volume(attachedVol, ec2.VolumeStateInUse, runningID)}
-	rig.instances.out = listing(map[string]string{runningID: ec2.InstanceStateNameRunning})
-	rig.teardown.outstanding = nil
-	rig.tags.ids = []string{attachedVol, runningID}
-
-	rig.run(t, false)
-
-	assert.Empty(t, rig.tags.deleted)
-}
-
-// An owner that cannot be read keeps its entries; absence is not shown.
-func TestS7_CleanupStorageUnresolvedOwnerKeepsItsTags(t *testing.T) {
-	for _, force := range []bool{false, true} {
-		rig := newStorageRig()
-		rig.volumes.vols = nil
-		rig.volumes.failAt = 3 // the tag-ownership read
-		rig.spot.err = errors.New("spot scan failed")
-		rig.tags.ids = []string{"vol-0x", "sir-0x"}
-
-		res := rig.run(t, force)
-
-		assert.Empty(t, rig.tags.deleted, "ADR-0003:S7: unresolved owners keep their tag entries (force=%v)", force)
-		assert.Contains(t, res.IncompleteReasons, ReasonTagOwnerUnresolved)
-		assert.Equal(t, StatusIncomplete, res.Status)
-	}
-}
-
-func TestCleanupStorage_TagDeleteFailureKeepsRunIncomplete(t *testing.T) {
-	rig := newStorageRig()
-	rig.volumes.vols = nil
-	rig.tags.ids = []string{"vol-0gone"}
-	rig.tags.deleteErr = map[string]error{"vol-0gone": errors.New("tag store down")}
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, DispositionFailed, itemFor(res, "vol-0gone").Disposition)
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-func TestCleanupStorage_TagStillPresentAfterDeleteIsFailed(t *testing.T) {
-	rig := newStorageRig()
-	rig.volumes.vols = nil
-	rig.tags.ids = []string{"vol-0gone"}
-	rig.tags.keep = map[string]bool{"vol-0gone": true}
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, DispositionFailed, itemFor(res, "vol-0gone").Disposition)
-	assert.Equal(t, ReasonStillPresent, itemFor(res, "vol-0gone").Reason)
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-func TestCleanupStorage_TagRescanFailureIsPending(t *testing.T) {
-	rig := newStorageRig()
-	rig.volumes.vols = nil
-	rig.tags.ids = []string{"vol-0gone"}
-	rig.tags.rescanErr = errors.New("rescan failed")
-
-	res := rig.run(t, false)
-
-	assert.Equal(t, DispositionPending, itemFor(res, "vol-0gone").Disposition)
-	assert.Equal(t, StatusIncomplete, res.Status)
-}
-
-func TestCleanupStorage_UnreadableTagIndexKeepsRunIncomplete(t *testing.T) {
-	rig := newStorageRig()
-	rig.tags.scanErr = errors.New("tag index unreadable")
-
-	res := rig.run(t, false)
-
-	assert.Contains(t, res.IncompleteReasons, ReasonTagIndexUnread)
-	assert.Equal(t, DispositionDeleted, itemFor(res, freeVol).Disposition, "volumes are not held up by the tag index")
-}
-
-func TestCleanupStorage_RejectsBadRequestAndNilDependency(t *testing.T) {
-	rig := newStorageRig()
-	c := rig.cleanup()
-	for _, req := range []*StorageRequest{nil, {}, {AccountID: "123"}} {
-		_, err := c.CleanupStorage(t.Context(), req)
-		assert.Error(t, err)
-	}
-	_, err := NewStorage(rig.volumes, rig.deleter, rig.instances, rig.teardown, rig.spot, nil).
-		CleanupStorage(t.Context(), &StorageRequest{AccountID: testAccount})
-	require.Error(t, err)
-	assert.Empty(t, rig.deleter.deleted)
-	assert.Zero(t, rig.volumes.reads, "nothing is read before the wiring is proven")
-}
-
-// ADR-0003:S7. Unreadable teardown deletes no volume; a terminated record with
-// unfinished teardown may still have a QEMU holding a volume no record names.
-func TestS7_CleanupStorageUnprovenTeardownDeletesNoVolume(t *testing.T) {
-	cases := map[string]func(*storageRig){
-		"teardown unreadable": func(r *storageRig) { r.teardown.err = errors.New("store down") },
-		"terminated record with failed teardown": func(r *storageRig) {
-			r.teardown.outstanding = []ec2instance.OutstandingInstance{{InstanceID: runningID, State: ec2.InstanceStateNameTerminated, Teardown: map[string]string{"qemu": "failed"}}}
-		},
-	}
-	for name, arrange := range cases {
-		for _, force := range []bool{false, true} {
-			t.Run(name, func(t *testing.T) {
-				rig := newStorageRig()
-				arrange(rig)
-
-				res := rig.run(t, force)
-
-				assert.Empty(t, rig.deleter.deleted, "ADR-0003:S7: unproven teardown must not allow a volume delete (force=%v)", force)
-				assert.Equal(t, StatusIncomplete, res.Status)
-			})
-		}
-	}
-}
-
-// A volume that first shows up in the re-read stays visible.
-func TestCleanupStorage_VolumeAppearingDuringCleanupIsReported(t *testing.T) {
-	rig := newStorageRig()
-	rig.deleter.reclaim = "gone"
-	rig.deleter.onDelete = func() {
-		rig.volumes.vols = append(rig.volumes.vols, volume(deletingVol, ec2.VolumeStateDeleting, ""))
-	}
-
-	res := rig.run(t, false)
-
-	item := itemFor(res, deletingVol)
-	require.NotNil(t, item)
-	assert.Equal(t, DispositionPending, item.Disposition)
-	assert.Equal(t, ReasonAppearedDuringCleanup, item.Reason)
-	assert.Equal(t, StatusIncomplete, res.Status)
-}

spinifex/domains/ec2/accountcleanup/types.go

@@ -47,8 +47,6 @@ type Item struct {
 	Disposition string `json:"disposition"`
 	Reason      string `json:"reason,omitempty"`
 	MemberCount int    `json:"member_count,omitempty"`
-	// State is the lifecycle state a compute item was last seen in.
-	State string `json:"state,omitempty"`
 }
 
 // Count is the number of records of one class with one disposition, for

spinifex/domains/ec2/ebs/obligation/obligation.go

@@ -1,185 +0,0 @@
-// Package ebsobligation records the volume exports a detach left in place.
-//
-// A force detach, or a terminate whose unmount did not complete, clears a
-// volume's attachment while the node that served it may still export it. The
-// request cannot reach that node, so it records an UnpublishObligation here and
-// returns; the named node settles it once it answers.
-package ebsobligation
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"regexp"
-	"strconv"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Bucket holds one obligation per publish a cleared attachment left behind.
-// It has no TTL: an obligation ends only when it is settled or closed.
-const Bucket = "spinifex-ebs-unpublish-obligations"
-
-// BucketVersion is the bucket schema version RunKV stamps.
-const BucketVersion = 1
-
-// SchemaVersion is the version of the UnpublishObligation record.
-const SchemaVersion uint16 = 1
-
-// UnpublishObligation names a publish a cleared attachment left on a node: the
-// node, volume and handle, and the attachment it served. The attachment is what
-// tells it apart from a later publish of the same volume on the same node.
-type UnpublishObligation struct {
-	SchemaVersion uint16    `json:"schema_version"`
-	VolumeID      string    `json:"volume_id"`
-	AccountID     string    `json:"account_id"`
-	Handle        string    `json:"handle,omitempty"` // ProviderHandle when the attachment was cleared
-	NodeID        string    `json:"node_id"`          // the node holding the publish
-	InstanceID    string    `json:"instance_id"`      // the attachment that was cleared
-	AttachedAt    time.Time `json:"attached_at"`
-	RecordedAt    time.Time `json:"recorded_at"`
-	Attempts      int       `json:"attempts"`
-	Error         string    `json:"error,omitempty"`
-}
-
-// keyToken is one NATS subject token a key segment may be: the key is three
-// tokens, so a segment holding a dot would split it and escape the filters.
-var keyToken = regexp.MustCompile(`^[-_=a-zA-Z0-9]+$`)
-
-// Key is the obligation's key, <node>.<volume-id>.<attached-at>, with the
-// attachment time in Unix nanoseconds and an unknown time as 0.
-func Key(o UnpublishObligation) (string, error) {
-	if !keyToken.MatchString(o.NodeID) {
-		return "", fmt.Errorf("unpublish obligation: node %q is not a valid key token", o.NodeID)
-	}
-	if !keyToken.MatchString(o.VolumeID) {
-		return "", fmt.Errorf("unpublish obligation: volume %q is not a valid key token", o.VolumeID)
-	}
-	var nanos int64
-	if !o.AttachedAt.IsZero() {
-		nanos = o.AttachedAt.UnixNano()
-	}
-	return o.NodeID + "." + o.VolumeID + "." + strconv.FormatInt(nanos, 10), nil
-}
-
-// allKeys matches every obligation key and not the bucket's version stamp,
-// which is a single token.
-const allKeys = "*.*.*"
-
-func bucketConfig() kvstore.Config {
-	return kvstore.Config{
-		Name:        Bucket,
-		Description: "EBS exports a cleared attachment left on a node, until that node unpublishes them",
-		History:     1,
-		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
-			return migrate.DefaultRegistry.RunKV(ctx, Bucket, kv, BucketVersion)
-		},
-		Missing: "EBS unpublish obligation bucket not initialized",
-	}
-}
-
-// Store reads and writes the obligation bucket.
-type Store struct {
-	s *kvstore.Store[UnpublishObligation]
-}
-
-// NewStore returns a Store over js. The bucket is created on first use.
-func NewStore(js jetstream.JetStream) *Store {
-	return &Store{s: kvstore.New[UnpublishObligation](js, bucketConfig())}
-}
-
-// Open creates or opens the bucket now, so a node that cannot reach it fails
-// at startup rather than on its first force detach.
-func (s *Store) Open(ctx context.Context) error {
-	_, err := s.s.KV(ctx)
-	return err
-}
-
-// Record writes o if no obligation with its key exists. An existing key is the
-// same obligation written by an earlier attempt, so it counts as written.
-func (s *Store) Record(ctx context.Context, o UnpublishObligation) error {
-	key, err := Key(o)
-	if err != nil {
-		return err
-	}
-	o.SchemaVersion = SchemaVersion
-	if _, err := s.s.Create(ctx, key, &o); err != nil && !errors.Is(err, kvstore.ErrExists) {
-		return fmt.Errorf("record unpublish obligation %s: %w", key, err)
-	}
-	return nil
-}
-
-// ListNode returns the open obligations naming node.
-func (s *Store) ListNode(ctx context.Context, node string) ([]UnpublishObligation, error) {
-	if !keyToken.MatchString(node) {
-		return nil, fmt.Errorf("unpublish obligation: node %q is not a valid key token", node)
-	}
-	return s.list(ctx, node+".*.*")
-}
-
-// List returns every open obligation.
-func (s *Store) List(ctx context.Context) ([]UnpublishObligation, error) {
-	return s.list(ctx, allKeys)
-}
-
-func (s *Store) list(ctx context.Context, filter string) ([]UnpublishObligation, error) {
-	items, _, err := s.s.Snapshot(ctx, filter)
-	if err != nil {
-		return nil, err
-	}
-	out := make([]UnpublishObligation, 0, len(items))
-	for _, item := range items {
-		out = append(out, item.Value)
-	}
-	return out, nil
-}
-
-// Close removes o. Removing one already gone is not an error.
-func (s *Store) Close(ctx context.Context, o UnpublishObligation) error {
-	key, err := Key(o)
-	if err != nil {
-		return err
-	}
-	return s.s.Delete(ctx, key)
-}
-
-// RecordFailure counts a failed settlement on o and keeps its error, so an
-// operator listing the obligations sees why it is still open.
-func (s *Store) RecordFailure(ctx context.Context, o UnpublishObligation, cause error) error {
-	key, err := Key(o)
-	if err != nil {
-		return err
-	}
-	err = s.s.Mutate(ctx, key, func(cur *UnpublishObligation) (bool, error) {
-		cur.Attempts++
-		cur.Error = cause.Error()
-		return true, nil
-	})
-	if errors.Is(err, kvstore.ErrNotFound) {
-		return nil
-	}
-	return err
-}
-
-// ListOpen reads the open obligations for an operator tool without creating
-// the bucket: a cluster that has never force-detached a volume has none, and
-// that is not an error.
-func ListOpen(ctx context.Context, nc *nats.Conn) ([]UnpublishObligation, error) {
-	js, err := jetstream.New(nc)
-	if err != nil {
-		return nil, fmt.Errorf("jetstream: %w", err)
-	}
-	if _, err := js.KeyValue(ctx, Bucket); errors.Is(err, jetstream.ErrBucketNotFound) {
-		return nil, nil
-	} else if err != nil {
-		return nil, fmt.Errorf("unpublish obligation bucket: %w", err)
-	}
-	cfg := bucketConfig()
-	cfg.AttachOnly = true
-	cfg.OnOpen = nil
-	return (&Store{s: kvstore.New[UnpublishObligation](js, cfg)}).List(ctx)
-}

spinifex/domains/ec2/ebs/obligation/obligation_test.go

@@ -1,141 +0,0 @@
-package ebsobligation_test
-
-import (
-	"errors"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func newStore(t *testing.T) *ebsobligation.Store {
-	t.Helper()
-	_, _, js := testutil.StartTestJetStream(t)
-	store := ebsobligation.NewStore(js)
-	require.NoError(t, store.Open(t.Context()))
-	return store
-}
-
-func obligation(node, volumeID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
-	return ebsobligation.UnpublishObligation{
-		VolumeID: volumeID, AccountID: "123456789012", NodeID: node,
-		InstanceID: "i-1", AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
-	}
-}
-
-func TestKey_NamesNodeVolumeAndAttachment(t *testing.T) {
-	at := time.Unix(1_700_000_000, 5)
-	key, err := ebsobligation.Key(obligation("node-a", "vol-1", at))
-	require.NoError(t, err)
-	assert.Equal(t, "node-a.vol-1.1700000000000000005", key)
-
-	key, err = ebsobligation.Key(obligation("node-a", "vol-1", time.Time{}))
-	require.NoError(t, err)
-	assert.Equal(t, "node-a.vol-1.0", key, "an unknown attachment time is 0, not a huge negative")
-}
-
-// A segment holding a subject separator or wildcard would split the key or
-// let one node's listing match another's.
-func TestKey_RejectsSegmentsThatAreNotOneToken(t *testing.T) {
-	for _, o := range []ebsobligation.UnpublishObligation{
-		obligation("node.a", "vol-1", time.Time{}),
-		obligation("node-a", "vol.1", time.Time{}),
-		obligation("*", "vol-1", time.Time{}),
-		obligation("node-a", ">", time.Time{}),
-		obligation("", "vol-1", time.Time{}),
-	} {
-		_, err := ebsobligation.Key(o)
-		assert.Error(t, err, "%s/%s", o.NodeID, o.VolumeID)
-	}
-}
-
-func TestStore_RecordIsIdempotentPerAttachment(t *testing.T) {
-	store := newStore(t)
-	at := time.Unix(1_700_000_000, 0).UTC()
-	first := obligation("node-a", "vol-1", at)
-	first.Handle = "h1"
-	require.NoError(t, store.Record(t.Context(), first))
-	retry := first
-	retry.RecordedAt = at.Add(time.Hour)
-	require.NoError(t, store.Record(t.Context(), retry), "a retry of the same obligation is not an error")
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at.Add(time.Second))),
-		"a different attachment of the same volume is a different obligation")
-
-	open, err := store.List(t.Context())
-	require.NoError(t, err)
-	require.Len(t, open, 2)
-	for _, o := range open {
-		assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
-		if o.AttachedAt.Equal(at) {
-			assert.True(t, o.RecordedAt.Equal(at.Add(time.Minute)), "the first write is kept")
-		}
-	}
-}
-
-// The bucket's version stamp is a key in the same bucket; listing must not
-// return it as an obligation.
-func TestStore_ListSkipsTheVersionStampAndListNodeIsolatesNodes(t *testing.T) {
-	store := newStore(t)
-	at := time.Unix(1_700_000_000, 0).UTC()
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at)))
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-2", at)))
-	require.NoError(t, store.Record(t.Context(), obligation("node-ab", "vol-3", at)))
-
-	all, err := store.List(t.Context())
-	require.NoError(t, err)
-	assert.Len(t, all, 3)
-
-	mine, err := store.ListNode(t.Context(), "node-a")
-	require.NoError(t, err)
-	volumes := []string{}
-	for _, o := range mine {
-		volumes = append(volumes, o.VolumeID)
-	}
-	assert.ElementsMatch(t, []string{"vol-1", "vol-2"}, volumes, "a node whose name prefixes another's sees only its own")
-
-	_, err = store.ListNode(t.Context(), "*")
-	assert.Error(t, err)
-}
-
-func TestStore_CloseAndRecordFailure(t *testing.T) {
-	store := newStore(t)
-	o := obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())
-	require.NoError(t, store.Record(t.Context(), o))
-
-	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("first")))
-	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("second")))
-	open, err := store.List(t.Context())
-	require.NoError(t, err)
-	require.Len(t, open, 1)
-	assert.Equal(t, 2, open[0].Attempts)
-	assert.Equal(t, "second", open[0].Error)
-
-	require.NoError(t, store.Close(t.Context(), o))
-	require.NoError(t, store.Close(t.Context(), o), "closing one already gone is not an error")
-	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("late")),
-		"a failure recorded after another sweep closed it does not resurrect it")
-	open, err = store.List(t.Context())
-	require.NoError(t, err)
-	assert.Empty(t, open)
-}
-
-func TestListOpen_MissingBucketIsEmptyAndNotCreated(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-
-	open, err := ebsobligation.ListOpen(t.Context(), nc)
-	require.NoError(t, err)
-	assert.Empty(t, open)
-	_, err = js.KeyValue(t.Context(), ebsobligation.Bucket)
-	assert.Error(t, err, "an operator listing must not create the bucket")
-
-	store := ebsobligation.NewStore(js)
-	require.NoError(t, store.Open(t.Context()))
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())))
-	open, err = ebsobligation.ListOpen(t.Context(), nc)
-	require.NoError(t, err)
-	require.Len(t, open, 1)
-	assert.Equal(t, "vol-1", open[0].VolumeID)
-}

spinifex/domains/ec2/eip/service_impl.go

@@ -879,7 +879,8 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco
 // against instanceID, across all accounts. Idempotent backstop for system-VM
 // teardown paths that lose the cached allocation ID — when the VM is already
 // gone from the manager the per-instance release never runs, so an
-// internet-facing ALB's EIP would otherwise orphan. No-op when nothing matches.
+// internet-facing ALB's EIP would otherwise orphan. It touches only addresses
+// tagged as system-managed, and is a no-op when nothing matches.
 func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error {
 	ctx := context.Background()
 	if instanceID == "" {
@@ -908,6 +909,13 @@ func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error {
 		if record.InstanceId != instanceID || record.AllocationId == "" {
 			continue
 		}
+		// Only an address a system VM launch allocated, so a wrong instance ID
+		// can never release a customer's own address.
+		if !awstags.IsSystemManaged(record.Tags[awstags.ManagedByKey]) {
+			slog.Warn("ReleaseAddressByInstanceID: address is not system-managed, left alone",
+				"instanceId", instanceID, "allocationId", record.AllocationId)
+			continue
+		}
 		// Key is "{accountID}.{allocID}"; the account scopes the release calls.
 		accountID, _, found := strings.Cut(k, ".")
 		if !found {

spinifex/domains/ec2/eip/service_impl_test.go

@@ -179,7 +179,12 @@ func TestEIP_ReleaseWhileAssociated(t *testing.T) {
 func TestEIP_ReleaseByInstanceID_ReclaimsAssociatedEIP(t *testing.T) {
 	svc, _, _ := setupTestEIP(t)
 
-	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{
+		TagSpecifications: []*ec2.TagSpecification{{
+			ResourceType: aws.String("elastic-ip"),
+			Tags:         []*ec2.Tag{{Key: aws.String("spinifex:managed-by"), Value: aws.String("elbv2")}},
+		}},
+	}, testAccountID)
 	require.NoError(t, err)
 	allocID := *out.AllocationId
 
@@ -861,3 +866,26 @@ func TestEIP_DisassociateByENI_ClearsEveryEIPOnTheInterface(t *testing.T) {
 	require.NoError(t, err)
 	assert.False(t, found)
 }
+
+func TestEIP_ReleaseByInstanceID_LeavesUntaggedAddress(t *testing.T) {
+	svc, _, _ := setupTestEIP(t)
+
+	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+	require.NoError(t, err)
+	key := testAccountID + "." + *out.AllocationId
+	entry, err := svc.eipKV.Get(t.Context(), key)
+	require.NoError(t, err)
+	var record EIPRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &record))
+	record.State = "associated"
+	record.InstanceId = "i-customer"
+	data, err := json.Marshal(record)
+	require.NoError(t, err)
+	_, err = svc.eipKV.Update(t.Context(), key, data, entry.Revision())
+	require.NoError(t, err)
+
+	require.NoError(t, svc.ReleaseAddressByInstanceID("i-customer"))
+
+	_, err = svc.eipKV.Get(t.Context(), key)
+	assert.NoError(t, err, "an address no system launch allocated is never released by instance ID")
+}

spinifex/domains/ec2/instance/service.go

@@ -141,17 +141,6 @@ type InstanceTagWriter interface {
 	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
 }
 
-// OwedUnpublish names the publish a terminate leaves in place when its unmount
-// did not complete: the node still exporting the volume and the instance it
-// served. The zero value means the unmount completed and nothing is owed.
-type OwedUnpublish struct {
-	NodeID     string
-	InstanceID string
-}
-
-// Owed reports whether a publish is left in place.
-func (o OwedUnpublish) Owed() bool { return o.NodeID != "" }
-
 // VolumeDeleter deletes EBS volumes. Implemented by domains/ec2/volume's
 // VolumeServiceImpl (used by the daemon).
 type VolumeDeleter interface {
@@ -160,16 +149,14 @@ type VolumeDeleter interface {
 	// DeleteVolumeOnTerminate deletes a DeleteOnTermination volume as part of
 	// an instance terminate: it clears any stale attachment (terminate
 	// implies detach) before deleting, so a still-attached boot volume left
-	// over from Stop is not rejected by DeleteVolume's in-use guard. When owed
-	// names a publish, an unpublish obligation is recorded before the clear.
-	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
+	// over from Stop is not rejected by DeleteVolume's in-use guard.
+	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
 
 	// DetachVolumeOnTerminate clears a still-attached volume's attachment on
 	// terminate without deleting it, matching AWS semantics for a
 	// DeleteOnTermination=false volume: terminate still implies detach, it
 	// just leaves the volume behind as available rather than deleting it.
-	// When owed names a publish, an unpublish obligation is recorded first.
-	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
+	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
 }
 
 // VolumeCreator creates the volumes a launch names beyond the root device, and

spinifex/domains/ec2/instance/service_impl.go

@@ -3048,15 +3048,6 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 	// stopped-path delete failure.
 	var lastErr error
 
-	// A volume whose seal failed at stop is still exported on the node that
-	// stopped it, so clearing its attachment owes that node an unpublish.
-	owedFor := func(volumeID string) OwedUnpublish {
-		if instance.LastNode == "" || !slices.Contains(instance.UnpublishOwed, volumeID) {
-			return OwedUnpublish{}
-		}
-		return OwedUnpublish{NodeID: instance.LastNode, InstanceID: instance.ID}
-	}
-
 	for _, ebsRequest := range instance.EBSRequests.Requests {
 		// Internal volumes (EFI) are always cleaned up via ebs.delete.
 		if ebsRequest.EFI {
@@ -3087,7 +3078,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 				continue
 			}
 			// A volume already gone is the detach's own goal, not a failure.
-			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil &&
+			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil &&
 				!awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
 				slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to detach volume", "name", ebsRequest.Name, "err", err)
 				lastErr = err
@@ -3106,7 +3097,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 		// clears a Boot volume, runtime/daemon/vm_adapters.go) before deleting, so
 		// DeleteVolume's in-use guard does not reject an otherwise-legitimate
 		// delete. The resulting error is surfaced into lastErr, not swallowed.
-		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil {
+		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil {
 			slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to delete volume", "name", ebsRequest.Name, "err", err)
 			lastErr = err
 		}

spinifex/domains/ec2/instance/service_impl_test.go

@@ -1734,7 +1734,6 @@ type fakeVolumeDeleter struct {
 	calls    []string
 	deleted  []string
 	detached []string
-	owed     []OwedUnpublish
 	err      error
 }
 
@@ -1750,8 +1749,7 @@ func (f *fakeVolumeDeleter) DeleteVolume(_ context.Context, input *ec2.DeleteVol
 
 // DeleteVolumeOnTerminate mirrors DeleteVolume's call/error bookkeeping; the
 // stopped-instance terminate path calls this instead of DeleteVolume.
-func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
-	f.owed = append(f.owed, owed)
+func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -1763,8 +1761,7 @@ func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID,
 // DetachVolumeOnTerminate mirrors DeleteVolumeOnTerminate's call/error
 // bookkeeping but records into detached instead of deleted; the non-DoT
 // Boot-volume branch calls this instead of deleting.
-func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
-	f.owed = append(f.owed, owed)
+func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -2012,45 +2009,6 @@ func TestTerminateStoppedInstance_UserVolumeDeleted(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, []string{"vol-user-001"}, vd.deleted, "only DeleteOnTermination=true volumes deleted")
 	assert.Equal(t, []string{"vol-keep-001"}, vd.detached, "terminate implies detach even for a volume that survives it")
-	assert.Equal(t, []OwedUnpublish{{}, {}}, vd.owed, "a stop whose unmount completed owes no unpublish")
-}
-
-// A stop whose seal failed left those volumes exported on the node that
-// stopped the instance, so terminate names that node for each of them alone.
-func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *testing.T) {
-	id := "i-seal-failed"
-	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", LastNode: "node-a",
-		UnpublishOwed: []string{"vol-root-001", "vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
-		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
-		{Name: "vol-keep-001", DeleteOnTermination: false},
-		{Name: "vol-sealed-001", DeleteOnTermination: false},
-	}
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
-	vd := &fakeVolumeDeleter{}
-	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
-
-	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
-	require.NoError(t, err)
-
-	owed := OwedUnpublish{NodeID: "node-a", InstanceID: id}
-	assert.Equal(t, []string{"vol-root-001", "vol-keep-001", "vol-sealed-001"}, vd.calls)
-	assert.Equal(t, []OwedUnpublish{owed, owed, {}}, vd.owed)
-}
-
-// With no node recorded there is no node to name; the terminate owes nothing
-// rather than naming an empty node.
-func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
-	id := "i-seal-failed-nonode"
-	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", UnpublishOwed: []string{"vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-keep-001"}}
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
-	vd := &fakeVolumeDeleter{}
-	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
-
-	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
-	require.NoError(t, err)
-	assert.Equal(t, []OwedUnpublish{{}}, vd.owed)
 }
 
 // TestTerminateStoppedInstance_StampsTeardownVolumesDone locks the

spinifex/domains/ec2/instance/start_volumes.go

@@ -18,9 +18,7 @@ import (
 // reconcileStartVolumes drops the data volumes a claimed instance can no longer
 // have, and fails the start when its boot volume is gone.
 func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instance *vm.VM) error {
-	// A direct-boot system guest boots from a kernel, not a root volume, and its
-	// one data volume, if any, is its own store rather than a customer's.
-	if s.metadata == nil || instance.DirectBoot {
+	if s.metadata == nil {
 		return nil
 	}
 

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -123,20 +123,3 @@ func TestReconcileStartVolumes_ReattachesStillAvailableUnattachedVolume(t *testi
 	assert.Equal(t, []string{"vol-root", "vol-released"}, requestedNames(inst))
 	assert.Equal(t, []string{"vol-root", "vol-released"}, mappedNames(inst))
 }
-
-// ALB and SQS guests boot from a kernel with no root volume, so a start or a
-// recovery must not demand one; an SQS store's data volume stays named.
-func TestReconcileStartVolumes_DirectBootNeedsNoRootVolume(t *testing.T) {
-	t.Parallel()
-	svc := startVolumesService(t)
-
-	alb := &vm.VM{ID: "i-alb", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
-		Instance: &ec2.Instance{InstanceId: aws.String("i-alb")}}
-	require.NoError(t, svc.reconcileStartVolumes(context.Background(), alb))
-
-	sqs := &vm.VM{ID: "i-sqs", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
-		Instance: &ec2.Instance{InstanceId: aws.String("i-sqs")}}
-	sqs.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
-	require.NoError(t, svc.reconcileStartVolumes(context.Background(), sqs))
-	assert.Equal(t, []string{"vol-store-gone"}, requestedNames(sqs), "a system store volume is never pruned")
-}

spinifex/domains/ec2/volume/account_termination_current_behaviour_test.go

@@ -9,7 +9,6 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -60,7 +59,7 @@ func TestCurrentBehaviour_AccountTermination_SnapshottedDeleteOnTerminationVolum
 	require.NoError(t, err)
 	require.NoError(t, volumes.UpdateVolumeState(accountID, volumeID, "in-use", instanceID, "/dev/sda1"))
 
-	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID, ec2instance.OwedUnpublish{})
+	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID)
 	assert.EqualError(t, err, awserrors.ErrorVolumeInUse, "the snapshot index refuses the terminate delete")
 
 	survivor, err := metadata.GetVolume(ctx, accountID, volumeID)

spinifex/domains/ec2/volume/delete_volume_unreadable_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -195,7 +194,7 @@ func TestDeleteVolumeOnTerminate_DeleteReadFailureIsNotSuccess(t *testing.T) {
 	svc.store = failing
 	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
 
-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
 
 	require.Error(t, err, "a delete-read failure after a successful attachment clear must not be reported as success")
 	assert.False(t, awserrors.IsNotFound(err),

spinifex/domains/ec2/volume/force_detach_test.go

@@ -20,7 +20,6 @@ func newForceDetachService(t *testing.T) *VolumeServiceImpl {
 	t.Helper()
 	svc := newTestVolumeService("ap-southeast-2a")
 	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-stuck0000000000": "node-a"}}, "node-a", "node-b")
 	return svc
 }
 
@@ -48,11 +47,7 @@ func TestForceDetachVolumeClearsTheAttachment(t *testing.T) {
 	// The previous instance is reported back: it is the only record of what
 	// the volume was torn away from.
 	assert.Equal(t, "i-stuck0000000000", aws.StringValue(attachment.InstanceId))
-	// The export is still up on the node that served it, so the attachment is
-	// reported mid-teardown, as AWS reports a detach it has not finished.
-	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
-	assert.Equal(t, "/dev/sdf", aws.StringValue(attachment.Device))
-	assert.False(t, aws.TimeValue(attachment.AttachTime).IsZero())
+	assert.Equal(t, "detached", aws.StringValue(attachment.State))
 
 	meta, err := svc.GetVolumeMetadata("000000000011", "vol-forcedetach1")
 	require.NoError(t, err)
@@ -117,8 +112,4 @@ func TestForceDetachVolumeOnAnUnattachedVolume(t *testing.T) {
 	require.NoError(t, err)
 
 	assert.Empty(t, aws.StringValue(attachment.InstanceId))
-	assert.Equal(t, "detached", aws.StringValue(attachment.State))
-	open, err := svc.obligations.List(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, open, "an unattached volume has no publish to undo")
 }

spinifex/domains/ec2/volume/service_impl.go

@@ -20,7 +20,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -60,12 +59,6 @@ type VolumeServiceImpl struct {
 	snapshotKV jetstream.KeyValue
 	provider   ebsprovider.EBSProvider
 	metadata   *ebsmetadata.Store
-
-	// The unpublish obligations a cleared attachment records, and what names
-	// the node holding the publish. Unset, a detach that leaves one fails.
-	obligations   *ebsobligation.Store
-	instanceNodes InstanceNodeLocator
-	members       func() []string
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane. It
@@ -1003,23 +996,8 @@ func (s *VolumeServiceImpl) UpdateVolumeState(accountID, volumeID, state, attach
 	return nil
 }
 
-// ModifyVolume modifies an EBS volume (grow-only, refused while attached).
+// ModifyVolume modifies an EBS volume (grow-only, requires stopped instance).
 func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID string) (*ec2.ModifyVolumeOutput, error) {
-	return s.modifyVolume(ctx, input, accountID, "")
-}
-
-// ExpandAttachedToStopped grows a volume still attached to instanceID, which the
-// caller has stopped and holds exclusively. Not reachable from the AWS API.
-func (s *VolumeServiceImpl) ExpandAttachedToStopped(ctx context.Context, accountID, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error) {
-	if instanceID == "" {
-		return nil, errors.New(awserrors.ErrorMissingParameter)
-	}
-	return s.modifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: aws.String(volumeID), Size: aws.Int64(sizeGiB)}, accountID, instanceID)
-}
-
-// modifyVolume grows a volume. An empty stoppedOwner refuses any attached
-// volume; otherwise only a volume attached to that instance is accepted.
-func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID, stoppedOwner string) (*ec2.ModifyVolumeOutput, error) {
 	if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
 		return nil, errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
 	}
@@ -1039,9 +1017,7 @@ func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyV
 	if input.Size == nil || *input.Size <= originalSize {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
-	attached := meta.AttachedInstance != "" && meta.State == "in-use"
-	if meta.State == string(ebsprovider.VolumeStateCreating) || attached != (stoppedOwner != "") ||
-		(attached && meta.AttachedInstance != stoppedOwner) {
+	if (meta.AttachedInstance != "" && meta.State == "in-use") || meta.State == string(ebsprovider.VolumeStateCreating) {
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 	originalType := meta.VolumeType
@@ -1107,7 +1083,7 @@ func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyV
 // terminating instance's QEMU has already been asked to shut down — so this
 // is always a metadata-only clear, never a QMP call. Errors from either step
 // are returned, not swallowed.
-func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
+func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error {
 	// Clearing the attachment would overwrite a recorded deletion, so a volume
 	// already owed is left alone: the reclaimer owns it from here, and resetting
 	// it to available would let a second teardown run against the same data.
@@ -1115,9 +1091,6 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 	if err == nil && meta.State == volumeStateDeleting {
 		return nil
 	}
-	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
-		return err
-	}
 
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
@@ -1137,10 +1110,7 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 // available rather than deleting it. Metadata-only, no QMP — same rationale
 // as DeleteVolumeOnTerminate: there is no live QEMU to hot-unplug on either
 // terminate path.
-func (s *VolumeServiceImpl) DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
-	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
-		return err
-	}
+func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID, accountID string) error {
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
 		// contract, so absence here is success, not a teardown failure.
@@ -1159,12 +1129,8 @@ func (s *VolumeServiceImpl) DetachVolumeOnTerminate(ctx context.Context, volumeI
 // The ordinary DetachVolume routes to ec2.cmd.{instanceID} for the QMP unplug,
 // so a volume attached to an instance whose host stopped answering can never be
 // detached and therefore never deleted. This exists for that deadlock and for
-// nothing else.
-//
-// That host still exports the volume, and the unpublish has to go to it, so
-// it cannot be made here. The request records an unpublish obligation naming
-// the publish before it clears the attachment, and returns; the node settles
-// it once it answers. A failed record clears nothing.
+// nothing else: it leaves a live guest holding a device the control plane no
+// longer believes in, which is only safe once that guest is being destroyed.
 func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
 	if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -1188,48 +1154,18 @@ func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.De
 	}
 
 	previous := meta.AttachedInstance
-	if previous != "" {
-		nodes, err := s.publishingNodes(previous)
-		if err != nil {
-			slog.ErrorContext(ctx, "ForceDetachVolume: cannot name the node holding the publish, clearing nothing",
-				"volumeId", volumeID, "instanceId", previous, "err", err)
-			return nil, errors.New(awserrors.ErrorInternalError)
-		}
-		if err := s.recordObligations(ctx, meta, nodes); err != nil {
-			slog.ErrorContext(ctx, "ForceDetachVolume: failed to record the unpublish obligation, clearing nothing",
-				"volumeId", volumeID, "instanceId", previous, "err", err)
-			return nil, errors.New(awserrors.ErrorInternalError)
-		}
-	}
-
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		slog.ErrorContext(ctx, "ForceDetachVolume: failed to clear attachment", "volumeId", volumeID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-
-	if previous == "" {
-		return &ec2.VolumeAttachment{
-			VolumeId: aws.String(volumeID),
-			State:    aws.String("detached"),
-		}, nil
-	}
-	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared, unpublish left to the node holding it",
+	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared in the control plane only",
 		"volumeId", volumeID, "previousInstance", previous, "accountId", accountID)
 
-	// AWS answers a detach with the attachment it is tearing down, in state
-	// detaching; the export here goes when the node settles the obligation.
-	attachment := &ec2.VolumeAttachment{
+	return &ec2.VolumeAttachment{
 		VolumeId:   aws.String(volumeID),
 		InstanceId: aws.String(previous),
-		State:      aws.String("detaching"),
-	}
-	if meta.DeviceName != "" {
-		attachment.Device = aws.String(meta.DeviceName)
-	}
-	if !meta.AttachedAt.IsZero() {
-		attachment.AttachTime = aws.Time(meta.AttachedAt)
-	}
-	return attachment, nil
+		State:      aws.String("detached"),
+	}, nil
 }
 
 // DeleteVolume records a durable obligation to destroy an EBS volume and

spinifex/domains/ec2/volume/service_impl_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit/ebsfake"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
@@ -1469,60 +1468,6 @@ func TestModifyVolume_AttachedInUse(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorIncorrectState, err.Error())
 }
 
-func TestExpandAttachedToStopped_GrowsTheNamedInstancesVolume(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
-	createVolumeInStoreWithMeta(t, svc, store, "vol-stopped", ebsmetadata.Volume{
-		VolumeID:         "vol-stopped",
-		CapacityGiB:      20,
-		State:            "in-use",
-		AttachedInstance: "i-owner",
-		DeviceName:       "/dev/sdf",
-	})
-
-	out, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-stopped", "i-owner", 30)
-	require.NoError(t, err)
-	assert.Equal(t, int64(30), aws.Int64Value(out.VolumeModification.TargetSize))
-
-	meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-stopped")
-	require.NoError(t, err)
-	assert.Equal(t, uint64(30), meta.CapacityGiB)
-	assert.Equal(t, "in-use", meta.State, "the grow must not change the attachment")
-	assert.Equal(t, "i-owner", meta.AttachedInstance)
-	assert.Equal(t, "/dev/sdf", meta.DeviceName)
-}
-
-func TestExpandAttachedToStopped_Refusals(t *testing.T) {
-	cases := []struct {
-		name     string
-		meta     ebsmetadata.Volume
-		instance string
-		want     string
-	}{
-		{"attached to another instance", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-other"}, "i-owner", awserrors.ErrorIncorrectState},
-		{"not attached", ebsmetadata.Volume{State: "available"}, "i-owner", awserrors.ErrorIncorrectState},
-		{"still creating", ebsmetadata.Volume{State: string(ebsprovider.VolumeStateCreating)}, "i-owner", awserrors.ErrorIncorrectState},
-		{"no instance named", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-owner"}, "", awserrors.ErrorMissingParameter},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			store := objectstore.NewMemoryObjectStore()
-			svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
-			tc.meta.VolumeID = "vol-x"
-			tc.meta.CapacityGiB = 20
-			createVolumeInStoreWithMeta(t, svc, store, "vol-x", tc.meta)
-
-			_, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-x", tc.instance, 30)
-			require.Error(t, err)
-			assert.Equal(t, tc.want, err.Error())
-
-			meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-x")
-			require.NoError(t, err)
-			assert.Equal(t, uint64(20), meta.CapacityGiB, "a refused grow must leave the size unchanged")
-		})
-	}
-}
-
 func TestModifyVolume_SuccessfulGrow(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
@@ -1844,7 +1789,7 @@ func TestDeleteVolumeOnTerminate_ClearsAttachmentThenDeletes(t *testing.T) {
 		AttachedInstance: "i-stopped",
 	})
 
-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
 	require.NoError(t, err, "terminate implies detach: a stale attachment must not block the terminate delete")
 
 	deleting, err := svc.GetVolumeMetadata(testVolAccountID, volumeID)
@@ -1871,7 +1816,7 @@ func TestDeleteVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc.snapshotKV = kv
 
 	// No metadata doc is ever written for this volume ID.
-	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }
 
@@ -1884,7 +1829,7 @@ func TestDetachVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
 	svc.snapshotKV = kv
 
-	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }
 
@@ -1968,7 +1913,7 @@ func TestDeleteVolume_SecondDeleteIsAcceptedWithoutASecondTeardown(t *testing.T)
 
 	// DeleteOnTermination arriving for a volume already owed must not reset it
 	// to available, which would let a second teardown through.
-	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{}))
+	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID))
 	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
 	require.NoError(t, err)
 	assert.Equal(t, "deleting", meta.State)
@@ -2016,7 +1961,7 @@ func TestDeleteVolumeOnTerminate_SurfacesDeleteFailure(t *testing.T) {
 	_, err = kv.Put(t.Context(), volumeID, snapData)
 	require.NoError(t, err)
 
-	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
+	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
 	require.Error(t, err, "a DeleteVolume failure must be surfaced, not swallowed")
 	assert.Contains(t, err.Error(), awserrors.ErrorVolumeInUse)
 

spinifex/domains/ec2/volume/unpublish_obligation.go

@@ -1,365 +0,0 @@
-package volume
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-	"time"
-
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/metric"
-)
-
-// InstanceNodeLocator reports the node that last ran an instance. found is
-// false when no record of the instance exists.
-type InstanceNodeLocator interface {
-	InstanceLastNode(instanceID string) (node string, found bool, err error)
-}
-
-// SetUnpublishObligations wires the obligation store a cleared attachment
-// records into, the locator that names the node holding the publish, and the
-// cluster members to name when the instance has no record.
-func (s *VolumeServiceImpl) SetUnpublishObligations(store *ebsobligation.Store, locator InstanceNodeLocator, members func() []string) {
-	s.obligations = store
-	s.instanceNodes = locator
-	s.members = members
-}
-
-// publishingNodes names the node holding instanceID's publish. With no record
-// of the instance there is no node to name, so every member is named and each
-// settles its own.
-func (s *VolumeServiceImpl) publishingNodes(instanceID string) ([]string, error) {
-	if s.instanceNodes == nil {
-		return nil, errors.New("no instance node locator configured")
-	}
-	node, found, err := s.instanceNodes.InstanceLastNode(instanceID)
-	if err != nil {
-		return nil, fmt.Errorf("read instance %s: %w", instanceID, err)
-	}
-	if found && node != "" {
-		return []string{node}, nil
-	}
-	var members []string
-	if s.members != nil {
-		members = s.members()
-	}
-	if len(members) == 0 {
-		return nil, fmt.Errorf("instance %s has no record and no cluster member is known", instanceID)
-	}
-	return members, nil
-}
-
-// recordObligations writes one obligation per node for the attachment meta
-// still carries. Every write must land before the caller clears it.
-func (s *VolumeServiceImpl) recordObligations(ctx context.Context, meta ebsmetadata.Volume, nodes []string) error {
-	if s.obligations == nil {
-		return errors.New("no unpublish obligation store configured")
-	}
-	now := time.Now()
-	for _, node := range nodes {
-		if err := s.obligations.Record(ctx, ebsobligation.UnpublishObligation{
-			VolumeID:   meta.VolumeID,
-			AccountID:  meta.TenantID,
-			Handle:     meta.ProviderHandle,
-			NodeID:     node,
-			InstanceID: meta.AttachedInstance,
-			AttachedAt: meta.AttachedAt,
-			RecordedAt: now,
-		}); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// recordTerminateObligation records the obligation for a terminate that clears
-// owed's attachment with its publish still in place. A volume no longer
-// attached to that instance has nothing of it left to clear.
-func (s *VolumeServiceImpl) recordTerminateObligation(ctx context.Context, accountID, volumeID string, owed ec2instance.OwedUnpublish) error {
-	if !owed.Owed() {
-		return nil
-	}
-	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
-	if objectstore.IsNoSuchKeyError(err) {
-		return nil
-	}
-	if err != nil {
-		return fmt.Errorf("read volume %s before recording its unpublish obligation: %w", volumeID, err)
-	}
-	if meta.AttachedInstance != owed.InstanceID {
-		return nil
-	}
-	if err := s.recordObligations(ctx, meta, []string{owed.NodeID}); err != nil {
-		return fmt.Errorf("record unpublish obligation for %s: %w", volumeID, err)
-	}
-	return nil
-}
-
-// ObligationUnpublisher is this node's mounter as the obligation reaper drives
-// it. Its lock is the one every publish and unpublish on this node takes.
-type ObligationUnpublisher interface {
-	// HoldVolume takes volumeID's lock without waiting. false means a publish
-	// or unpublish of it is in flight here, and the sweep comes back later.
-	HoldVolume(volumeID string) (VolumeHold, bool)
-}
-
-// VolumeHold is one volume's publish lock on this node.
-type VolumeHold interface {
-	// PublishedSince reports whether this process published the volume after
-	// t and has not unpublished it since.
-	PublishedSince(t time.Time) bool
-	// Unpublish tears down this node's export of the volume. A volume the
-	// provider holds no export of counts as done.
-	Unpublish(ctx context.Context, accountID, handle string, noSeal bool) error
-	Release()
-}
-
-// obligationAction is what settling an obligation does with the volume as it
-// is now.
-type obligationAction int
-
-const (
-	// obligationWait leaves the obligation open: the attachment it names has
-	// not been cleared yet, or a publish here is newer than it.
-	obligationWait obligationAction = iota
-	// obligationSuperseded closes it without a provider call: a newer
-	// attachment owns the export.
-	obligationSuperseded
-	// obligationSeal unpublishes with a seal: nobody else writes the volume,
-	// so the copy here is the last owner's and its writes are kept.
-	obligationSeal
-	// obligationNoSeal unpublishes without sealing a volume being deleted.
-	obligationNoSeal
-)
-
-func (a obligationAction) String() string {
-	switch a {
-	case obligationSuperseded:
-		return "superseded"
-	case obligationSeal:
-		return "seal"
-	case obligationNoSeal:
-		return "no-seal"
-	default:
-		return "wait"
-	}
-}
-
-// decideObligation is the settlement policy over the volume's current record;
-// meta is nil when the volume has none. It never unpublishes an attachment the
-// volume still carries.
-func decideObligation(o ebsobligation.UnpublishObligation, meta *ebsmetadata.Volume) obligationAction {
-	if meta == nil || meta.State == volumeStateDeleting {
-		return obligationNoSeal
-	}
-	if meta.AttachedInstance != "" {
-		if meta.AttachedInstance != o.InstanceID || meta.AttachedAt.After(o.AttachedAt) {
-			return obligationSuperseded
-		}
-		return obligationWait
-	}
-	return obligationSeal
-}
-
-// obligationSince is the time a publish on this node must follow to be newer
-// than the one an obligation undoes.
-func obligationSince(o ebsobligation.UnpublishObligation) time.Time {
-	if o.AttachedAt.IsZero() {
-		return o.RecordedAt
-	}
-	return o.AttachedAt
-}
-
-// unpublishObligationReaper settles the obligations naming its node through
-// that node's mounter, once the node answers.
-type unpublishObligationReaper struct {
-	svc         *VolumeServiceImpl
-	node        string
-	unpublisher ObligationUnpublisher
-}
-
-var _ vm.Reaper = (*unpublishObligationReaper)(nil)
-
-// NewUnpublishObligationReaper builds the reaper that settles node's
-// obligations through unpublisher, which must be the node's own mounter.
-func (s *VolumeServiceImpl) NewUnpublishObligationReaper(node string, unpublisher ObligationUnpublisher) vm.Reaper {
-	return &unpublishObligationReaper{svc: s, node: node, unpublisher: unpublisher}
-}
-
-func (r *unpublishObligationReaper) Class() string { return "ebs-unpublish-obligation" }
-
-// Scope is node-local: an obligation is settled only by the node it names,
-// because only that node can tear its own export down.
-func (r *unpublishObligationReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
-
-// Sweep settles what it can and reports how many obligations it closed. A
-// failure is recorded on the obligation and retried on the next sweep.
-func (r *unpublishObligationReaper) Sweep(ctx context.Context) (int, error) {
-	if r.svc.obligations == nil || r.unpublisher == nil {
-		return 0, nil
-	}
-	open, err := r.svc.obligations.ListNode(ctx, r.node)
-	if err != nil {
-		return 0, err
-	}
-	closed := 0
-	for _, o := range open {
-		if err := ctx.Err(); err != nil {
-			return closed, err
-		}
-		done, err := r.settle(ctx, o)
-		if err != nil {
-			slog.WarnContext(ctx, "ebs-unpublish-obligation: settlement failed, retrying next sweep",
-				"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attempts", o.Attempts+1, "err", err)
-			if recErr := r.svc.obligations.RecordFailure(ctx, o, err); recErr != nil {
-				slog.WarnContext(ctx, "ebs-unpublish-obligation: could not record the failure",
-					"volumeId", o.VolumeID, "node", o.NodeID, "err", recErr)
-			}
-			continue
-		}
-		if done {
-			closed++
-		}
-	}
-	return closed, nil
-}
-
-// settle acts on one obligation under the volume's lock, so no publish on this
-// node falls between reading the volume and unpublishing it.
-func (r *unpublishObligationReaper) settle(ctx context.Context, o ebsobligation.UnpublishObligation) (bool, error) {
-	hold, ok := r.unpublisher.HoldVolume(o.VolumeID)
-	if !ok {
-		return false, nil
-	}
-	defer hold.Release()
-
-	var current *ebsmetadata.Volume
-	meta, err := r.svc.metadata.GetVolume(ctx, o.AccountID, o.VolumeID)
-	switch {
-	case err == nil:
-		current = &meta
-	case objectstore.IsNoSuchKeyError(err):
-	default:
-		return false, fmt.Errorf("read volume %s: %w", o.VolumeID, err)
-	}
-
-	action := decideObligation(o, current)
-	// A newer publish here may own the export, so only a superseded obligation,
-	// which sends nothing, closes past it.
-	if action != obligationSuperseded && hold.PublishedSince(obligationSince(o)) {
-		return false, nil
-	}
-	switch action {
-	case obligationWait:
-		return false, nil
-	case obligationSuperseded:
-		slog.WarnContext(ctx, "ebs-unpublish-obligation: a newer attachment owns the volume, closing without unpublishing",
-			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attachedInstance", current.AttachedInstance)
-	case obligationSeal, obligationNoSeal:
-		if err := hold.Unpublish(ctx, o.AccountID, o.Handle, action == obligationNoSeal); err != nil {
-			return false, err
-		}
-		slog.InfoContext(ctx, "ebs-unpublish-obligation: unpublished the export a cleared attachment left",
-			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "noSeal", action == obligationNoSeal)
-	}
-	if err := r.svc.obligations.Close(ctx, o); err != nil {
-		return false, fmt.Errorf("close obligation: %w", err)
-	}
-	recordObligationClosed(o.NodeID, action.String())
-	return true, nil
-}
-
-// NodeDeparted reports whether node has left the cluster, rather than being
-// down, stale or partitioned, which must not count.
-type NodeDeparted func(ctx context.Context, node string) (bool, error)
-
-// unpublishObligationCloser closes, without a provider call, the obligations of
-// nodes removed from the cluster: such a node holds nothing a member can reach.
-type unpublishObligationCloser struct {
-	svc      *VolumeServiceImpl
-	departed NodeDeparted
-}
-
-var _ vm.Reaper = (*unpublishObligationCloser)(nil)
-
-// NewUnpublishObligationCloser builds the cluster-wide pass that closes the
-// obligations of nodes departed reports as gone.
-func (s *VolumeServiceImpl) NewUnpublishObligationCloser(departed NodeDeparted) vm.Reaper {
-	return &unpublishObligationCloser{svc: s, departed: departed}
-}
-
-func (c *unpublishObligationCloser) Class() string { return "ebs-unpublish-obligation-close" }
-
-// Scope is cluster-wide: membership is the cluster's, and one pass is enough.
-func (c *unpublishObligationCloser) Scope() vm.ReaperScope { return vm.ScopeClusterWide }
-
-func (c *unpublishObligationCloser) Sweep(ctx context.Context) (int, error) {
-	if c.svc.obligations == nil || c.departed == nil {
-		return 0, nil
-	}
-	open, err := c.svc.obligations.List(ctx)
-	if err != nil {
-		return 0, err
-	}
-	gone := map[string]bool{}
-	closed := 0
-	for _, o := range open {
-		departed, seen := gone[o.NodeID]
-		if !seen {
-			departed, err = c.departed(ctx, o.NodeID)
-			if err != nil {
-				slog.WarnContext(ctx, "ebs-unpublish-obligation: cannot tell whether a node left the cluster, keeping its obligations",
-					"node", o.NodeID, "err", err)
-				departed = false
-			}
-			gone[o.NodeID] = departed
-		}
-		if !departed {
-			continue
-		}
-		if err := c.svc.obligations.Close(ctx, o); err != nil {
-			slog.WarnContext(ctx, "ebs-unpublish-obligation: failed to close a departed node's obligation",
-				"volumeId", o.VolumeID, "node", o.NodeID, "err", err)
-			continue
-		}
-		slog.WarnContext(ctx, "ebs-unpublish-obligation: closed for a node removed from the cluster",
-			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID)
-		recordObligationClosed(o.NodeID, "departed")
-		closed++
-	}
-	return closed, nil
-}
-
-const obligationMeterName = "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume/obligation"
-
-var (
-	obligationClosedOnce    sync.Once
-	obligationClosedCounter metric.Int64Counter
-)
-
-// recordObligationClosed counts one closed obligation by the node it named
-// and how it closed.
-func recordObligationClosed(node, outcome string) {
-	obligationClosedOnce.Do(func() {
-		var err error
-		obligationClosedCounter, err = otel.Meter(obligationMeterName).Int64Counter("mulga.ebs.unpublish_obligation.closed",
-			metric.WithDescription("Unpublish obligations closed, by the node they named and how they closed."),
-			metric.WithUnit("{obligation}"))
-		if err != nil {
-			otel.Handle(err)
-		}
-	})
-	if obligationClosedCounter == nil {
-		return
-	}
-	obligationClosedCounter.Add(context.Background(), 1, metric.WithAttributes(
-		attribute.String("node", node), attribute.String("outcome", outcome)))
-}

spinifex/domains/ec2/volume/unpublish_obligation_test.go

@@ -1,596 +0,0 @@
-package volume
-
-//test:in-package — the obligation reaper and its policy are unexported, and
-// seeding a volume's metadata document is how an attachment is set up.
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeNodeLocator struct {
-	nodes map[string]string
-	err   error
-}
-
-var _ InstanceNodeLocator = (*fakeNodeLocator)(nil)
-
-func (f *fakeNodeLocator) InstanceLastNode(instanceID string) (string, bool, error) {
-	if f.err != nil {
-		return "", false, f.err
-	}
-	node, found := f.nodes[instanceID]
-	return node, found, nil
-}
-
-// wireObligations gives svc an obligation bucket on a JetStream of its own and
-// returns the connection under it, so a test can cut it.
-func wireObligations(t *testing.T, svc *VolumeServiceImpl, locator InstanceNodeLocator, members ...string) *nats.Conn {
-	t.Helper()
-	_, nc, js := testutil.StartTestJetStream(t)
-	store := ebsobligation.NewStore(js)
-	require.NoError(t, store.Open(t.Context()))
-	svc.SetUnpublishObligations(store, locator, func() []string { return members })
-	return nc
-}
-
-type unpublishCall struct {
-	volumeID, accountID, handle string
-	noSeal                      bool
-}
-
-// fakeUnpublisher stands in for a node's mounter: busy volumes refuse the
-// hold, publishedAt is the last publish this process made of each volume.
-type fakeUnpublisher struct {
-	mu          sync.Mutex
-	busy        map[string]bool
-	publishedAt map[string]time.Time
-	err         error
-	calls       []unpublishCall
-	held        int
-}
-
-var _ ObligationUnpublisher = (*fakeUnpublisher)(nil)
-
-func (f *fakeUnpublisher) HoldVolume(volumeID string) (VolumeHold, bool) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if f.busy[volumeID] {
-		return nil, false
-	}
-	f.held++
-	return &fakeHold{f: f, volumeID: volumeID}, true
-}
-
-func (f *fakeUnpublisher) unpublishes() []unpublishCall {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return append([]unpublishCall(nil), f.calls...)
-}
-
-type fakeHold struct {
-	f        *fakeUnpublisher
-	volumeID string
-}
-
-var _ VolumeHold = (*fakeHold)(nil)
-
-func (h *fakeHold) PublishedSince(t time.Time) bool {
-	h.f.mu.Lock()
-	defer h.f.mu.Unlock()
-	at, ok := h.f.publishedAt[h.volumeID]
-	return ok && at.After(t)
-}
-
-func (h *fakeHold) Unpublish(_ context.Context, accountID, handle string, noSeal bool) error {
-	h.f.mu.Lock()
-	defer h.f.mu.Unlock()
-	h.f.calls = append(h.f.calls, unpublishCall{volumeID: h.volumeID, accountID: accountID, handle: handle, noSeal: noSeal})
-	return h.f.err
-}
-
-func (h *fakeHold) Release() {
-	h.f.mu.Lock()
-	defer h.f.mu.Unlock()
-	h.f.held--
-}
-
-// putAttachment writes volumeID's document with an explicit attachment, so a
-// test controls AttachedAt rather than racing the clock.
-func putAttachment(t *testing.T, svc *VolumeServiceImpl, volumeID, state, instanceID string, attachedAt time.Time) ebsmetadata.Volume {
-	t.Helper()
-	meta := ebsmetadata.Volume{
-		VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 1,
-		State: state, AvailabilityZone: "ap-southeast-2a", VolumeType: "gp3",
-		AttachedInstance: instanceID, AttachedAt: attachedAt, ProviderHandle: "handle-" + volumeID,
-	}
-	if instanceID != "" {
-		meta.DeviceName = "/dev/sdf"
-	}
-	require.NoError(t, svc.metadata.PutVolume(context.Background(), meta))
-	return meta
-}
-
-func readVolume(t *testing.T, svc *VolumeServiceImpl, volumeID string) ebsmetadata.Volume {
-	t.Helper()
-	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
-	require.NoError(t, err)
-	return meta
-}
-
-func openObligations(t *testing.T, svc *VolumeServiceImpl) []ebsobligation.UnpublishObligation {
-	t.Helper()
-	open, err := svc.obligations.List(context.Background())
-	require.NoError(t, err)
-	return open
-}
-
-func forceDetach(t *testing.T, svc *VolumeServiceImpl, volumeID string) (*ec2.VolumeAttachment, error) {
-	t.Helper()
-	return svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{
-		VolumeId: aws.String(volumeID), Force: aws.Bool(true),
-	}, testVolAccountID)
-}
-
-// --- ForceDetachVolume records the obligation ---
-
-func TestForceDetach_RecordsOneObligationNamingThePublish(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a", "node-b")
-	attachedAt := time.Unix(1_700_000_000, 0).UTC()
-	putAttachment(t, svc, "vol-fd-once", "in-use", "i-old", attachedAt)
-
-	_, err := forceDetach(t, svc, "vol-fd-once")
-	require.NoError(t, err)
-
-	open := openObligations(t, svc)
-	require.Len(t, open, 1, "only the node that last ran the instance is named")
-	o := open[0]
-	assert.Equal(t, "vol-fd-once", o.VolumeID)
-	assert.Equal(t, testVolAccountID, o.AccountID)
-	assert.Equal(t, "handle-vol-fd-once", o.Handle)
-	assert.Equal(t, "node-a", o.NodeID)
-	assert.Equal(t, "i-old", o.InstanceID)
-	assert.True(t, attachedAt.Equal(o.AttachedAt), "the obligation names the attachment it undoes")
-	assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
-}
-
-// A caller retrying a force detach whose clear failed writes the same key, so
-// the retry leaves one obligation, not two.
-func TestForceDetach_RetryLeavesOneObligation(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
-	meta := putAttachment(t, svc, "vol-fd-retry", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
-
-	require.NoError(t, svc.recordObligations(context.Background(), meta, []string{"node-a"}))
-	_, err := forceDetach(t, svc, "vol-fd-retry")
-	require.NoError(t, err)
-
-	assert.Len(t, openObligations(t, svc), 1)
-}
-
-// With no record of the instance there is no node to name, so every member is
-// named and each one settles its own.
-func TestForceDetach_UnknownInstanceNamesEveryMember(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b", "node-c")
-	putAttachment(t, svc, "vol-fd-every", "in-use", "i-gone", time.Unix(1_700_000_000, 0).UTC())
-
-	_, err := forceDetach(t, svc, "vol-fd-every")
-	require.NoError(t, err)
-
-	nodes := []string{}
-	for _, o := range openObligations(t, svc) {
-		nodes = append(nodes, o.NodeID)
-	}
-	assert.ElementsMatch(t, []string{"node-a", "node-b", "node-c"}, nodes)
-}
-
-// The attachment is the only record of where the export is. If the obligation
-// cannot be written, clearing it would forget an export, so nothing is cleared.
-func TestForceDetach_ObligationWriteFailureClearsNothing(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	nc := wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
-	putAttachment(t, svc, "vol-fd-nowrite", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	nc.Close()
-
-	_, err := forceDetach(t, svc, "vol-fd-nowrite")
-
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInternalError, err.Error())
-	meta := readVolume(t, svc, "vol-fd-nowrite")
-	assert.Equal(t, "in-use", meta.State)
-	assert.Equal(t, "i-old", meta.AttachedInstance)
-}
-
-func TestForceDetach_UnnameableNodeClearsNothing(t *testing.T) {
-	for name, locator := range map[string]InstanceNodeLocator{
-		"locator fails":            &fakeNodeLocator{err: errors.New("instance bucket unreachable")},
-		"no record and no members": &fakeNodeLocator{},
-		"no locator configured":    nil,
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, locator)
-			putAttachment(t, svc, "vol-fd-noname", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
-
-			_, err := forceDetach(t, svc, "vol-fd-noname")
-
-			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorInternalError, err.Error())
-			assert.Equal(t, "i-old", readVolume(t, svc, "vol-fd-noname").AttachedInstance)
-			assert.Empty(t, openObligations(t, svc))
-		})
-	}
-}
-
-// providerCallCounter fails the test's expectations if the request path makes
-// any provider call: an unpublish waits on a node that may never answer.
-type providerCallCounter struct {
-	ebsprovider.EBSProvider
-
-	mu        sync.Mutex
-	unpublish int
-	publish   int
-}
-
-func (p *providerCallCounter) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
-	p.mu.Lock()
-	p.unpublish++
-	p.mu.Unlock()
-	<-ctx.Done()
-	return nil, ctx.Err()
-}
-
-func (p *providerCallCounter) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
-	p.mu.Lock()
-	p.publish++
-	p.mu.Unlock()
-	<-ctx.Done()
-	return nil, ctx.Err()
-}
-
-// TestS8_ForceDetachRecordsTheObligationAndReturns pins that a force detach
-// never waits on the node holding the export: it decides its errors, records
-// the obligation and returns, and the unpublish is left to that node's reaper.
-func TestS8_ForceDetachRecordsTheObligationAndReturns(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	provider := &providerCallCounter{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
-	svc.SetEBSProvider(provider)
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-wedged": "node-unreachable"}}, "node-unreachable")
-	putAttachment(t, svc, "vol-s8-forcedetach", "in-use", "i-wedged", time.Unix(1_700_000_000, 0).UTC())
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	attachment, err := svc.ForceDetachVolume(ctx, &ec2.DetachVolumeInput{
-		VolumeId: aws.String("vol-s8-forcedetach"), Force: aws.Bool(true),
-	}, testVolAccountID)
-	require.NoError(t, err)
-	require.NoError(t, ctx.Err(), "the request returned on its own, not on the deadline")
-
-	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
-	provider.mu.Lock()
-	assert.Zero(t, provider.unpublish, "the request path makes no unpublish call")
-	assert.Zero(t, provider.publish)
-	provider.mu.Unlock()
-
-	open := openObligations(t, svc)
-	require.Len(t, open, 1, "the unpublish the request did not wait for is recorded")
-	assert.Equal(t, "node-unreachable", open[0].NodeID)
-	assert.Empty(t, readVolume(t, svc, "vol-s8-forcedetach").AttachedInstance)
-}
-
-// --- Terminate paths ---
-
-func TestTerminate_OwedUnpublishRecordsOneObligationBeforeTheClear(t *testing.T) {
-	for name, terminate := range map[string]func(*VolumeServiceImpl, string, ec2instance.OwedUnpublish) error{
-		"detach": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
-			return svc.DetachVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
-		},
-		"delete": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
-			return svc.DeleteVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			svc.snapshotKV = setupTestVolumeKV(t)
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			seedTestProviderVolume(t, svc, "vol-term-owed", 1)
-			putAttachment(t, svc, "vol-term-owed", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
-
-			require.NoError(t, terminate(svc, "vol-term-owed", ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}))
-
-			open := openObligations(t, svc)
-			require.Len(t, open, 1)
-			assert.Equal(t, "node-a", open[0].NodeID)
-			assert.Equal(t, "i-term", open[0].InstanceID)
-		})
-	}
-}
-
-// A terminate whose unmount completed owes nothing, and an attachment that is
-// already some other instance's is not this terminate's to undo.
-func TestTerminate_NothingOwedRecordsNoObligation(t *testing.T) {
-	for name, tc := range map[string]struct {
-		attachedTo string
-		owed       ec2instance.OwedUnpublish
-	}{
-		"unmount completed":   {attachedTo: "i-term", owed: ec2instance.OwedUnpublish{}},
-		"attached to another": {attachedTo: "i-other", owed: ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			putAttachment(t, svc, "vol-term-none", "in-use", tc.attachedTo, time.Unix(1_700_000_000, 0).UTC())
-
-			require.NoError(t, svc.DetachVolumeOnTerminate(context.Background(), "vol-term-none", testVolAccountID, tc.owed))
-
-			assert.Empty(t, openObligations(t, svc))
-		})
-	}
-}
-
-func TestTerminate_ObligationWriteFailureClearsNothing(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	nc := wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-term-nowrite", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
-	nc.Close()
-
-	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-term-nowrite", testVolAccountID,
-		ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"})
-
-	require.Error(t, err)
-	assert.Equal(t, "i-term", readVolume(t, svc, "vol-term-nowrite").AttachedInstance)
-}
-
-// --- The reaper ---
-
-func recordObligation(t *testing.T, svc *VolumeServiceImpl, node, volumeID, instanceID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
-	t.Helper()
-	o := ebsobligation.UnpublishObligation{
-		VolumeID: volumeID, AccountID: testVolAccountID, Handle: "handle-" + volumeID,
-		NodeID: node, InstanceID: instanceID, AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
-	}
-	require.NoError(t, svc.obligations.Record(context.Background(), o))
-	return o
-}
-
-func sweepObligations(t *testing.T, svc *VolumeServiceImpl, node string, unpublisher ObligationUnpublisher) int {
-	t.Helper()
-	closed, err := svc.NewUnpublishObligationReaper(node, unpublisher).Sweep(context.Background())
-	require.NoError(t, err)
-	return closed
-}
-
-// TestS7_ObligationReaperNeverUnpublishesACurrentAttachment pins that the
-// reaper, which runs with no customer asking, never tears down an export a
-// live attachment uses and never removes or rewrites the volume.
-func TestS7_ObligationReaperNeverUnpublishesACurrentAttachment(t *testing.T) {
-	cleared := time.Unix(1_700_000_000, 0).UTC()
-	for name, tc := range map[string]struct {
-		state       string
-		attachedTo  string
-		attachedAt  time.Time
-		publishedAt time.Time // a publish this process made, zero if none
-		stillOpen   bool
-	}{
-		// The volume moved to another instance, possibly on this very node.
-		"attached to another instance": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour)},
-		// The same instance was attached again after the clear.
-		"same instance attached again": {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour)},
-		// The clear has not landed: the obligation was written first.
-		"attachment not yet cleared": {state: "in-use", attachedTo: "i-old", attachedAt: cleared, stillOpen: true},
-		// A hot attach published here and has not yet written in-use.
-		"newer publish here, record not yet in-use": {state: "available", publishedAt: cleared.Add(time.Hour), stillOpen: true},
-		// Force detach, then attach again on this node: the new publish is here.
-		"attached again here after a newer publish": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
-		"same instance attached again here":         {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			before := putAttachment(t, svc, "vol-s7-current", tc.state, tc.attachedTo, tc.attachedAt)
-			recordObligation(t, svc, "node-a", "vol-s7-current", "i-old", cleared)
-			unpublisher := &fakeUnpublisher{publishedAt: map[string]time.Time{}}
-			if !tc.publishedAt.IsZero() {
-				unpublisher.publishedAt["vol-s7-current"] = tc.publishedAt
-			}
-
-			sweepObligations(t, svc, "node-a", unpublisher)
-
-			assert.Empty(t, unpublisher.unpublishes(), "a current attachment's export must never be unpublished")
-			assert.Zero(t, unpublisher.held, "every hold is released")
-			after := readVolume(t, svc, "vol-s7-current")
-			assert.Equal(t, before.State, after.State, "the reaper never rewrites the volume")
-			assert.Equal(t, before.AttachedInstance, after.AttachedInstance)
-			if tc.stillOpen {
-				assert.Len(t, openObligations(t, svc), 1, "the obligation waits for its attachment to be gone")
-			} else {
-				assert.Empty(t, openObligations(t, svc), "a superseded obligation closes without a provider call")
-			}
-		})
-	}
-}
-
-// TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes pins that
-// the reaper keeps a surviving volume's writes and only drops them unsealed
-// for a volume that is being deleted or is gone.
-func TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes(t *testing.T) {
-	cleared := time.Unix(1_700_000_000, 0).UTC()
-	for name, tc := range map[string]struct {
-		state  string // empty for no document
-		noSeal bool
-	}{
-		"available keeps its writes": {state: "available", noSeal: false},
-		"deleting is dropped":        {state: volumeStateDeleting, noSeal: true},
-		"absent is dropped":          {state: "", noSeal: true},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			if tc.state != "" {
-				putAttachment(t, svc, "vol-s7-settle", tc.state, "", time.Time{})
-			}
-			recordObligation(t, svc, "node-a", "vol-s7-settle", "i-old", cleared)
-			unpublisher := &fakeUnpublisher{}
-
-			assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
-
-			assert.Equal(t, []unpublishCall{{volumeID: "vol-s7-settle", accountID: testVolAccountID, handle: "handle-vol-s7-settle", noSeal: tc.noSeal}},
-				unpublisher.unpublishes())
-			assert.Empty(t, openObligations(t, svc))
-			if tc.state != "" {
-				assert.Equal(t, tc.state, readVolume(t, svc, "vol-s7-settle").State, "the reaper never rewrites the volume")
-			}
-		})
-	}
-}
-
-// TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone pins that a read
-// failing for a reason other than absence leaves the obligation open: taking
-// it for a deleted volume would drop live writes unsealed.
-func TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone(t *testing.T) {
-	memory := objectstore.NewMemoryObjectStore()
-	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-s7-unread", "available", "", time.Time{})
-	recordObligation(t, svc, "node-a", "vol-s7-unread", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	failing := &getFailingStore{MemoryObjectStore: memory, failKey: metadataKey(t, "vol-s7-unread")}
-	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
-	unpublisher := &fakeUnpublisher{}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes())
-	open := openObligations(t, svc)
-	require.Len(t, open, 1)
-	assert.Equal(t, 1, open[0].Attempts, "the failure is counted where an operator can see it")
-	assert.Contains(t, open[0].Error, "reconstruction failed")
-}
-
-// End to end through the service: a force detach followed by a new attachment
-// of the same volume on the same node must leave that new export alone.
-func TestS7_ForceDetachThenReattachOnTheSameNodeKeepsTheNewExport(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a", "i-new": "node-a"}}, "node-a")
-	putAttachment(t, svc, "vol-s7-reattach", "in-use", "i-old", time.Now().Add(-time.Hour).UTC())
-
-	_, err := forceDetach(t, svc, "vol-s7-reattach")
-	require.NoError(t, err)
-	require.NoError(t, svc.UpdateVolumeState(testVolAccountID, "vol-s7-reattach", "in-use", "i-new", "/dev/sdg"))
-	unpublisher := &fakeUnpublisher{}
-
-	assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes(), "the new attachment's export must survive the old obligation")
-	assert.Equal(t, "i-new", readVolume(t, svc, "vol-s7-reattach").AttachedInstance)
-	assert.Empty(t, openObligations(t, svc))
-}
-
-func TestObligationReaper_SettlesOnlyItsOwnNode(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b")
-	putAttachment(t, svc, "vol-own", "available", "", time.Time{})
-	recordObligation(t, svc, "node-b", "vol-own", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	unpublisher := &fakeUnpublisher{}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes(), "only the node holding the export can tear it down")
-	assert.Len(t, openObligations(t, svc), 1)
-}
-
-// A publish or unpublish of the volume in flight on this node holds its lock;
-// the sweep skips it rather than waiting behind a possibly wedged call.
-func TestObligationReaper_SkipsABusyVolume(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-busy", "available", "", time.Time{})
-	recordObligation(t, svc, "node-a", "vol-busy", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	unpublisher := &fakeUnpublisher{busy: map[string]bool{"vol-busy": true}}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes())
-	open := openObligations(t, svc)
-	require.Len(t, open, 1)
-	assert.Zero(t, open[0].Attempts, "a busy volume is not a failure")
-}
-
-func TestObligationReaper_FailedUnpublishStaysOpenAndCounts(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-fail", "available", "", time.Time{})
-	recordObligation(t, svc, "node-a", "vol-fail", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	unpublisher := &fakeUnpublisher{err: errors.New("seal refused")}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	open := openObligations(t, svc)
-	require.Len(t, open, 1)
-	assert.Equal(t, 2, open[0].Attempts)
-	assert.Equal(t, "seal refused", open[0].Error)
-	assert.Zero(t, unpublisher.held, "a failed unpublish still releases the hold")
-}
-
-// An obligation with no attachment time falls back to when it was recorded, so
-// a publish here after the clear still counts as newer.
-func TestObligationSince_FallsBackToRecordedAt(t *testing.T) {
-	recorded := time.Unix(1_700_000_000, 0)
-	assert.Equal(t, recorded, obligationSince(ebsobligation.UnpublishObligation{RecordedAt: recorded}))
-	attached := recorded.Add(-time.Hour)
-	assert.Equal(t, attached, obligationSince(ebsobligation.UnpublishObligation{AttachedAt: attached, RecordedAt: recorded}))
-}
-
-// --- The cluster-wide close ---
-
-func TestObligationCloser_ClosesOnlyDepartedNodes(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	at := time.Unix(1_700_000_000, 0).UTC()
-	recordObligation(t, svc, "node-gone", "vol-c1", "i-1", at)
-	recordObligation(t, svc, "node-gone", "vol-c2", "i-2", at)
-	recordObligation(t, svc, "node-down", "vol-c3", "i-3", at)
-	recordObligation(t, svc, "node-unknown", "vol-c4", "i-4", at)
-	asked := map[string]int{}
-	departed := func(_ context.Context, node string) (bool, error) {
-		asked[node]++
-		switch node {
-		case "node-gone":
-			return true, nil
-		case "node-unknown":
-			return false, errors.New("heartbeat bucket unreachable")
-		default:
-			return false, nil
-		}
-	}
-
-	closed, err := svc.NewUnpublishObligationCloser(departed).Sweep(context.Background())
-	require.NoError(t, err)
-
-	assert.Equal(t, 2, closed)
-	remaining := []string{}
-	for _, o := range openObligations(t, svc) {
-		remaining = append(remaining, o.NodeID)
-	}
-	assert.ElementsMatch(t, []string{"node-down", "node-unknown"}, remaining,
-		"a node that is down, or whose membership cannot be read, keeps its obligations")
-	assert.Equal(t, 1, asked["node-gone"], "membership is asked once per node per sweep")
-}

spinifex/domains/ecr/awsapi/create_repository.go

@@ -20,18 +20,16 @@ type createRepositoryRequest struct {
 	RepositoryName             string                           `json:"repositoryName"`
 	RegistryID                 string                           `json:"registryId"`
 	ImageTagMutability         string                           `json:"imageTagMutability"`
-	ExclusionFilters           []exclusionFilter                `json:"imageTagMutabilityExclusionFilters"`
 	Tags                       []*ecr.Tag                       `json:"tags"`
 	EncryptionConfiguration    *encryptionConfigurationInput    `json:"encryptionConfiguration"`
 	ImageScanningConfiguration *imageScanningConfigurationInput `json:"imageScanningConfiguration"`
 }
 
 // encryptionConfigurationInput is the camelCase input shape for
-// encryptionConfiguration. kmsKey is length-checked and then always refused:
-// KMS is rejected outright and AES256 takes no key.
+// encryptionConfiguration. kmsKey is not decoded: KMS is rejected outright,
+// so any key supplied alongside it is never used.
 type encryptionConfigurationInput struct {
-	EncryptionType string  `json:"encryptionType"`
-	KMSKey         *string `json:"kmsKey"`
+	EncryptionType string `json:"encryptionType"`
 }
 
 // CreateRepository creates only the account-scoped ECR metadata record. The
@@ -59,9 +57,6 @@ func CreateRepository(ctx context.Context, store RepositoryStore, endpoint Repos
 	if err != nil {
 		return nil, err
 	}
-	if err := validateExclusionFilters(req.ExclusionFilters); err != nil {
-		return nil, err
-	}
 	encryptionType, err := normalizeEncryptionType(req.EncryptionConfiguration)
 	if err != nil {
 		return nil, err
@@ -120,19 +115,10 @@ func normalizeEncryptionType(cfg *encryptionConfigurationInput) (string, error)
 	if cfg == nil {
 		return ecrdomain.EncryptionTypeAES256, nil
 	}
-	if cfg.KMSKey != nil {
-		if err := ValidateTextLength("encryptionConfiguration.kmsKey", *cfg.KMSKey, 1, maxKMSKeyLength); err != nil {
-			return "", err
-		}
-	}
 	switch cfg.EncryptionType {
 	case "":
 		return "", RequiredParameterError("encryptionConfiguration.encryptionType")
 	case ecrdomain.EncryptionTypeAES256:
-		if cfg.KMSKey != nil {
-			return "", ConstraintError("encryptionConfiguration",
-				"Invalid encryption configuration: Encryption key should not be set for AES256 encryption.")
-		}
 		return cfg.EncryptionType, nil
 	case ecrdomain.EncryptionTypeKMS:
 		return "", awserrors.Errorf(awserrors.ErrorECRInvalidParameter,

spinifex/domains/ecr/awsapi/describe_images.go

@@ -25,7 +25,6 @@ type describeImagesRequest struct {
 	RepositoryName string            `json:"repositoryName"`
 	RegistryID     string            `json:"registryId"`
 	ImageIDs       []imageIdentifier `json:"imageIds"`
-	MaxResults     *int64            `json:"maxResults"`
 	// Filter's tagStatus is enum-checked but not yet applied; its semantics
 	// are a separate ECR compatibility decision.
 	Filter *tagStatusFilter `json:"filter"`
@@ -45,9 +44,6 @@ func DescribeImages(ctx context.Context, catalog ImageCatalog, accountID string,
 	if err := validateImageIDs(req.ImageIDs, false); err != nil {
 		return nil, err
 	}
-	if err := validateMaxResults(req.MaxResults); err != nil {
-		return nil, err
-	}
 	if err := req.Filter.validate(); err != nil {
 		return nil, err
 	}

spinifex/domains/ecr/awsapi/describe_repositories.go

@@ -18,7 +18,6 @@ import (
 type describeRepositoriesRequest struct {
 	RegistryID      string   `json:"registryId"`
 	RepositoryNames []string `json:"repositoryNames"`
-	MaxResults      *int64   `json:"maxResults"`
 }
 
 // DescribeRepositories lists repositories visible in the caller's account.
@@ -49,10 +48,6 @@ func DescribeRepositories(ctx context.Context, store RepositoryStore, endpoint R
 		return nil, ConstraintError("RepositoryNameList", fmt.Sprintf("Must have length less than or equal to %d", maxRepositoryNameList))
 	}
 
-	if err := validateMaxResults(req.MaxResults); err != nil {
-		return nil, err
-	}
-
 	names := req.RepositoryNames
 	if len(names) == 0 {
 		var err error

spinifex/domains/ecr/awsapi/list_images.go

@@ -12,27 +12,20 @@ import (
 type listImagesRequest struct {
 	RepositoryName string           `json:"repositoryName"`
 	RegistryID     string           `json:"registryId"`
-	MaxResults     *int64           `json:"maxResults"`
 	Filter         *tagStatusFilter `json:"filter"`
 }
 
 type tagStatusFilter struct {
-	TagStatus   string  `json:"tagStatus"`
-	ImageStatus *string `json:"imageStatus"`
+	TagStatus string `json:"tagStatus"`
 }
 
-// validate refuses a tagStatus or imageStatus outside its enum; an omitted
-// filter or member is valid. imageStatus is checked but not applied.
+// validate refuses a tagStatus outside the TagStatus enum; an omitted filter
+// or tagStatus is valid.
 func (f *tagStatusFilter) validate() error {
-	switch {
-	case f == nil:
+	if f == nil || f.TagStatus == "" || slices.Contains(TagStatusValues, f.TagStatus) {
 		return nil
-	case f.TagStatus != "" && !slices.Contains(TagStatusValues, f.TagStatus):
-		return EnumValueError("filter.tagStatus", TagStatusValues...)
-	case f.ImageStatus != nil && !slices.Contains(ImageStatusValues, *f.ImageStatus):
-		return EnumValueError("filter.imageStatus", ImageStatusValues...)
 	}
-	return nil
+	return EnumValueError("filter.tagStatus", TagStatusValues...)
 }
 
 // ListImages returns every image identifier in the caller's repository,
@@ -46,9 +39,6 @@ func ListImages(ctx context.Context, catalog ImageCatalog, accountID string, bod
 	if err := ValidateRepositoryScope(req.RepositoryName, req.RegistryID, accountID); err != nil {
 		return nil, err
 	}
-	if err := validateMaxResults(req.MaxResults); err != nil {
-		return nil, err
-	}
 	if err := req.Filter.validate(); err != nil {
 		return nil, err
 	}

spinifex/domains/ecr/awsapi/registration.go

@@ -116,9 +116,6 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 			slog.Error("ECR authorization-token action: capabilities not configured")
 			return errors.New(awserrors.ErrorServerInternal)
 		}
-		if err := ValidateAuthorizationTokenRequest(body); err != nil {
-			return err
-		}
 		principal, perr := tokenPrincipal(inv)
 		if perr != nil {
 			slog.Error("GetAuthorizationToken: cannot build canonical caller ARN", "err", perr)

spinifex/domains/ecr/awsapi/registration_test.go

@@ -322,15 +322,6 @@ func TestDispatch_GetAuthorizationTokenMintsForTheVerifiedActor(t *testing.T) {
 	assert.Equal(t, envelope.JSONContentType, w.Header().Get("Content-Type"))
 }
 
-// A registryIds break is refused before any token is minted.
-func TestDispatch_GetAuthorizationTokenRefusesRegistryIDsOutsideModel(t *testing.T) {
-	f := &fakeInvocation{actor: dispatch.Actor{AccountID: regAccount}}
-	be := &backend{log: &f.log}
-	_, err := dispatchWith(t, be.deps(), f.invocation(TargetPrefix+".GetAuthorizationToken", `{"registryIds":["abc"]}`))
-	requireECRInvalidParameter(t, err, "Invalid parameter at 'registryIds' failed to satisfy constraint: 'Member must satisfy constraint: [Member must satisfy regular expression pattern: [0-9]{12}]'")
-	assert.Empty(t, be.minted)
-}
-
 // Only a role session names a role, and the ID is the part of aws:userid before the
 // session name. Every other caller gets a token without the claim.
 func TestDispatch_GetAuthorizationTokenCarriesTheRoleIDOfARoleSessionOnly(t *testing.T) {

spinifex/domains/ecr/awsapi/request_member_validation_test.go

@@ -1,100 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"strings"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
-	"github.com/stretchr/testify/require"
-)
-
-// The messages below are AWS's, captured from real ECR refusals.
-const (
-	awsMaxResultsLow   = "Invalid parameter at 'maxResults' failed to satisfy constraint: 'Member must have value greater than or equal to 1'"
-	awsMaxResultsHigh  = "Invalid parameter at 'maxResults' failed to satisfy constraint: 'Member must have value less than or equal to 1000'"
-	awsImageStatusEnum = "Invalid parameter at 'filter.imageStatus' failed to satisfy constraint: 'Member must satisfy enum value set: [ARCHIVED, ACTIVE, ANY, ACTIVATING]'"
-	awsExclusionField  = "Invalid parameter at 'imageTagMutabilityExclusionFilters"
-	awsExclusionRegex  = "Member must satisfy regular expression pattern: ^[0-9a-zA-Z._*-]{1,128}$"
-)
-
-func TestListActions_RefuseMaxResultsOutsideRange(t *testing.T) {
-	store := ecr.NewNATSMetaStore(newRepositoryActionTestConn(t))
-	for value, want := range map[string]string{"0": awsMaxResultsLow, "1001": awsMaxResultsHigh} {
-		repoBody := []byte(`{"repositoryName":"team/app","maxResults":` + value + `}`)
-		_, err := ListImages(context.Background(), nil, "123456789012", repoBody)
-		requireECRInvalidParameter(t, err, want)
-		_, err = DescribeImages(context.Background(), nil, "123456789012", repoBody)
-		requireECRInvalidParameter(t, err, want)
-		_, err = DescribeRepositories(context.Background(), store, repositoryActionEndpoint(), repositoryActionTestAccount, []byte(`{"maxResults":`+value+`}`))
-		requireECRInvalidParameter(t, err, want)
-	}
-	for _, value := range []string{"1", "1000"} {
-		_, err := DescribeRepositories(context.Background(), store, repositoryActionEndpoint(), repositoryActionTestAccount, []byte(`{"maxResults":`+value+`}`))
-		require.NoError(t, err, value)
-	}
-}
-
-func TestImageListActions_RefuseImageStatusOutsideEnum(t *testing.T) {
-	body := []byte(`{"repositoryName":"team/app","filter":{"imageStatus":"BOGUS"}}`)
-	_, err := ListImages(context.Background(), nil, "123456789012", body)
-	requireECRInvalidParameter(t, err, awsImageStatusEnum)
-	_, err = DescribeImages(context.Background(), nil, "123456789012", body)
-	requireECRInvalidParameter(t, err, awsImageStatusEnum)
-}
-
-func TestTagMutabilityActions_RefuseExclusionFiltersOutsideModel(t *testing.T) {
-	nc := newRepositoryActionTestConn(t)
-	store := ecr.NewNATSMetaStore(nc)
-	six := strings.TrimSuffix(strings.Repeat(`{"filterType":"WILDCARD","filter":"v*"},`, 6), ",")
-	member := awsExclusionField + ".1.member."
-	cases := map[string]string{
-		`[]`:                                     awsExclusionField + "' failed to satisfy constraint: 'Member must have length greater than or equal to 1'",
-		`[` + six + `]`:                          awsExclusionField + "' failed to satisfy constraint: 'Member must have length less than or equal to 5'",
-		`[{"filterType":"REGEX","filter":"v*"}]`: member + "filterType' failed to satisfy constraint: 'Member must satisfy enum value set: [WILDCARD]'",
-		`[{"filterType":"WILDCARD","filter":"v/1"}]`:                              member + "filter' failed to satisfy constraint: '" + awsExclusionRegex + "'",
-		`[{"filterType":"WILDCARD","filter":""}]`:                                 member + "filter' failed to satisfy constraint: '" + awsExclusionRegex + "; Member must have length greater than or equal to 1'",
-		`[{"filterType":"WILDCARD","filter":"` + strings.Repeat("a", 129) + `"}]`: member + "filter' failed to satisfy constraint: '" + awsExclusionRegex + "; Member must have length less than or equal to 128'",
-	}
-	for filters, want := range cases {
-		_, err := CreateRepository(context.Background(), store, repositoryActionEndpoint(), repositoryActionTestAccount,
-			[]byte(`{"repositoryName":"team/app","imageTagMutabilityExclusionFilters":`+filters+`}`))
-		requireECRInvalidParameter(t, err, want)
-		_, err = PutImageTagMutability(context.Background(), nc, repositoryActionTestAccount,
-			[]byte(`{"repositoryName":"team/app","imageTagMutability":"MUTABLE","imageTagMutabilityExclusionFilters":`+filters+`}`))
-		requireECRInvalidParameter(t, err, want)
-	}
-	five := strings.Repeat(`{"filterType":"WILDCARD","filter":"`+strings.Repeat("v", 128)+`"},`, 4) + `{"filterType":"WILDCARD","filter":"v*"}`
-	_, err := CreateRepository(context.Background(), store, repositoryActionEndpoint(), repositoryActionTestAccount,
-		[]byte(`{"repositoryName":"team/app","imageTagMutabilityExclusionFilters":[`+five+`]}`))
-	require.NoError(t, err)
-}
-
-func TestCreateRepository_RefusesEveryKMSKey(t *testing.T) {
-	store := ecr.NewNATSMetaStore(newRepositoryActionTestConn(t))
-	cases := map[string]string{
-		`{"encryptionType":"AES256","kmsKey":""}`:                                  "Invalid parameter at 'encryptionConfiguration.kmsKey' failed to satisfy constraint: 'Member must have length greater than or equal to 1'",
-		`{"encryptionType":"AES256","kmsKey":"` + strings.Repeat("k", 2049) + `"}`: "Invalid parameter at 'encryptionConfiguration.kmsKey' failed to satisfy constraint: 'Member must have length less than or equal to 2048'",
-		`{"encryptionType":"AES256","kmsKey":"alias/app"}`:                         "Invalid parameter at 'encryptionConfiguration' failed to satisfy constraint: 'Invalid encryption configuration: Encryption key should not be set for AES256 encryption.'",
-	}
-	for cfg, want := range cases {
-		_, err := CreateRepository(context.Background(), store, repositoryActionEndpoint(), repositoryActionTestAccount,
-			[]byte(`{"repositoryName":"team/app","encryptionConfiguration":`+cfg+`}`))
-		requireECRInvalidParameter(t, err, want)
-	}
-}
-
-func TestValidateAuthorizationTokenRequest_RefusesRegistryIDsOutsideModel(t *testing.T) {
-	eleven := strings.TrimSuffix(strings.Repeat(`"123456789012",`, 11), ",")
-	cases := map[string]string{
-		`[]`:               "Invalid parameter at 'registryIds' failed to satisfy constraint: 'Member must have length greater than or equal to 1'",
-		`[` + eleven + `]`: "Invalid parameter at 'registryIds' failed to satisfy constraint: 'Member must have length less than or equal to 10'",
-		`["abc"]`:          "Invalid parameter at 'registryIds' failed to satisfy constraint: 'Member must satisfy constraint: [Member must satisfy regular expression pattern: [0-9]{12}]'",
-	}
-	for ids, want := range cases {
-		requireECRInvalidParameter(t, ValidateAuthorizationTokenRequest([]byte(`{"registryIds":`+ids+`}`)), want)
-	}
-	for _, body := range []string{``, `{}`, `{"registryIds":["123456789012"]}`} {
-		require.NoError(t, ValidateAuthorizationTokenRequest([]byte(body)), body)
-	}
-}

spinifex/domains/ecr/awsapi/tag_mutability.go

@@ -15,10 +15,9 @@ import (
 
 // putImageTagMutabilityRequest is the camelCase AWS JSON 1.1 input shape.
 type putImageTagMutabilityRequest struct {
-	RepositoryName     string            `json:"repositoryName"`
-	RegistryID         string            `json:"registryId"`
-	ImageTagMutability string            `json:"imageTagMutability"`
-	ExclusionFilters   []exclusionFilter `json:"imageTagMutabilityExclusionFilters"`
+	RepositoryName     string `json:"repositoryName"`
+	RegistryID         string `json:"registryId"`
+	ImageTagMutability string `json:"imageTagMutability"`
 }
 
 // PutImageTagMutability changes a repository between MUTABLE and IMMUTABLE.
@@ -42,9 +41,6 @@ func PutImageTagMutability(ctx context.Context, nc *nats.Conn, accountID string,
 	default:
 		return nil, EnumValueError("imageTagMutability", ImageTagMutabilityValues...)
 	}
-	if err := validateExclusionFilters(req.ExclusionFilters); err != nil {
-		return nil, err
-	}
 
 	store := ecrdomain.NewNATSMetaStore(nc)
 	meta, err := store.GetRepo(ctx, accountID, req.RepositoryName)

spinifex/domains/ecr/awsapi/validation.go

@@ -1,9 +1,7 @@
 package awsapi
 
 import (
-	"encoding/json"
 	"fmt"
-	"regexp"
 	"strings"
 	"unicode/utf8"
 
@@ -25,18 +23,6 @@ const (
 	maxImageTagLength     = 300
 	maxMediaTypeListItems = 100
 	maxRepositoryNameList = 100
-
-	minMaxResults           = 1
-	maxMaxResults           = 1000
-	maxKMSKeyLength         = 2048
-	maxExclusionFilter      = 5
-	maxExclusionFilterValue = 128
-	maxRegistryIDList       = 10
-)
-
-var (
-	exclusionFilterPattern = regexp.MustCompile(`^[0-9a-zA-Z._*-]{1,128}$`)
-	registryIDPattern      = regexp.MustCompile(`^[0-9]{12}$`)
 )
 
 // ValidateRepositoryName returns InvalidParameterException naming the first
@@ -168,89 +154,3 @@ var ImageTagMutabilityValues = []string{"IMMUTABLE", "MUTABLE", "MUTABLE_WITH_EX
 
 // TagStatusValues is the filter tagStatus enum AWS's model declares.
 var TagStatusValues = []string{"TAGGED", "UNTAGGED", "ANY"}
-
-// ImageStatusValues is the filter imageStatus enum, in the order AWS lists it
-// when refusing a value.
-var ImageStatusValues = []string{"ARCHIVED", "ACTIVE", "ANY", "ACTIVATING"}
-
-// validateMaxResults refuses a maxResults outside MaxResults' range of
-// 1-1000; an omitted value is valid.
-func validateMaxResults(n *int64) error {
-	switch {
-	case n == nil:
-		return nil
-	case *n < minMaxResults:
-		return ConstraintError("maxResults", fmt.Sprintf("Member must have value greater than or equal to %d", minMaxResults))
-	case *n > maxMaxResults:
-		return ConstraintError("maxResults", fmt.Sprintf("Member must have value less than or equal to %d", maxMaxResults))
-	}
-	return nil
-}
-
-// exclusionFilter is one imageTagMutabilityExclusionFilters member.
-type exclusionFilter struct {
-	FilterType *string `json:"filterType"`
-	Filter     *string `json:"filter"`
-}
-
-// validateExclusionFilters checks imageTagMutabilityExclusionFilters against
-// its model: 1-5 members, each a WILDCARD filterType and a filter matching the
-// model's pattern. The filters are checked only, never applied.
-func validateExclusionFilters(filters []exclusionFilter) error {
-	const field = "imageTagMutabilityExclusionFilters"
-	if filters == nil {
-		return nil
-	}
-	if err := validateListLength(field, len(filters), 1, maxExclusionFilter); err != nil {
-		return err
-	}
-	for i, f := range filters {
-		member := fmt.Sprintf("%s.%d.member", field, i+1)
-		switch {
-		case f.FilterType == nil:
-			return RequiredParameterError(member + ".filterType")
-		case *f.FilterType != "WILDCARD":
-			return EnumValueError(member+".filterType", "WILDCARD")
-		case f.Filter == nil:
-			return RequiredParameterError(member + ".filter")
-		}
-		if exclusionFilterPattern.MatchString(*f.Filter) {
-			continue
-		}
-		constraint := "Member must satisfy regular expression pattern: " + exclusionFilterPattern.String()
-		switch n := utf8.RuneCountInString(*f.Filter); {
-		case n < 1:
-			constraint += "; Member must have length greater than or equal to 1"
-		case n > maxExclusionFilterValue:
-			constraint += fmt.Sprintf("; Member must have length less than or equal to %d", maxExclusionFilterValue)
-		}
-		return ConstraintError(member+".filter", constraint)
-	}
-	return nil
-}
-
-// ValidateAuthorizationTokenRequest checks GetAuthorizationToken's deprecated
-// registryIds against its model: 1-10 members, each 12 digits. The ids carry
-// no meaning, as on AWS, so they are only checked.
-func ValidateAuthorizationTokenRequest(body []byte) error {
-	var req struct {
-		RegistryIDs []string `json:"registryIds"`
-	}
-	if len(body) > 0 {
-		if err := json.Unmarshal(body, &req); err != nil {
-			return MalformedBodyError()
-		}
-	}
-	if req.RegistryIDs == nil {
-		return nil
-	}
-	if err := validateListLength("registryIds", len(req.RegistryIDs), 1, maxRegistryIDList); err != nil {
-		return err
-	}
-	for _, id := range req.RegistryIDs {
-		if !registryIDPattern.MatchString(id) {
-			return ConstraintError("registryIds", "Member must satisfy constraint: [Member must satisfy regular expression pattern: [0-9]{12}]")
-		}
-	}
-	return nil
-}

spinifex/domains/ecs/awsapi/actions.go

@@ -38,10 +38,6 @@ func checkTaskLaunchRoles(ctx context.Context, svc ecsdomain.ECSService, account
 		return nil
 	}
 	out, err := svc.DescribeTaskDefinition(ctx, &ecs.DescribeTaskDefinitionInput{TaskDefinition: taskDefinition}, accountID)
-	if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorECSClientException {
-		// DescribeTaskDefinition's not-found message is not the one launches get.
-		return ecsdomain.TaskDefinitionNotFound()
-	}
 	if err != nil {
 		return err
 	}
@@ -60,9 +56,6 @@ func CreateCluster(ctx context.Context, nc *nats.Conn, accountID string, body []
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
-	if err := checkCreateCluster(input); err != nil {
-		return nil, err
-	}
 	return ecsdomain.NewNATSECSService(nc).CreateCluster(ctx, input, accountID)
 }
 
@@ -285,9 +278,6 @@ func runTask(ctx context.Context, svc ecsdomain.ECSService, accountID string, in
 	if err := checkTaskLaunchRoles(ctx, svc, accountID, input.TaskDefinition, passRoleCheck); err != nil {
 		return nil, err
 	}
-	if err := checkCapacityProviderStrategy(input.CapacityProviderStrategy); err != nil {
-		return nil, err
-	}
 	if input.Overrides != nil {
 		if err := checkTaskDefinitionRoles(overrideRoleCheck(ctx), input.Overrides.TaskRoleArn, input.Overrides.ExecutionRoleArn); err != nil {
 			return nil, err
@@ -373,9 +363,6 @@ func createService(ctx context.Context, svc ecsdomain.ECSService, accountID stri
 	if err := checkTaskLaunchRoles(ctx, svc, accountID, input.TaskDefinition, passRoleCheck); err != nil {
 		return nil, err
 	}
-	if err := checkCapacityProviderStrategy(input.CapacityProviderStrategy); err != nil {
-		return nil, err
-	}
 	return svc.CreateService(ctx, input, accountID)
 }
 
@@ -516,9 +503,6 @@ func PutClusterCapacityProviders(ctx context.Context, nc *nats.Conn, accountID s
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
-	if err := checkCapacityProviderStrategy(input.DefaultCapacityProviderStrategy); err != nil {
-		return nil, err
-	}
 	return ecsdomain.NewNATSECSService(nc).PutClusterCapacityProviders(ctx, input, accountID)
 }
 
@@ -530,9 +514,6 @@ func CreateCapacityProvider(ctx context.Context, nc *nats.Conn, accountID string
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
-	if err := checkAutoScalingGroupProvider(input.AutoScalingGroupProvider); err != nil {
-		return nil, err
-	}
 	return ecsdomain.NewNATSECSService(nc).CreateCapacityProvider(ctx, input, accountID)
 }
 

spinifex/domains/ecs/awsapi/actions_test.go

@@ -272,14 +272,3 @@ func TestUpdateService_NoTaskDefinitionSkipsPassRole(t *testing.T) {
 	require.NoError(t, err)
 	assert.True(t, svc.updateCalled)
 }
-
-// DescribeTaskDefinition's not-found answer reaches a launch as the launch's
-// own not-found message, as on AWS.
-func TestCheckTaskLaunchRoles_UnknownTaskDefinitionIsNotFound(t *testing.T) {
-	svc := &fakeECSService{describeErr: errors.New(awserrors.ErrorECSClientException)}
-	err := checkTaskLaunchRoles(context.Background(), svc, "123456789012", aws.String("ghost:1"), allowCheck(t, testTaskRoleARN))
-	code, msg, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok)
-	assert.Equal(t, awserrors.ErrorECSClientException, code)
-	assert.Equal(t, "TaskDefinition not found.", msg)
-}

spinifex/domains/ecs/awsapi/validate_capacityprovider.go

@@ -1,59 +0,0 @@
-package awsapi
-
-import (
-	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// Messages AWS answers, as ClientException, for each autoScalingGroupProvider
-// break, before it looks up the Auto Scaling group.
-const (
-	msgManagedDraining              = "The specified managed draining setting is invalid. Specify ENABLED or DISABLED and try again."
-	msgManagedTerminationProtection = "The specified managed termination protection setting is invalid. Specify ENABLED or DISABLED and try again."
-	msgManagedScalingStatus         = "The specified managed scaling status setting is invalid. Specify ENABLED or DISABLED and try again."
-	msgTargetCapacity               = "The managed scaling target capacity value is invalid. Specify an integer greater than 0 and less than or equal to 100 and try again."
-	msgMinimumScalingStepSize       = "The specified minimum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
-	msgMaximumScalingStepSize       = "The specified maximum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
-	msgInstanceWarmupPeriod         = "The managed scaling instance warmup period value is invalid. Specify an integer greater than or equal to 0 and less than or equal to 9999 and try again."
-)
-
-func validEnabledDisabled(v *string) bool {
-	return v == nil || *v == "ENABLED" || *v == "DISABLED"
-}
-
-func validRange(v *int64, lo, hi int64) bool {
-	return v == nil || (*v >= lo && *v <= hi)
-}
-
-// checkAutoScalingGroupProvider refuses an enum or range break inside
-// autoScalingGroupProvider. AWS refuses an instanceWarmupPeriod of 10000 although
-// the model allows it, so the upper bound here is 9999.
-func checkAutoScalingGroupProvider(p *ecs.AutoScalingGroupProvider) error {
-	if p == nil {
-		return nil
-	}
-	fail := func(msg string) error { return awserrors.Errorf(awserrors.ErrorECSClientException, "%s", msg) }
-	if !validEnabledDisabled(p.ManagedDraining) {
-		return fail(msgManagedDraining)
-	}
-	if !validEnabledDisabled(p.ManagedTerminationProtection) {
-		return fail(msgManagedTerminationProtection)
-	}
-	ms := p.ManagedScaling
-	if ms == nil {
-		return nil
-	}
-	switch {
-	case !validEnabledDisabled(ms.Status):
-		return fail(msgManagedScalingStatus)
-	case !validRange(ms.TargetCapacity, 1, 100):
-		return fail(msgTargetCapacity)
-	case !validRange(ms.MinimumScalingStepSize, 1, 10000):
-		return fail(msgMinimumScalingStepSize)
-	case !validRange(ms.MaximumScalingStepSize, 1, 10000):
-		return fail(msgMaximumScalingStepSize)
-	case !validRange(ms.InstanceWarmupPeriod, 0, 9999):
-		return fail(msgInstanceWarmupPeriod)
-	}
-	return nil
-}

spinifex/domains/ecs/awsapi/validate_capacityprovider_test.go

@@ -1,51 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// Each break answers AWS's ClientException and message before dispatch. A valid
-// request reaches dispatch, which fails here only because nc is nil.
-func TestCreateCapacityProviderAutoScalingGroupProvider(t *testing.T) {
-	body := func(asg string) []byte {
-		return []byte(`{"name":"cp","autoScalingGroupProvider":{"autoScalingGroupArn":"arn:aws:autoscaling:x"` + asg + `}}`)
-	}
-	refused := []struct {
-		name, asg, msg string
-	}{
-		{"managedDraining", `,"managedDraining":"BOGUS"`, msgManagedDraining},
-		{"managedTerminationProtection", `,"managedTerminationProtection":"enabled"`, msgManagedTerminationProtection},
-		{"status", `,"managedScaling":{"status":"BOGUS"}`, msgManagedScalingStatus},
-		{"targetCapacity/0", `,"managedScaling":{"targetCapacity":0}`, msgTargetCapacity},
-		{"targetCapacity/101", `,"managedScaling":{"targetCapacity":101}`, msgTargetCapacity},
-		{"minimumScalingStepSize/0", `,"managedScaling":{"minimumScalingStepSize":0}`, msgMinimumScalingStepSize},
-		{"maximumScalingStepSize/10001", `,"managedScaling":{"maximumScalingStepSize":10001}`, msgMaximumScalingStepSize},
-		{"instanceWarmupPeriod/-1", `,"managedScaling":{"instanceWarmupPeriod":-1}`, msgInstanceWarmupPeriod},
-		{"instanceWarmupPeriod/10000", `,"managedScaling":{"instanceWarmupPeriod":10000}`, msgInstanceWarmupPeriod},
-	}
-	for _, tc := range refused {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(tc.asg), nil)
-			code, msg, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok, "err = %v", err)
-			assert.Equal(t, awserrors.ErrorECSClientException, code)
-			assert.Equal(t, tc.msg, msg)
-		})
-	}
-
-	valid := []string{
-		``,
-		`,"managedDraining":"ENABLED","managedTerminationProtection":"DISABLED","managedScaling":{"status":"ENABLED","targetCapacity":1,"minimumScalingStepSize":1,"maximumScalingStepSize":10000,"instanceWarmupPeriod":0}`,
-		`,"managedDraining":"DISABLED","managedTerminationProtection":"ENABLED","managedScaling":{"status":"DISABLED","targetCapacity":100,"minimumScalingStepSize":10000,"maximumScalingStepSize":1,"instanceWarmupPeriod":9999}`,
-	}
-	for _, asg := range valid {
-		_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(asg), nil)
-		_, _, refusedErr := awserrors.ResolveErrorDetail(err)
-		assert.False(t, refusedErr, "asg %q: err = %v", asg, err)
-	}
-}

spinifex/domains/ecs/awsapi/validate_cluster.go

@@ -1,54 +0,0 @@
-package awsapi
-
-import (
-	"slices"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-var (
-	executeCommandLoggingTypes = []string{"NONE", "DEFAULT", "OVERRIDE"}
-	clusterContainerInsights   = []string{"enabled", "disabled", "enhanced"}
-)
-
-// checkCreateCluster refuses settings, logging and strategy values outside the
-// model's sets and ranges with the messages AWS answers.
-func checkCreateCluster(input *ecs.CreateClusterInput) error {
-	for _, st := range input.Settings {
-		if st == nil {
-			continue
-		}
-		if aws.StringValue(st.Name) != "containerInsights" {
-			return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "Invalid setting 'name'")
-		}
-		if !slices.Contains(clusterContainerInsights, aws.StringValue(st.Value)) {
-			return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "Invalid setting 'value'")
-		}
-	}
-	if cfg := input.Configuration; cfg != nil && cfg.ExecuteCommandConfiguration != nil {
-		if l := cfg.ExecuteCommandConfiguration.Logging; l != nil && !slices.Contains(executeCommandLoggingTypes, *l) {
-			return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "Unknown executeCommandLoggingType '%s'", *l)
-		}
-	}
-	return checkCapacityProviderStrategy(input.DefaultCapacityProviderStrategy)
-}
-
-// checkCapacityProviderStrategy range-checks each item's weight and base.
-func checkCapacityProviderStrategy(items []*ecs.CapacityProviderStrategyItem) error {
-	for _, item := range items {
-		if item == nil {
-			continue
-		}
-		if w := item.Weight; w != nil && (*w < 0 || *w > 1000) {
-			return awserrors.Errorf(awserrors.ErrorECSInvalidParameter,
-				"The specified weight value '%d' is invalid. Specify an integer between 0 and 1000 and try again.", *w)
-		}
-		if b := item.Base; b != nil && (*b < 0 || *b > 100000) {
-			return awserrors.Errorf(awserrors.ErrorECSInvalidParameter,
-				"The specified base value '%d' is invalid. Specify an integer between 0 and 100000 and try again.", *b)
-		}
-	}
-	return nil
-}

spinifex/domains/ecs/awsapi/validate_cluster_test.go

@@ -1,141 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func assertInvalidParameter(t *testing.T, err error, wantMsg string) {
-	t.Helper()
-	code, msg, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok, "err = %v", err)
-	assert.Equal(t, awserrors.ErrorECSInvalidParameter, code)
-	assert.Equal(t, wantMsg, msg)
-}
-
-const (
-	weightMsg     = "The specified weight value '1001' is invalid. Specify an integer between 0 and 1000 and try again."
-	baseHighMsg   = "The specified base value '100001' is invalid. Specify an integer between 0 and 100000 and try again."
-	baseNegMsg    = "The specified base value '-1' is invalid. Specify an integer between 0 and 100000 and try again."
-	validStrategy = `[{"capacityProvider":"cp","weight":1000,"base":100000},{"capacityProvider":"cp2","weight":0,"base":0}]`
-)
-
-// Out-of-model CreateCluster and PutClusterCapacityProviders values are
-// refused before dispatch; valid ones reach dispatch, which fails here only
-// because nc is nil.
-func TestClusterRequestValidation(t *testing.T) {
-	cases := []struct {
-		name    string
-		handler describeHandler
-		body    string
-		wantMsg string
-	}{
-		{"CreateCluster/logging", CreateCluster,
-			`{"configuration":{"executeCommandConfiguration":{"logging":"BOGUS"}}}`, "Unknown executeCommandLoggingType 'BOGUS'"},
-		{"CreateCluster/settingName", CreateCluster,
-			`{"settings":[{"name":"BOGUS","value":"enabled"}]}`, "Invalid setting 'name'"},
-		{"CreateCluster/settingValue", CreateCluster,
-			`{"settings":[{"name":"containerInsights","value":"BOGUS"}]}`, "Invalid setting 'value'"},
-		{"CreateCluster/weight", CreateCluster,
-			`{"defaultCapacityProviderStrategy":[{"capacityProvider":"cp","weight":1001}]}`, weightMsg},
-		{"CreateCluster/base", CreateCluster,
-			`{"defaultCapacityProviderStrategy":[{"capacityProvider":"cp","base":100001}]}`, baseHighMsg},
-		{"PutClusterCapacityProviders/weight", PutClusterCapacityProviders,
-			`{"cluster":"c","capacityProviders":[],"defaultCapacityProviderStrategy":[{"capacityProvider":"cp","weight":1001}]}`, weightMsg},
-		{"PutClusterCapacityProviders/base", PutClusterCapacityProviders,
-			`{"cluster":"c","capacityProviders":[],"defaultCapacityProviderStrategy":[{"capacityProvider":"cp","base":-1}]}`, baseNegMsg},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := tc.handler(context.Background(), nil, "123456789012", []byte(tc.body), nil)
-			assertInvalidParameter(t, err, tc.wantMsg)
-		})
-	}
-
-	valid := []struct {
-		name    string
-		handler describeHandler
-		body    string
-	}{
-		{"CreateCluster/logging/NONE", CreateCluster, `{"configuration":{"executeCommandConfiguration":{"logging":"NONE"}}}`},
-		{"CreateCluster/logging/DEFAULT", CreateCluster, `{"configuration":{"executeCommandConfiguration":{"logging":"DEFAULT"}}}`},
-		{"CreateCluster/logging/OVERRIDE", CreateCluster, `{"configuration":{"executeCommandConfiguration":{"logging":"OVERRIDE"}}}`},
-		{"CreateCluster/settings", CreateCluster,
-			`{"settings":[{"name":"containerInsights","value":"enabled"},{"name":"containerInsights","value":"disabled"},{"name":"containerInsights","value":"enhanced"}]}`},
-		{"CreateCluster/strategy", CreateCluster, `{"defaultCapacityProviderStrategy":` + validStrategy + `}`},
-		{"PutClusterCapacityProviders/strategy", PutClusterCapacityProviders,
-			`{"cluster":"c","capacityProviders":["cp","cp2"],"defaultCapacityProviderStrategy":` + validStrategy + `}`},
-	}
-	for _, tc := range valid {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := tc.handler(context.Background(), nil, "123456789012", []byte(tc.body), nil)
-			_, _, refused := awserrors.ResolveErrorDetail(err)
-			assert.False(t, refused, "err = %v", err)
-		})
-	}
-}
-
-func strategy(weight, base int64) []*ecs.CapacityProviderStrategyItem {
-	return []*ecs.CapacityProviderStrategyItem{{CapacityProvider: aws.String("cp"), Weight: aws.Int64(weight), Base: aws.Int64(base)}}
-}
-
-func TestRunTask_CapacityProviderStrategyRange(t *testing.T) {
-	svc := &fakeECSService{describeOut: taskDefWithoutRole(), runOut: &ecs.RunTaskOutput{}}
-	_, err := runTask(context.Background(), svc, "123456789012",
-		&ecs.RunTaskInput{TaskDefinition: aws.String("web:1"), CapacityProviderStrategy: strategy(1001, 0)}, allowCheck(t, testTaskRoleARN))
-	assertInvalidParameter(t, err, weightMsg)
-	_, err = runTask(context.Background(), svc, "123456789012",
-		&ecs.RunTaskInput{TaskDefinition: aws.String("web:1"), CapacityProviderStrategy: strategy(1, 100001)}, allowCheck(t, testTaskRoleARN))
-	assertInvalidParameter(t, err, baseHighMsg)
-	assert.False(t, svc.runCalled)
-
-	_, err = runTask(context.Background(), svc, "123456789012",
-		&ecs.RunTaskInput{TaskDefinition: aws.String("web:1"), CapacityProviderStrategy: strategy(1000, 100000)}, allowCheck(t, testTaskRoleARN))
-	require.NoError(t, err)
-	assert.True(t, svc.runCalled)
-}
-
-func TestCreateService_CapacityProviderStrategyRange(t *testing.T) {
-	svc := &fakeECSService{describeOut: taskDefWithoutRole(), createOut: &ecs.CreateServiceOutput{}}
-	_, err := createService(context.Background(), svc, "123456789012",
-		&ecs.CreateServiceInput{TaskDefinition: aws.String("web:1"), CapacityProviderStrategy: strategy(1001, 0)}, allowCheck(t, testTaskRoleARN))
-	assertInvalidParameter(t, err, weightMsg)
-	_, err = createService(context.Background(), svc, "123456789012",
-		&ecs.CreateServiceInput{TaskDefinition: aws.String("web:1"), CapacityProviderStrategy: strategy(0, -1)}, allowCheck(t, testTaskRoleARN))
-	assertInvalidParameter(t, err, baseNegMsg)
-	assert.False(t, svc.createCalled)
-
-	_, err = createService(context.Background(), svc, "123456789012",
-		&ecs.CreateServiceInput{TaskDefinition: aws.String("web:1"), CapacityProviderStrategy: strategy(0, 0)}, allowCheck(t, testTaskRoleARN))
-	require.NoError(t, err)
-	assert.True(t, svc.createCalled)
-}
-
-// AWS resolves the task definition before range-checking the strategy, so a
-// missing task definition wins over a bad weight.
-func TestLaunch_TaskDefinitionResolvedBeforeStrategy(t *testing.T) {
-	svc := &fakeECSService{describeErr: errors.New(awserrors.ErrorECSClientException)}
-	_, err := runTask(context.Background(), svc, "123456789012",
-		&ecs.RunTaskInput{TaskDefinition: aws.String("ghost:1"), CapacityProviderStrategy: strategy(1001, 0)}, allowCheck(t, testTaskRoleARN))
-	assertTaskDefinitionNotFound(t, err)
-	_, err = createService(context.Background(), svc, "123456789012",
-		&ecs.CreateServiceInput{TaskDefinition: aws.String("ghost:1"), CapacityProviderStrategy: strategy(1001, 0)}, allowCheck(t, testTaskRoleARN))
-	assertTaskDefinitionNotFound(t, err)
-	assert.False(t, svc.runCalled)
-	assert.False(t, svc.createCalled)
-}
-
-func assertTaskDefinitionNotFound(t *testing.T, err error) {
-	t.Helper()
-	code, msg, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok, "err = %v", err)
-	assert.Equal(t, awserrors.ErrorECSClientException, code)
-	assert.Equal(t, "TaskDefinition not found.", msg)
-}

spinifex/domains/ecs/capacity_providers.go

@@ -53,9 +53,6 @@ func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateC
 		aws.StringValue(input.AutoScalingGroupProvider.AutoScalingGroupArn) == "" {
 		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 	}
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	name := aws.StringValue(input.Name)
 	kv, err := s.bucket(ctx, accountID)
 	if err != nil {

spinifex/domains/ecs/instances.go

@@ -64,9 +64,6 @@ func applyRuntimeCapacity(r *InstanceRecord, cpu, memoryMiB int) {
 // normally registers over the Layer-2 bus; this keeps API parity by writing the
 // same record shape from an explicit call.
 func (s *Service) RegisterContainerInstance(ctx context.Context, input *ecs.RegisterContainerInstanceInput, accountID string) (*ecs.RegisterContainerInstanceOutput, error) {
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	cluster := ClusterShortName(aws.StringValue(input.Cluster))
 	instanceID := aws.StringValue(input.InstanceIdentityDocument)
 	if instanceID == "" {

spinifex/domains/ecs/service.go

@@ -8,12 +8,10 @@ import (
 	"errors"
 	"log/slog"
 	"maps"
-	"regexp"
 	"slices"
 	"strconv"
 	"strings"
 	"time"
-	"unicode/utf8"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -192,9 +190,6 @@ func deleteKeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix str
 // CreateCluster persists a cluster meta record. Idempotent: re-creating an
 // existing cluster returns the existing ACTIVE record.
 func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInput, accountID string) (*ecs.CreateClusterOutput, error) {
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	name := aws.StringValue(input.ClusterName)
 	if name == "" {
 		name = defaultCluster
@@ -221,7 +216,12 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 			return nil, err
 		}
 		rec.Settings = settings
-		rec.Tags = tagsToMap(input.Tags)
+		if len(input.Tags) > 0 {
+			rec.Tags = map[string]string{}
+			for _, t := range input.Tags {
+				rec.Tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+			}
+		}
 		if err := taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(name), &rec); err != nil {
 			return nil, err
 		}
@@ -463,56 +463,6 @@ func tagsToMap(tags []*ecs.Tag) map[string]string {
 	return out
 }
 
-// maxTags is the most tags an ECS resource can carry.
-const maxTags = 50
-
-var tagPattern = regexp.MustCompile(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`)
-
-// validateTags applies the model's tag constraints with AWS's messages.
-func validateTags(tags []*ecs.Tag) error {
-	if len(tags) > maxTags {
-		return errTagLimitExceeded()
-	}
-	for _, t := range tags {
-		if t == nil {
-			return validateTagKey("")
-		}
-		if err := validateTagKey(aws.StringValue(t.Key)); err != nil {
-			return err
-		}
-		value := aws.StringValue(t.Value)
-		if utf8.RuneCountInString(value) > 256 {
-			return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "Tag value must have length less than or equal to 256")
-		}
-		if !tagPattern.MatchString(value) {
-			return errInvalidTagCharacters()
-		}
-	}
-	return nil
-}
-
-func validateTagKey(key string) error {
-	if key == "" {
-		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "Tag key can not be null or empty.")
-	}
-	if utf8.RuneCountInString(key) > 128 {
-		return awserrors.Errorf(awserrors.ErrorECSInvalidParameter, "Tag key must have length less than or equal to 128")
-	}
-	if !tagPattern.MatchString(key) {
-		return errInvalidTagCharacters()
-	}
-	return nil
-}
-
-func errTagLimitExceeded() error {
-	return awserrors.Errorf(awserrors.ErrorECSClientException, "Tag Limit Exceeded.")
-}
-
-func errInvalidTagCharacters() error {
-	return awserrors.Errorf(awserrors.ErrorECSInvalidParameter,
-		"Some tags contain invalid characters. Valid characters: UTF-8 letters, spaces, numbers and _ . / = + - : @.")
-}
-
 // --- Task definition ---
 
 // RegisterTaskDefinition stores a new revision of a family, bumping latest-rev.
@@ -524,9 +474,6 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
 	if err := validateContainerDefs(input.ContainerDefinitions, aws.StringValue(input.Memory)); err != nil {
 		return nil, err
 	}
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	warnUnsupportedLogDrivers(ctx, family, input.ContainerDefinitions)
 	kv, err := s.bucket(ctx, accountID)
 	if err != nil {
@@ -664,8 +611,8 @@ func (s *Service) DescribeTaskDefinition(ctx context.Context, input *ecs.Describ
 	if family, _ := taskdefinition.ParseRef(ref); family == "" {
 		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 	}
-	// AWS answers an unknown revision with ClientException; launches get the
-	// same code with TaskDefinitionNotFound's message instead.
+	// AWS answers an unknown revision with ClientException, not the
+	// InvalidParameterException that launches get for the same reference.
 	rec, err := s.taskDefs.Resolve(ctx, kv, ref)
 	if err != nil && err.Error() == awserrors.ErrorECSInvalidParameter {
 		return nil, errors.New(awserrors.ErrorECSClientException)
@@ -707,22 +654,9 @@ type taskDefinitionResolver interface {
 
 var _ taskDefinitionResolver = (*taskdefinition.Owner)(nil)
 
-// TaskDefinitionNotFound is AWS's answer when RunTask, StartTask, CreateService
-// or UpdateService names a well-formed but unknown task definition.
-func TaskDefinitionNotFound() error {
-	return awserrors.Errorf(awserrors.ErrorECSClientException, "TaskDefinition not found.")
-}
-
 // resolveTaskDef is how task and service launches reach task definitions.
-// A malformed reference (no family) keeps InvalidParameterException.
 func (s *Service) resolveTaskDef(ctx context.Context, kv jetstream.KeyValue, ref string) (*taskdefinition.Record, error) {
-	rec, err := s.launchDefs.Resolve(ctx, kv, ref)
-	if err != nil && err.Error() == awserrors.ErrorECSInvalidParameter {
-		if family, _ := taskdefinition.ParseRef(ref); family != "" {
-			return nil, TaskDefinitionNotFound()
-		}
-	}
-	return rec, err
+	return s.launchDefs.Resolve(ctx, kv, ref)
 }
 
 func taskDefToAWS(r *taskdefinition.Record) *ecs.TaskDefinition {

spinifex/domains/ecs/services.go

@@ -131,9 +131,6 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp
 	if name == "" {
 		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 	}
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	if strategy := aws.StringValue(input.SchedulingStrategy); strategy != "" && strategy != SchedulingStrategyReplica {
 		return nil, errors.New(awserrors.ErrorECSInvalidParameter) // DAEMON unsupported (Q15)
 	}

spinifex/domains/ecs/tagging_current_behaviour_test.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"strings"
 	"sync"
 	"testing"
 
@@ -64,12 +65,12 @@ func TestCurrentBehaviour_Tagging_ECSTagWriteRevertsInterleavedRecordWrite(t *te
 		require.NoError(t, taskdefinition.PutJSON(ctx, kv, key, &rec))
 	}}
 
-	err = svc.mutateResourceTags(ctx, wrapped, ecsResourceService, "web", "api", func(existing map[string]string) (map[string]string, error) {
+	err = svc.mutateResourceTags(ctx, wrapped, ecsResourceService, "web", "api", func(existing map[string]string) map[string]string {
 		if existing == nil {
 			existing = map[string]string{}
 		}
 		existing["team"] = "infra"
-		return existing, nil
+		return existing
 	})
 	require.NoError(t, err)
 
@@ -105,8 +106,9 @@ func TestCurrentBehaviour_Tagging_ECSForeignAccountARNTagsCallerResource(t *test
 	assert.Equal(t, "foreign-arn", aws.StringValue(listed.Tags[0].Value))
 }
 
-// TagResource applies no aws: prefix rule: the reserved key below is stored.
-func TestCurrentBehaviour_Tagging_ECSTagResourceAcceptsAWSPrefix(t *testing.T) {
+// TagResource applies no tag-count, key-length, value-length or aws: prefix
+// rule: all 52 tags below are stored.
+func TestCurrentBehaviour_Tagging_ECSTagResourceAppliesNoTagLimits(t *testing.T) {
 	svc, _ := newTestService(t)
 	ctx := context.Background()
 
@@ -114,11 +116,17 @@ func TestCurrentBehaviour_Tagging_ECSTagResourceAcceptsAWSPrefix(t *testing.T) {
 	require.NoError(t, err)
 	arn := aws.StringValue(created.Cluster.ClusterArn)
 
-	tags := []*ecs.Tag{{Key: aws.String("aws:reserved"), Value: aws.String("v")}}
+	tags := []*ecs.Tag{
+		{Key: aws.String("aws:reserved"), Value: aws.String("v")},
+		{Key: aws.String(strings.Repeat("k", 129)), Value: aws.String(strings.Repeat("v", 257))},
+	}
+	for i := range 50 {
+		tags = append(tags, &ecs.Tag{Key: aws.String("k" + strings.Repeat("x", i+1)), Value: aws.String("v")})
+	}
 	_, err = svc.TagResource(ctx, &ecs.TagResourceInput{ResourceArn: aws.String(arn), Tags: tags}, testAccountID)
 	require.NoError(t, err)
 
 	listed, err := svc.ListTagsForResource(ctx, &ecs.ListTagsForResourceInput{ResourceArn: aws.String(arn)}, testAccountID)
 	require.NoError(t, err)
-	assert.Len(t, listed.Tags, 1)
+	assert.Len(t, listed.Tags, 52)
 }

spinifex/domains/ecs/tags.go

@@ -138,7 +138,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 // mutateResourceTags reads the ARN-identified resource, applies mutate to its
 // tag map, and re-puts the record. Used by TagResource (merge) and
 // UntagResource (delete).
-func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue, kind ecsResourceKind, cluster, id string, mutate func(map[string]string) (map[string]string, error)) error {
+func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue, kind ecsResourceKind, cluster, id string, mutate func(map[string]string) map[string]string) error {
 	switch kind {
 	case ecsResourceCluster:
 		var rec ClusterRecord
@@ -149,11 +149,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 		if !found {
 			return errors.New(awserrors.ErrorECSClusterNotFound)
 		}
-		tags, err := mutate(rec.Tags)
-		if err != nil {
-			return err
-		}
-		rec.Tags = tags
+		rec.Tags = mutate(rec.Tags)
 		return taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(id), &rec)
 	case ecsResourceTaskDefinition:
 		family, rev := taskdefinition.ParseRef(id)
@@ -168,11 +164,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 		if !found {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
-		tags, err := mutate(rec.Tags)
-		if err != nil {
-			return err
-		}
-		rec.Tags = tags
+		rec.Tags = mutate(rec.Tags)
 		return taskdefinition.PutJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
 	case ecsResourceService:
 		var rec ServiceRecord
@@ -183,11 +175,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 		if !found {
 			return errors.New(awserrors.ErrorECSServiceNotFound)
 		}
-		tags, err := mutate(rec.Tags)
-		if err != nil {
-			return err
-		}
-		rec.Tags = tags
+		rec.Tags = mutate(rec.Tags)
 		return taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, id), &rec)
 	case ecsResourceTask:
 		var rec TaskRecord
@@ -198,11 +186,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 		if !found {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
-		tags, err := mutate(rec.Tags)
-		if err != nil {
-			return err
-		}
-		rec.Tags = tags
+		rec.Tags = mutate(rec.Tags)
 		return taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, id), &rec)
 	case ecsResourceContainerInstance:
 		var rec InstanceRecord
@@ -213,11 +197,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 		if !found {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
-		tags, err := mutate(rec.Tags)
-		if err != nil {
-			return err
-		}
-		rec.Tags = tags
+		rec.Tags = mutate(rec.Tags)
 		return taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 	default:
 		return errors.New(awserrors.ErrorECSInvalidParameter)
@@ -257,19 +237,13 @@ func (s *Service) TagResource(ctx context.Context, input *ecs.TagResourceInput,
 	if err != nil {
 		return nil, err
 	}
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	add := tagsToMap(input.Tags)
-	err = s.mutateResourceTags(ctx, kv, kind, cluster, id, func(existing map[string]string) (map[string]string, error) {
+	err = s.mutateResourceTags(ctx, kv, kind, cluster, id, func(existing map[string]string) map[string]string {
 		if existing == nil {
 			existing = map[string]string{}
 		}
 		maps.Copy(existing, add)
-		if len(existing) > maxTags {
-			return nil, errTagLimitExceeded()
-		}
-		return existing, nil
+		return existing
 	})
 	if err != nil {
 		return nil, err
@@ -290,17 +264,12 @@ func (s *Service) UntagResource(ctx context.Context, input *ecs.UntagResourceInp
 	if err != nil {
 		return nil, err
 	}
-	for _, k := range input.TagKeys {
-		if err := validateTagKey(aws.StringValue(k)); err != nil {
-			return nil, err
-		}
-	}
 	keys := awsStringSlice(input.TagKeys)
-	err = s.mutateResourceTags(ctx, kv, kind, cluster, id, func(existing map[string]string) (map[string]string, error) {
+	err = s.mutateResourceTags(ctx, kv, kind, cluster, id, func(existing map[string]string) map[string]string {
 		for _, k := range keys {
 			delete(existing, k)
 		}
-		return existing, nil
+		return existing
 	})
 	if err != nil {
 		return nil, err

spinifex/domains/ecs/tags_test.go

@@ -2,8 +2,6 @@ package ecs
 
 import (
 	"context"
-	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -240,151 +238,3 @@ func TestService_RunTask_TagsPersisted(t *testing.T) {
 	require.Len(t, described.Tasks, 1)
 	require.Len(t, described.Tasks[0].Tags, 1)
 }
-
-func tagN(n int) []*ecs.Tag {
-	tags := make([]*ecs.Tag, n)
-	for i := range tags {
-		tags[i] = &ecs.Tag{Key: aws.String(fmt.Sprintf("k%d", i)), Value: aws.String("v")}
-	}
-	return tags
-}
-
-// tagBreaks are the model's tag constraints with the error AWS answers each with.
-var tagBreaks = []struct {
-	name string
-	tags []*ecs.Tag
-	want string
-}{
-	{"empty key", []*ecs.Tag{{Key: aws.String(""), Value: aws.String("v")}}, "Tag key can not be null or empty.: InvalidParameterException"},
-	{"key over 128", []*ecs.Tag{{Key: aws.String(strings.Repeat("é", 129)), Value: aws.String("v")}}, "Tag key must have length less than or equal to 128: InvalidParameterException"},
-	{"key pattern", []*ecs.Tag{{Key: aws.String("a*b"), Value: aws.String("v")}}, "Some tags contain invalid characters. Valid characters: UTF-8 letters, spaces, numbers and _ . / = + - : @.: InvalidParameterException"},
-	{"value over 256", []*ecs.Tag{{Key: aws.String("k"), Value: aws.String(strings.Repeat("é", 257))}}, "Tag value must have length less than or equal to 256: InvalidParameterException"},
-	{"value pattern", []*ecs.Tag{{Key: aws.String("k"), Value: aws.String("a#b")}}, "Some tags contain invalid characters. Valid characters: UTF-8 letters, spaces, numbers and _ . / = + - : @.: InvalidParameterException"},
-	{"51 tags", tagN(51), "Tag Limit Exceeded.: ClientException"},
-}
-
-func TestService_CreateCluster_RefusesTagBreaks(t *testing.T) {
-	svc, _ := newTestService(t)
-	ctx := context.Background()
-	for _, tc := range tagBreaks {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web"), Tags: tc.tags}, testAccountID)
-			require.Error(t, err)
-			assert.Equal(t, tc.want, err.Error())
-		})
-	}
-	listed, err := svc.ListClusters(ctx, &ecs.ListClustersInput{}, testAccountID)
-	require.NoError(t, err)
-	assert.Empty(t, listed.ClusterArns)
-
-	_, err = svc.CreateCluster(ctx, &ecs.CreateClusterInput{
-		ClusterName: aws.String("web"),
-		Tags: append(tagN(48),
-			&ecs.Tag{Key: aws.String(strings.Repeat("é", 128)), Value: aws.String("ü _.:/=+-@ 1")},
-			&ecs.Tag{Key: aws.String("long"), Value: aws.String(strings.Repeat("é", 256))}),
-	}, testAccountID)
-	require.NoError(t, err)
-}
-
-func TestService_TagResource_RefusesTagBreaks(t *testing.T) {
-	svc, _ := newTestService(t)
-	ctx := context.Background()
-	created, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
-	require.NoError(t, err)
-	arn := created.Cluster.ClusterArn
-	for _, tc := range tagBreaks {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := svc.TagResource(ctx, &ecs.TagResourceInput{ResourceArn: arn, Tags: tc.tags}, testAccountID)
-			require.Error(t, err)
-			assert.Equal(t, tc.want, err.Error())
-		})
-	}
-	listed, err := svc.ListTagsForResource(ctx, &ecs.ListTagsForResourceInput{ResourceArn: arn}, testAccountID)
-	require.NoError(t, err)
-	assert.Empty(t, listed.Tags)
-}
-
-// The limit counts the resource's tags after the merge: overwriting an
-// existing key does not add to it, a new key past 50 does.
-func TestService_TagResource_RefusesMergedTotalOver50(t *testing.T) {
-	svc, _ := newTestService(t)
-	ctx := context.Background()
-	created, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web"), Tags: tagN(50)}, testAccountID)
-	require.NoError(t, err)
-	arn := created.Cluster.ClusterArn
-
-	_, err = svc.TagResource(ctx, &ecs.TagResourceInput{ResourceArn: arn, Tags: []*ecs.Tag{{Key: aws.String("k0"), Value: aws.String("new")}}}, testAccountID)
-	require.NoError(t, err)
-
-	_, err = svc.TagResource(ctx, &ecs.TagResourceInput{ResourceArn: arn, Tags: []*ecs.Tag{{Key: aws.String("extra"), Value: aws.String("v")}}}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, "Tag Limit Exceeded.: ClientException", err.Error())
-
-	listed, err := svc.ListTagsForResource(ctx, &ecs.ListTagsForResourceInput{ResourceArn: arn}, testAccountID)
-	require.NoError(t, err)
-	assert.Len(t, listed.Tags, 50)
-}
-
-func TestService_UntagResource_RefusesBadKey(t *testing.T) {
-	svc, _ := newTestService(t)
-	ctx := context.Background()
-	created, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web"), Tags: tagN(1)}, testAccountID)
-	require.NoError(t, err)
-	for _, tc := range tagBreaks[:3] {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := svc.UntagResource(ctx, &ecs.UntagResourceInput{
-				ResourceArn: created.Cluster.ClusterArn,
-				TagKeys:     []*string{aws.String("k0"), tc.tags[0].Key},
-			}, testAccountID)
-			require.Error(t, err)
-			assert.Equal(t, tc.want, err.Error())
-		})
-	}
-	listed, err := svc.ListTagsForResource(ctx, &ecs.ListTagsForResourceInput{ResourceArn: created.Cluster.ClusterArn}, testAccountID)
-	require.NoError(t, err)
-	assert.Len(t, listed.Tags, 1)
-}
-
-// Every other call that takes tags refuses a break before acting on it.
-func TestService_TaggedCreates_RefuseTagBreaks(t *testing.T) {
-	svc, _ := newTestService(t)
-	ctx := context.Background()
-	bad := []*ecs.Tag{{Key: aws.String("a*b"), Value: aws.String("v")}}
-	calls := map[string]func() error{
-		"RegisterTaskDefinition": func() error {
-			_, err := svc.RegisterTaskDefinition(ctx, &ecs.RegisterTaskDefinitionInput{Family: aws.String("f"), Tags: bad}, testAccountID)
-			return err
-		},
-		"CreateService": func() error {
-			_, err := svc.CreateService(ctx, &ecs.CreateServiceInput{ServiceName: aws.String("s"), Tags: bad}, testAccountID)
-			return err
-		},
-		"RunTask": func() error {
-			_, err := svc.RunTask(ctx, &ecs.RunTaskInput{Tags: bad}, testAccountID)
-			return err
-		},
-		"StartTask": func() error {
-			_, err := svc.StartTask(ctx, &ecs.StartTaskInput{Tags: bad}, testAccountID)
-			return err
-		},
-		"RegisterContainerInstance": func() error {
-			_, err := svc.RegisterContainerInstance(ctx, &ecs.RegisterContainerInstanceInput{Tags: bad}, testAccountID)
-			return err
-		},
-		"CreateCapacityProvider": func() error {
-			_, err := svc.CreateCapacityProvider(ctx, &ecs.CreateCapacityProviderInput{
-				Name:                     aws.String("cp"),
-				AutoScalingGroupProvider: &ecs.AutoScalingGroupProvider{AutoScalingGroupArn: aws.String("arn:aws:autoscaling:::x")},
-				Tags:                     bad,
-			}, testAccountID)
-			return err
-		},
-	}
-	for name, call := range calls {
-		t.Run(name, func(t *testing.T) {
-			err := call()
-			require.Error(t, err)
-			assert.Equal(t, tagBreaks[2].want, err.Error())
-		})
-	}
-}

spinifex/domains/ecs/task_stop.go

@@ -52,9 +52,6 @@ func (s *Service) StopTask(ctx context.Context, input *ecs.StopTaskInput, accoun
 // StartTask places one task per named container instance (explicit placement, no
 // scheduler bin-pack). Mirrors RunTask's reserve → record → ENI → assign flow.
 func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, accountID string) (*ecs.StartTaskOutput, error) {
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	cluster := ClusterShortName(aws.StringValue(input.Cluster))
 	kv, err := s.bucket(ctx, accountID)
 	if err != nil {

spinifex/domains/ecs/taskdef_lifecycle_test.go

@@ -194,50 +194,3 @@ func TestCreateService_PinsTheResolvedRevision(t *testing.T) {
 	require.Len(t, d.Services, 1)
 	assert.Equal(t, pinned, aws.StringValue(d.Services[0].TaskDefinition))
 }
-
-// Every launch answers an unknown family or revision with AWS's ClientException
-// "TaskDefinition not found.", while a reference with no family stays
-// InvalidParameterException.
-func TestLaunches_UnknownTaskDefinitionIsNotFound(t *testing.T) {
-	svc, _, _ := serviceTestRig(t)
-	_, err := svc.CreateService(context.Background(), &ecs.CreateServiceInput{
-		Cluster: aws.String("web"), ServiceName: aws.String("web"), TaskDefinition: aws.String("app"),
-	}, testAccountID)
-	require.NoError(t, err)
-
-	launches := map[string]func(ref string) error{
-		"RunTask": func(ref string) error {
-			_, err := svc.RunTask(context.Background(), &ecs.RunTaskInput{Cluster: aws.String("web"), TaskDefinition: aws.String(ref)}, testAccountID)
-			return err
-		},
-		"StartTask": func(ref string) error {
-			_, err := svc.StartTask(context.Background(), &ecs.StartTaskInput{
-				Cluster: aws.String("web"), TaskDefinition: aws.String(ref), ContainerInstances: []*string{aws.String("i-1")},
-			}, testAccountID)
-			return err
-		},
-		"CreateService": func(ref string) error {
-			_, err := svc.CreateService(context.Background(), &ecs.CreateServiceInput{
-				Cluster: aws.String("web"), ServiceName: aws.String("other"), TaskDefinition: aws.String(ref),
-			}, testAccountID)
-			return err
-		},
-		"UpdateService": func(ref string) error {
-			_, err := svc.UpdateService(context.Background(), &ecs.UpdateServiceInput{
-				Cluster: aws.String("web"), Service: aws.String("web"), TaskDefinition: aws.String(ref),
-			}, testAccountID)
-			return err
-		},
-	}
-	for name, launch := range launches {
-		for _, ref := range []string{"ghost", "app:9", taskdefinition.ARN(testRegion, testAccountID, "app", 9)} {
-			code, msg, ok := awserrors.ResolveErrorDetail(launch(ref))
-			require.True(t, ok, "%s %s", name, ref)
-			assert.Equal(t, awserrors.ErrorECSClientException, code, "%s %s", name, ref)
-			assert.Equal(t, "TaskDefinition not found.", msg, "%s %s", name, ref)
-		}
-		err := launch(":1")
-		require.Error(t, err, name)
-		assert.Equal(t, awserrors.ErrorECSInvalidParameter, err.Error(), name)
-	}
-}

spinifex/domains/ecs/tasks.go

@@ -27,9 +27,6 @@ const reservePlacementRetries = 5
 // publishes an assign on the Layer-2 bus for each. Placement failures for a task
 // are returned as RunTask failures; already-placed tasks in the same call stay.
 func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountID string) (*ecs.RunTaskOutput, error) {
-	if err := validateTags(input.Tags); err != nil {
-		return nil, err
-	}
 	cluster := ClusterShortName(aws.StringValue(input.Cluster))
 	kv, err := s.bucket(ctx, accountID)
 	if err != nil {

spinifex/domains/lambda/awsapi/create.go

@@ -110,7 +110,7 @@ func (d Deps) createFunction(ctx context.Context, c *call) (reply, error) {
 
 // rollback makes one compare-and-swap attempt to free the name. It cannot undo
 // a create that completed, because the revision it holds has moved on. The code
-// object, if one landed, stays in place under its owed upload record.
+// object, if one landed, is left in place and recorded as a cleanup obligation.
 func (d Deps) rollback(ctx context.Context, c *call, name string, createRev uint64, ref codestore.Ref, why string, cause error) {
 	slog.ErrorContext(ctx, "Lambda: create did not complete", "reason", why, "function_name", name,
 		"function_id", ref.FunctionID, "err", cause)

spinifex/domains/lambda/awsapi/get.go

@@ -40,12 +40,6 @@ func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Recor
 		}
 		q = query
 	}
-	return d.lookupTarget(ctx, c, name, q)
-}
-
-// lookupTarget reads the record for an already resolved name and qualifier;
-// only $LATEST exists.
-func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (function.Record, uint64, error) {
 	if q != "" && q != "$LATEST" {
 		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}

spinifex/domains/lambda/awsapi/handler_test.go

@@ -45,7 +45,6 @@ type localClient struct {
 	svc *function.MetaService
 
 	mu             sync.Mutex
-	forceActive    bool
 	failCodeStored bool
 	passOwnerClock bool
 	rollbacks      int
@@ -69,11 +68,6 @@ func (c *localClient) Get(ctx context.Context, account, region, name string) (fu
 	if !r.Found {
 		return function.Record{}, 0, function.ErrNotFound
 	}
-	c.mu.Lock()
-	if c.forceActive {
-		r.Record.State = function.StateActive
-	}
-	c.mu.Unlock()
 	return r.Record, r.Revision, nil
 }
 
@@ -110,20 +104,11 @@ type memStore struct {
 	objects map[codestore.Ref][]byte
 	putErr  error
 	urlErr  error
-	// beforePut runs once, unlocked, before the next write lands.
-	beforePut func()
 }
 
 var _ awsapi.CodeStore = (*memStore)(nil)
 
 func (m *memStore) Put(_ context.Context, ref codestore.Ref, code []byte) (string, error) {
-	m.mu.Lock()
-	hook := m.beforePut
-	m.beforePut = nil
-	m.mu.Unlock()
-	if hook != nil {
-		hook()
-	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if m.putErr != nil {
@@ -179,40 +164,6 @@ func newHarness(t *testing.T, statements ...iamdomain.Statement) (*gwtest.Harnes
 	return gwtest.Scoped(statements...), meta, store, kv, key
 }
 
-// uploads returns every upload record in the account bucket, by function ID.
-func (e *env) uploads(t *testing.T) map[string]function.Upload {
-	t.Helper()
-	kv := e.kv(t)
-	keys, err := kv.ListKeys(t.Context())
-	require.NoError(t, err)
-	out := map[string]function.Upload{}
-	for k := range keys.Keys() {
-		if !strings.HasPrefix(k, "upload.") {
-			continue
-		}
-		entry, err := kv.Get(t.Context(), k)
-		require.NoError(t, err)
-		var up function.Upload
-		require.NoError(t, json.Unmarshal(entry.Value(), &up))
-		require.Equal(t, function.KVUploadKey(up.Region, up.FunctionID, up.RevisionID), k)
-		out[up.FunctionID] = up
-	}
-	return out
-}
-
-// requireTracked asserts every code copy in the store has an upload record.
-func (e *env) requireTracked(t *testing.T) {
-	t.Helper()
-	ups := e.uploads(t)
-	e.store.mu.Lock()
-	defer e.store.mu.Unlock()
-	for ref := range e.store.objects {
-		up, ok := ups[ref.FunctionID]
-		require.True(t, ok, "code copy %s has no upload record", ref.FunctionID)
-		require.Equal(t, ref.RevisionID, up.RevisionID)
-	}
-}
-
 type env struct {
 	h     *gwtest.Harness
 	meta  *localClient
@@ -324,15 +275,6 @@ func TestCreateFunction_Success(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotContains(t, string(entry.Value()), "hunter2", "the environment is never stored in plaintext")
 	assert.Contains(t, string(entry.Value()), "envCipher")
-
-	e.requireTracked(t)
-	ups := e.uploads(t)
-	require.Len(t, ups, 1)
-	for _, up := range ups {
-		assert.Equal(t, function.UploadAttached, up.State)
-		assert.Equal(t, entry.Revision(), up.AttachedRevision)
-		assert.Equal(t, "hello", up.FunctionName)
-	}
 }
 
 func TestCreateFunction_RendersTheStoredTimestamps(t *testing.T) {
@@ -365,8 +307,6 @@ func TestCreateFunction_DuplicateNameLeavesFirstUntouched(t *testing.T) {
 	assert.Contains(t, err.Error(), "Function already exist: hello")
 
 	assert.Equal(t, 1, e.store.count(), "the loser wrote no code")
-	assert.Len(t, e.uploads(t), 1, "the refused create leaves no upload record")
-	e.requireTracked(t)
 	after, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.NoError(t, err)
 	assert.Equal(t, first, after)
@@ -381,11 +321,6 @@ func TestCreateFunction_CodePutFailureFreesTheName(t *testing.T) {
 	assert.Equal(t, 1, e.meta.rollbacks)
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound)
-	ups := e.uploads(t)
-	require.Len(t, ups, 1)
-	for _, up := range ups {
-		assert.Equal(t, function.UploadOwed, up.State, "a failed write may still have landed")
-	}
 
 	e.store.putErr = nil
 	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
@@ -402,51 +337,6 @@ func TestCreateFunction_CodeStoredFailureRollsBackOnce(t *testing.T) {
 	assert.Equal(t, 1, e.store.count(), "the code object is left in place, never deleted here")
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound, "the still-storing claim was withdrawn")
-	e.requireTracked(t)
-	for _, up := range e.uploads(t) {
-		assert.Equal(t, function.UploadOwed, up.State, "the unnamed copy stays owed")
-	}
-}
-
-// The first create's write stalls past its claim, a second create takes the
-// name over, then the first write lands and its completion is refused.
-func TestCreateFunction_LateWriterAfterTakeoverStaysTracked(t *testing.T) {
-	e := newEnv(t)
-	key := function.KVFunctionKey(gwtest.Region, "hello")
-	e.store.beforePut = func() {
-		entry, err := e.kv(t).Get(t.Context(), key)
-		require.NoError(t, err)
-		var rec function.Record
-		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
-		rec.ClaimExpiresAt = time.Now().Add(-time.Second)
-		data, err := json.Marshal(rec)
-		require.NoError(t, err)
-		_, err = e.kv(t).Update(t.Context(), key, data, entry.Revision())
-		require.NoError(t, err)
-
-		rec2, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
-			b["Code"] = map[string]any{"ZipFile": mustZip(t, "second")}
-		}))
-		require.NoError(t, err)
-		require.Equal(t, http.StatusCreated, rec2.Code)
-	}
-
-	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.Equal(t, 2, e.store.count(), "both copies landed")
-	e.requireTracked(t)
-
-	winner, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
-	require.NoError(t, err)
-	ups := e.uploads(t)
-	require.Len(t, ups, 2)
-	for id, up := range ups {
-		if id == winner.FunctionID {
-			assert.Equal(t, function.UploadAttached, up.State)
-			continue
-		}
-		assert.Equal(t, function.UploadOwed, up.State, "the late writer's copy is never inferred complete")
-	}
 }
 
 func TestCreateFunction_RefusesUnsupportedInput(t *testing.T) {
@@ -765,7 +655,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "ListVersionsByFunction"}, reg.Inventory.Registered)
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))