fix: release only system-tagged addresses in the terminate backstop

An internet-facing ALB's address is now tagged managed-by at allocation, and
ReleaseAddressByInstanceID leaves any untagged record alone with a warning, so
a wrong instance ID can no longer release a customer's own address.

.gitignore

@@ -27,7 +27,6 @@ build/microvm/sqs-worker/.stage.*/
 build/microvm/lambda-runtime/vmlinuz
 build/microvm/lambda-runtime/initramfs.cpio.gz
 build/microvm/lambda-runtime/.stage.*/
-build/microvm/lambda-testlink-runtime/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -82,35 +82,6 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
-# Controlled-test only: builds the tagged daemon and its hard-kill initializer
-# together, pinning the initializer's digest into that daemon build. The daemon
-# keeps its symbols so the pin can be read back from the binary.
-# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
-# build-lambda-testlink-image prints; it is read raw and validated before use.
-LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
-LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
-LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
-LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
-build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
-build-lambda-testlink:
-	@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
-	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
-	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
-	@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
-	@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
-	@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
-	cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
-	scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
-	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
-	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
-	echo "initializer sha256 $$digest"; \
-	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
-	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
-	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
-	@mkdir -p $(LAMBDA_TESTLINK_DIR)
-	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
-	@rmdir $(LAMBDA_TESTLINK_DIR).tmp
-
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
 	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -226,24 +197,6 @@ bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
 
 build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
-	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
-
-# Controlled-test only: a separately built guest whose agent has the
-# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
-# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
-LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
-build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
-	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
-	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
-	$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
-	@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
-	GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
-	./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
-	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
-	install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
-	@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
-	[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
-	echo "LAMBDA_IMAGE_DIGEST=$$digest"
 
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -299,10 +252,6 @@ test-build-scripts:
 	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
 		$(_RUNSH); \
 	done
-	@$(_HDR) "\n....Running scripts/terraform/latitude-spx/*_test.sh...."
-	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find scripts/terraform/latitude-spx -maxdepth 1 -name '*_test.sh' | sort); do \
-		$(_RUNSH); \
-	done
 	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
 	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
 		$(_RUNSH); \
@@ -380,10 +329,7 @@ test-cover:
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
 	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -594,7 +540,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

cmd/lambda-agent/connect.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 )
 
@@ -30,20 +29,20 @@ func defaultStartup() startup {
 
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
-func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {
+func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
 	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
 	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
-		return nil, "", exitRefused, errors.Join(loadErr, err)
+		return nil, exitRefused, errors.Join(loadErr, err)
 	}
 	if loadErr != nil {
-		return nil, "", exitRefused, loadErr
+		return nil, exitRefused, loadErr
 	}
 	conn, err := hostlink.Dial(ctx, id)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
 		}
-		return nil, "", exitLink, err
+		return nil, exitLink, err
 	}
-	return conn, id.Scope, 0, nil
+	return conn, 0, nil
 }

cmd/lambda-agent/grant_testlink.go

@@ -1,66 +0,0 @@
-//go:build lambdatestlink
-
-package main
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
-)
-
-const entryGrant = "test-grant"
-
-// testGrant is the grant the host launched this VM under.
-type testGrant struct {
-	grantID, attemptID, environmentID string
-	generation                        uint64
-}
-
-// readGrant reads the test-grant fw_cfg entry: grant ID, attempt ID,
-// environment ID and generation, one per line.
-func readGrant(fwCfgDir string) (testGrant, error) {
-	b, err := os.ReadFile(filepath.Join(fwCfgDir, entryGrant, "raw"))
-	if err != nil {
-		return testGrant{}, fmt.Errorf("read %s: %w", entryGrant, err)
-	}
-	lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n")
-	if len(lines) != 4 {
-		return testGrant{}, errors.New(entryGrant + " entry malformed")
-	}
-	g := testGrant{grantID: lines[0], attemptID: lines[1], environmentID: lines[2]}
-	g.generation, err = strconv.ParseUint(lines[3], 10, 64)
-	if err != nil || g.generation == 0 || g.grantID == "" || g.attemptID == "" || g.environmentID == "" {
-		return testGrant{}, errors.New(entryGrant + " entry malformed")
-	}
-	return g, nil
-}
-
-// checkBinding refuses a run unless the grant names the authenticated environment
-// generation, the request carries that grant and its deadline is still ahead. The
-// clock check is a local guard only: it cannot detect skew or a superseded grant.
-func checkBinding(scope lambdav1.Scope, g testGrant, req hostlink.Request, now time.Time) error {
-	parts := strings.Split(string(scope), "/")
-	if len(parts) != 5 {
-		return errors.New("authenticated scope malformed")
-	}
-	if parts[3] != g.environmentID || parts[4] != strconv.FormatUint(g.generation, 10) {
-		return errors.New("grant is not for the authenticated environment generation")
-	}
-	if req.GrantID != g.grantID || req.AttemptID != g.attemptID {
-		return errors.New("request does not carry this VM's grant")
-	}
-	if req.DeadlineMs <= 0 {
-		return errors.New("request has no deadline")
-	}
-	if !now.Before(time.UnixMilli(req.DeadlineMs)) {
-		return errors.New("request deadline already passed")
-	}
-	return nil
-}

cmd/lambda-agent/helpers_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/1"
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
 
 // guest is a fake fw_cfg tree; unload removes it as unloading the module would.
 type guest struct {

cmd/lambda-agent/package_testlink.go

@@ -18,7 +18,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-	"unicode"
 
 	"golang.org/x/sys/unix"
 )
@@ -29,7 +28,6 @@ import (
 const (
 	entryPackageSize   = "test-package-size"
 	entryPackageSHA256 = "test-package-sha256"
-	entryHandler       = "test-handler"
 
 	devMaxZipBytes    = 16 << 20
 	devMaxUnzipBytes  = 64 << 20
@@ -39,9 +37,8 @@ const (
 )
 
 type packageSpec struct {
-	size    int64
-	sha256  [sha256.Size]byte
-	handler string
+	size   int64
+	sha256 [sha256.Size]byte
 }
 
 type unpackLimits struct {
@@ -79,13 +76,6 @@ func readPackageSpec(fwCfgDir string) (packageSpec, error) {
 		return spec, errors.New("package sha256 is not 64 hex digits")
 	}
 	copy(spec.sha256[:], b)
-	if spec.handler, err = read(entryHandler); err != nil {
-		return spec, err
-	}
-	if spec.handler == "" || len(spec.handler) > 128 || strings.IndexFunc(spec.handler, unicode.IsSpace) >= 0 ||
-		strings.ContainsRune(spec.handler, 0) {
-		return spec, errors.New("handler must be 1..128 characters with no whitespace")
-	}
 	return spec, nil
 }
 

cmd/lambda-agent/package_testlink_test.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 	"syscall"
 	"testing"
 
@@ -166,23 +165,19 @@ func TestUnpackChecksSizeAndHash(t *testing.T) {
 func TestReadPackageSpecBounds(t *testing.T) {
 	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 	cases := map[string]struct {
-		size, hash, handler string
-		want                string
+		size, hash string
+		want       string
 	}{
-		"valid":            {"10", sum, "bootstrap.h", ""},
-		"zero":             {"0", sum, "bootstrap.h", "outside"},
-		"too large":        {"16777217", sum, "bootstrap.h", "outside"},
-		"not hex":          {"10", "zz", "bootstrap.h", "64 hex digits"},
-		"short":            {"10", sum[:62], "bootstrap.h", "64 hex digits"},
-		"empty handler":    {"10", sum, "", "handler"},
-		"handler space":    {"10", sum, "a b", "handler"},
-		"128-char handler": {"10", sum, strings.Repeat("h", 128), ""},
-		"129-char handler": {"10", sum, strings.Repeat("h", 129), "handler"},
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			dir := t.TempDir()
-			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash, entryHandler: tc.handler} {
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
 				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
 			}
@@ -190,7 +185,6 @@ func TestReadPackageSpecBounds(t *testing.T) {
 			if tc.want == "" {
 				require.NoError(t, err)
 				assert.Equal(t, int64(10), spec.size)
-				assert.Equal(t, tc.handler, spec.handler)
 				return
 			}
 			assert.ErrorContains(t, err, tc.want)

cmd/lambda-agent/start_tcp.go

@@ -25,7 +25,7 @@ func start(ctx context.Context) int {
 
 // run authenticates to the host and holds the connection until it closes.
 func (s startup) run(ctx context.Context) (int, error) {
-	conn, _, code, err := s.connect(ctx)
+	conn, code, err := s.connect(ctx)
 	if err != nil {
 		return code, err
 	}

cmd/lambda-agent/start_testlink.go

@@ -5,6 +5,7 @@ package main
 import (
 	"context"
 	"errors"
+	"flag"
 	"fmt"
 	"log/slog"
 	"net"
@@ -23,6 +24,8 @@ const (
 	taskRoot    = "/var/task"
 )
 
+var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
+
 // testlink relays one invocation over the test carrier on the authenticated
 // connection; the carrier is not the guest protocol.
 type testlink struct {
@@ -51,14 +54,13 @@ func start(ctx context.Context) int {
 func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
-	grant, grantErr := readGrant(t.fwCfgDir)
-	conn, scope, code, err := t.connect(parent)
+	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, errors.Join(err, specErr, grantErr)
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
-	if err := errors.Join(specErr, grantErr); err != nil {
-		return exitRefused, err
+	if specErr != nil {
+		return exitRefused, specErr
 	}
 	link := hostlink.NewTestStream(conn)
 
@@ -74,10 +76,6 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
-	// Nothing of the tenant's is installed or run for an unbound request.
-	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
-		return exitRefused, err
-	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -95,7 +93,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	defer cancel(nil)
 	cmd, err := t.startBootstrap(ctx, []string{
 		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
-		"_HANDLER=" + spec.handler,
+		"_HANDLER=" + *handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
 		"PATH=/bin:/sbin:/usr/bin",
 	})

cmd/lambda-agent/start_testlink_test.go

@@ -25,18 +25,12 @@ import (
 
 const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"
 
-// helperHandlerOut names the file the helper bootstrap writes its _HANDLER to.
-const helperHandlerOut = "LAMBDA_AGENT_TEST_HANDLER_OUT"
-
 // TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
 // echoes one invocation through the Runtime API, then waits for the next.
 func TestHelperBootstrap(t *testing.T) {
 	if os.Getenv(helperEnv) != "1" {
 		return
 	}
-	if out := os.Getenv(helperHandlerOut); out != "" {
-		require.NoError(t, os.WriteFile(out, []byte(os.Getenv("_HANDLER")), 0o600))
-	}
 	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
 	resp, err := http.Get(base + "next")
 	require.NoError(t, err)
@@ -50,19 +44,11 @@ func TestHelperBootstrap(t *testing.T) {
 	_, _ = http.Get(base + "next")
 }
 
-// testGrantEntry grants g-1 for attempt a-1 on the fixture scope's env-1/1.
-const testGrantEntry = "g-1\na-1\nenv-1\n1\n"
-
-// testHandler is the handler the fixture's host configures.
-const testHandler = "bootstrap.fixture"
-
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
 	task    string
 	sealed  bool
-	// handlerOut receives the _HANDLER the bootstrap process itself saw.
-	handlerOut string
 }
 
 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -75,16 +61,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	sum := sha256.Sum256(zipBytes)
 	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
 	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
-	if _, ok := entries[entryHandler]; !ok {
-		entries[entryHandler] = []byte(testHandler)
-	}
-	if _, ok := entries[entryGrant]; !ok {
-		entries[entryGrant] = []byte(testGrantEntry)
-	}
 	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
-	b.handlerOut = filepath.Join(t.TempDir(), "handler")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -98,7 +77,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			b.n++
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
-			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
+			cmd.Env = append(env, helperEnv+"=1")
 			return cmd, cmd.Start()
 		},
 	}
@@ -153,9 +132,6 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
-	seen, err := os.ReadFile(b.handlerOut)
-	require.NoError(t, err, "bootstrap process did not report its handler")
-	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
@@ -242,8 +218,7 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
 
 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
-		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
 	assert.ErrorContains(t, res.err, "install package")
@@ -269,87 +244,3 @@ func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
 	_, err := os.Stat(tl.fwCfgRoot)
 	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
 }
-
-// A request that does not carry this VM's grant, or a grant for another
-// environment generation than the one authenticated, installs and runs
-// nothing of the tenant's.
-func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
-	cases := map[string]struct {
-		grant            string
-		grantID, attempt string
-		deadlineMs       int64
-		want             string
-	}{
-		"frame with no deadline":      {testGrantEntry, "g-1", "a-1", -1, "no deadline"},
-		"frame with expired deadline": {testGrantEntry, "g-1", "a-1", time.Now().Add(-time.Second).UnixMilli(), "deadline already passed"},
-		"frame for another grant":     {testGrantEntry, "g-2", "a-1", 0, "does not carry this VM's grant"},
-		"frame for another attempt":   {testGrantEntry, "g-1", "a-2", 0, "does not carry this VM's grant"},
-		"frame with no grant":         {testGrantEntry, "", "", 0, "does not carry this VM's grant"},
-		"grant for another env":       {"g-1\na-1\nenv-2\n1\n", "g-1", "a-1", 0, "not for the authenticated environment"},
-		"grant for another gen":       {"g-1\na-1\nenv-1\n2\n", "g-1", "a-1", 0, "not for the authenticated environment"},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			entries := h.entries()
-			entries[entryGrant] = []byte(tc.grant)
-			done := runTestlinkAsync(ctx, b.testlink(t, entries))
-			conn := h.guest(t)
-			defer conn.Close()
-			deadline := time.Now().Add(5 * time.Second).UnixMilli()
-			switch {
-			case tc.deadlineMs < 0:
-				deadline = 0
-			case tc.deadlineMs > 0:
-				deadline = tc.deadlineMs
-			}
-			req := hostlink.Request{GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
-				DeadlineMs: deadline, Payload: []byte(`{}`)}
-			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
-
-			res := wait(t, done, "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, tc.want)
-			assert.Zero(t, b.n)
-			assert.False(t, b.sealed)
-			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
-		})
-	}
-}
-
-// A missing or malformed grant entry refuses before any frame is awaited.
-func TestTestlinkBadGrantEntryRefuses(t *testing.T) {
-	cases := map[string]*string{
-		"missing":       nil,
-		"too few lines": ptr("g-1\na-1\nenv-1\n"),
-		"generation 0":  ptr("g-1\na-1\nenv-1\n0\n"),
-		"empty grant":   ptr("\na-1\nenv-1\n1\n"),
-	}
-	for name, grant := range cases {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			entries := h.entries()
-			if grant != nil {
-				entries[entryGrant] = []byte(*grant)
-			}
-			tl := b.testlink(t, entries)
-			if grant == nil {
-				require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryGrant)))
-			}
-			res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, entryGrant)
-			assert.Zero(t, b.n)
-		})
-	}
-}
-
-func ptr(s string) *string { return &s }

cmd/lambda-image-agent-tag/main.go

@@ -1,221 +0,0 @@
-// Command lambda-image-agent-tag checks whether the lambda-agent archived in a
-// built Lambda guest initramfs carries the lambdatestlink build tag. Build
-// information is recorded by the build; it is not proof of how the agent behaves.
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"compress/gzip"
-	"debug/buildinfo"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"slices"
-	"strconv"
-	"strings"
-)
-
-const (
-	agentPath    = "usr/local/bin/lambda-agent"
-	maxArchive   = 1 << 30
-	maxAgent     = 256 << 20
-	maxName      = 4096
-	newcHeader   = 110
-	trailerName  = "TRAILER!!!"
-	modeTypeMask = 0o170000
-	modeDir      = 0o040000
-	modeRegular  = 0o100000
-)
-
-var agentParents = []string{"usr", "usr/local", "usr/local/bin"}
-
-func main() {
-	if len(os.Args) != 3 || (os.Args[2] != "present" && os.Args[2] != "absent") {
-		fmt.Fprintln(os.Stderr, "usage: lambda-image-agent-tag <initramfs.cpio.gz> present|absent")
-		os.Exit(2)
-	}
-	if err := run(os.Args[1], os.Args[2] == "present"); err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[1], err)
-		os.Exit(1)
-	}
-}
-
-func run(path string, want bool) error {
-	fi, err := os.Lstat(path)
-	if err != nil {
-		return err
-	}
-	if !fi.Mode().IsRegular() {
-		return errors.New("not a regular file")
-	}
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	agent, err := archivedAgent(f)
-	if err != nil {
-		return err
-	}
-	return checkTag(agent, want)
-}
-
-// archivedAgent returns the single agent entry after checking listed names,
-// entry types, the agent's link count and zero-only padding after the trailer.
-// It is not a full kernel-equivalent parser; 070702 CRCs, for one, are unchecked.
-func archivedAgent(r io.Reader) ([]byte, error) {
-	zr, err := gzip.NewReader(bufio.NewReader(r))
-	if err != nil {
-		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
-	}
-	raw, err := io.ReadAll(io.LimitReader(zr, maxArchive+1))
-	if err != nil {
-		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
-	}
-	if len(raw) > maxArchive {
-		return nil, fmt.Errorf("archive larger than %d bytes", maxArchive)
-	}
-
-	seen := map[string]bool{}
-	var agent []byte
-	off := 0
-	for {
-		e, next, err := parseEntry(raw, off)
-		if err != nil {
-			return nil, fmt.Errorf("entry at offset %d: %w", off, err)
-		}
-		off = next
-		if e.name == trailerName {
-			break
-		}
-		if !canonical(e.name) {
-			return nil, fmt.Errorf("entry name %q is not canonical", e.name)
-		}
-		if seen[e.name] {
-			return nil, fmt.Errorf("entry %q appears more than once", e.name)
-		}
-		seen[e.name] = true
-		for _, p := range agentParents {
-			if e.name == p && e.mode&modeTypeMask != modeDir {
-				return nil, fmt.Errorf("%s is not a directory", p)
-			}
-		}
-		if e.name == agentPath {
-			for _, p := range agentParents {
-				if !seen[p] {
-					return nil, fmt.Errorf("%s has no %s directory entry before it", agentPath, p)
-				}
-			}
-			if e.mode&modeTypeMask != modeRegular {
-				return nil, fmt.Errorf("%s is not a regular file", agentPath)
-			}
-			if e.nlink != 1 {
-				return nil, fmt.Errorf("%s has %d links, want 1", agentPath, e.nlink)
-			}
-			if len(e.data) > maxAgent {
-				return nil, fmt.Errorf("%s is larger than %d bytes", agentPath, maxAgent)
-			}
-			agent = e.data
-		}
-	}
-	if i := bytes.IndexFunc(raw[off:], func(r rune) bool { return r != 0 }); i >= 0 {
-		return nil, fmt.Errorf("data after the archive trailer at offset %d", off+i)
-	}
-	if agent == nil {
-		return nil, fmt.Errorf("no %s entry", agentPath)
-	}
-	return agent, nil
-}
-
-type entry struct {
-	name  string
-	mode  uint64
-	nlink uint64
-	data  []byte
-}
-
-func parseEntry(raw []byte, off int) (entry, int, error) {
-	if len(raw)-off < newcHeader {
-		return entry{}, 0, errors.New("truncated header")
-	}
-	h := raw[off : off+newcHeader]
-	if m := string(h[:6]); m != "070701" && m != "070702" {
-		return entry{}, 0, fmt.Errorf("magic %q is not newc", m)
-	}
-	field := func(i int) (uint64, error) {
-		return strconv.ParseUint(string(h[6+8*i:14+8*i]), 16, 32)
-	}
-	mode, err1 := field(1)
-	nlink, err2 := field(4)
-	size, err3 := field(6)
-	nameSize, err4 := field(11)
-	if err := errors.Join(err1, err2, err3, err4); err != nil {
-		return entry{}, 0, fmt.Errorf("bad header field: %w", err)
-	}
-	if nameSize < 2 || nameSize > maxName {
-		return entry{}, 0, fmt.Errorf("name size %d out of range", nameSize)
-	}
-	nameEnd := off + newcHeader + int(nameSize)
-	dataOff := align4(nameEnd)
-	dataEnd := dataOff + int(size)
-	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
-		return entry{}, 0, errors.New("truncated entry")
-	}
-	name := raw[off+newcHeader : nameEnd]
-	if name[len(name)-1] != 0 || bytes.IndexByte(name[:len(name)-1], 0) >= 0 {
-		return entry{}, 0, errors.New("name is not one NUL-terminated string")
-	}
-	return entry{name: string(name[:len(name)-1]), mode: mode, nlink: nlink, data: raw[dataOff:dataEnd]}, align4(dataEnd), nil
-}
-
-func align4(n int) int { return (n + 3) &^ 3 }
-
-// canonical accepts "." and relative names with no empty, "." or ".." component.
-func canonical(name string) bool {
-	if name == "." {
-		return true
-	}
-	if name == "" || strings.HasPrefix(name, "/") {
-		return false
-	}
-	for c := range strings.SplitSeq(name, "/") {
-		if c == "" || c == "." || c == ".." {
-			return false
-		}
-	}
-	return true
-}
-
-func checkTag(agent []byte, want bool) error {
-	info, err := buildinfo.Read(bytes.NewReader(agent))
-	if err != nil {
-		return fmt.Errorf("no Go build information in %s: %w", agentPath, err)
-	}
-	return decide(info, want)
-}
-
-// decide refuses build information with no recorded settings, since their
-// absence does not show the tag was absent.
-func decide(info *buildinfo.BuildInfo, want bool) error {
-	if len(info.Settings) == 0 {
-		return fmt.Errorf("%s records no build settings", agentPath)
-	}
-	if hasTag(info, "lambdatestlink") != want {
-		return fmt.Errorf("%s: lambdatestlink build tag present=%v, want %v", agentPath, !want, want)
-	}
-	return nil
-}
-
-func hasTag(info *buildinfo.BuildInfo, tag string) bool {
-	for _, s := range info.Settings {
-		if s.Key != "-tags" {
-			continue
-		}
-		if slices.Contains(strings.Split(s.Value, ","), tag) {
-			return true
-		}
-	}
-	return false
-}

cmd/lambda-image-agent-tag/main_test.go

@@ -1,154 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"debug/buildinfo"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime/debug"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-type fileEntry struct {
-	name  string
-	mode  uint32
-	nlink uint32
-	data  string
-}
-
-func dir(name string) fileEntry { return fileEntry{name: name, mode: modeDir | 0o755, nlink: 2} }
-func file(name, data string) fileEntry {
-	return fileEntry{name: name, mode: modeRegular | 0o755, nlink: 1, data: data}
-}
-func symlink(name, target string) fileEntry {
-	return fileEntry{name: name, mode: 0o120777, nlink: 1, data: target}
-}
-
-// newc writes entries and a trailer in the format the kernel unpacks.
-func newc(entries ...fileEntry) []byte {
-	var b bytes.Buffer
-	for i, e := range append(entries, fileEntry{name: trailerName, nlink: 1}) {
-		fmt.Fprintf(&b, "070701%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X",
-			i+1, e.mode, 0, 0, e.nlink, 0, len(e.data), 0, 0, 0, 0, len(e.name)+1, 0)
-		b.WriteString(e.name)
-		b.WriteByte(0)
-		pad(&b)
-		b.WriteString(e.data)
-		pad(&b)
-	}
-	return b.Bytes()
-}
-
-func pad(b *bytes.Buffer) {
-	for b.Len()%4 != 0 {
-		b.WriteByte(0)
-	}
-}
-
-func gz(parts ...[]byte) []byte {
-	var b bytes.Buffer
-	for _, p := range parts {
-		w := gzip.NewWriter(&b)
-		_, _ = w.Write(p)
-		_ = w.Close()
-	}
-	return b.Bytes()
-}
-
-func withAgent(agent fileEntry) []byte {
-	return newc(dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent, file("init", "#!/bin/sh"))
-}
-
-func TestArchivedAgent_ReturnsTheOneArchivedAgent(t *testing.T) {
-	got, err := archivedAgent(bytes.NewReader(gz(withAgent(file(agentPath, "agent-bytes")))))
-	require.NoError(t, err)
-	require.Equal(t, "agent-bytes", string(got))
-
-	padded := append(withAgent(file(agentPath, "agent-bytes")), make([]byte, 1024)...)
-	got, err = archivedAgent(bytes.NewReader(gz(padded)))
-	require.NoError(t, err, "zero padding after the trailer")
-	require.Equal(t, "agent-bytes", string(got))
-}
-
-func TestArchivedAgent_Refuses(t *testing.T) {
-	parents := []fileEntry{dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin")}
-	agent := file(agentPath, "untagged")
-	small := newc(append(parents, agent)...)
-	second := newc(append(parents[1:], file(agentPath, "tagged"))...)
-	// The second archive starts inside what would be the first one's final block.
-	sameBlock := append(append([]byte{}, small...), second...)
-	for len(sameBlock)%512 != 0 {
-		sameBlock = append(sameBlock, 0)
-	}
-
-	for name, archive := range map[string][]byte{
-		"aliased duplicate":            gz(newc(append(parents, agent, file("././"+agentPath, "tagged"))...)),
-		"leading slash alias":          gz(newc(append(parents, agent, file("/"+agentPath, "tagged"))...)),
-		"dot-dot component":            gz(newc(append(parents, agent, file("usr/local/../local/bin/lambda-agent", "x"))...)),
-		"empty component":              gz(newc(append(parents, agent, file("usr//local/bin/lambda-agent", "x"))...)),
-		"exact duplicate":              gz(newc(append(parents, agent, file(agentPath, "tagged"))...)),
-		"duplicate directory":          gz(newc(dir("usr"), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent)),
-		"missing directory entry":      gz(newc(dir("usr"), dir("usr/local/bin"), agent)),
-		"directory after the agent":    gz(newc(dir("usr"), dir("usr/local"), agent, dir("usr/local/bin"))),
-		"parent is a symlink":          gz(newc(dir("usr"), dir("usr/local"), symlink("usr/local/bin", "elsewhere"), agent)),
-		"agent is a symlink":           gz(newc(append(parents, symlink(agentPath, "/bin/sh"))...)),
-		"hard-linked agent":            gz(newc(append(parents, fileEntry{name: agentPath, mode: modeRegular | 0o755, nlink: 2, data: "x"})...)),
-		"no agent":                     gz(newc(parents...)),
-		"second archive appended":      gz(small, second),
-		"second archive in same block": gz(sameBlock),
-		"non-zero byte after trailer":  gz(append(append([]byte{}, small...), 0, 0, 1)),
-		"truncated archive":            gz(small[:len(small)-20]),
-		"not newc":                     gz([]byte("not cpio at all, but long enough to fill one newc header: " + string(make([]byte, 120)))),
-		"not gzip":                     small,
-	} {
-		t.Run(name, func(t *testing.T) {
-			_, err := archivedAgent(bytes.NewReader(archive))
-			require.Error(t, err)
-		})
-	}
-}
-
-func TestRun_RefusesASymlinkedImagePath(t *testing.T) {
-	d := t.TempDir()
-	img := filepath.Join(d, "initramfs.cpio.gz")
-	require.NoError(t, os.WriteFile(img, gz(withAgent(file(agentPath, "x"))), 0o644))
-	require.NoError(t, os.Symlink(img, filepath.Join(d, "link")))
-	require.ErrorContains(t, run(filepath.Join(d, "link"), false), "not a regular file")
-}
-
-func TestHasTag_MatchesWholeTagsOnly(t *testing.T) {
-	info := func(tags string) *buildinfo.BuildInfo {
-		return &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}, {Key: "-tags", Value: tags}}}
-	}
-	require.True(t, hasTag(info("lambdatestlink,fips140v1.0"), "lambdatestlink"))
-	require.False(t, hasTag(info("fips140v1.0"), "lambdatestlink"))
-	require.False(t, hasTag(info("lambdatestlinkx,xlambdatestlink"), "lambdatestlink"))
-	require.False(t, hasTag(&buildinfo.BuildInfo{}, "lambdatestlink"))
-}
-
-// The test binary carries real build information, so checkTag reads it and
-// refuses the opposite answer; bytes with no build information are refused.
-func TestCheckTag_ReadsRealBuildInformation(t *testing.T) {
-	exe, err := os.Executable()
-	require.NoError(t, err)
-	raw, err := os.ReadFile(exe)
-	require.NoError(t, err)
-	info, err := buildinfo.Read(bytes.NewReader(raw))
-	require.NoError(t, err)
-	tagged := hasTag(info, "lambdatestlink")
-	require.NoError(t, checkTag(raw, tagged))
-	require.Error(t, checkTag(raw, !tagged))
-	require.Error(t, checkTag([]byte("not a Go binary"), false))
-}
-
-func TestDecide_RefusesMissingBuildSettings(t *testing.T) {
-	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, false), "no build settings")
-	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, true), "no build settings")
-	untagged := &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}}}
-	require.NoError(t, decide(untagged, false))
-	require.Error(t, decide(untagged, true))
-}

cmd/lambda-image-digest/main.go

@@ -1,26 +0,0 @@
-//go:build lambdatestlink
-
-// Command lambda-image-digest prints the test-link runtime image digest of one
-// directory, selecting the boot files the same way the daemon does. The digest
-// identifies the bytes; it does not authenticate where they came from.
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
-)
-
-func main() {
-	if len(os.Args) != 2 {
-		fmt.Fprintln(os.Stderr, "usage: lambda-image-digest <image-dir>")
-		os.Exit(2)
-	}
-	digest, err := testlink.DirDigest(os.Args[1])
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-	fmt.Println(digest)
-}

scripts/build-lambda-runtime-microvm-image.sh

@@ -3,28 +3,24 @@
 # + initramfs: a guest running the Lambda guest agent, with virtio-net for its
 # host control connection. It is not shipped.
 #
-# Usage: build-lambda-runtime-microvm-image.sh [agent-binary [output-dir]]
-#
 # Inputs:
 #   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
-#   agent binary, default $REPO_ROOT/bin/lambda-agent (make bin/lambda-agent)
+#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
 #
-# Outputs, in output-dir (default $REPO_ROOT/build/microvm/lambda-runtime):
-#   vmlinuz
-#   initramfs.cpio.gz
+# Outputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
+#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
 #
 # Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
 # shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-INPUT_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
-INIT_SH="$INPUT_DIR/init.sh"
-INITTAB="$INPUT_DIR/inittab"
-RUN_AGENT="$INPUT_DIR/run-agent"
-LAMBDA_AGENT_BIN="${1:-$REPO_ROOT/bin/lambda-agent}"
-# The library stages beside BUILD_DIR, so it names the output, not the inputs.
-BUILD_DIR="${2:-$INPUT_DIR}"
+BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+RUN_AGENT="$BUILD_DIR/run-agent"
+LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
 TAG="[build-lambda-runtime-microvm-image]"
 
 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
@@ -42,8 +38,7 @@ MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
 source "$REPO_ROOT/scripts/lib/microvm-image.sh"
 
 echo "$TAG repo root: $REPO_ROOT"
-echo "$TAG input dir: $INPUT_DIR"
-echo "$TAG output dir: $BUILD_DIR"
+echo "$TAG build dir: $BUILD_DIR"
 
 microvm_check_tools
 
@@ -62,9 +57,6 @@ if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
     exit 1
 fi
 
-# Absolute, because packing the initramfs runs from inside the chroot.
-mkdir -p "$BUILD_DIR"
-BUILD_DIR=$(cd "$BUILD_DIR" && pwd)
 microvm_init_dirs
 microvm_make_rootfs
 microvm_empty_dev

scripts/build-lambda-runtime-microvm-image_test.sh

@@ -1,81 +0,0 @@
-#!/bin/sh
-# build-lambda-runtime-microvm-image_test.sh — a build into a chosen output
-# directory stages and publishes there, and leaves the default input directory
-# untouched.
-#
-# The script runs in a fixture repo against the real library, with only the
-# rootfs, kernel and packing steps replaced so no image is downloaded.
-set -eu
-
-SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-echo "-- build-lambda-runtime-microvm-image.sh output directory"
-
-REPO="$TMP/repo"
-IN="$REPO/build/microvm/lambda-runtime"
-mkdir -p "$REPO/scripts/lib" "$IN" "$REPO/bin" "$TMP/bin"
-cp "$SCRIPTS_DIR/build-lambda-runtime-microvm-image.sh" "$REPO/scripts/"
-for f in init.sh inittab run-agent; do printf '%s\n' "$f" > "$IN/$f"; done
-printf 'default image\n' > "$IN/vmlinuz"
-printf '#!/bin/sh\n' > "$REPO/bin/agent-testlink"
-chmod +x "$REPO/bin/agent-testlink"
-printf '#!/bin/sh\necho "ELF 64-bit, statically linked"\n' > "$TMP/bin/file"
-chmod +x "$TMP/bin/file"
-BEFORE=$(cd "$IN" && ls -A | cksum)
-
-# The real library, with the steps that need a rootfs or kernel replaced.
-# Packing runs from inside the chroot as the real step does, and records where
-# the stage is while it still exists.
-cat > "$REPO/scripts/lib/microvm-image.sh" <<EOF
-. "$SCRIPTS_DIR/lib/microvm-image.sh"
-microvm_check_tools() { :; }
-microvm_make_rootfs() { mkdir -p "\$CHROOT_DIR/etc"; }
-microvm_empty_dev() { :; }
-microvm_locate_kernel() { :; }
-microvm_strip_modules() { :; }
-microvm_strip_rootfs() { :; }
-microvm_stage_kernel() { printf 'kernel\n' > "\$STAGE_DIR/vmlinuz"; }
-microvm_pack_initramfs() {
-    (cd "\$CHROOT_DIR" && printf 'initramfs\n' | gzip > "\$STAGE_DIR/initramfs.cpio.gz")
-    dirname "\$STAGE_DIR" > "$TMP/stage-parent"
-    ls -A "$IN" > "$TMP/input-during"
-}
-EOF
-
-# The output is given relative to the repo root, as make passes it.
-OUT="$REPO/build/microvm/lambda-testlink-runtime"
-if ! (cd "$REPO" && PATH="$TMP/bin:$PATH" bash scripts/build-lambda-runtime-microvm-image.sh bin/agent-testlink build/microvm/lambda-testlink-runtime) > "$TMP/log" 2>&1; then
-    fail "build into output dir: $(tr '\n' ' ' < "$TMP/log")"
-else
-    if [ "$(cat "$TMP/stage-parent")" = "$OUT" ]; then
-        pass "stage created in the output directory"
-    else
-        fail "stage created in $(cat "$TMP/stage-parent"), want $OUT"
-    fi
-    if grep -q '^\.stage\.' "$TMP/input-during"; then
-        fail "stage present in the default input directory during the build"
-    else
-        pass "no stage in the default input directory during the build"
-    fi
-    if [ "$(cd "$IN" && ls -A | cksum)" = "$BEFORE" ] && grep -qx 'default image' "$IN/vmlinuz"; then
-        pass "default input directory unchanged"
-    else
-        fail "default input directory changed: $(cd "$IN" && ls -A | tr '\n' ' ')"
-    fi
-    if grep -qx kernel "$OUT/vmlinuz" && gzip -t "$OUT/initramfs.cpio.gz" && [ "$(ls -A "$OUT" | wc -l)" -eq 2 ]; then
-        pass "output directory holds exactly the published pair"
-    else
-        fail "output directory: $(ls -A "$OUT" | tr '\n' ' ')"
-    fi
-fi
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi

scripts/install-node.sh

@@ -260,14 +260,6 @@ out() {
     ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@" 2>/dev/null || true
 }
 
-# remote_join_cmd prints the command that joins one node. The token is not in it: it arrives on
-# stdin and is read after sudo has logged its command, so sudo's log never holds it. The other
-# values are positional arguments of a literal script, never spliced into it.
-remote_join_cmd() {
-    local script='exec spx admin join --force --node "$1" --bind "$2" --cluster-bind "$2" --advertise "$3" --host "$4" --token "$(cat)" --region "$5" --az "$6"'
-    printf 'sudo bash -c %q _ %q %q %q %q %q %q' "$script" "$@"
-}
-
 SETUP_OVN="/usr/local/share/spinifex/setup-ovn.sh"
 
 # --- Preflight -------------------------------------------------------------
@@ -660,18 +652,14 @@ fi
 [ -n "$EXT_IFACE" ] && init_args+=(--external-iface="$EXT_IFACE")
 [ -n "$EMAIL" ] && init_args+=(--email="$EMAIL")
 
-# init prints the join token, so its log is root-only and outside the world-writable /tmp.
-INIT_LOG=/var/log/spx-init.log
-
-# redact_tokens hides join tokens in text about to be shown to the operator.
-redact_tokens() { sed -E 's/spx_join_[A-Za-z0-9_-]+/spx_join_[redacted]/g'; }
+INIT_LOG=/tmp/spx-init.log
 
 log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s) to join)"
 
 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid sh -c 'spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
     exit 0" || fail "${HOSTS[0]}: could not start spx admin init"
 
 TOKEN=""
@@ -687,7 +675,7 @@ else
         elapsed=$((elapsed + 5))
     done
     if [ -z "$TOKEN" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: spx admin init produced no join token within ${TOKEN_TIMEOUT}s"
     fi
     log "  join token issued"
@@ -703,9 +691,11 @@ fi
 # waiting on a shell that node 2 is still holding.
 JOIN_PIDS=()
 for i in $(seq 1 $((N - 1))); do
-    # The token goes to the node on stdin, so it is in no ssh argument and not in sudo's log.
-    # spx admin join takes it only as --token, so it is still in that process's arguments.
-    on "${HOSTS[$i]}" "$(remote_join_cmd "${NODE_NAMES[$i]}" "${LAN_IPS[$i]}" "${WAN_IPS[$i]}" "${LAN_IPS[0]}:$PORT" "$REGION" "$AZ")" <<<"$TOKEN" &
+    on "${HOSTS[$i]}" "sudo spx admin join --force \
+        --node ${NODE_NAMES[$i]} \
+        --bind ${LAN_IPS[$i]} --cluster-bind ${LAN_IPS[$i]} --advertise ${WAN_IPS[$i]} \
+        --host ${LAN_IPS[0]}:$PORT --token $TOKEN \
+        --region $REGION --az $AZ" &
     JOIN_PIDS+=("$!")
 done
 
@@ -734,7 +724,7 @@ if ! $DRY_RUN; then
         elapsed=$((elapsed + 5))
     done
     if [ "$elapsed" -ge "$FORMATION_TIMEOUT" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: formation did not complete within ${FORMATION_TIMEOUT}s"
     fi
     log "  formation complete"

scripts/install-node_test.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
-# sudo's log (which holds the command it was given) never contains it, and the other values must
-# arrive as plain arguments even when they hold shell syntax.
-# preflight runs this with sh
-[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
-set -euo pipefail
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-mkdir "$work/bin"
-cd "$work"
-
-cat > "$work/bin/sudo" <<'SH'
-#!/bin/sh
-echo "$*" >> "$FAKE_SUDO_LOG"
-exec "$@"
-SH
-cat > "$work/bin/spx" <<'SH'
-#!/bin/sh
-for a in "$@"; do printf '<%s>' "$a"; done > "$FAKE_SPX_ARGV"
-SH
-chmod +x "$work/bin/sudo" "$work/bin/spx"
-export FAKE_SUDO_LOG="$work/sudo.log" FAKE_SPX_ARGV="$work/spx.argv" PATH="$work/bin:$PATH"
-
-eval "$(sed -n '/^remote_join_cmd()/,/^}/p' "$here/install-node.sh")"
-
-fails=0
-check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-
-TOKEN=spx_join_FAKETOKEN-abc_1
-cmd="$(remote_join_cmd node2 10.0.0.2 203.0.113.2 '10.0.0.1:4432;touch PWNED' us-east-1 'az $(touch PWNED2)')"
-bash -c "$cmd" <<<"$TOKEN"
-
-check "the token is not in the command" "$(printf '%s' "$cmd" | grep -c FAKETOKEN || true)" 0
-check "the token is not in sudo's log" "$(grep -c FAKETOKEN "$FAKE_SUDO_LOG" || true)" 0
-check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--token><$TOKEN>"
-check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
-check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
-
-[ "$fails" -eq 0 ]

scripts/lib/lambda-testlink-build_test.sh

@@ -1,111 +0,0 @@
-#!/bin/sh
-# lambda-testlink-build_test.sh — the test-link make targets refuse a malformed
-# image digest and an unsupported host or GOARCH before touching any output.
-#
-# Every case is refused on the recipe's first lines, so nothing is compiled.
-set -eu
-
-LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-REPO_ROOT=$(dirname -- "$(dirname -- "$LIB_DIR")")
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-echo "-- make build-lambda-testlink / build-lambda-testlink-image guards"
-
-PWN="$TMP/pwn"
-OUT="$TMP/out"
-HEX=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-mkdir -p "$OUT"
-printf 'old daemon\n' > "$OUT/spx"
-printf 'old initializer\n' > "$OUT/hardkill.so"
-PAIR=$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)
-
-# unchanged <name> — the existing pair is intact, no scratch was made and no
-# injected command ran.
-unchanged() {
-    if [ -e "$PWN" ]; then
-        fail "$1: injected command ran"
-        rm -f "$PWN"
-    elif [ -e "$OUT.tmp" ]; then
-        fail "$1: scratch directory created"
-    elif [ "$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)" != "$PAIR" ]; then
-        fail "$1: existing pair replaced"
-    else
-        pass "$1"
-    fi
-}
-
-# refused <name> <want> <log> <status> — make failed with the expected message.
-refused() {
-    if [ "$4" -eq 0 ]; then
-        fail "$1: make succeeded"
-    elif ! grep -q "$2" "$3"; then
-        fail "$1: wrong refusal: $(tr '\n' ' ' < "$3")"
-    else
-        unchanged "$1"
-    fi
-}
-
-testlink() {
-    make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" "$@" > "$TMP/log" 2>&1
-}
-
-DIGEST_MSG="LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"
-NL='
-'
-for v in '' "\$(shell touch $PWN)" "\$\$(touch $PWN)" "\`touch $PWN\`" "$HEX; touch $PWN" \
-    "$HEX\" -X main.x=\"y" "$HEX -X main.x=y" "$HEX${NL}touch $PWN" \
-    0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF "${HEX%?}" "${HEX}0"; do
-    # GNU make itself expands a command-line $(shell ...) while parsing, for
-    # any target, so that one literal is tested only from the environment.
-    case "$v" in
-    "\$(shell"*) ;;
-    *)
-        status=0
-        testlink LAMBDA_IMAGE_DIGEST="$v" || status=$?
-        refused "command-line digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
-        ;;
-    esac
-    status=0
-    LAMBDA_IMAGE_DIGEST="$v" testlink || status=$?
-    refused "environment digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
-done
-
-status=0
-env -u LAMBDA_IMAGE_DIGEST make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" > "$TMP/log" 2>&1 || status=$?
-refused "digest unset" "$DIGEST_MSG" "$TMP/log" "$status"
-
-ARCH_MSG="supports only an x86_64 host building linux/amd64"
-SHIM="$TMP/shim"
-mkdir -p "$SHIM"
-printf '#!/bin/sh\necho aarch64\n' > "$SHIM/uname"
-chmod +x "$SHIM/uname"
-
-status=0
-GOARCH=arm64 testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
-refused "GOARCH=arm64" "$ARCH_MSG" "$TMP/log" "$status"
-status=0
-PATH="$SHIM:$PATH" testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
-refused "aarch64 host" "$ARCH_MSG" "$TMP/log" "$status"
-
-# The image target refuses before building qboot or any agent binary.
-for c in "GOARCH=arm64" "PATH=$SHIM:$PATH"; do
-    status=0
-    env "$c" make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink-image > "$TMP/log" 2>&1 || status=$?
-    if [ "$status" -eq 0 ]; then
-        fail "image target [${c%%=*}]: make succeeded"
-    elif ! grep -q "$ARCH_MSG" "$TMP/log" || grep -q "qboot\|go build" "$TMP/log"; then
-        fail "image target [${c%%=*}]: did not refuse first: $(tr '\n' ' ' < "$TMP/log")"
-    else
-        pass "image target [${c%%=*}] refused first"
-    fi
-done
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi

scripts/terraform/latitude-spx/.gitignore

@@ -1,10 +0,0 @@
-.terraform/
-terraform.tfstate
-terraform.tfstate.*
-*.tfplan
-*.auto.tfvars
-crash.log
-terraform.tfvars
-*.pem
-*.log
-.terraform.lock.hcl

scripts/terraform/latitude-spx/README.md

@@ -1,137 +0,0 @@
-# latitude-spx
-
-Builds a Spinifex deployment on latitude.sh bare metal from your laptop with `deploy-latitude.sh`.
-The commands, flags, step markers and state layout follow `vultr-spx`.
-
-**Status: not yet run end to end.**
-The script was written from the plan doc findings (`docs/development/feature/latitude-bm.md`, L1 to L26) and from the provider schema.
-No latitude.sh API call or host run was made while building it, so treat the first `up` as the test.
-One node has been the only shape run live, and only in part (see Unverified). The three-node flow, including the floating pool, has not been run: Ben takes that over.
-
-## What it creates
-
-| Resource | Notes |
-|---|---|
-| Project | `spinifex-<name>`, unless `--project` names an existing one (then it is never deleted) |
-| SSH key | `spinifex-<name>`, account scoped |
-| User data | `spinifex-<name>`, host preparation only (no `{{ }}`, latitude.sh substitutes them) |
-| Servers | `m4-metal-medium`, site `DAL`, Ubuntu 26.04, hourly billing, hostnames `spx-<name>-nodeN` |
-| Elastic IP | one by default (`elastic_ip_count` in `variables.tf`, 1 to 5; set `TF_VAR_elastic_ip_count` to change it). With one node it is a static pool address and the count must stay 1, because a second pool is never tried when the first is full; with three nodes the addresses form the `latitude` floating pool, which Spinifex moves between servers. `nginx-webserver` needs one address and runs on one node; `nginx-alb` needs two public addresses and is a three-node workbook |
-| Virtual network | three nodes only: `spinifex-<name>`, one VLAN assignment per server |
-
-## Requirements
-
-OpenTofu (or Terraform) 1.5 or later, `aws` CLI, `python3`, `curl`, `ssh`, an SSH key pair.
-
-| Variable | When | Meaning |
-|---|---|---|
-| `LATITUDESH_AUTH_TOKEN` | always | the account API token; used by the provider and this script only, never sent to a node |
-| `LATITUDESH_NODE_TOKEN` | `--nodes 3` | a second, different, writable API key, copied to each node (`/etc/spinifex/latitude/api-token`, owned by `spinifex-daemon` with mode 0400, in a root-owned 0711 directory; the other services share the `spinifex` group, so no group may read it; ownership and mode are set on every `up`; a symlink or any other owner there is refused) so the daemon can move Elastic IPs |
-
-The script refuses a node key it can see is read-only (it lists `/auth/api_keys` and matches the key's last five characters). If the key is not among the account's keys, write authority is first proven by the first Elastic IP move.
-
-Exposure: a latitude.sh API key is team-wide. It is not scoped to Elastic IPs, so anyone who reads it from a node can do whatever the key's owner can in the team (the API exposes `read_only` and `allowed_ips` on a key, and no narrower scope). Create the node key with `allowed_ips` set to the nodes' public addresses, and revoke it after `destroy`.
-
-Nodes are reached as `ubuntu` with the deployment's SSH key, and remote commands run through passwordless `sudo`: latitude.sh puts the key on the image's login user and root login is not enabled.
-
-## Usage
-
-```
-export LATITUDESH_AUTH_TOKEN=...        # from the latitude.sh dashboard
-./deploy-latitude.sh up --name demo     # one node
-
-export LATITUDESH_NODE_TOKEN=...        # a second, writable key
-./deploy-latitude.sh up --name demo3 --nodes 3
-./deploy-latitude.sh status --name demo
-./deploy-latitude.sh ssh --name demo node1 'systemctl status spinifex.target'
-./deploy-latitude.sh credentials --name demo
-./deploy-latitude.sh stop node1 --name demo
-./deploy-latitude.sh start node1 --name demo
-./deploy-latitude.sh destroy --name demo
-```
-
-Options: `--name`, `--site` (alias `--region`), `--plan`, `--os`, `--project ID|SLUG`, `--distro TARBALL`, `--channel`, `--spx-region`, `--spx-az`, `--ssh-key`, `--image`, `--data-disk-id`, `--vpc-subnet` (the VLAN's /24, three nodes), `--nodes 1|3`, `--yes`.
-`./deploy-latitude.sh --help` lists them with defaults.
-
-Three nodes need `--distro` with a tarball built from this branch (`make distro-amd64`) until a release carries the latitude provider.
-A released spx rejects `source="latitude"` and its daemon restarts in a loop, so `up --nodes 3` without `--distro` stops in preflight.
-Before the pool is written, each node's `/usr/local/bin/spx` is checked for the `latitude_elastic_ip_ids` key name; spx has no validate subcommand, so this is a string search of the binary, not a run of its config check.
-
-## What `up` does
-
-`terraform` then, per node: `host` (wait for SSH as `ubuntu` and cloud-init, `/dev/kvm`), `disk`, `vlan` (three nodes), `install`, `form`, `start`; then `pool` (one node) or `cluster` (three nodes: `install-node.sh` forms the cluster); three nodes then install the node token on every `up` and add the `latitude` pool (`pool`); then `restart` (three nodes wait for the current daemon to log "floating pool ready" for the pool), `public`, `image` and the AWS profile.
-Each step writes `step.<name>[.nodeN]` in the state directory, and a rerun resumes at the first step without one.
-
-- **Data disk.** `host-data-disk.sh` picks the blank NVMe by content and by-id EUI, never by `nvmeXn1` (L1, L11).
-A candidate is at least 100 GB with no signature, partitions, holders or mounts, and not in the root filesystem's ancestry; with none it stops and lists the disks.
-With several (LON2 has two 1.7 TB and one spare 447 GB blank NVMe next to the root disk) it takes the largest, the lowest by-id path among equals, prints what it chose and the blank disks it left, and never touches the others (`--data-disk-id` chooses another).
-It mounts by UUID with `nofail` and the script records the UUID in `data_uuid.<node>`.
-- **Mount check.** Before anything writes under `/var/lib/spinifex`, and on every run, the mounted filesystem UUID must equal the recorded one.
-The units that write there also get a `RequiresMountsFor=/var/lib/spinifex` drop-in.
-The mount is by UUID in fstab, so renamed disks do not change it, and there is no check at service start.
-- **Firewall.** `setup.sh --firewall on` runs before Spinifex first starts.
-- **One node.** The Elastic IP is a static pool (`gateway_ip = "192.0.2.1"`, the node's next hop as `gateway`, no `prefix_len`) with the IMDS addresses moved off the ones latitude.sh routes.
-- **Three nodes.** The `vlan` step makes `eno2.<vid>` with `ip link` and persists it as systemd-networkd files for that link plus a drop-in on the `eno2` unit.
-It never runs `netplan apply`, refuses a subnet that overlaps the SSH peer or a route, and removes the VLAN again if the public interface's address, default route or the route to the SSH peer differ afterwards.
-`form_cluster` then puts the cluster and Geneve planes on that VLAN with `install-node.sh` (`--user ubuntu`); its output is not kept, as it can hold the join token.
-- **Floating pool (three nodes).** The `latitude` pool names the Elastic IP IDs and the project ID from the Terraform outputs, and each node's `provider_host_id` is its server ID. `spinifex.toml` is edited idempotently and read back. Failover moves an address with one `PATCH /elastic_ips/{id}` and waits for it to be active; latitude.sh has no unassign, so a node that loses an address is not told to release it.
-- **State** is in `~/.spinifex/latitude/<name>/`: Terraform state, `ids` (every `latitudesh_*` object in the state, merged on each apply and before each destroy), `step.*` markers, logs and the data disk UUIDs.
-- **AWS profile** `spinifex-latitude-<name>` is replaced on each `up`, with the CA at `~/.spinifex/latitude-<name>-ca.pem`.
-The secret key is written to `~/.aws/credentials` (0600, other profiles kept, the old file saved beside it) and never appears in a process argument.
-
-## Token handling
-
-`LATITUDESH_AUTH_TOKEN` is read from the environment by the provider and by the script's API calls.
-It is not a Terraform variable, is on no command line (curl reads it from stdin), is not sent to a node, and is written to no file.
-`LATITUDESH_NODE_TOKEN` is checked with curl reading it from stdin and reaches each node on stdin of the install script, so it is in no argument on either side.
-`TF_LOG` is unset by the script.
-The join token that `install-node.sh` creates is shared with Vultr and OCI: its init log is root-only, and `spx admin join` still takes it as `--token` on the joining node (tracked separately).
-
-## Recovery
-
-If SSH is lost once the firewall is armed, use the serial console.
-`up` enables the server's out-of-band connection with the deployment's SSH key and prints the `ssh` command it returns before it arms the firewall, falling back to the Out-of-Band tab in the dashboard.
-Logging in there needs an OS user password, which you set yourself while SSH still works.
-A fresh deployment that cannot be recovered is cheaper to `destroy` and `up` again.
-The VLAN step prints the console command first.
-It refuses a subnet that holds the SSH peer or overlaps an existing route, and removes the VLAN again if the route to the SSH peer or the public interface changes.
-
-## destroy
-
-`destroy` asks you to type the deployment name (`--yes` skips only that), then records every ID in the Terraform state and stops without destroying anything if it cannot.
-It runs `terraform destroy`, fetches every recorded ID by itself (2xx present, 404 gone, anything else fails and keeps the records), and deletes what remains of those.
-Servers, keys, user data, virtual networks and projects whose names match the deployment but that are not recorded are listed by ID and deleted only after you type `delete-untracked` for exactly those IDs, which `--yes` never supplies; an ID that shows up later needs its own confirmation, and without a terminal they are kept and `destroy` exits 1.
-Elastic IPs are touched only by recorded ID.
-Elastic IP release is asynchronous and only works once the address is active or in error, so the sweep retries.
-Because the Elastic IP list ignores pagination, `destroy` ends by asking you to check the dashboard for leftovers and type `checked`.
-An `up` that failed while creating can leave an object nothing recorded, such as an Elastic IP; that dashboard check is the only thing that finds it.
-
-## Unverified
-
-Nothing in the three-node flow has run against real servers. For one node, only the parts the person ran have been exercised, and the script has changed since.
-
-- The OS slug `ubuntu_26_04_x64_lts` (the `latitudesh_operating_system` data source fails the plan if it is wrong).
-- That the deploy key is on `ubuntu` and `sudo -n` works for it, and that `sudo` resets nothing the script needs (the SSH peer address is read before `sudo` for that reason).
-- The power action request (`POST /servers/{id}/actions` with `power_off` and `power_on`).
-- Resolving `--project` as a slug through `GET /projects/{slug}`, and the `in_stock` shape of `GET /plans` used by the stock check, which only warns when it cannot read it.
-- The `GET` and `DELETE` paths for `/user_data/{id}` and `/virtual_networks/{id}`.
-- That the Spinifex API certificate names the node's public address, which the profile check needs.
-- That the private NIC is `eno2`, its tagging and MTU (the script uses 1500), and that the networkd files persist the VLAN across a reboot.
-- `GET /auth/api_keys` and its `token_last_slice` and `read_only` fields, which the node key check relies on; and that a key with `allowed_ips` set to the nodes' addresses still works from them.
-- The Elastic IP move itself: `PATCH /elastic_ips/{id}` with `server_id`, its 202 then `active`, the error codes, and how long a move takes; the client waits up to its move timeout and makes one write per call.
-- That the daemon reads the node token and logs "floating pool ready", then fails over when a node stops (`stop node1`) and the address lands on another node.
-- `install-node.sh` as `--user ubuntu` on latitude.sh servers.
-
-## Not exercised
-
-Any real API call from the three-node flow, `destroy`, `host-data-disk.sh` on a host and the VLAN step on a server.
-Guest launch, public reachability of the Elastic IP, the Terraform workbooks and failover are not part of `up`; prove them by hand.
-`e2e-latitude.sh` and removing the old `install-node.sh` in this directory are not done.
-
-## Tests
-
-`sh node-scripts_test.sh` checks the node token install (install, unchanged, rotation, modes, an older install converging, symlinks and owners refused; run it as root too, with `sudo sh node-scripts_test.sh`, to check that only the daemon's user can read the file) and the readiness check (old success then current failure) with stand-in tools.
-`bash deploy-argv_test.sh` checks that neither API token, nor the five characters used to find the node key, is an argument of any external command in the node token paths, and that the SSH peer address comes from the same connection after sudo's environment reset.
-`scripts/install-node_test.sh` (part of `make preflight`) checks that the join token reaches `spx admin join` only on stdin, so sudo's log never holds it.
-The first two are not part of `make preflight`.
-The Go side: `cd spinifex && GOWORK=off go test ./providers/cloud/latitude/ ./domains/network/external/floating/ ./bootstrap/config/ ./runtime/daemon/`.

scripts/terraform/latitude-spx/cloud-init/user-data.yaml

@@ -1,23 +0,0 @@
-#cloud-config
-# Host preparation only. Spinifex is installed and formed over SSH by deploy-latitude.sh.
-# latitude.sh substitutes double-brace variables in user data, so this file holds none.
-write_files:
-  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
-    content: |
-      network: {config: disabled}
-  # The image's netplan lists eno2 with no address and without optional, so boot waits two minutes
-  # for it. --any returns once one link, eno1, is online; it changes the wait, not the network.
-  # UNVERIFIED: the binary path is the Ubuntu one and the drop-in first applies on the next boot.
-  - path: /etc/systemd/system/systemd-networkd-wait-online.service.d/10-spinifex-any.conf
-    content: |
-      [Service]
-      ExecStart=
-      ExecStart=/usr/lib/systemd/systemd-networkd-wait-online --any
-# runcmd carries on past a failing command, so each line stands alone.
-runcmd:
-  - [sh, -c, "command -v ufw >/dev/null && ufw --force disable || true"]
-  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
-  # The image ships empty apt lists and no git, which the workbooks need.
-  - [sh, -c, "apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git || true"]
-  - [sh, -c, "systemctl daemon-reload || true"]
-  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]

scripts/terraform/latitude-spx/compute.tf

@@ -1,70 +0,0 @@
-locals {
-  hostname_prefix = "spx-${var.deployment_name}-node"
-  project         = var.project != "" ? var.project : latitudesh_project.deploy[0].slug
-  project_id      = var.project != "" ? var.project_id : latitudesh_project.deploy[0].id
-}
-
-# Both lookups fail the plan when a slug does not exist, which is the only check of the OS slug.
-data "latitudesh_operating_system" "os" {
-  slug = var.operating_system
-}
-
-data "latitudesh_plan" "plan" {
-  slug = var.plan
-}
-
-resource "latitudesh_project" "deploy" {
-  count             = var.project == "" ? 1 : 0
-  name              = "spinifex-${var.deployment_name}"
-  description       = "Spinifex deployment ${var.deployment_name}"
-  environment       = "Development"
-  provisioning_type = "on_demand"
-}
-
-# The key is team scoped. The server login user is ubuntu, not root.
-resource "latitudesh_ssh_key" "deploy" {
-  name       = "spinifex-${var.deployment_name}"
-  public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
-}
-
-# Host preparation only. Latitude substitutes double-brace variables inside user data, so the file
-# must hold none. Spinifex is installed over SSH by deploy-latitude.sh.
-resource "latitudesh_user_data" "host" {
-  description = "spinifex-${var.deployment_name}"
-  content     = filebase64("${path.module}/cloud-init/user-data.yaml")
-}
-
-# billing must be hourly: the default is monthly, which bills upfront with no refund.
-resource "latitudesh_server" "node" {
-  count            = var.node_count
-  project          = local.project
-  hostname         = "${local.hostname_prefix}${count.index + 1}"
-  site             = var.site
-  plan             = data.latitudesh_plan.plan.slug
-  operating_system = data.latitudesh_operating_system.os.slug
-  billing          = "hourly"
-  ssh_keys         = [latitudesh_ssh_key.deploy.id]
-  user_data        = latitudesh_user_data.host.id
-
-  timeouts = {
-    create = "60m"
-  }
-}
-
-# Elastic IPs are bought here and never created or deleted by Spinifex. Each starts on a server, spread
-# over the nodes, and server_id is ignored afterwards so a move by Spinifex is not undone by a later apply.
-resource "latitudesh_elastic_ip" "pool" {
-  count = var.elastic_ip_count
-  # The provider documents ID or slug but posts the value as project_id unresolved: a slug gave 404.
-  project   = local.project_id
-  server_id = latitudesh_server.node[count.index % var.node_count].id
-
-  lifecycle {
-    ignore_changes = [server_id]
-  }
-}
-
-moved {
-  from = latitudesh_elastic_ip.node1
-  to   = latitudesh_elastic_ip.pool[0]
-}

scripts/terraform/latitude-spx/deploy-argv_test.sh

@@ -1,228 +0,0 @@
-#!/usr/bin/env bash
-# Checks that neither API token is ever an argument of an external command in the node token
-# paths of deploy-latitude.sh. External commands are stand-ins that record their arguments and
-# their standard input; the test fails if a token is in the first and proves it is in the second.
-# preflight runs this with sh
-[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
-set -euo pipefail
-exec </dev/null
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-mkdir "$work/bin" "$work/home"
-real_python="$(command -v python3)"
-
-AUTH=FAKEAUTHTOKEN9
-NODE=FAKENODETOKEN7
-
-# Every command the script runs outside bash records "name args" and, for the first line of
-# standard input, "name stdin: ...".
-for c in curl ssh scp aws tofu ssh-keygen; do
-    cat > "$work/bin/$c" <<'SH'
-#!/bin/sh
-echo "$(basename "$0") $*" >> "$FAKE_ARGV"
-if [ ! -t 0 ]; then
-    in="$(cat)"
-    [ -z "$in" ] || echo "$(basename "$0") stdin: $in" >> "$FAKE_STDIN"
-fi
-case "$(basename "$0")" in
-    curl) printf '{"data":[{"attributes":{"token_last_slice":"%s","read_only":false}}]}\n200' "$(printf '%s' "$FAKE_NODE" | tail -c 5)" ;;
-    ssh)
-        if [ -n "${FAKE_SSH_EVAL:-}" ]; then
-            # Behave like a login on a node: skip the options and host, run the rest with the
-            # connection variable of this connection, and let sudo reset the environment.
-            while [ "$#" -gt 0 ]; do case "$1" in ubuntu@*) shift; break ;; *) shift ;; esac; done
-            SSH_CONNECTION="$FAKE_SSH_CONNECTION" bash -c "$*"
-        else
-            echo installed
-        fi ;;
-esac
-SH
-    chmod +x "$work/bin/$c"
-done
-cat > "$work/bin/sudo" <<'SH'
-#!/bin/sh
-while [ "$#" -gt 0 ]; do case "$1" in -*) shift ;; *) break ;; esac; done
-exec env -i PATH="$PATH" "$@"
-SH
-chmod +x "$work/bin/sudo"
-cat > "$work/bin/python3" <<SH
-#!/bin/sh
-echo "python3 \$*" >> "\$FAKE_ARGV"
-exec "$real_python" "\$@"
-SH
-chmod +x "$work/bin/python3"
-
-export FAKE_ARGV="$work/argv" FAKE_STDIN="$work/stdin" FAKE_NODE="$NODE" PATH="$work/bin:$PATH" HOME="$work/home"
-: > "$FAKE_ARGV"
-: > "$FAKE_STDIN"
-export LATITUDESH_AUTH_TOKEN="$AUTH" LATITUDESH_NODE_TOKEN="$NODE"
-
-# shellcheck source=deploy-latitude.sh
-source "$here/deploy-latitude.sh"
-
-# shellcheck disable=SC2034 # read by the sourced functions
-NODES=3 IP=192.0.2.10 NODE_NAME=node1
-check_node_token >/dev/null
-install_key >/dev/null 2>&1
-
-fails=0
-for tok in "$AUTH" "$NODE" "${NODE: -5}"; do
-    if grep -q "$tok" "$FAKE_ARGV"; then
-        echo "FAIL token $tok is in a command's arguments:"
-        grep "$tok" "$FAKE_ARGV"
-        fails=$((fails + 1))
-    else
-        echo "ok   $tok is in no argument"
-    fi
-done
-for what in "curl stdin:.*Bearer $NODE" "ssh stdin: $NODE"; do
-    if grep -q "$what" "$FAKE_STDIN"; then
-        echo "ok   the check can see the token where it is sent ($what)"
-    else
-        echo "FAIL the token never reached stdin ($what): the test would pass without checking anything"
-        fails=$((fails + 1))
-    fi
-done
-
-# The peer address comes from the same connection that runs the command, whichever source
-# address that connection has, and survives sudo's environment reset.
-export FAKE_SSH_EVAL=1
-for src in 198.51.100.7 203.0.113.99; do
-    export FAKE_SSH_CONNECTION="$src 51000 192.0.2.10 22"
-    got="$(on 'printf %s "$SSH_PEER"')"
-    if [ "$got" = "$src" ]; then echo "ok   peer $src reaches the command"; else echo "FAIL peer for $src was '$got'"; fails=$((fails + 1)); fi
-done
-
-# A three-node run needs a build that knows the latitude source: a release would be refused by its
-# own config check. The guard needs --distro, and the node is checked before it gets the pool.
-check_result() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-check_result "three nodes without --distro stop" "$( (NODES=3 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-check_result "three nodes with --distro go on" "$( (NODES=3 DISTRO=/tmp/x.tar.gz require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node without --distro goes on" "$( (NODES=1 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with one address goes on" "$( (NODES=1 TF_VAR_elastic_ip_count=1 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with no setting goes on" "$( (NODES=1 TF_VAR_elastic_ip_count="" require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with two addresses stops" "$( (NODES=1 TF_VAR_elastic_ip_count=2 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-check_result "three nodes with three addresses go on" "$( (NODES=3 TF_VAR_elastic_ip_count=3 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-printf 'binary with latitude_elastic_ip_ids inside\n' > "$work/spx-new"
-printf 'binary of a release\n' > "$work/spx-old"
-SPX_BIN="$work/spx-new"
-check_result "a build that knows latitude is accepted" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-# shellcheck disable=SC2034 # read by the sourced functions
-SPX_BIN="$work/spx-old"
-check_result "a release is refused" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-# The profile spx writes under sudo is in the ubuntu user's home, so node-side reads name that
-# directory instead of relying on root's HOME.
-mkdir "$work/awsdir"
-printf '[other]\naws_access_key_id = X\n[spinifex]\naws_access_key_id = AK1\naws_secret_access_key = SK1\n[z]\nq = 1\n' > "$work/awsdir/credentials"
-node_ip() { printf 192.0.2.10; }
-# shellcheck disable=SC2034 # read by the sourced functions
-NODE_AWS_DIR="$work/awsdir"
-check_result "credentials come from the node's AWS dir" "$(cmd_credentials | tr '\n' ' ')" "[spinifex] aws_access_key_id = AK1 aws_secret_access_key = SK1 "
-
-# A failed API call logs its status and a documented code, never the response text, which can
-# echo the request's headers.
-mkdir "$work/bin2"
-cat > "$work/bin2/curl" <<'SH'
-#!/bin/sh
-cat > /dev/null
-printf '{"errors":[{"code":"%s","detail":"Authorization: Bearer %s"}]}\n401' "$FAKE_CODE" "$FAKE_LEAK"
-SH
-chmod +x "$work/bin2/curl"
-logged="$(FAKE_CODE=UNAUTHORIZED FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
-check_result "a failed call names its status and code" "$(grep -c 'returned 401 (UNAUTHORIZED)' <<<"$logged" || true)" 1
-check_result "a failed call never logs the token" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
-logged="$(FAKE_CODE="$AUTH" FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
-check_result "an undocumented code is not logged" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
-
-# A recorded ID that now names another deployment's object is not ours to delete.
-# shellcheck disable=SC2034 # read by the sourced functions
-NAME=demo
-latitude() { printf '%s' "$FAKE_LAT"; }
-orig_expected_project_id="$(declare -f expected_project_id)"
-expected_project_id() { echo proj_1; }
-srv() {
-    local frag=',"project":{"id":"proj_1"}'
-    [ "$#" -lt 2 ] || frag="$2"
-    printf '{"data":{"id":"sv_1","attributes":{"hostname":"%s"%s}}}' "$1" "$frag"
-}
-FAKE_LAT="$(srv spx-demo-node2)"
-check_result "a recorded server with our hostname is ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" ours
-FAKE_LAT="$(srv spx-other-node1)"
-check_result "a recorded server with another hostname is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo2-node1)"
-check_result "a hostname that only starts like ours is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
-check_result "our hostname in another project is not ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo-node1 '')"
-check_result "a server with no project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{}')"
-check_result "a project with no id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-FAKE_LAT="$(srv spx-demo-node1 ',"project":null')"
-check_result "a null project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-expected_project_id() { return 1; }
-FAKE_LAT="$(srv spx-demo-node1)"
-check_result "no known project id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-expected_project_id() { echo proj_1; }
-FAKE_LAT='not json'
-check_result "an unreadable object is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-KNOWN_IDS="$work/ids"
-printf 'server sv_1\n' > "$KNOWN_IDS"
-lat_exists() { return 0; }
-FAKE_LAT="$(srv spx-other-node1)"
-check_result "destroy stops before Terraform when a recorded ID was reused" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
-check_result "destroy stops when the same hostname is in another project" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1 '')"
-check_result "destroy stops when the project cannot be read" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1)"
-check_result "destroy goes on when the recorded server is ours" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" went
-leftovers_out="$( (lat_match() { return 0; }; FAKE_LAT="$(srv spx-other-node1)" leftovers) 2>/dev/null || true)"
-check_result "a reused ID is not listed as a leftover to delete" "$(grep -c sv_1 <<<"$leftovers_out" || true)" 0
-
-# After terraform destroy the state's project output is gone; the remembered ID still lets the
-# sweep delete a matching leftover and keep a foreign one.
-eval "$orig_expected_project_id"
-TF_BIN="$work/tofu"
-printf '#!/bin/sh\nif [ -f "%s/gone" ]; then exit 1; fi\nprintf proj_1\n' "$work" > "$TF_BIN"
-chmod +x "$TF_BIN"
-EXPECTED_PROJECT=""
-remember_project_id
-check_result "the project ID is remembered while the state holds it" "$EXPECTED_PROJECT" proj_1
-: > "$work/gone"
-check_result "the state no longer answers after destroy" "$("$TF_BIN" output >/dev/null 2>&1 && echo up || echo gone)" gone
-printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
-lat_match() { return 1; }
-latitude() { if [ "$2" = /servers/sv_1 ]; then srv spx-demo-node1; else srv spx-other-node1 ',"project":{"id":"proj_other"}'; fi; }
-swept="$(leftovers 2>/dev/null || true)"
-check_result "after destroy a matching leftover is still listed" "$(grep -c 'server sv_1 ' <<<"$swept" || true)" 1
-check_result "after destroy a foreign one is kept out" "$(grep -c 'server sv_2 ' <<<"$swept" || true)" 0
-EXPECTED_PROJECT=""
-swept="$(leftovers 2>/dev/null || true)"
-check_result "without the remembered ID a leftover is unknown, not matched" "$(grep -c 'server sv_1 sv_1 (unknown)' <<<"$swept" || true)" 1
-expected_project_id() { echo proj_1; }
-
-# Cleanup never deletes an object whose ownership could not be read, recorded or not.
-deleted="$work/deleted"
-: > "$deleted"
-latitude() { printf '%s %s\n' "$1" "$2" >> "$deleted"; }
-printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
-delete_left $'server sv_1 sv_1 (unknown)\nserver sv_2 spx-demo-node2\n' "" >/dev/null 2>&1
-check_result "an unreadable recorded object is not deleted" "$(grep -c sv_1 "$deleted" || true)" 0
-check_result "a verified recorded object is deleted" "$(grep -c 'DELETE .*sv_2' "$deleted" || true)" 1
-
-# Both deployment sizes write the metadata remap.
-cfg="$work/spinifex.toml"
-extract() { sed -n "/^$1()/,/^}/p" "$here/deploy-latitude.sh" | sed -n "/<<'PY'/,/^PY\$/p" | sed '1d;$d' | sed "s|/etc/spinifex/spinifex.toml|$cfg|"; }
-printf '[network]\nexternal_mode = "nat"\n\n[bootstrap]\ny = 1\n' > "$cfg"
-extract configure_pool | python3 -I - 203.0.113.1 203.0.113.254
-check_result "one node gets the metadata remap" "$(grep -c 'imds_host_meta_ip = "169.254.42.254"' "$cfg")" 1
-printf '[network]\nexternal_mode = "nat"\n\n[nodes.node1]\nx = 1\n\n[nodes.node2]\nx = 1\n\n[nodes.node3]\nx = 1\n\n[bootstrap]\ny = 1\n' > "$cfg"
-extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
-check_result "three nodes get the metadata remap" "$(grep -c 'imds_host_dns_ip  = "169.254.42.253"' "$cfg")" 1
-check_result "three nodes get the latitude pool" "$(grep -c 'source                  = "latitude"' "$cfg")" 1
-extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
-check_result "writing the config again changes nothing" "$(grep -c 'imds_host_meta_ip' "$cfg")" 1
-unset FAKE_SSH_EVAL
-
-[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/host-data-disk.sh

@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-# host-data-disk.sh — format and mount the blank data disk at /var/lib/spinifex. Run as root ON the host.
-#
-#   host-data-disk.sh [/dev/disk/by-id/nvme-eui.<id>]
-#
-# The disk is chosen by content, never by its nvmeXn1 name, which swaps between boots. A candidate is a
-# whole-disk by-id EUI link whose device is not in the root filesystem's ancestry, has no partitions,
-# holders, swap or mounts, no filesystem signature and no partition table. With none the script stops
-# and lists the disks. With several it takes the largest, and the lowest by-id path among equals, and
-# prints what it chose and the blank disks it left alone; pass the by-id path to choose another.
-# It never reformats: a rerun on a host already done prints the UUID and exits. The last line of stdout
-# is DATA_UUID=<uuid>, which the later install phases need.
-
-set -euo pipefail
-
-MP=/var/lib/spinifex
-LABEL=spinifex-data
-FSTAB_BAK=/etc/fstab.pre-spinifex
-MIN_BYTES="${MIN_DATA_BYTES:-100000000000}"
-WANT="${1:-}"
-
-die() { echo "STOP: $*" >&2; exit 1; }
-[ "$(id -u)" = 0 ] || die "run as root"
-
-# Resume: the mount is already there and is ours.
-if mp_uuid="$(findmnt --mountpoint "$MP" -no UUID 2>/dev/null)" && [ -n "$mp_uuid" ]; then
-    src="$(findmnt --mountpoint "$MP" -no SOURCE)"
-    [ "$(blkid -s LABEL -o value "$src")" = "$LABEL" ] || die "$MP is mounted from $src, which is not labelled $LABEL; not touching it"
-    grep -q "UUID=${mp_uuid}[[:space:]]" /etc/fstab || die "$MP is mounted but /etc/fstab has no UUID=$mp_uuid entry"
-    echo "already done: $MP is mounted from $src (UUID $mp_uuid)"
-    echo "DATA_UUID=$mp_uuid"
-    exit 0
-fi
-if grep -q "$MP" /etc/fstab; then die "/etc/fstab mentions $MP but nothing is mounted there; fix that by hand first"; fi
-[ ! -e "$MP" ] || [ -z "$(ls -A "$MP")" ] || die "$MP exists and is not empty"
-
-root_src="$(findmnt -no SOURCE /)" || die "cannot find the root source"
-root_anc="$(lsblk -s -nro NAME "$root_src")" || die "cannot read the root ancestry"
-[ -n "$root_anc" ] || die "empty root ancestry"
-
-# blank_reason ID prints why the disk is not a candidate and returns 1, or returns 0.
-blank_reason() {
-    local id="$1" dev name size kids hold sw mnt rc w
-    dev="$(readlink -e "$id")" || { echo "link does not resolve"; return 1; }
-    [ -b "$dev" ] || { echo "not a block device"; return 1; }
-    name="$(basename "$dev")"
-    if grep -qx "$name" <<<"$root_anc"; then echo "in the root ancestry"; return 1; fi
-    size="$(blockdev --getsize64 "$id")" || { echo "size unreadable"; return 1; }
-    [ "$size" -ge "$MIN_BYTES" ] || { echo "only $size bytes"; return 1; }
-    kids="$(lsblk -nro NAME "$id")" || { echo "children unreadable"; return 1; }
-    [ "$kids" = "$name" ] || { echo "has children"; return 1; }
-    hold="$(ls -A "/sys/block/$name/holders/")" || { echo "holders unreadable"; return 1; }
-    [ -z "$hold" ] || { echo "has holders"; return 1; }
-    sw="$(swapon --show=NAME --noheadings)" || { echo "swapon failed"; return 1; }
-    if grep -q "$name" <<<"$sw"; then echo "is swap"; return 1; fi
-    mnt="$(lsblk -nro MOUNTPOINTS "$id")" || { echo "mounts unreadable"; return 1; }
-    [ -z "${mnt//[[:space:]]/}" ] || { echo "is mounted"; return 1; }
-    set +e; blkid -p "$id" >/dev/null 2>&1; rc=$?; set -e
-    [ "$rc" -eq 2 ] || { echo "blkid -p says it has a signature (rc $rc)"; return 1; }
-    w="$(wipefs -n "$id")" || { echo "wipefs failed"; return 1; }
-    [ -z "$w" ] || { echo "wipefs finds signatures"; return 1; }
-    return 0
-}
-
-# choose_data_disk ID... prints the largest disk's ID, the lowest path among equal sizes, and says on
-# stderr which it left. The choice depends only on size and path, so every node of one plan agrees.
-choose_data_disk() {
-    local chosen rest
-    chosen="$(for id in "$@"; do printf '%s %s\n' "$(blockdev --getsize64 "$id")" "$id"; done |
-        sort -k1,1nr -k2,2 | head -n1)"
-    chosen="${chosen#* }"
-    if [ "$#" -gt 1 ]; then
-        rest="$(printf '%s\n' "$@" | grep -vxF "$chosen" | sort | tr '\n' ' ')"
-        echo "chose $chosen ($(blockdev --getsize64 "$chosen") bytes), the largest of $# blank disks; left untouched: ${rest% }" >&2
-    fi
-    printf '%s\n' "$chosen"
-}
-
-candidates=()
-shopt -s nullglob
-for id in /dev/disk/by-id/nvme-eui.*; do
-    case "$id" in *-part*) continue ;; esac
-    if [ -n "$WANT" ] && [ "$id" != "$WANT" ]; then continue; fi
-    if why="$(blank_reason "$id")"; then
-        candidates+=("$id")
-    else
-        echo "skip $id: $why"
-    fi
-done
-shopt -u nullglob
-
-if [ "${#candidates[@]}" -eq 0 ]; then
-    lsblk -o NAME,SIZE,TYPE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS >&2 || true
-    die "no blank data disk among the by-id EUI disks"
-fi
-ID="$(choose_data_disk "${candidates[@]}")"
-DEV="$(readlink -e "$ID")"
-echo "CHECKS PASSED: $ID -> $DEV"
-
-mkdir -p "$MP"
-mkfs.ext4 -q -L "$LABEL" "$ID"
-U="$(blkid -p -o value -s UUID "$ID")" || die "blkid UUID failed"
-[ -n "$U" ] || die "no UUID after mkfs"
-[ "$(blkid -p -o value -s TYPE "$ID")" = ext4 ] || die "not ext4 after mkfs"
-
-NEW="$(mktemp /etc/fstab.spinifex.XXXXXX)"
-cp /etc/fstab "$NEW"
-echo "UUID=$U $MP ext4 defaults,nofail,x-systemd.device-timeout=30s 0 2" >> "$NEW"
-findmnt --verify --tab-file "$NEW" || { rm -f "$NEW"; die "staged fstab failed verify, /etc/fstab untouched"; }
-BAK="$FSTAB_BAK.$(date +%s)"
-cp -a /etc/fstab "$BAK"
-trap 'cp -a "$BAK" /etc/fstab; systemctl daemon-reload; echo "ROLLED BACK /etc/fstab from $BAK" >&2' EXIT
-chmod 644 "$NEW"
-mv "$NEW" /etc/fstab
-systemctl daemon-reload
-mount "$MP"
-[ "$(findmnt --mountpoint "$MP" -no UUID)" = "$U" ] || die "wrong UUID at $MP"
-[ "$(findmnt --mountpoint "$MP" -no FSTYPE)" = ext4 ] || die "wrong fstype at $MP"
-[ "$(findmnt --mountpoint "$MP" -no MAJ:MIN | tr -d ' ')" = "$(lsblk -dno MAJ:MIN "$ID" | tr -d ' ')" ] || die "wrong device at $MP"
-trap - EXIT
-findmnt --mountpoint "$MP" -o TARGET,SOURCE,MAJ:MIN,UUID,FSTYPE,OPTIONS
-echo "STORAGE OK"
-echo "DATA_UUID=$U"

scripts/terraform/latitude-spx/network.tf

@@ -1,14 +0,0 @@
-# A private network needs two or more servers, so a single node has none. The VLAN carries the cluster
-# and Geneve traffic; its address is set over SSH, since the VID is only known after apply.
-resource "latitudesh_virtual_network" "lan" {
-  count       = var.node_count > 1 ? 1 : 0
-  project     = local.project_id
-  site        = var.site
-  description = "spinifex-${var.deployment_name}"
-}
-
-resource "latitudesh_vlan_assignment" "node" {
-  count              = var.node_count > 1 ? var.node_count : 0
-  server_id          = latitudesh_server.node[count.index].id
-  virtual_network_id = latitudesh_virtual_network.lan[0].id
-}

scripts/terraform/latitude-spx/node-install-api-token.sh

@@ -1,48 +0,0 @@
-#!/bin/sh
-# Runs as root on a node. Reads the API token on stdin and installs it for the daemon.
-# The directory belongs to root and is not writable by anyone else, so no service can create or
-# swap an entry in it. The file belongs to the daemon's user alone (mode 0400): the other
-# services share the spinifex group, so group access would hand them the team-wide token. The
-# temporary file has an unpredictable name and is created exclusively. A symlink, or an owner
-# other than root or the daemon's user, where the directory or the file should be is refused.
-# Ownership and mode are set on every run, so an older install converges. The file's content is
-# replaced only when the token differs. Prints installed or unchanged.
-set -eu
-dir="${API_TOKEN_DIR:-/etc/spinifex/latitude}"
-root="${API_TOKEN_ROOT-root}"
-rootgroup="${API_TOKEN_ROOT_GROUP-root}"
-user="${API_TOKEN_USER-spinifex-daemon}"
-umask 077
-
-fail() { echo "$*" >&2; exit 1; }
-owner_of() { stat -c %U "$1"; }
-
-[ ! -L "$dir" ] || fail "$dir is a symlink"
-if [ ! -e "$dir" ]; then mkdir -p "$dir"; fi
-[ -d "$dir" ] || fail "$dir is not a directory"
-[ "$(owner_of "$dir")" = "$root" ] || fail "$dir is not owned by $root"
-chown "$root:$rootgroup" "$dir"
-chmod 0711 "$dir"
-
-target="$dir/api-token"
-[ ! -L "$target" ] || fail "$target is a symlink"
-if [ -e "$target" ]; then
-    [ -f "$target" ] || fail "$target is not a regular file"
-    holder="$(owner_of "$target")"
-    [ "$holder" = "$root" ] || [ "$holder" = "$user" ] || fail "$target is owned by $holder"
-fi
-
-tmp="$(mktemp "$dir/.api-token.XXXXXX")"
-trap 'rm -f "$tmp"' EXIT
-cat > "$tmp"
-[ -s "$tmp" ] || fail "empty token on stdin"
-if [ -f "$target" ] && cmp -s "$tmp" "$target"; then
-    chown "$user" "$target"
-    chmod 0400 "$target"
-    echo unchanged
-    exit 0
-fi
-chown "$user" "$tmp"
-chmod 0400 "$tmp"
-mv -f "$tmp" "$target"
-echo installed

scripts/terraform/latitude-spx/node-pool-state.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-# Runs as root on a node with the pool name as $1. Reads only the journal of the daemon's
-# current invocation, so a success logged before a restart cannot hide a failure after it.
-# Prints ready, refused or waiting.
-set -eu
-pool="${1:?pool name}"
-id="$(systemctl show -p InvocationID --value spinifex-daemon)"
-[ -n "$id" ] || { echo waiting; exit 0; }
-log="$(journalctl "_SYSTEMD_INVOCATION_ID=$id" -u spinifex-daemon --no-pager -o cat 2>/dev/null || true)"
-named() { grep -F -e "pool=$pool" -e "\"pool\":\"$pool\""; }
-if printf '%s\n' "$log" | grep -F "refused the pool's first call" | named >/dev/null; then
-    echo refused
-elif printf '%s\n' "$log" | grep -F "floating pool ready" | named >/dev/null; then
-    echo ready
-else
-    echo waiting
-fi

scripts/terraform/latitude-spx/node-scripts_test.sh

@@ -1,119 +0,0 @@
-#!/bin/sh
-# Tests node-install-api-token.sh and node-pool-state.sh with stand-in tools.
-set -eu
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-fails=0
-check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-
-# token install: A, A again, then B across a rotation; the file ends as B, owned by the daemon
-# user alone (0400) in a 0711 directory that only root may change
-me="$(id -un)"
-export API_TOKEN_DIR="$work/tok" API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
-install_token() { sh "$here/node-install-api-token.sh"; }
-check "first install" "$(printf 'TOKENA' | install_token)" installed
-check "same token" "$(printf 'TOKENA' | install_token)" unchanged
-check "rotation" "$(printf 'TOKENB' | install_token)" installed
-check "file holds B" "$(cat "$work/tok/api-token")" TOKENB
-check "file mode and owner" "$(stat -c '%U %a' "$work/tok/api-token")" "$me 400"
-check "dir mode and owner" "$(stat -c '%U %a' "$work/tok")" "$me 711"
-check "no temp left after install" "$(ls -A "$work/tok")" api-token
-check "empty refused" "$(printf '' | install_token 2>/dev/null || echo failed)" failed
-check "file kept after refusal" "$(cat "$work/tok/api-token")" TOKENB
-
-# an older install converges on the unchanged path too
-chmod 0640 "$work/tok/api-token"
-chmod 0755 "$work/tok"
-check "unchanged path repairs" "$(printf 'TOKENB' | install_token)" unchanged
-check "mode repaired on unchanged" "$(stat -c %a "$work/tok/api-token")" 400
-check "dir mode repaired on unchanged" "$(stat -c %a "$work/tok")" 711
-
-# who can read it: only the daemon's identity. This needs two real identities, so it runs as root only.
-if [ "$(id -u)" = 0 ]; then
-    chmod 0711 "$work"
-    export API_TOKEN_ROOT=root API_TOKEN_ROOT_GROUP=root API_TOKEN_USER=daemon
-    for mode in installed unchanged; do
-        [ "$mode" = installed ] && rm -f "$work/tok/api-token"
-        check "$mode path" "$(printf 'TOKENB' | install_token)" "$mode"
-        check "daemon reads ($mode)" "$(setpriv --reuid=daemon --regid=daemon --clear-groups cat "$work/tok/api-token")" TOKENB
-        check "another identity in the shared group cannot read ($mode)" \
-            "$(setpriv --reuid=nobody --regid=daemon --clear-groups cat "$work/tok/api-token" 2>/dev/null || echo denied)" denied
-    done
-    export API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
-else
-    echo "skip reading as the daemon and as another service: needs root"
-fi
-
-# a link planted where the file or the directory belongs is refused, and nothing it points at is touched
-echo SENTINEL > "$work/sentinel"
-rm -f "$work/tok/api-token"
-ln -s "$work/sentinel" "$work/tok/api-token"
-check "symlinked file refused" "$(printf 'TOKENC' | sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
-check "symlink target untouched" "$(cat "$work/sentinel")" SENTINEL
-ln -s "$work/tok" "$work/tok-link"
-check "symlinked dir refused" "$(printf 'TOKENC' | API_TOKEN_DIR="$work/tok-link" sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
-check "dir owned by someone else refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=nobody-such install_token 2>/dev/null || echo refused)" refused
-rm -f "$work/tok/api-token"
-printf 'TOKENB' | install_token >/dev/null
-check "file owned by a stranger refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=$me API_TOKEN_USER=nobody-such install_token 2>/dev/null || echo refused)" refused
-check "no temp left" "$(ls -A "$work/tok")" api-token
-
-# data disk choice: the largest blank disk, the lowest path among equals, the rest left alone
-choose_dir="$work/choose"
-mkdir -p "$choose_dir/bin"
-cat > "$choose_dir/bin/blockdev" <<'SH'
-#!/bin/sh
-# blockdev --getsize64 <id>: the fixture's size for that disk
-cat "$CHOOSE_DIR/size.$(basename "$2")"
-SH
-chmod +x "$choose_dir/bin/blockdev"
-eval "$(sed -n '/^choose_data_disk()/,/^}/p' "$here/host-data-disk.sh")"
-size_of() { printf '%s' "$2" > "$choose_dir/size.$1"; }
-pick() { CHOOSE_DIR="$choose_dir" PATH="$choose_dir/bin:$PATH" choose_data_disk "$@" 2>"$choose_dir/err"; }
-size_of nvme-eui.0b 1700000000000
-size_of nvme-eui.0a 1700000000000
-size_of nvme-eui.09 447000000000
-size_of nvme-eui.0c 1900000000000
-check "largest of three wins" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0c)" /dev/disk/by-id/nvme-eui.0c
-check "equal sizes: the lowest path" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a)" /dev/disk/by-id/nvme-eui.0a
-check "order of the list does not matter" "$(pick /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.0a
-check "what was left is said" "$(grep -c 'left untouched: /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b' "$choose_dir/err")" 1
-check "one disk is just chosen" "$(pick /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.09
-check "one disk says nothing" "$(wc -c < "$choose_dir/err" | tr -d ' ')" 0
-
-# pool state: the stand-in journal serves only the lines of the invocation asked for
-mkdir "$work/bin"
-cat > "$work/bin/systemctl" <<'SH'
-#!/bin/sh
-cat "$FAKE_DIR/invocation"
-SH
-cat > "$work/bin/journalctl" <<'SH'
-#!/bin/sh
-for a in "$@"; do
-    case "$a" in _SYSTEMD_INVOCATION_ID=*) id="${a#_SYSTEMD_INVOCATION_ID=}" ;; esac
-done
-cat "$FAKE_DIR/journal.$id" 2>/dev/null
-SH
-chmod +x "$work/bin/systemctl" "$work/bin/journalctl"
-export FAKE_DIR="$work" PATH="$work/bin:$PATH"
-state() { sh "$here/node-pool-state.sh" latitude; }
-
-echo old > "$work/invocation"
-echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.old"
-check "current success" "$(state)" ready
-
-echo new > "$work/invocation"
-echo 'ERROR floating: the provider refused the pool'"'"'s first call; public addresses will fail until fixed pool=latitude' > "$work/journal.new"
-check "old success then current failure" "$(state)" refused
-
-echo 'INFO starting' > "$work/journal.new"
-check "current invocation not yet ready" "$(state)" waiting
-
-echo 'INFO floating pool ready pool=other source=latitude' > "$work/journal.new"
-check "another pool does not count" "$(state)" waiting
-
-echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.new"
-check "current success" "$(state)" ready
-
-[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/outputs.tf

@@ -1,56 +0,0 @@
-output "deployment_name" {
-  value = var.deployment_name
-}
-
-output "site" {
-  value = var.site
-}
-
-output "project" {
-  value = local.project
-}
-
-# provider_host_id is the server ID. public_ip is the server's own /31 address, which SSH and the
-# Spinifex API use; the Elastic IP is separate.
-output "nodes" {
-  value = [for n in latitudesh_server.node : {
-    name             = trimprefix(n.hostname, "spx-${var.deployment_name}-")
-    provider_host_id = n.id
-    public_ip        = n.primary_ipv4
-  }]
-}
-
-# The first address, for a single node's static pool.
-output "elastic_ip" {
-  value = latitudesh_elastic_ip.pool[0].address
-}
-
-# IDs as well as addresses: destroy deletes only these IDs, and the floating pool lists them.
-output "elastic_ips" {
-  value = [for e in latitudesh_elastic_ip.pool : { id = e.id, address = e.address }]
-}
-
-# The VLAN ID of the cluster network; null for one node, which has no private network.
-output "vid" {
-  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].vid
-}
-
-output "ssh_key_id" {
-  value = latitudesh_ssh_key.deploy.id
-}
-
-output "user_data_id" {
-  value = latitudesh_user_data.host.id
-}
-
-output "project_id" {
-  value = local.project_id
-}
-
-output "project_created" {
-  value = length(latitudesh_project.deploy) > 0
-}
-
-output "virtual_network_id" {
-  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].id
-}

scripts/terraform/latitude-spx/provider.tf

@@ -1,4 +0,0 @@
-# The token comes from LATITUDESH_AUTH_TOKEN in the environment, which the provider reads
-# itself (its auth_token argument documents that variable). It is never a variable here, so it
-# is not written to the state or to a tfvars file. The default project is set per resource.
-provider "latitudesh" {}

scripts/terraform/latitude-spx/terraform.tfvars.example

@@ -1,5 +0,0 @@
-deployment_name     = "l1"
-site                = "DAL"
-plan                = "m4-metal-medium"
-node_count          = 1
-ssh_public_key_path = "~/.ssh/id_ed25519.pub"

scripts/terraform/latitude-spx/variables.tf

@@ -1,63 +0,0 @@
-variable "deployment_name" {
-  type        = string
-  description = "Names every resource and the server hostnames, so two deployments in one account never collide."
-  validation {
-    # A hostname is at most 32 characters: "spx-" + name + "-node1".
-    condition     = can(regex("^[a-z0-9][a-z0-9-]{0,15}$", var.deployment_name))
-    error_message = "deployment_name must be 1-16 lowercase letters, digits or hyphens."
-  }
-}
-
-variable "project" {
-  type        = string
-  default     = ""
-  description = "ID or slug of an existing latitude.sh project to deploy into. Empty creates a project spinifex-<deployment_name> and removes it on destroy."
-}
-
-variable "project_id" {
-  type        = string
-  default     = ""
-  description = "ID of the project named by project. The Elastic IP create sends it as project_id without resolving a slug (provider 4.8.0), so it must be an ID. Unused when project is empty."
-}
-
-variable "site" {
-  type        = string
-  default     = "DAL"
-  description = "latitude.sh site slug. Pick one with stock for the plan and Elastic IP support; not SAO."
-}
-
-variable "plan" {
-  type        = string
-  default     = "m4-metal-medium"
-  description = "Server plan slug (the pricing page calls it m4.metal.medium)."
-}
-
-variable "operating_system" {
-  type        = string
-  default     = "ubuntu_26_04_x64_lts"
-  description = "OS slug. UNVERIFIED: the 26.04 slug follows the 24.04 pattern (ubuntu_24_04_x64_lts) and is checked by a data source at plan time."
-}
-
-variable "node_count" {
-  type    = number
-  default = 1
-  validation {
-    condition     = contains([1, 3], var.node_count)
-    error_message = "node_count must be 1 or 3."
-  }
-}
-
-variable "elastic_ip_count" {
-  type        = number
-  default     = 1
-  description = "Elastic IPs to buy, each 4 USD a month. Every one starts on a server and keeps it unless Spinifex moves it. latitude.sh allows 5 per team."
-  validation {
-    condition     = var.elastic_ip_count >= 1 && var.elastic_ip_count <= 5 && floor(var.elastic_ip_count) == var.elastic_ip_count
-    error_message = "elastic_ip_count must be a whole number from 1 to 5."
-  }
-}
-
-variable "ssh_public_key_path" {
-  type    = string
-  default = "~/.ssh/id_ed25519.pub"
-}

scripts/terraform/latitude-spx/versions.tf

@@ -1,7 +0,0 @@
-terraform {
-  required_version = ">= 1.5.0"
-  required_providers {
-    # Pinned exactly: 4.8.0 is the version whose schema these files were written against.
-    latitudesh = { source = "latitudesh/latitudesh", version = "= 4.8.0" }
-  }
-}

spinifex/agents/lambda/runtimeapi/server.go

@@ -212,7 +212,7 @@ func (s *Server) response(w http.ResponseWriter, r *http.Request) {
 	}
 	if len(body) > MaxResponseBytes {
 		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
-		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeUnhandled})
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
 		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
 		return
 	}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -170,7 +170,7 @@ func TestOversizeResponse(t *testing.T) {
 
 	got := <-out
 	require.NoError(t, got.err)
-	assert.Equal(t, "Unhandled", got.res.FunctionError)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
 	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
 }
 

spinifex/bootstrap/config/config.go

@@ -111,7 +111,7 @@ type AWSConfig struct {
 // ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
 type ExternalPool struct {
 	Name       string   `mapstructure:"name"`        // Pool identifier (e.g., "wan", "dc1-primary")
-	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale", "vultr" or "latitude"
+	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale" or "vultr"
 	BindBridge string   `mapstructure:"bind_bridge"` // Linux bridge for DHCP DORA (source=dhcp only)
 	DHCPMAC    string   `mapstructure:"dhcp_mac"`    // DHCP client MAC strategy: "derived" (default) or "interface" (source=dhcp only)
 	RangeStart string   `mapstructure:"range_start"` // First IP in range (static source only)
@@ -155,17 +155,6 @@ type ExternalPool struct {
 	VultrRegion  string `mapstructure:"vultr_region"`   // Vultr region ID, e.g. "ord"; never derived from the Spinifex region
 	VultrKeyFile string `mapstructure:"vultr_key_file"` // API key file (optional; see vultr.DefaultKeyFile)
 	LabelPrefix  string `mapstructure:"label_prefix"`   // Marks this deployment's addresses, e.g. "spinifex-prod"
-
-	// Latitude names the project, token file and fixed set of Elastic IPs
-	// (source=latitude only). Each node's server ID is provider_host_id on the node.
-	LatitudeProjectID    string   `mapstructure:"latitude_project_id"`     // latitude.sh project ID the Elastic IPs belong to
-	LatitudeKeyFile      string   `mapstructure:"latitude_key_file"`       // API token file (optional; see latitude.DefaultKeyFile)
-	LatitudeElasticIPIDs []string `mapstructure:"latitude_elastic_ip_ids"` // Elastic IP IDs the pool hands out
-}
-
-// hasLatitudeKeys reports whether any latitude_* key is set.
-func (p ExternalPool) hasLatitudeKeys() bool {
-	return p.LatitudeProjectID != "" || p.LatitudeKeyFile != "" || len(p.LatitudeElasticIPIDs) > 0
 }
 
 // hasVultrKeys reports whether any vultr_* key, or the label prefix only a
@@ -262,7 +251,7 @@ type Config struct {
 
 	// ProviderHostID is this node's instance ID at its cloud provider, which a
 	// floating public address is attached to. Required on every node when a
-	// source="vultr" or source="latitude" pool exists.
+	// source="vultr" pool exists.
 	ProviderHostID string `json:"ProviderHostID,omitempty" mapstructure:"provider_host_id"`
 
 	Daemon      DaemonConfig      `json:"Daemon" mapstructure:"daemon"`
@@ -818,53 +807,6 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
-// MaxLatitudeElasticIPs bounds a latitude pool: latitude.sh allows five Elastic IPs per team, and
-// every allocation reads each configured address from the API.
-const MaxLatitudeElasticIPs = 5
-
-// validateLatitudePool checks a source="latitude" pool. The Elastic IPs are
-// bought ahead of time and named here, the API picks nothing, so the range,
-// gateway and label keys mean nothing, and every node must name its server.
-func validateLatitudePool(cc *ClusterConfig, p ExternalPool) error {
-	if p.LatitudeProjectID == "" || len(p.LatitudeElasticIPIDs) == 0 {
-		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires latitude_project_id and latitude_elastic_ip_ids", p.Name)
-	}
-	if len(p.LatitudeElasticIPIDs) > MaxLatitudeElasticIPs {
-		return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids lists %d addresses, at most %d are allowed", p.Name, len(p.LatitudeElasticIPIDs), MaxLatitudeElasticIPs)
-	}
-	seen := map[string]struct{}{}
-	for _, id := range p.LatitudeElasticIPIDs {
-		if id == "" {
-			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids has an empty entry", p.Name)
-		}
-		if _, dup := seen[id]; dup {
-			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids lists %q twice", p.Name, id)
-		}
-		seen[id] = struct{}{}
-	}
-	if p.LatitudeKeyFile != "" && !filepath.IsAbs(p.LatitudeKeyFile) {
-		return fmt.Errorf("config: [[network.external_pools]] %q: latitude_key_file must be an absolute path, got %q", p.Name, p.LatitudeKeyFile)
-	}
-	if p.BindBridge != "" || p.DHCPMAC != "" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)
-	}
-	if p.RangeStart != "" || p.RangeEnd != "" || p.Gateway != "" || p.GatewayIP != "" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: range_start/range_end/gateway/gateway_ip not allowed with source=\"latitude\" (the Elastic IPs are listed by ID)", p.Name)
-	}
-	if p.GwLrpRangeStart != "" || p.GwLrpRangeEnd != "" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: gw_lrp_range_start/gw_lrp_range_end not allowed with source=\"latitude\" (a gateway address per VPC would use up the Elastic IP limit)", p.Name)
-	}
-	if cc.Network.ExternalMode != "nat" {
-		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires [network] external_mode = \"nat\" (routed); a routed Elastic IP is delivered to the server's own MAC", p.Name)
-	}
-	for name, n := range cc.Nodes {
-		if n.ProviderHostID == "" {
-			return fmt.Errorf("config: [nodes.%s] provider_host_id is required with a source=\"latitude\" pool (%q): it is the latitude.sh server ID the node's Elastic IPs move to", name, p.Name)
-		}
-	}
-	return nil
-}
-
 // validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
 // providers differ. A peer stanza without one is unknown, not the default: the
 // generated config omits it, and the cluster's KV lock is the authoritative check.
@@ -937,9 +879,6 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		end   netip.Addr
 	}
 	var ranges []poolRange
-	// latitudeIDs maps each Elastic IP ID to the pool naming it. Every pool keeps its own
-	// record of what it handed out, so an ID in two pools would be given to two guests.
-	latitudeIDs := map[string]string{}
 	for _, p := range cc.Network.ExternalPools {
 		if p.Source != "oci" && (p.OCICompartmentID != "" || p.OCIVNICID != "" || p.OCIVNICIface != "" || p.OCISubnetID != "" || p.OCIPublicIPPool != "" || p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
 			return fmt.Errorf("config: [[network.external_pools]] %q: oci_* keys are only valid with source=\"oci\"", p.Name)
@@ -947,9 +886,6 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if p.Source != "exoscale" && p.hasExoscaleKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: exoscale_* keys are only valid with source=\"exoscale\"", p.Name)
 		}
-		if p.Source != "latitude" && p.hasLatitudeKeys() {
-			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_* keys are only valid with source=\"latitude\"", p.Name)
-		}
 		if p.Source != "vultr" && p.hasVultrKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: vultr_* and label_prefix keys are only valid with source=\"vultr\"", p.Name)
 		}
@@ -1049,19 +985,8 @@ func validateClusterConfig(cc *ClusterConfig) error {
 				return err
 			}
 			continue
-		case "latitude":
-			if err := validateLatitudePool(cc, p); err != nil {
-				return err
-			}
-			for _, id := range p.LatitudeElasticIPIDs {
-				if other, dup := latitudeIDs[id]; dup {
-					return fmt.Errorf("config: [[network.external_pools]] %q: Elastic IP %q is also named by pool %q; each Elastic IP may belong to one pool", p.Name, id, other)
-				}
-				latitudeIDs[id] = p.Name
-			}
-			continue
 		default:
-			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\", \"vultr\" or \"latitude\"", p.Name, p.Source)
+			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\" or \"vultr\"", p.Name, p.Source)
 		}
 		if p.RangeStart == "" || p.RangeEnd == "" {
 			continue

spinifex/bootstrap/config/config_test.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/spf13/viper"
@@ -1551,83 +1550,6 @@ func TestLoadConfig_NetworkPoolVultrRejections(t *testing.T) {
 	}
 }
 
-const latitudePoolBase = `
-source = "latitude"
-latitude_project_id = "proj_1"
-latitude_elastic_ip_ids = ["eip_1", "eip_2"]
-`
-
-func TestLoadConfig_NetworkPoolLatitudeSourceAccepted(t *testing.T) {
-	cfg, err := loadVultrPool(t, "nat", latitudePoolBase+"latitude_key_file = \"/etc/spinifex/latitude/api-token\"\n",
-		[2]string{"sv_a", "sv_b"})
-	require.NoError(t, err)
-	p := cfg.Network.ExternalPools[0]
-	assert.Equal(t, "latitude", p.Source)
-	assert.Equal(t, "proj_1", p.LatitudeProjectID)
-	assert.Equal(t, []string{"eip_1", "eip_2"}, p.LatitudeElasticIPIDs)
-	assert.Equal(t, "/etc/spinifex/latitude/api-token", p.LatitudeKeyFile)
-	assert.Equal(t, "sv_a", cfg.Nodes["n1"].ProviderHostID)
-}
-
-func latitudeIDs(n int) string {
-	ids := make([]string, n)
-	for i := range ids {
-		ids[i] = fmt.Sprintf("%q", fmt.Sprintf("eip_%d", i))
-	}
-	return "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [" + strings.Join(ids, ", ") + "]\n"
-}
-
-func TestLoadConfig_NetworkPoolLatitudeSetIsCapped(t *testing.T) {
-	cfg, err := loadVultrPool(t, "nat", latitudeIDs(MaxLatitudeElasticIPs), [2]string{"a", "b"})
-	require.NoError(t, err)
-	assert.Len(t, cfg.Network.ExternalPools[0].LatitudeElasticIPIDs, MaxLatitudeElasticIPs)
-
-	_, err = loadVultrPool(t, "nat", latitudeIDs(MaxLatitudeElasticIPs+1), [2]string{"a", "b"})
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "at most 5 are allowed")
-}
-
-func TestLoadConfig_NetworkPoolLatitudeRejections(t *testing.T) {
-	both := [2]string{"a", "b"}
-	cases := []struct {
-		name, mode, body string
-		nodes            [2]string
-		want             string
-	}{
-		{"pool mode", "pool", latitudePoolBase, both, `requires [network] external_mode = "nat"`},
-		{"no project", "nat", "source = \"latitude\"\nlatitude_elastic_ip_ids = [\"e\"]\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
-		{"no ids", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
-		{"duplicate id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e\", \"e\"]\n", both, "twice"},
-		{"empty id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"\"]\n", both, "empty entry"},
-		{"relative key file", "nat", latitudePoolBase + "latitude_key_file = \"token\"\n", both, "absolute path"},
-		{"label prefix", "nat", latitudePoolBase + "label_prefix = \"x\"\n", both, "only valid with source=\"vultr\""},
-		{"range", "nat", latitudePoolBase + "range_start = \"1.2.3.4\"\nrange_end = \"1.2.3.4\"\n", both, "listed by ID"},
-		{"a node without its server id", "nat", latitudePoolBase, [2]string{"a", ""}, "[nodes.n2] provider_host_id is required"},
-		{"latitude keys on static", "nat", "latitude_project_id = \"p\"\n", both, `only valid with source="latitude"`},
-		{"vultr keys on latitude", "nat", latitudePoolBase + "vultr_region = \"ord\"\n", both, `only valid with source="vultr"`},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := loadVultrPool(t, tc.mode, tc.body, tc.nodes)
-			require.Error(t, err)
-			assert.Contains(t, err.Error(), tc.want)
-		})
-	}
-}
-
-func TestLoadConfig_NetworkPoolLatitudeIDInTwoPoolsIsRejected(t *testing.T) {
-	resetViper(t)
-	body := "node = \"n1\"\n\n[network]\nexternal_mode = \"nat\"\n\n" +
-		"[[network.external_pools]]\nname = \"a\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e1\", \"e2\"]\n\n" +
-		"[[network.external_pools]]\nname = \"b\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e3\", \"e2\"]\n\n" +
-		"[nodes.n1]\nregion = \"us-east-1\"\nprovider_host_id = \"sv_1\"\n"
-	path := filepath.Join(t.TempDir(), "spinifex.toml")
-	require.NoError(t, os.WriteFile(path, []byte(body), 0600))
-	_, err := LoadConfig(path)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), `"e2" is also named by pool "a"`)
-}
-
 func TestLoadConfig_NorthstarNameservers(t *testing.T) {
 	load := func(t *testing.T, block string) (*ClusterConfig, error) {
 		t.Helper()

spinifex/domains/ec2/ebs/obligation/obligation.go

@@ -1,185 +0,0 @@
-// Package ebsobligation records the volume exports a detach left in place.
-//
-// A force detach, or a terminate whose unmount did not complete, clears a
-// volume's attachment while the node that served it may still export it. The
-// request cannot reach that node, so it records an UnpublishObligation here and
-// returns; the named node settles it once it answers.
-package ebsobligation
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"regexp"
-	"strconv"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Bucket holds one obligation per publish a cleared attachment left behind.
-// It has no TTL: an obligation ends only when it is settled or closed.
-const Bucket = "spinifex-ebs-unpublish-obligations"
-
-// BucketVersion is the bucket schema version RunKV stamps.
-const BucketVersion = 1
-
-// SchemaVersion is the version of the UnpublishObligation record.
-const SchemaVersion uint16 = 1
-
-// UnpublishObligation names a publish a cleared attachment left on a node: the
-// node, volume and handle, and the attachment it served. The attachment is what
-// tells it apart from a later publish of the same volume on the same node.
-type UnpublishObligation struct {
-	SchemaVersion uint16    `json:"schema_version"`
-	VolumeID      string    `json:"volume_id"`
-	AccountID     string    `json:"account_id"`
-	Handle        string    `json:"handle,omitempty"` // ProviderHandle when the attachment was cleared
-	NodeID        string    `json:"node_id"`          // the node holding the publish
-	InstanceID    string    `json:"instance_id"`      // the attachment that was cleared
-	AttachedAt    time.Time `json:"attached_at"`
-	RecordedAt    time.Time `json:"recorded_at"`
-	Attempts      int       `json:"attempts"`
-	Error         string    `json:"error,omitempty"`
-}
-
-// keyToken is one NATS subject token a key segment may be: the key is three
-// tokens, so a segment holding a dot would split it and escape the filters.
-var keyToken = regexp.MustCompile(`^[-_=a-zA-Z0-9]+$`)
-
-// Key is the obligation's key, <node>.<volume-id>.<attached-at>, with the
-// attachment time in Unix nanoseconds and an unknown time as 0.
-func Key(o UnpublishObligation) (string, error) {
-	if !keyToken.MatchString(o.NodeID) {
-		return "", fmt.Errorf("unpublish obligation: node %q is not a valid key token", o.NodeID)
-	}
-	if !keyToken.MatchString(o.VolumeID) {
-		return "", fmt.Errorf("unpublish obligation: volume %q is not a valid key token", o.VolumeID)
-	}
-	var nanos int64
-	if !o.AttachedAt.IsZero() {
-		nanos = o.AttachedAt.UnixNano()
-	}
-	return o.NodeID + "." + o.VolumeID + "." + strconv.FormatInt(nanos, 10), nil
-}
-
-// allKeys matches every obligation key and not the bucket's version stamp,
-// which is a single token.
-const allKeys = "*.*.*"
-
-func bucketConfig() kvstore.Config {
-	return kvstore.Config{
-		Name:        Bucket,
-		Description: "EBS exports a cleared attachment left on a node, until that node unpublishes them",
-		History:     1,
-		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
-			return migrate.DefaultRegistry.RunKV(ctx, Bucket, kv, BucketVersion)
-		},
-		Missing: "EBS unpublish obligation bucket not initialized",
-	}
-}
-
-// Store reads and writes the obligation bucket.
-type Store struct {
-	s *kvstore.Store[UnpublishObligation]
-}
-
-// NewStore returns a Store over js. The bucket is created on first use.
-func NewStore(js jetstream.JetStream) *Store {
-	return &Store{s: kvstore.New[UnpublishObligation](js, bucketConfig())}
-}
-
-// Open creates or opens the bucket now, so a node that cannot reach it fails
-// at startup rather than on its first force detach.
-func (s *Store) Open(ctx context.Context) error {
-	_, err := s.s.KV(ctx)
-	return err
-}
-
-// Record writes o if no obligation with its key exists. An existing key is the
-// same obligation written by an earlier attempt, so it counts as written.
-func (s *Store) Record(ctx context.Context, o UnpublishObligation) error {
-	key, err := Key(o)
-	if err != nil {
-		return err
-	}
-	o.SchemaVersion = SchemaVersion
-	if _, err := s.s.Create(ctx, key, &o); err != nil && !errors.Is(err, kvstore.ErrExists) {
-		return fmt.Errorf("record unpublish obligation %s: %w", key, err)
-	}
-	return nil
-}
-
-// ListNode returns the open obligations naming node.
-func (s *Store) ListNode(ctx context.Context, node string) ([]UnpublishObligation, error) {
-	if !keyToken.MatchString(node) {
-		return nil, fmt.Errorf("unpublish obligation: node %q is not a valid key token", node)
-	}
-	return s.list(ctx, node+".*.*")
-}
-
-// List returns every open obligation.
-func (s *Store) List(ctx context.Context) ([]UnpublishObligation, error) {
-	return s.list(ctx, allKeys)
-}
-
-func (s *Store) list(ctx context.Context, filter string) ([]UnpublishObligation, error) {
-	items, _, err := s.s.Snapshot(ctx, filter)
-	if err != nil {
-		return nil, err
-	}
-	out := make([]UnpublishObligation, 0, len(items))
-	for _, item := range items {
-		out = append(out, item.Value)
-	}
-	return out, nil
-}
-
-// Close removes o. Removing one already gone is not an error.
-func (s *Store) Close(ctx context.Context, o UnpublishObligation) error {
-	key, err := Key(o)
-	if err != nil {
-		return err
-	}
-	return s.s.Delete(ctx, key)
-}
-
-// RecordFailure counts a failed settlement on o and keeps its error, so an
-// operator listing the obligations sees why it is still open.
-func (s *Store) RecordFailure(ctx context.Context, o UnpublishObligation, cause error) error {
-	key, err := Key(o)
-	if err != nil {
-		return err
-	}
-	err = s.s.Mutate(ctx, key, func(cur *UnpublishObligation) (bool, error) {
-		cur.Attempts++
-		cur.Error = cause.Error()
-		return true, nil
-	})
-	if errors.Is(err, kvstore.ErrNotFound) {
-		return nil
-	}
-	return err
-}
-
-// ListOpen reads the open obligations for an operator tool without creating
-// the bucket: a cluster that has never force-detached a volume has none, and
-// that is not an error.
-func ListOpen(ctx context.Context, nc *nats.Conn) ([]UnpublishObligation, error) {
-	js, err := jetstream.New(nc)
-	if err != nil {
-		return nil, fmt.Errorf("jetstream: %w", err)
-	}
-	if _, err := js.KeyValue(ctx, Bucket); errors.Is(err, jetstream.ErrBucketNotFound) {
-		return nil, nil
-	} else if err != nil {
-		return nil, fmt.Errorf("unpublish obligation bucket: %w", err)
-	}
-	cfg := bucketConfig()
-	cfg.AttachOnly = true
-	cfg.OnOpen = nil
-	return (&Store{s: kvstore.New[UnpublishObligation](js, cfg)}).List(ctx)
-}

spinifex/domains/ec2/ebs/obligation/obligation_test.go

@@ -1,141 +0,0 @@
-package ebsobligation_test
-
-import (
-	"errors"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func newStore(t *testing.T) *ebsobligation.Store {
-	t.Helper()
-	_, _, js := testutil.StartTestJetStream(t)
-	store := ebsobligation.NewStore(js)
-	require.NoError(t, store.Open(t.Context()))
-	return store
-}
-
-func obligation(node, volumeID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
-	return ebsobligation.UnpublishObligation{
-		VolumeID: volumeID, AccountID: "123456789012", NodeID: node,
-		InstanceID: "i-1", AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
-	}
-}
-
-func TestKey_NamesNodeVolumeAndAttachment(t *testing.T) {
-	at := time.Unix(1_700_000_000, 5)
-	key, err := ebsobligation.Key(obligation("node-a", "vol-1", at))
-	require.NoError(t, err)
-	assert.Equal(t, "node-a.vol-1.1700000000000000005", key)
-
-	key, err = ebsobligation.Key(obligation("node-a", "vol-1", time.Time{}))
-	require.NoError(t, err)
-	assert.Equal(t, "node-a.vol-1.0", key, "an unknown attachment time is 0, not a huge negative")
-}
-
-// A segment holding a subject separator or wildcard would split the key or
-// let one node's listing match another's.
-func TestKey_RejectsSegmentsThatAreNotOneToken(t *testing.T) {
-	for _, o := range []ebsobligation.UnpublishObligation{
-		obligation("node.a", "vol-1", time.Time{}),
-		obligation("node-a", "vol.1", time.Time{}),
-		obligation("*", "vol-1", time.Time{}),
-		obligation("node-a", ">", time.Time{}),
-		obligation("", "vol-1", time.Time{}),
-	} {
-		_, err := ebsobligation.Key(o)
-		assert.Error(t, err, "%s/%s", o.NodeID, o.VolumeID)
-	}
-}
-
-func TestStore_RecordIsIdempotentPerAttachment(t *testing.T) {
-	store := newStore(t)
-	at := time.Unix(1_700_000_000, 0).UTC()
-	first := obligation("node-a", "vol-1", at)
-	first.Handle = "h1"
-	require.NoError(t, store.Record(t.Context(), first))
-	retry := first
-	retry.RecordedAt = at.Add(time.Hour)
-	require.NoError(t, store.Record(t.Context(), retry), "a retry of the same obligation is not an error")
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at.Add(time.Second))),
-		"a different attachment of the same volume is a different obligation")
-
-	open, err := store.List(t.Context())
-	require.NoError(t, err)
-	require.Len(t, open, 2)
-	for _, o := range open {
-		assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
-		if o.AttachedAt.Equal(at) {
-			assert.True(t, o.RecordedAt.Equal(at.Add(time.Minute)), "the first write is kept")
-		}
-	}
-}
-
-// The bucket's version stamp is a key in the same bucket; listing must not
-// return it as an obligation.
-func TestStore_ListSkipsTheVersionStampAndListNodeIsolatesNodes(t *testing.T) {
-	store := newStore(t)
-	at := time.Unix(1_700_000_000, 0).UTC()
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at)))
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-2", at)))
-	require.NoError(t, store.Record(t.Context(), obligation("node-ab", "vol-3", at)))
-
-	all, err := store.List(t.Context())
-	require.NoError(t, err)
-	assert.Len(t, all, 3)
-
-	mine, err := store.ListNode(t.Context(), "node-a")
-	require.NoError(t, err)
-	volumes := []string{}
-	for _, o := range mine {
-		volumes = append(volumes, o.VolumeID)
-	}
-	assert.ElementsMatch(t, []string{"vol-1", "vol-2"}, volumes, "a node whose name prefixes another's sees only its own")
-
-	_, err = store.ListNode(t.Context(), "*")
-	assert.Error(t, err)
-}
-
-func TestStore_CloseAndRecordFailure(t *testing.T) {
-	store := newStore(t)
-	o := obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())
-	require.NoError(t, store.Record(t.Context(), o))
-
-	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("first")))
-	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("second")))
-	open, err := store.List(t.Context())
-	require.NoError(t, err)
-	require.Len(t, open, 1)
-	assert.Equal(t, 2, open[0].Attempts)
-	assert.Equal(t, "second", open[0].Error)
-
-	require.NoError(t, store.Close(t.Context(), o))
-	require.NoError(t, store.Close(t.Context(), o), "closing one already gone is not an error")
-	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("late")),
-		"a failure recorded after another sweep closed it does not resurrect it")
-	open, err = store.List(t.Context())
-	require.NoError(t, err)
-	assert.Empty(t, open)
-}
-
-func TestListOpen_MissingBucketIsEmptyAndNotCreated(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-
-	open, err := ebsobligation.ListOpen(t.Context(), nc)
-	require.NoError(t, err)
-	assert.Empty(t, open)
-	_, err = js.KeyValue(t.Context(), ebsobligation.Bucket)
-	assert.Error(t, err, "an operator listing must not create the bucket")
-
-	store := ebsobligation.NewStore(js)
-	require.NoError(t, store.Open(t.Context()))
-	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())))
-	open, err = ebsobligation.ListOpen(t.Context(), nc)
-	require.NoError(t, err)
-	require.Len(t, open, 1)
-	assert.Equal(t, "vol-1", open[0].VolumeID)
-}

spinifex/domains/ec2/eip/service_impl.go

@@ -879,7 +879,8 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco
 // against instanceID, across all accounts. Idempotent backstop for system-VM
 // teardown paths that lose the cached allocation ID — when the VM is already
 // gone from the manager the per-instance release never runs, so an
-// internet-facing ALB's EIP would otherwise orphan. No-op when nothing matches.
+// internet-facing ALB's EIP would otherwise orphan. It touches only addresses
+// tagged as system-managed, and is a no-op when nothing matches.
 func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error {
 	ctx := context.Background()
 	if instanceID == "" {
@@ -908,6 +909,13 @@ func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error {
 		if record.InstanceId != instanceID || record.AllocationId == "" {
 			continue
 		}
+		// Only an address a system VM launch allocated, so a wrong instance ID
+		// can never release a customer's own address.
+		if !awstags.IsSystemManaged(record.Tags[awstags.ManagedByKey]) {
+			slog.Warn("ReleaseAddressByInstanceID: address is not system-managed, left alone",
+				"instanceId", instanceID, "allocationId", record.AllocationId)
+			continue
+		}
 		// Key is "{accountID}.{allocID}"; the account scopes the release calls.
 		accountID, _, found := strings.Cut(k, ".")
 		if !found {

spinifex/domains/ec2/eip/service_impl_test.go

@@ -179,7 +179,12 @@ func TestEIP_ReleaseWhileAssociated(t *testing.T) {
 func TestEIP_ReleaseByInstanceID_ReclaimsAssociatedEIP(t *testing.T) {
 	svc, _, _ := setupTestEIP(t)
 
-	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{
+		TagSpecifications: []*ec2.TagSpecification{{
+			ResourceType: aws.String("elastic-ip"),
+			Tags:         []*ec2.Tag{{Key: aws.String("spinifex:managed-by"), Value: aws.String("elbv2")}},
+		}},
+	}, testAccountID)
 	require.NoError(t, err)
 	allocID := *out.AllocationId
 
@@ -861,3 +866,26 @@ func TestEIP_DisassociateByENI_ClearsEveryEIPOnTheInterface(t *testing.T) {
 	require.NoError(t, err)
 	assert.False(t, found)
 }
+
+func TestEIP_ReleaseByInstanceID_LeavesUntaggedAddress(t *testing.T) {
+	svc, _, _ := setupTestEIP(t)
+
+	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+	require.NoError(t, err)
+	key := testAccountID + "." + *out.AllocationId
+	entry, err := svc.eipKV.Get(t.Context(), key)
+	require.NoError(t, err)
+	var record EIPRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &record))
+	record.State = "associated"
+	record.InstanceId = "i-customer"
+	data, err := json.Marshal(record)
+	require.NoError(t, err)
+	_, err = svc.eipKV.Update(t.Context(), key, data, entry.Revision())
+	require.NoError(t, err)
+
+	require.NoError(t, svc.ReleaseAddressByInstanceID("i-customer"))
+
+	_, err = svc.eipKV.Get(t.Context(), key)
+	assert.NoError(t, err, "an address no system launch allocated is never released by instance ID")
+}

spinifex/domains/ec2/instance/service.go

@@ -141,17 +141,6 @@ type InstanceTagWriter interface {
 	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
 }
 
-// OwedUnpublish names the publish a terminate leaves in place when its unmount
-// did not complete: the node still exporting the volume and the instance it
-// served. The zero value means the unmount completed and nothing is owed.
-type OwedUnpublish struct {
-	NodeID     string
-	InstanceID string
-}
-
-// Owed reports whether a publish is left in place.
-func (o OwedUnpublish) Owed() bool { return o.NodeID != "" }
-
 // VolumeDeleter deletes EBS volumes. Implemented by domains/ec2/volume's
 // VolumeServiceImpl (used by the daemon).
 type VolumeDeleter interface {
@@ -160,16 +149,14 @@ type VolumeDeleter interface {
 	// DeleteVolumeOnTerminate deletes a DeleteOnTermination volume as part of
 	// an instance terminate: it clears any stale attachment (terminate
 	// implies detach) before deleting, so a still-attached boot volume left
-	// over from Stop is not rejected by DeleteVolume's in-use guard. When owed
-	// names a publish, an unpublish obligation is recorded before the clear.
-	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
+	// over from Stop is not rejected by DeleteVolume's in-use guard.
+	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
 
 	// DetachVolumeOnTerminate clears a still-attached volume's attachment on
 	// terminate without deleting it, matching AWS semantics for a
 	// DeleteOnTermination=false volume: terminate still implies detach, it
 	// just leaves the volume behind as available rather than deleting it.
-	// When owed names a publish, an unpublish obligation is recorded first.
-	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
+	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
 }
 
 // VolumeCreator creates the volumes a launch names beyond the root device, and

spinifex/domains/ec2/instance/service_impl.go

@@ -3048,15 +3048,6 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 	// stopped-path delete failure.
 	var lastErr error
 
-	// A volume whose seal failed at stop is still exported on the node that
-	// stopped it, so clearing its attachment owes that node an unpublish.
-	owedFor := func(volumeID string) OwedUnpublish {
-		if instance.LastNode == "" || !slices.Contains(instance.UnpublishOwed, volumeID) {
-			return OwedUnpublish{}
-		}
-		return OwedUnpublish{NodeID: instance.LastNode, InstanceID: instance.ID}
-	}
-
 	for _, ebsRequest := range instance.EBSRequests.Requests {
 		// Internal volumes (EFI) are always cleaned up via ebs.delete.
 		if ebsRequest.EFI {
@@ -3087,7 +3078,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 				continue
 			}
 			// A volume already gone is the detach's own goal, not a failure.
-			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil &&
+			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil &&
 				!awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
 				slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to detach volume", "name", ebsRequest.Name, "err", err)
 				lastErr = err
@@ -3106,7 +3097,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 		// clears a Boot volume, runtime/daemon/vm_adapters.go) before deleting, so
 		// DeleteVolume's in-use guard does not reject an otherwise-legitimate
 		// delete. The resulting error is surfaced into lastErr, not swallowed.
-		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil {
+		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil {
 			slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to delete volume", "name", ebsRequest.Name, "err", err)
 			lastErr = err
 		}

spinifex/domains/ec2/instance/service_impl_test.go

@@ -1734,7 +1734,6 @@ type fakeVolumeDeleter struct {
 	calls    []string
 	deleted  []string
 	detached []string
-	owed     []OwedUnpublish
 	err      error
 }
 
@@ -1750,8 +1749,7 @@ func (f *fakeVolumeDeleter) DeleteVolume(_ context.Context, input *ec2.DeleteVol
 
 // DeleteVolumeOnTerminate mirrors DeleteVolume's call/error bookkeeping; the
 // stopped-instance terminate path calls this instead of DeleteVolume.
-func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
-	f.owed = append(f.owed, owed)
+func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -1763,8 +1761,7 @@ func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID,
 // DetachVolumeOnTerminate mirrors DeleteVolumeOnTerminate's call/error
 // bookkeeping but records into detached instead of deleted; the non-DoT
 // Boot-volume branch calls this instead of deleting.
-func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
-	f.owed = append(f.owed, owed)
+func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -2012,45 +2009,6 @@ func TestTerminateStoppedInstance_UserVolumeDeleted(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, []string{"vol-user-001"}, vd.deleted, "only DeleteOnTermination=true volumes deleted")
 	assert.Equal(t, []string{"vol-keep-001"}, vd.detached, "terminate implies detach even for a volume that survives it")
-	assert.Equal(t, []OwedUnpublish{{}, {}}, vd.owed, "a stop whose unmount completed owes no unpublish")
-}
-
-// A stop whose seal failed left those volumes exported on the node that
-// stopped the instance, so terminate names that node for each of them alone.
-func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *testing.T) {
-	id := "i-seal-failed"
-	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", LastNode: "node-a",
-		UnpublishOwed: []string{"vol-root-001", "vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
-		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
-		{Name: "vol-keep-001", DeleteOnTermination: false},
-		{Name: "vol-sealed-001", DeleteOnTermination: false},
-	}
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
-	vd := &fakeVolumeDeleter{}
-	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
-
-	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
-	require.NoError(t, err)
-
-	owed := OwedUnpublish{NodeID: "node-a", InstanceID: id}
-	assert.Equal(t, []string{"vol-root-001", "vol-keep-001", "vol-sealed-001"}, vd.calls)
-	assert.Equal(t, []OwedUnpublish{owed, owed, {}}, vd.owed)
-}
-
-// With no node recorded there is no node to name; the terminate owes nothing
-// rather than naming an empty node.
-func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
-	id := "i-seal-failed-nonode"
-	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", UnpublishOwed: []string{"vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-keep-001"}}
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
-	vd := &fakeVolumeDeleter{}
-	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
-
-	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
-	require.NoError(t, err)
-	assert.Equal(t, []OwedUnpublish{{}}, vd.owed)
 }
 
 // TestTerminateStoppedInstance_StampsTeardownVolumesDone locks the

spinifex/domains/ec2/instance/start_volumes.go

@@ -18,9 +18,7 @@ import (
 // reconcileStartVolumes drops the data volumes a claimed instance can no longer
 // have, and fails the start when its boot volume is gone.
 func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instance *vm.VM) error {
-	// A direct-boot system guest boots from a kernel, not a root volume, and its
-	// one data volume, if any, is its own store rather than a customer's.
-	if s.metadata == nil || instance.DirectBoot {
+	if s.metadata == nil {
 		return nil
 	}
 

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -123,20 +123,3 @@ func TestReconcileStartVolumes_ReattachesStillAvailableUnattachedVolume(t *testi
 	assert.Equal(t, []string{"vol-root", "vol-released"}, requestedNames(inst))
 	assert.Equal(t, []string{"vol-root", "vol-released"}, mappedNames(inst))
 }
-
-// ALB and SQS guests boot from a kernel with no root volume, so a start or a
-// recovery must not demand one; an SQS store's data volume stays named.
-func TestReconcileStartVolumes_DirectBootNeedsNoRootVolume(t *testing.T) {
-	t.Parallel()
-	svc := startVolumesService(t)
-
-	alb := &vm.VM{ID: "i-alb", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
-		Instance: &ec2.Instance{InstanceId: aws.String("i-alb")}}
-	require.NoError(t, svc.reconcileStartVolumes(context.Background(), alb))
-
-	sqs := &vm.VM{ID: "i-sqs", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
-		Instance: &ec2.Instance{InstanceId: aws.String("i-sqs")}}
-	sqs.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
-	require.NoError(t, svc.reconcileStartVolumes(context.Background(), sqs))
-	assert.Equal(t, []string{"vol-store-gone"}, requestedNames(sqs), "a system store volume is never pruned")
-}

spinifex/domains/ec2/volume/account_termination_current_behaviour_test.go

@@ -9,7 +9,6 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -60,7 +59,7 @@ func TestCurrentBehaviour_AccountTermination_SnapshottedDeleteOnTerminationVolum
 	require.NoError(t, err)
 	require.NoError(t, volumes.UpdateVolumeState(accountID, volumeID, "in-use", instanceID, "/dev/sda1"))
 
-	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID, ec2instance.OwedUnpublish{})
+	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID)
 	assert.EqualError(t, err, awserrors.ErrorVolumeInUse, "the snapshot index refuses the terminate delete")
 
 	survivor, err := metadata.GetVolume(ctx, accountID, volumeID)

spinifex/domains/ec2/volume/delete_volume_unreadable_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -195,7 +194,7 @@ func TestDeleteVolumeOnTerminate_DeleteReadFailureIsNotSuccess(t *testing.T) {
 	svc.store = failing
 	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
 
-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
 
 	require.Error(t, err, "a delete-read failure after a successful attachment clear must not be reported as success")
 	assert.False(t, awserrors.IsNotFound(err),

spinifex/domains/ec2/volume/force_detach_test.go

@@ -20,7 +20,6 @@ func newForceDetachService(t *testing.T) *VolumeServiceImpl {
 	t.Helper()
 	svc := newTestVolumeService("ap-southeast-2a")
 	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-stuck0000000000": "node-a"}}, "node-a", "node-b")
 	return svc
 }
 
@@ -48,11 +47,7 @@ func TestForceDetachVolumeClearsTheAttachment(t *testing.T) {
 	// The previous instance is reported back: it is the only record of what
 	// the volume was torn away from.
 	assert.Equal(t, "i-stuck0000000000", aws.StringValue(attachment.InstanceId))
-	// The export is still up on the node that served it, so the attachment is
-	// reported mid-teardown, as AWS reports a detach it has not finished.
-	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
-	assert.Equal(t, "/dev/sdf", aws.StringValue(attachment.Device))
-	assert.False(t, aws.TimeValue(attachment.AttachTime).IsZero())
+	assert.Equal(t, "detached", aws.StringValue(attachment.State))
 
 	meta, err := svc.GetVolumeMetadata("000000000011", "vol-forcedetach1")
 	require.NoError(t, err)
@@ -117,8 +112,4 @@ func TestForceDetachVolumeOnAnUnattachedVolume(t *testing.T) {
 	require.NoError(t, err)
 
 	assert.Empty(t, aws.StringValue(attachment.InstanceId))
-	assert.Equal(t, "detached", aws.StringValue(attachment.State))
-	open, err := svc.obligations.List(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, open, "an unattached volume has no publish to undo")
 }

spinifex/domains/ec2/volume/service_impl.go

@@ -20,7 +20,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -60,12 +59,6 @@ type VolumeServiceImpl struct {
 	snapshotKV jetstream.KeyValue
 	provider   ebsprovider.EBSProvider
 	metadata   *ebsmetadata.Store
-
-	// The unpublish obligations a cleared attachment records, and what names
-	// the node holding the publish. Unset, a detach that leaves one fails.
-	obligations   *ebsobligation.Store
-	instanceNodes InstanceNodeLocator
-	members       func() []string
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane. It
@@ -1003,23 +996,8 @@ func (s *VolumeServiceImpl) UpdateVolumeState(accountID, volumeID, state, attach
 	return nil
 }
 
-// ModifyVolume modifies an EBS volume (grow-only, refused while attached).
+// ModifyVolume modifies an EBS volume (grow-only, requires stopped instance).
 func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID string) (*ec2.ModifyVolumeOutput, error) {
-	return s.modifyVolume(ctx, input, accountID, "")
-}
-
-// ExpandAttachedToStopped grows a volume still attached to instanceID, which the
-// caller has stopped and holds exclusively. Not reachable from the AWS API.
-func (s *VolumeServiceImpl) ExpandAttachedToStopped(ctx context.Context, accountID, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error) {
-	if instanceID == "" {
-		return nil, errors.New(awserrors.ErrorMissingParameter)
-	}
-	return s.modifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: aws.String(volumeID), Size: aws.Int64(sizeGiB)}, accountID, instanceID)
-}
-
-// modifyVolume grows a volume. An empty stoppedOwner refuses any attached
-// volume; otherwise only a volume attached to that instance is accepted.
-func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID, stoppedOwner string) (*ec2.ModifyVolumeOutput, error) {
 	if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
 		return nil, errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
 	}
@@ -1039,9 +1017,7 @@ func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyV
 	if input.Size == nil || *input.Size <= originalSize {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
-	attached := meta.AttachedInstance != "" && meta.State == "in-use"
-	if meta.State == string(ebsprovider.VolumeStateCreating) || attached != (stoppedOwner != "") ||
-		(attached && meta.AttachedInstance != stoppedOwner) {
+	if (meta.AttachedInstance != "" && meta.State == "in-use") || meta.State == string(ebsprovider.VolumeStateCreating) {
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 	originalType := meta.VolumeType
@@ -1107,7 +1083,7 @@ func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyV
 // terminating instance's QEMU has already been asked to shut down — so this
 // is always a metadata-only clear, never a QMP call. Errors from either step
 // are returned, not swallowed.
-func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
+func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error {
 	// Clearing the attachment would overwrite a recorded deletion, so a volume
 	// already owed is left alone: the reclaimer owns it from here, and resetting
 	// it to available would let a second teardown run against the same data.
@@ -1115,9 +1091,6 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 	if err == nil && meta.State == volumeStateDeleting {
 		return nil
 	}
-	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
-		return err
-	}
 
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
@@ -1137,10 +1110,7 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
 // available rather than deleting it. Metadata-only, no QMP — same rationale
 // as DeleteVolumeOnTerminate: there is no live QEMU to hot-unplug on either
 // terminate path.
-func (s *VolumeServiceImpl) DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed ec2instance.OwedUnpublish) error {
-	if err := s.recordTerminateObligation(ctx, accountID, volumeID, owed); err != nil {
-		return err
-	}
+func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID, accountID string) error {
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		// No metadata doc means the volume is already gone, per GetVolume's own
 		// contract, so absence here is success, not a teardown failure.
@@ -1159,12 +1129,8 @@ func (s *VolumeServiceImpl) DetachVolumeOnTerminate(ctx context.Context, volumeI
 // The ordinary DetachVolume routes to ec2.cmd.{instanceID} for the QMP unplug,
 // so a volume attached to an instance whose host stopped answering can never be
 // detached and therefore never deleted. This exists for that deadlock and for
-// nothing else.
-//
-// That host still exports the volume, and the unpublish has to go to it, so
-// it cannot be made here. The request records an unpublish obligation naming
-// the publish before it clears the attachment, and returns; the node settles
-// it once it answers. A failed record clears nothing.
+// nothing else: it leaves a live guest holding a device the control plane no
+// longer believes in, which is only safe once that guest is being destroyed.
 func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
 	if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -1188,48 +1154,18 @@ func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.De
 	}
 
 	previous := meta.AttachedInstance
-	if previous != "" {
-		nodes, err := s.publishingNodes(previous)
-		if err != nil {
-			slog.ErrorContext(ctx, "ForceDetachVolume: cannot name the node holding the publish, clearing nothing",
-				"volumeId", volumeID, "instanceId", previous, "err", err)
-			return nil, errors.New(awserrors.ErrorInternalError)
-		}
-		if err := s.recordObligations(ctx, meta, nodes); err != nil {
-			slog.ErrorContext(ctx, "ForceDetachVolume: failed to record the unpublish obligation, clearing nothing",
-				"volumeId", volumeID, "instanceId", previous, "err", err)
-			return nil, errors.New(awserrors.ErrorInternalError)
-		}
-	}
-
 	if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
 		slog.ErrorContext(ctx, "ForceDetachVolume: failed to clear attachment", "volumeId", volumeID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-
-	if previous == "" {
-		return &ec2.VolumeAttachment{
-			VolumeId: aws.String(volumeID),
-			State:    aws.String("detached"),
-		}, nil
-	}
-	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared, unpublish left to the node holding it",
+	slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared in the control plane only",
 		"volumeId", volumeID, "previousInstance", previous, "accountId", accountID)
 
-	// AWS answers a detach with the attachment it is tearing down, in state
-	// detaching; the export here goes when the node settles the obligation.
-	attachment := &ec2.VolumeAttachment{
+	return &ec2.VolumeAttachment{
 		VolumeId:   aws.String(volumeID),
 		InstanceId: aws.String(previous),
-		State:      aws.String("detaching"),
-	}
-	if meta.DeviceName != "" {
-		attachment.Device = aws.String(meta.DeviceName)
-	}
-	if !meta.AttachedAt.IsZero() {
-		attachment.AttachTime = aws.Time(meta.AttachedAt)
-	}
-	return attachment, nil
+		State:      aws.String("detached"),
+	}, nil
 }
 
 // DeleteVolume records a durable obligation to destroy an EBS volume and

spinifex/domains/ec2/volume/service_impl_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit/ebsfake"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
@@ -1469,60 +1468,6 @@ func TestModifyVolume_AttachedInUse(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorIncorrectState, err.Error())
 }
 
-func TestExpandAttachedToStopped_GrowsTheNamedInstancesVolume(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
-	createVolumeInStoreWithMeta(t, svc, store, "vol-stopped", ebsmetadata.Volume{
-		VolumeID:         "vol-stopped",
-		CapacityGiB:      20,
-		State:            "in-use",
-		AttachedInstance: "i-owner",
-		DeviceName:       "/dev/sdf",
-	})
-
-	out, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-stopped", "i-owner", 30)
-	require.NoError(t, err)
-	assert.Equal(t, int64(30), aws.Int64Value(out.VolumeModification.TargetSize))
-
-	meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-stopped")
-	require.NoError(t, err)
-	assert.Equal(t, uint64(30), meta.CapacityGiB)
-	assert.Equal(t, "in-use", meta.State, "the grow must not change the attachment")
-	assert.Equal(t, "i-owner", meta.AttachedInstance)
-	assert.Equal(t, "/dev/sdf", meta.DeviceName)
-}
-
-func TestExpandAttachedToStopped_Refusals(t *testing.T) {
-	cases := []struct {
-		name     string
-		meta     ebsmetadata.Volume
-		instance string
-		want     string
-	}{
-		{"attached to another instance", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-other"}, "i-owner", awserrors.ErrorIncorrectState},
-		{"not attached", ebsmetadata.Volume{State: "available"}, "i-owner", awserrors.ErrorIncorrectState},
-		{"still creating", ebsmetadata.Volume{State: string(ebsprovider.VolumeStateCreating)}, "i-owner", awserrors.ErrorIncorrectState},
-		{"no instance named", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-owner"}, "", awserrors.ErrorMissingParameter},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			store := objectstore.NewMemoryObjectStore()
-			svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
-			tc.meta.VolumeID = "vol-x"
-			tc.meta.CapacityGiB = 20
-			createVolumeInStoreWithMeta(t, svc, store, "vol-x", tc.meta)
-
-			_, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-x", tc.instance, 30)
-			require.Error(t, err)
-			assert.Equal(t, tc.want, err.Error())
-
-			meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-x")
-			require.NoError(t, err)
-			assert.Equal(t, uint64(20), meta.CapacityGiB, "a refused grow must leave the size unchanged")
-		})
-	}
-}
-
 func TestModifyVolume_SuccessfulGrow(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
@@ -1844,7 +1789,7 @@ func TestDeleteVolumeOnTerminate_ClearsAttachmentThenDeletes(t *testing.T) {
 		AttachedInstance: "i-stopped",
 	})
 
-	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
 	require.NoError(t, err, "terminate implies detach: a stale attachment must not block the terminate delete")
 
 	deleting, err := svc.GetVolumeMetadata(testVolAccountID, volumeID)
@@ -1871,7 +1816,7 @@ func TestDeleteVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc.snapshotKV = kv
 
 	// No metadata doc is ever written for this volume ID.
-	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DeleteVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }
 
@@ -1884,7 +1829,7 @@ func TestDetachVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
 	svc.snapshotKV = kv
 
-	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID, ec2instance.OwedUnpublish{})
+	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
 	require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
 }
 
@@ -1968,7 +1913,7 @@ func TestDeleteVolume_SecondDeleteIsAcceptedWithoutASecondTeardown(t *testing.T)
 
 	// DeleteOnTermination arriving for a volume already owed must not reset it
 	// to available, which would let a second teardown through.
-	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{}))
+	require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID))
 	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
 	require.NoError(t, err)
 	assert.Equal(t, "deleting", meta.State)
@@ -2016,7 +1961,7 @@ func TestDeleteVolumeOnTerminate_SurfacesDeleteFailure(t *testing.T) {
 	_, err = kv.Put(t.Context(), volumeID, snapData)
 	require.NoError(t, err)
 
-	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID, ec2instance.OwedUnpublish{})
+	err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
 	require.Error(t, err, "a DeleteVolume failure must be surfaced, not swallowed")
 	assert.Contains(t, err.Error(), awserrors.ErrorVolumeInUse)
 

spinifex/domains/ec2/volume/unpublish_obligation.go

@@ -1,365 +0,0 @@
-package volume
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-	"time"
-
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/metric"
-)
-
-// InstanceNodeLocator reports the node that last ran an instance. found is
-// false when no record of the instance exists.
-type InstanceNodeLocator interface {
-	InstanceLastNode(instanceID string) (node string, found bool, err error)
-}
-
-// SetUnpublishObligations wires the obligation store a cleared attachment
-// records into, the locator that names the node holding the publish, and the
-// cluster members to name when the instance has no record.
-func (s *VolumeServiceImpl) SetUnpublishObligations(store *ebsobligation.Store, locator InstanceNodeLocator, members func() []string) {
-	s.obligations = store
-	s.instanceNodes = locator
-	s.members = members
-}
-
-// publishingNodes names the node holding instanceID's publish. With no record
-// of the instance there is no node to name, so every member is named and each
-// settles its own.
-func (s *VolumeServiceImpl) publishingNodes(instanceID string) ([]string, error) {
-	if s.instanceNodes == nil {
-		return nil, errors.New("no instance node locator configured")
-	}
-	node, found, err := s.instanceNodes.InstanceLastNode(instanceID)
-	if err != nil {
-		return nil, fmt.Errorf("read instance %s: %w", instanceID, err)
-	}
-	if found && node != "" {
-		return []string{node}, nil
-	}
-	var members []string
-	if s.members != nil {
-		members = s.members()
-	}
-	if len(members) == 0 {
-		return nil, fmt.Errorf("instance %s has no record and no cluster member is known", instanceID)
-	}
-	return members, nil
-}
-
-// recordObligations writes one obligation per node for the attachment meta
-// still carries. Every write must land before the caller clears it.
-func (s *VolumeServiceImpl) recordObligations(ctx context.Context, meta ebsmetadata.Volume, nodes []string) error {
-	if s.obligations == nil {
-		return errors.New("no unpublish obligation store configured")
-	}
-	now := time.Now()
-	for _, node := range nodes {
-		if err := s.obligations.Record(ctx, ebsobligation.UnpublishObligation{
-			VolumeID:   meta.VolumeID,
-			AccountID:  meta.TenantID,
-			Handle:     meta.ProviderHandle,
-			NodeID:     node,
-			InstanceID: meta.AttachedInstance,
-			AttachedAt: meta.AttachedAt,
-			RecordedAt: now,
-		}); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// recordTerminateObligation records the obligation for a terminate that clears
-// owed's attachment with its publish still in place. A volume no longer
-// attached to that instance has nothing of it left to clear.
-func (s *VolumeServiceImpl) recordTerminateObligation(ctx context.Context, accountID, volumeID string, owed ec2instance.OwedUnpublish) error {
-	if !owed.Owed() {
-		return nil
-	}
-	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
-	if objectstore.IsNoSuchKeyError(err) {
-		return nil
-	}
-	if err != nil {
-		return fmt.Errorf("read volume %s before recording its unpublish obligation: %w", volumeID, err)
-	}
-	if meta.AttachedInstance != owed.InstanceID {
-		return nil
-	}
-	if err := s.recordObligations(ctx, meta, []string{owed.NodeID}); err != nil {
-		return fmt.Errorf("record unpublish obligation for %s: %w", volumeID, err)
-	}
-	return nil
-}
-
-// ObligationUnpublisher is this node's mounter as the obligation reaper drives
-// it. Its lock is the one every publish and unpublish on this node takes.
-type ObligationUnpublisher interface {
-	// HoldVolume takes volumeID's lock without waiting. false means a publish
-	// or unpublish of it is in flight here, and the sweep comes back later.
-	HoldVolume(volumeID string) (VolumeHold, bool)
-}
-
-// VolumeHold is one volume's publish lock on this node.
-type VolumeHold interface {
-	// PublishedSince reports whether this process published the volume after
-	// t and has not unpublished it since.
-	PublishedSince(t time.Time) bool
-	// Unpublish tears down this node's export of the volume. A volume the
-	// provider holds no export of counts as done.
-	Unpublish(ctx context.Context, accountID, handle string, noSeal bool) error
-	Release()
-}
-
-// obligationAction is what settling an obligation does with the volume as it
-// is now.
-type obligationAction int
-
-const (
-	// obligationWait leaves the obligation open: the attachment it names has
-	// not been cleared yet, or a publish here is newer than it.
-	obligationWait obligationAction = iota
-	// obligationSuperseded closes it without a provider call: a newer
-	// attachment owns the export.
-	obligationSuperseded
-	// obligationSeal unpublishes with a seal: nobody else writes the volume,
-	// so the copy here is the last owner's and its writes are kept.
-	obligationSeal
-	// obligationNoSeal unpublishes without sealing a volume being deleted.
-	obligationNoSeal
-)
-
-func (a obligationAction) String() string {
-	switch a {
-	case obligationSuperseded:
-		return "superseded"
-	case obligationSeal:
-		return "seal"
-	case obligationNoSeal:
-		return "no-seal"
-	default:
-		return "wait"
-	}
-}
-
-// decideObligation is the settlement policy over the volume's current record;
-// meta is nil when the volume has none. It never unpublishes an attachment the
-// volume still carries.
-func decideObligation(o ebsobligation.UnpublishObligation, meta *ebsmetadata.Volume) obligationAction {
-	if meta == nil || meta.State == volumeStateDeleting {
-		return obligationNoSeal
-	}
-	if meta.AttachedInstance != "" {
-		if meta.AttachedInstance != o.InstanceID || meta.AttachedAt.After(o.AttachedAt) {
-			return obligationSuperseded
-		}
-		return obligationWait
-	}
-	return obligationSeal
-}
-
-// obligationSince is the time a publish on this node must follow to be newer
-// than the one an obligation undoes.
-func obligationSince(o ebsobligation.UnpublishObligation) time.Time {
-	if o.AttachedAt.IsZero() {
-		return o.RecordedAt
-	}
-	return o.AttachedAt
-}
-
-// unpublishObligationReaper settles the obligations naming its node through
-// that node's mounter, once the node answers.
-type unpublishObligationReaper struct {
-	svc         *VolumeServiceImpl
-	node        string
-	unpublisher ObligationUnpublisher
-}
-
-var _ vm.Reaper = (*unpublishObligationReaper)(nil)
-
-// NewUnpublishObligationReaper builds the reaper that settles node's
-// obligations through unpublisher, which must be the node's own mounter.
-func (s *VolumeServiceImpl) NewUnpublishObligationReaper(node string, unpublisher ObligationUnpublisher) vm.Reaper {
-	return &unpublishObligationReaper{svc: s, node: node, unpublisher: unpublisher}
-}
-
-func (r *unpublishObligationReaper) Class() string { return "ebs-unpublish-obligation" }
-
-// Scope is node-local: an obligation is settled only by the node it names,
-// because only that node can tear its own export down.
-func (r *unpublishObligationReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
-
-// Sweep settles what it can and reports how many obligations it closed. A
-// failure is recorded on the obligation and retried on the next sweep.
-func (r *unpublishObligationReaper) Sweep(ctx context.Context) (int, error) {
-	if r.svc.obligations == nil || r.unpublisher == nil {
-		return 0, nil
-	}
-	open, err := r.svc.obligations.ListNode(ctx, r.node)
-	if err != nil {
-		return 0, err
-	}
-	closed := 0
-	for _, o := range open {
-		if err := ctx.Err(); err != nil {
-			return closed, err
-		}
-		done, err := r.settle(ctx, o)
-		if err != nil {
-			slog.WarnContext(ctx, "ebs-unpublish-obligation: settlement failed, retrying next sweep",
-				"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attempts", o.Attempts+1, "err", err)
-			if recErr := r.svc.obligations.RecordFailure(ctx, o, err); recErr != nil {
-				slog.WarnContext(ctx, "ebs-unpublish-obligation: could not record the failure",
-					"volumeId", o.VolumeID, "node", o.NodeID, "err", recErr)
-			}
-			continue
-		}
-		if done {
-			closed++
-		}
-	}
-	return closed, nil
-}
-
-// settle acts on one obligation under the volume's lock, so no publish on this
-// node falls between reading the volume and unpublishing it.
-func (r *unpublishObligationReaper) settle(ctx context.Context, o ebsobligation.UnpublishObligation) (bool, error) {
-	hold, ok := r.unpublisher.HoldVolume(o.VolumeID)
-	if !ok {
-		return false, nil
-	}
-	defer hold.Release()
-
-	var current *ebsmetadata.Volume
-	meta, err := r.svc.metadata.GetVolume(ctx, o.AccountID, o.VolumeID)
-	switch {
-	case err == nil:
-		current = &meta
-	case objectstore.IsNoSuchKeyError(err):
-	default:
-		return false, fmt.Errorf("read volume %s: %w", o.VolumeID, err)
-	}
-
-	action := decideObligation(o, current)
-	// A newer publish here may own the export, so only a superseded obligation,
-	// which sends nothing, closes past it.
-	if action != obligationSuperseded && hold.PublishedSince(obligationSince(o)) {
-		return false, nil
-	}
-	switch action {
-	case obligationWait:
-		return false, nil
-	case obligationSuperseded:
-		slog.WarnContext(ctx, "ebs-unpublish-obligation: a newer attachment owns the volume, closing without unpublishing",
-			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "attachedInstance", current.AttachedInstance)
-	case obligationSeal, obligationNoSeal:
-		if err := hold.Unpublish(ctx, o.AccountID, o.Handle, action == obligationNoSeal); err != nil {
-			return false, err
-		}
-		slog.InfoContext(ctx, "ebs-unpublish-obligation: unpublished the export a cleared attachment left",
-			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID, "noSeal", action == obligationNoSeal)
-	}
-	if err := r.svc.obligations.Close(ctx, o); err != nil {
-		return false, fmt.Errorf("close obligation: %w", err)
-	}
-	recordObligationClosed(o.NodeID, action.String())
-	return true, nil
-}
-
-// NodeDeparted reports whether node has left the cluster, rather than being
-// down, stale or partitioned, which must not count.
-type NodeDeparted func(ctx context.Context, node string) (bool, error)
-
-// unpublishObligationCloser closes, without a provider call, the obligations of
-// nodes removed from the cluster: such a node holds nothing a member can reach.
-type unpublishObligationCloser struct {
-	svc      *VolumeServiceImpl
-	departed NodeDeparted
-}
-
-var _ vm.Reaper = (*unpublishObligationCloser)(nil)
-
-// NewUnpublishObligationCloser builds the cluster-wide pass that closes the
-// obligations of nodes departed reports as gone.
-func (s *VolumeServiceImpl) NewUnpublishObligationCloser(departed NodeDeparted) vm.Reaper {
-	return &unpublishObligationCloser{svc: s, departed: departed}
-}
-
-func (c *unpublishObligationCloser) Class() string { return "ebs-unpublish-obligation-close" }
-
-// Scope is cluster-wide: membership is the cluster's, and one pass is enough.
-func (c *unpublishObligationCloser) Scope() vm.ReaperScope { return vm.ScopeClusterWide }
-
-func (c *unpublishObligationCloser) Sweep(ctx context.Context) (int, error) {
-	if c.svc.obligations == nil || c.departed == nil {
-		return 0, nil
-	}
-	open, err := c.svc.obligations.List(ctx)
-	if err != nil {
-		return 0, err
-	}
-	gone := map[string]bool{}
-	closed := 0
-	for _, o := range open {
-		departed, seen := gone[o.NodeID]
-		if !seen {
-			departed, err = c.departed(ctx, o.NodeID)
-			if err != nil {
-				slog.WarnContext(ctx, "ebs-unpublish-obligation: cannot tell whether a node left the cluster, keeping its obligations",
-					"node", o.NodeID, "err", err)
-				departed = false
-			}
-			gone[o.NodeID] = departed
-		}
-		if !departed {
-			continue
-		}
-		if err := c.svc.obligations.Close(ctx, o); err != nil {
-			slog.WarnContext(ctx, "ebs-unpublish-obligation: failed to close a departed node's obligation",
-				"volumeId", o.VolumeID, "node", o.NodeID, "err", err)
-			continue
-		}
-		slog.WarnContext(ctx, "ebs-unpublish-obligation: closed for a node removed from the cluster",
-			"volumeId", o.VolumeID, "node", o.NodeID, "instanceId", o.InstanceID)
-		recordObligationClosed(o.NodeID, "departed")
-		closed++
-	}
-	return closed, nil
-}
-
-const obligationMeterName = "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume/obligation"
-
-var (
-	obligationClosedOnce    sync.Once
-	obligationClosedCounter metric.Int64Counter
-)
-
-// recordObligationClosed counts one closed obligation by the node it named
-// and how it closed.
-func recordObligationClosed(node, outcome string) {
-	obligationClosedOnce.Do(func() {
-		var err error
-		obligationClosedCounter, err = otel.Meter(obligationMeterName).Int64Counter("mulga.ebs.unpublish_obligation.closed",
-			metric.WithDescription("Unpublish obligations closed, by the node they named and how they closed."),
-			metric.WithUnit("{obligation}"))
-		if err != nil {
-			otel.Handle(err)
-		}
-	})
-	if obligationClosedCounter == nil {
-		return
-	}
-	obligationClosedCounter.Add(context.Background(), 1, metric.WithAttributes(
-		attribute.String("node", node), attribute.String("outcome", outcome)))
-}

spinifex/domains/ec2/volume/unpublish_obligation_test.go

@@ -1,596 +0,0 @@
-package volume
-
-//test:in-package — the obligation reaper and its policy are unexported, and
-// seeding a volume's metadata document is how an attachment is set up.
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeNodeLocator struct {
-	nodes map[string]string
-	err   error
-}
-
-var _ InstanceNodeLocator = (*fakeNodeLocator)(nil)
-
-func (f *fakeNodeLocator) InstanceLastNode(instanceID string) (string, bool, error) {
-	if f.err != nil {
-		return "", false, f.err
-	}
-	node, found := f.nodes[instanceID]
-	return node, found, nil
-}
-
-// wireObligations gives svc an obligation bucket on a JetStream of its own and
-// returns the connection under it, so a test can cut it.
-func wireObligations(t *testing.T, svc *VolumeServiceImpl, locator InstanceNodeLocator, members ...string) *nats.Conn {
-	t.Helper()
-	_, nc, js := testutil.StartTestJetStream(t)
-	store := ebsobligation.NewStore(js)
-	require.NoError(t, store.Open(t.Context()))
-	svc.SetUnpublishObligations(store, locator, func() []string { return members })
-	return nc
-}
-
-type unpublishCall struct {
-	volumeID, accountID, handle string
-	noSeal                      bool
-}
-
-// fakeUnpublisher stands in for a node's mounter: busy volumes refuse the
-// hold, publishedAt is the last publish this process made of each volume.
-type fakeUnpublisher struct {
-	mu          sync.Mutex
-	busy        map[string]bool
-	publishedAt map[string]time.Time
-	err         error
-	calls       []unpublishCall
-	held        int
-}
-
-var _ ObligationUnpublisher = (*fakeUnpublisher)(nil)
-
-func (f *fakeUnpublisher) HoldVolume(volumeID string) (VolumeHold, bool) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if f.busy[volumeID] {
-		return nil, false
-	}
-	f.held++
-	return &fakeHold{f: f, volumeID: volumeID}, true
-}
-
-func (f *fakeUnpublisher) unpublishes() []unpublishCall {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return append([]unpublishCall(nil), f.calls...)
-}
-
-type fakeHold struct {
-	f        *fakeUnpublisher
-	volumeID string
-}
-
-var _ VolumeHold = (*fakeHold)(nil)
-
-func (h *fakeHold) PublishedSince(t time.Time) bool {
-	h.f.mu.Lock()
-	defer h.f.mu.Unlock()
-	at, ok := h.f.publishedAt[h.volumeID]
-	return ok && at.After(t)
-}
-
-func (h *fakeHold) Unpublish(_ context.Context, accountID, handle string, noSeal bool) error {
-	h.f.mu.Lock()
-	defer h.f.mu.Unlock()
-	h.f.calls = append(h.f.calls, unpublishCall{volumeID: h.volumeID, accountID: accountID, handle: handle, noSeal: noSeal})
-	return h.f.err
-}
-
-func (h *fakeHold) Release() {
-	h.f.mu.Lock()
-	defer h.f.mu.Unlock()
-	h.f.held--
-}
-
-// putAttachment writes volumeID's document with an explicit attachment, so a
-// test controls AttachedAt rather than racing the clock.
-func putAttachment(t *testing.T, svc *VolumeServiceImpl, volumeID, state, instanceID string, attachedAt time.Time) ebsmetadata.Volume {
-	t.Helper()
-	meta := ebsmetadata.Volume{
-		VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 1,
-		State: state, AvailabilityZone: "ap-southeast-2a", VolumeType: "gp3",
-		AttachedInstance: instanceID, AttachedAt: attachedAt, ProviderHandle: "handle-" + volumeID,
-	}
-	if instanceID != "" {
-		meta.DeviceName = "/dev/sdf"
-	}
-	require.NoError(t, svc.metadata.PutVolume(context.Background(), meta))
-	return meta
-}
-
-func readVolume(t *testing.T, svc *VolumeServiceImpl, volumeID string) ebsmetadata.Volume {
-	t.Helper()
-	meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
-	require.NoError(t, err)
-	return meta
-}
-
-func openObligations(t *testing.T, svc *VolumeServiceImpl) []ebsobligation.UnpublishObligation {
-	t.Helper()
-	open, err := svc.obligations.List(context.Background())
-	require.NoError(t, err)
-	return open
-}
-
-func forceDetach(t *testing.T, svc *VolumeServiceImpl, volumeID string) (*ec2.VolumeAttachment, error) {
-	t.Helper()
-	return svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{
-		VolumeId: aws.String(volumeID), Force: aws.Bool(true),
-	}, testVolAccountID)
-}
-
-// --- ForceDetachVolume records the obligation ---
-
-func TestForceDetach_RecordsOneObligationNamingThePublish(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a", "node-b")
-	attachedAt := time.Unix(1_700_000_000, 0).UTC()
-	putAttachment(t, svc, "vol-fd-once", "in-use", "i-old", attachedAt)
-
-	_, err := forceDetach(t, svc, "vol-fd-once")
-	require.NoError(t, err)
-
-	open := openObligations(t, svc)
-	require.Len(t, open, 1, "only the node that last ran the instance is named")
-	o := open[0]
-	assert.Equal(t, "vol-fd-once", o.VolumeID)
-	assert.Equal(t, testVolAccountID, o.AccountID)
-	assert.Equal(t, "handle-vol-fd-once", o.Handle)
-	assert.Equal(t, "node-a", o.NodeID)
-	assert.Equal(t, "i-old", o.InstanceID)
-	assert.True(t, attachedAt.Equal(o.AttachedAt), "the obligation names the attachment it undoes")
-	assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
-}
-
-// A caller retrying a force detach whose clear failed writes the same key, so
-// the retry leaves one obligation, not two.
-func TestForceDetach_RetryLeavesOneObligation(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
-	meta := putAttachment(t, svc, "vol-fd-retry", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
-
-	require.NoError(t, svc.recordObligations(context.Background(), meta, []string{"node-a"}))
-	_, err := forceDetach(t, svc, "vol-fd-retry")
-	require.NoError(t, err)
-
-	assert.Len(t, openObligations(t, svc), 1)
-}
-
-// With no record of the instance there is no node to name, so every member is
-// named and each one settles its own.
-func TestForceDetach_UnknownInstanceNamesEveryMember(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b", "node-c")
-	putAttachment(t, svc, "vol-fd-every", "in-use", "i-gone", time.Unix(1_700_000_000, 0).UTC())
-
-	_, err := forceDetach(t, svc, "vol-fd-every")
-	require.NoError(t, err)
-
-	nodes := []string{}
-	for _, o := range openObligations(t, svc) {
-		nodes = append(nodes, o.NodeID)
-	}
-	assert.ElementsMatch(t, []string{"node-a", "node-b", "node-c"}, nodes)
-}
-
-// The attachment is the only record of where the export is. If the obligation
-// cannot be written, clearing it would forget an export, so nothing is cleared.
-func TestForceDetach_ObligationWriteFailureClearsNothing(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	nc := wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a"}}, "node-a")
-	putAttachment(t, svc, "vol-fd-nowrite", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	nc.Close()
-
-	_, err := forceDetach(t, svc, "vol-fd-nowrite")
-
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInternalError, err.Error())
-	meta := readVolume(t, svc, "vol-fd-nowrite")
-	assert.Equal(t, "in-use", meta.State)
-	assert.Equal(t, "i-old", meta.AttachedInstance)
-}
-
-func TestForceDetach_UnnameableNodeClearsNothing(t *testing.T) {
-	for name, locator := range map[string]InstanceNodeLocator{
-		"locator fails":            &fakeNodeLocator{err: errors.New("instance bucket unreachable")},
-		"no record and no members": &fakeNodeLocator{},
-		"no locator configured":    nil,
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, locator)
-			putAttachment(t, svc, "vol-fd-noname", "in-use", "i-old", time.Unix(1_700_000_000, 0).UTC())
-
-			_, err := forceDetach(t, svc, "vol-fd-noname")
-
-			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorInternalError, err.Error())
-			assert.Equal(t, "i-old", readVolume(t, svc, "vol-fd-noname").AttachedInstance)
-			assert.Empty(t, openObligations(t, svc))
-		})
-	}
-}
-
-// providerCallCounter fails the test's expectations if the request path makes
-// any provider call: an unpublish waits on a node that may never answer.
-type providerCallCounter struct {
-	ebsprovider.EBSProvider
-
-	mu        sync.Mutex
-	unpublish int
-	publish   int
-}
-
-func (p *providerCallCounter) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
-	p.mu.Lock()
-	p.unpublish++
-	p.mu.Unlock()
-	<-ctx.Done()
-	return nil, ctx.Err()
-}
-
-func (p *providerCallCounter) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
-	p.mu.Lock()
-	p.publish++
-	p.mu.Unlock()
-	<-ctx.Done()
-	return nil, ctx.Err()
-}
-
-// TestS8_ForceDetachRecordsTheObligationAndReturns pins that a force detach
-// never waits on the node holding the export: it decides its errors, records
-// the obligation and returns, and the unpublish is left to that node's reaper.
-func TestS8_ForceDetachRecordsTheObligationAndReturns(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	provider := &providerCallCounter{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
-	svc.SetEBSProvider(provider)
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-wedged": "node-unreachable"}}, "node-unreachable")
-	putAttachment(t, svc, "vol-s8-forcedetach", "in-use", "i-wedged", time.Unix(1_700_000_000, 0).UTC())
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	attachment, err := svc.ForceDetachVolume(ctx, &ec2.DetachVolumeInput{
-		VolumeId: aws.String("vol-s8-forcedetach"), Force: aws.Bool(true),
-	}, testVolAccountID)
-	require.NoError(t, err)
-	require.NoError(t, ctx.Err(), "the request returned on its own, not on the deadline")
-
-	assert.Equal(t, "detaching", aws.StringValue(attachment.State))
-	provider.mu.Lock()
-	assert.Zero(t, provider.unpublish, "the request path makes no unpublish call")
-	assert.Zero(t, provider.publish)
-	provider.mu.Unlock()
-
-	open := openObligations(t, svc)
-	require.Len(t, open, 1, "the unpublish the request did not wait for is recorded")
-	assert.Equal(t, "node-unreachable", open[0].NodeID)
-	assert.Empty(t, readVolume(t, svc, "vol-s8-forcedetach").AttachedInstance)
-}
-
-// --- Terminate paths ---
-
-func TestTerminate_OwedUnpublishRecordsOneObligationBeforeTheClear(t *testing.T) {
-	for name, terminate := range map[string]func(*VolumeServiceImpl, string, ec2instance.OwedUnpublish) error{
-		"detach": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
-			return svc.DetachVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
-		},
-		"delete": func(svc *VolumeServiceImpl, id string, owed ec2instance.OwedUnpublish) error {
-			return svc.DeleteVolumeOnTerminate(context.Background(), id, testVolAccountID, owed)
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			svc.snapshotKV = setupTestVolumeKV(t)
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			seedTestProviderVolume(t, svc, "vol-term-owed", 1)
-			putAttachment(t, svc, "vol-term-owed", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
-
-			require.NoError(t, terminate(svc, "vol-term-owed", ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}))
-
-			open := openObligations(t, svc)
-			require.Len(t, open, 1)
-			assert.Equal(t, "node-a", open[0].NodeID)
-			assert.Equal(t, "i-term", open[0].InstanceID)
-		})
-	}
-}
-
-// A terminate whose unmount completed owes nothing, and an attachment that is
-// already some other instance's is not this terminate's to undo.
-func TestTerminate_NothingOwedRecordsNoObligation(t *testing.T) {
-	for name, tc := range map[string]struct {
-		attachedTo string
-		owed       ec2instance.OwedUnpublish
-	}{
-		"unmount completed":   {attachedTo: "i-term", owed: ec2instance.OwedUnpublish{}},
-		"attached to another": {attachedTo: "i-other", owed: ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"}},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			putAttachment(t, svc, "vol-term-none", "in-use", tc.attachedTo, time.Unix(1_700_000_000, 0).UTC())
-
-			require.NoError(t, svc.DetachVolumeOnTerminate(context.Background(), "vol-term-none", testVolAccountID, tc.owed))
-
-			assert.Empty(t, openObligations(t, svc))
-		})
-	}
-}
-
-func TestTerminate_ObligationWriteFailureClearsNothing(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	nc := wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-term-nowrite", "in-use", "i-term", time.Unix(1_700_000_000, 0).UTC())
-	nc.Close()
-
-	err := svc.DetachVolumeOnTerminate(context.Background(), "vol-term-nowrite", testVolAccountID,
-		ec2instance.OwedUnpublish{NodeID: "node-a", InstanceID: "i-term"})
-
-	require.Error(t, err)
-	assert.Equal(t, "i-term", readVolume(t, svc, "vol-term-nowrite").AttachedInstance)
-}
-
-// --- The reaper ---
-
-func recordObligation(t *testing.T, svc *VolumeServiceImpl, node, volumeID, instanceID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
-	t.Helper()
-	o := ebsobligation.UnpublishObligation{
-		VolumeID: volumeID, AccountID: testVolAccountID, Handle: "handle-" + volumeID,
-		NodeID: node, InstanceID: instanceID, AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
-	}
-	require.NoError(t, svc.obligations.Record(context.Background(), o))
-	return o
-}
-
-func sweepObligations(t *testing.T, svc *VolumeServiceImpl, node string, unpublisher ObligationUnpublisher) int {
-	t.Helper()
-	closed, err := svc.NewUnpublishObligationReaper(node, unpublisher).Sweep(context.Background())
-	require.NoError(t, err)
-	return closed
-}
-
-// TestS7_ObligationReaperNeverUnpublishesACurrentAttachment pins that the
-// reaper, which runs with no customer asking, never tears down an export a
-// live attachment uses and never removes or rewrites the volume.
-func TestS7_ObligationReaperNeverUnpublishesACurrentAttachment(t *testing.T) {
-	cleared := time.Unix(1_700_000_000, 0).UTC()
-	for name, tc := range map[string]struct {
-		state       string
-		attachedTo  string
-		attachedAt  time.Time
-		publishedAt time.Time // a publish this process made, zero if none
-		stillOpen   bool
-	}{
-		// The volume moved to another instance, possibly on this very node.
-		"attached to another instance": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour)},
-		// The same instance was attached again after the clear.
-		"same instance attached again": {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour)},
-		// The clear has not landed: the obligation was written first.
-		"attachment not yet cleared": {state: "in-use", attachedTo: "i-old", attachedAt: cleared, stillOpen: true},
-		// A hot attach published here and has not yet written in-use.
-		"newer publish here, record not yet in-use": {state: "available", publishedAt: cleared.Add(time.Hour), stillOpen: true},
-		// Force detach, then attach again on this node: the new publish is here.
-		"attached again here after a newer publish": {state: "in-use", attachedTo: "i-new", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
-		"same instance attached again here":         {state: "in-use", attachedTo: "i-old", attachedAt: cleared.Add(time.Hour), publishedAt: cleared.Add(time.Hour)},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			before := putAttachment(t, svc, "vol-s7-current", tc.state, tc.attachedTo, tc.attachedAt)
-			recordObligation(t, svc, "node-a", "vol-s7-current", "i-old", cleared)
-			unpublisher := &fakeUnpublisher{publishedAt: map[string]time.Time{}}
-			if !tc.publishedAt.IsZero() {
-				unpublisher.publishedAt["vol-s7-current"] = tc.publishedAt
-			}
-
-			sweepObligations(t, svc, "node-a", unpublisher)
-
-			assert.Empty(t, unpublisher.unpublishes(), "a current attachment's export must never be unpublished")
-			assert.Zero(t, unpublisher.held, "every hold is released")
-			after := readVolume(t, svc, "vol-s7-current")
-			assert.Equal(t, before.State, after.State, "the reaper never rewrites the volume")
-			assert.Equal(t, before.AttachedInstance, after.AttachedInstance)
-			if tc.stillOpen {
-				assert.Len(t, openObligations(t, svc), 1, "the obligation waits for its attachment to be gone")
-			} else {
-				assert.Empty(t, openObligations(t, svc), "a superseded obligation closes without a provider call")
-			}
-		})
-	}
-}
-
-// TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes pins that
-// the reaper keeps a surviving volume's writes and only drops them unsealed
-// for a volume that is being deleted or is gone.
-func TestS7_ObligationReaperSealsLiveDataAndDropsOnlyDeletingVolumes(t *testing.T) {
-	cleared := time.Unix(1_700_000_000, 0).UTC()
-	for name, tc := range map[string]struct {
-		state  string // empty for no document
-		noSeal bool
-	}{
-		"available keeps its writes": {state: "available", noSeal: false},
-		"deleting is dropped":        {state: volumeStateDeleting, noSeal: true},
-		"absent is dropped":          {state: "", noSeal: true},
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc := newTestVolumeService("ap-southeast-2a")
-			wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-			if tc.state != "" {
-				putAttachment(t, svc, "vol-s7-settle", tc.state, "", time.Time{})
-			}
-			recordObligation(t, svc, "node-a", "vol-s7-settle", "i-old", cleared)
-			unpublisher := &fakeUnpublisher{}
-
-			assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
-
-			assert.Equal(t, []unpublishCall{{volumeID: "vol-s7-settle", accountID: testVolAccountID, handle: "handle-vol-s7-settle", noSeal: tc.noSeal}},
-				unpublisher.unpublishes())
-			assert.Empty(t, openObligations(t, svc))
-			if tc.state != "" {
-				assert.Equal(t, tc.state, readVolume(t, svc, "vol-s7-settle").State, "the reaper never rewrites the volume")
-			}
-		})
-	}
-}
-
-// TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone pins that a read
-// failing for a reason other than absence leaves the obligation open: taking
-// it for a deleted volume would drop live writes unsealed.
-func TestS7_ObligationReaperNeverTreatsAnUnreadableVolumeAsGone(t *testing.T) {
-	memory := objectstore.NewMemoryObjectStore()
-	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-s7-unread", "available", "", time.Time{})
-	recordObligation(t, svc, "node-a", "vol-s7-unread", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	failing := &getFailingStore{MemoryObjectStore: memory, failKey: metadataKey(t, "vol-s7-unread")}
-	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
-	unpublisher := &fakeUnpublisher{}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes())
-	open := openObligations(t, svc)
-	require.Len(t, open, 1)
-	assert.Equal(t, 1, open[0].Attempts, "the failure is counted where an operator can see it")
-	assert.Contains(t, open[0].Error, "reconstruction failed")
-}
-
-// End to end through the service: a force detach followed by a new attachment
-// of the same volume on the same node must leave that new export alone.
-func TestS7_ForceDetachThenReattachOnTheSameNodeKeepsTheNewExport(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{nodes: map[string]string{"i-old": "node-a", "i-new": "node-a"}}, "node-a")
-	putAttachment(t, svc, "vol-s7-reattach", "in-use", "i-old", time.Now().Add(-time.Hour).UTC())
-
-	_, err := forceDetach(t, svc, "vol-s7-reattach")
-	require.NoError(t, err)
-	require.NoError(t, svc.UpdateVolumeState(testVolAccountID, "vol-s7-reattach", "in-use", "i-new", "/dev/sdg"))
-	unpublisher := &fakeUnpublisher{}
-
-	assert.Equal(t, 1, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes(), "the new attachment's export must survive the old obligation")
-	assert.Equal(t, "i-new", readVolume(t, svc, "vol-s7-reattach").AttachedInstance)
-	assert.Empty(t, openObligations(t, svc))
-}
-
-func TestObligationReaper_SettlesOnlyItsOwnNode(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a", "node-b")
-	putAttachment(t, svc, "vol-own", "available", "", time.Time{})
-	recordObligation(t, svc, "node-b", "vol-own", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	unpublisher := &fakeUnpublisher{}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes(), "only the node holding the export can tear it down")
-	assert.Len(t, openObligations(t, svc), 1)
-}
-
-// A publish or unpublish of the volume in flight on this node holds its lock;
-// the sweep skips it rather than waiting behind a possibly wedged call.
-func TestObligationReaper_SkipsABusyVolume(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-busy", "available", "", time.Time{})
-	recordObligation(t, svc, "node-a", "vol-busy", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	unpublisher := &fakeUnpublisher{busy: map[string]bool{"vol-busy": true}}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	assert.Empty(t, unpublisher.unpublishes())
-	open := openObligations(t, svc)
-	require.Len(t, open, 1)
-	assert.Zero(t, open[0].Attempts, "a busy volume is not a failure")
-}
-
-func TestObligationReaper_FailedUnpublishStaysOpenAndCounts(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	putAttachment(t, svc, "vol-fail", "available", "", time.Time{})
-	recordObligation(t, svc, "node-a", "vol-fail", "i-old", time.Unix(1_700_000_000, 0).UTC())
-	unpublisher := &fakeUnpublisher{err: errors.New("seal refused")}
-
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-	assert.Zero(t, sweepObligations(t, svc, "node-a", unpublisher))
-
-	open := openObligations(t, svc)
-	require.Len(t, open, 1)
-	assert.Equal(t, 2, open[0].Attempts)
-	assert.Equal(t, "seal refused", open[0].Error)
-	assert.Zero(t, unpublisher.held, "a failed unpublish still releases the hold")
-}
-
-// An obligation with no attachment time falls back to when it was recorded, so
-// a publish here after the clear still counts as newer.
-func TestObligationSince_FallsBackToRecordedAt(t *testing.T) {
-	recorded := time.Unix(1_700_000_000, 0)
-	assert.Equal(t, recorded, obligationSince(ebsobligation.UnpublishObligation{RecordedAt: recorded}))
-	attached := recorded.Add(-time.Hour)
-	assert.Equal(t, attached, obligationSince(ebsobligation.UnpublishObligation{AttachedAt: attached, RecordedAt: recorded}))
-}
-
-// --- The cluster-wide close ---
-
-func TestObligationCloser_ClosesOnlyDepartedNodes(t *testing.T) {
-	svc := newTestVolumeService("ap-southeast-2a")
-	wireObligations(t, svc, &fakeNodeLocator{}, "node-a")
-	at := time.Unix(1_700_000_000, 0).UTC()
-	recordObligation(t, svc, "node-gone", "vol-c1", "i-1", at)
-	recordObligation(t, svc, "node-gone", "vol-c2", "i-2", at)
-	recordObligation(t, svc, "node-down", "vol-c3", "i-3", at)
-	recordObligation(t, svc, "node-unknown", "vol-c4", "i-4", at)
-	asked := map[string]int{}
-	departed := func(_ context.Context, node string) (bool, error) {
-		asked[node]++
-		switch node {
-		case "node-gone":
-			return true, nil
-		case "node-unknown":
-			return false, errors.New("heartbeat bucket unreachable")
-		default:
-			return false, nil
-		}
-	}
-
-	closed, err := svc.NewUnpublishObligationCloser(departed).Sweep(context.Background())
-	require.NoError(t, err)
-
-	assert.Equal(t, 2, closed)
-	remaining := []string{}
-	for _, o := range openObligations(t, svc) {
-		remaining = append(remaining, o.NodeID)
-	}
-	assert.ElementsMatch(t, []string{"node-down", "node-unknown"}, remaining,
-		"a node that is down, or whose membership cannot be read, keeps its obligations")
-	assert.Equal(t, 1, asked["node-gone"], "membership is asked once per node per sweep")
-}

spinifex/domains/ecs/awsapi/actions.go

@@ -514,9 +514,6 @@ func CreateCapacityProvider(ctx context.Context, nc *nats.Conn, accountID string
 	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, err
 	}
-	if err := checkAutoScalingGroupProvider(input.AutoScalingGroupProvider); err != nil {
-		return nil, err
-	}
 	return ecsdomain.NewNATSECSService(nc).CreateCapacityProvider(ctx, input, accountID)
 }
 

spinifex/domains/ecs/awsapi/validate_capacityprovider.go

@@ -1,59 +0,0 @@
-package awsapi
-
-import (
-	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// Messages AWS answers, as ClientException, for each autoScalingGroupProvider
-// break, before it looks up the Auto Scaling group.
-const (
-	msgManagedDraining              = "The specified managed draining setting is invalid. Specify ENABLED or DISABLED and try again."
-	msgManagedTerminationProtection = "The specified managed termination protection setting is invalid. Specify ENABLED or DISABLED and try again."
-	msgManagedScalingStatus         = "The specified managed scaling status setting is invalid. Specify ENABLED or DISABLED and try again."
-	msgTargetCapacity               = "The managed scaling target capacity value is invalid. Specify an integer greater than 0 and less than or equal to 100 and try again."
-	msgMinimumScalingStepSize       = "The specified minimum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
-	msgMaximumScalingStepSize       = "The specified maximum scaling step size is invalid. Specify an integer between 1 and 10000 and try again."
-	msgInstanceWarmupPeriod         = "The managed scaling instance warmup period value is invalid. Specify an integer greater than or equal to 0 and less than or equal to 9999 and try again."
-)
-
-func validEnabledDisabled(v *string) bool {
-	return v == nil || *v == "ENABLED" || *v == "DISABLED"
-}
-
-func validRange(v *int64, lo, hi int64) bool {
-	return v == nil || (*v >= lo && *v <= hi)
-}
-
-// checkAutoScalingGroupProvider refuses an enum or range break inside
-// autoScalingGroupProvider. AWS refuses an instanceWarmupPeriod of 10000 although
-// the model allows it, so the upper bound here is 9999.
-func checkAutoScalingGroupProvider(p *ecs.AutoScalingGroupProvider) error {
-	if p == nil {
-		return nil
-	}
-	fail := func(msg string) error { return awserrors.Errorf(awserrors.ErrorECSClientException, "%s", msg) }
-	if !validEnabledDisabled(p.ManagedDraining) {
-		return fail(msgManagedDraining)
-	}
-	if !validEnabledDisabled(p.ManagedTerminationProtection) {
-		return fail(msgManagedTerminationProtection)
-	}
-	ms := p.ManagedScaling
-	if ms == nil {
-		return nil
-	}
-	switch {
-	case !validEnabledDisabled(ms.Status):
-		return fail(msgManagedScalingStatus)
-	case !validRange(ms.TargetCapacity, 1, 100):
-		return fail(msgTargetCapacity)
-	case !validRange(ms.MinimumScalingStepSize, 1, 10000):
-		return fail(msgMinimumScalingStepSize)
-	case !validRange(ms.MaximumScalingStepSize, 1, 10000):
-		return fail(msgMaximumScalingStepSize)
-	case !validRange(ms.InstanceWarmupPeriod, 0, 9999):
-		return fail(msgInstanceWarmupPeriod)
-	}
-	return nil
-}

spinifex/domains/ecs/awsapi/validate_capacityprovider_test.go

@@ -1,51 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// Each break answers AWS's ClientException and message before dispatch. A valid
-// request reaches dispatch, which fails here only because nc is nil.
-func TestCreateCapacityProviderAutoScalingGroupProvider(t *testing.T) {
-	body := func(asg string) []byte {
-		return []byte(`{"name":"cp","autoScalingGroupProvider":{"autoScalingGroupArn":"arn:aws:autoscaling:x"` + asg + `}}`)
-	}
-	refused := []struct {
-		name, asg, msg string
-	}{
-		{"managedDraining", `,"managedDraining":"BOGUS"`, msgManagedDraining},
-		{"managedTerminationProtection", `,"managedTerminationProtection":"enabled"`, msgManagedTerminationProtection},
-		{"status", `,"managedScaling":{"status":"BOGUS"}`, msgManagedScalingStatus},
-		{"targetCapacity/0", `,"managedScaling":{"targetCapacity":0}`, msgTargetCapacity},
-		{"targetCapacity/101", `,"managedScaling":{"targetCapacity":101}`, msgTargetCapacity},
-		{"minimumScalingStepSize/0", `,"managedScaling":{"minimumScalingStepSize":0}`, msgMinimumScalingStepSize},
-		{"maximumScalingStepSize/10001", `,"managedScaling":{"maximumScalingStepSize":10001}`, msgMaximumScalingStepSize},
-		{"instanceWarmupPeriod/-1", `,"managedScaling":{"instanceWarmupPeriod":-1}`, msgInstanceWarmupPeriod},
-		{"instanceWarmupPeriod/10000", `,"managedScaling":{"instanceWarmupPeriod":10000}`, msgInstanceWarmupPeriod},
-	}
-	for _, tc := range refused {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(tc.asg), nil)
-			code, msg, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok, "err = %v", err)
-			assert.Equal(t, awserrors.ErrorECSClientException, code)
-			assert.Equal(t, tc.msg, msg)
-		})
-	}
-
-	valid := []string{
-		``,
-		`,"managedDraining":"ENABLED","managedTerminationProtection":"DISABLED","managedScaling":{"status":"ENABLED","targetCapacity":1,"minimumScalingStepSize":1,"maximumScalingStepSize":10000,"instanceWarmupPeriod":0}`,
-		`,"managedDraining":"DISABLED","managedTerminationProtection":"ENABLED","managedScaling":{"status":"DISABLED","targetCapacity":100,"minimumScalingStepSize":10000,"maximumScalingStepSize":1,"instanceWarmupPeriod":9999}`,
-	}
-	for _, asg := range valid {
-		_, err := CreateCapacityProvider(context.Background(), nil, "123456789012", body(asg), nil)
-		_, _, refusedErr := awserrors.ResolveErrorDetail(err)
-		assert.False(t, refusedErr, "asg %q: err = %v", asg, err)
-	}
-}

spinifex/domains/lambda/awsapi/create.go

@@ -110,7 +110,7 @@ func (d Deps) createFunction(ctx context.Context, c *call) (reply, error) {
 
 // rollback makes one compare-and-swap attempt to free the name. It cannot undo
 // a create that completed, because the revision it holds has moved on. The code
-// object, if one landed, stays in place under its owed upload record.
+// object, if one landed, is left in place and recorded as a cleanup obligation.
 func (d Deps) rollback(ctx context.Context, c *call, name string, createRev uint64, ref codestore.Ref, why string, cause error) {
 	slog.ErrorContext(ctx, "Lambda: create did not complete", "reason", why, "function_name", name,
 		"function_id", ref.FunctionID, "err", cause)

spinifex/domains/lambda/awsapi/get.go

@@ -40,12 +40,6 @@ func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Recor
 		}
 		q = query
 	}
-	return d.lookupTarget(ctx, c, name, q)
-}
-
-// lookupTarget reads the record for an already resolved name and qualifier;
-// only $LATEST exists.
-func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (function.Record, uint64, error) {
 	if q != "" && q != "$LATEST" {
 		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}

spinifex/domains/lambda/awsapi/handler_test.go

@@ -45,7 +45,6 @@ type localClient struct {
 	svc *function.MetaService
 
 	mu             sync.Mutex
-	forceActive    bool
 	failCodeStored bool
 	passOwnerClock bool
 	rollbacks      int
@@ -69,11 +68,6 @@ func (c *localClient) Get(ctx context.Context, account, region, name string) (fu
 	if !r.Found {
 		return function.Record{}, 0, function.ErrNotFound
 	}
-	c.mu.Lock()
-	if c.forceActive {
-		r.Record.State = function.StateActive
-	}
-	c.mu.Unlock()
 	return r.Record, r.Revision, nil
 }
 
@@ -110,20 +104,11 @@ type memStore struct {
 	objects map[codestore.Ref][]byte
 	putErr  error
 	urlErr  error
-	// beforePut runs once, unlocked, before the next write lands.
-	beforePut func()
 }
 
 var _ awsapi.CodeStore = (*memStore)(nil)
 
 func (m *memStore) Put(_ context.Context, ref codestore.Ref, code []byte) (string, error) {
-	m.mu.Lock()
-	hook := m.beforePut
-	m.beforePut = nil
-	m.mu.Unlock()
-	if hook != nil {
-		hook()
-	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if m.putErr != nil {
@@ -179,40 +164,6 @@ func newHarness(t *testing.T, statements ...iamdomain.Statement) (*gwtest.Harnes
 	return gwtest.Scoped(statements...), meta, store, kv, key
 }
 
-// uploads returns every upload record in the account bucket, by function ID.
-func (e *env) uploads(t *testing.T) map[string]function.Upload {
-	t.Helper()
-	kv := e.kv(t)
-	keys, err := kv.ListKeys(t.Context())
-	require.NoError(t, err)
-	out := map[string]function.Upload{}
-	for k := range keys.Keys() {
-		if !strings.HasPrefix(k, "upload.") {
-			continue
-		}
-		entry, err := kv.Get(t.Context(), k)
-		require.NoError(t, err)
-		var up function.Upload
-		require.NoError(t, json.Unmarshal(entry.Value(), &up))
-		require.Equal(t, function.KVUploadKey(up.Region, up.FunctionID, up.RevisionID), k)
-		out[up.FunctionID] = up
-	}
-	return out
-}
-
-// requireTracked asserts every code copy in the store has an upload record.
-func (e *env) requireTracked(t *testing.T) {
-	t.Helper()
-	ups := e.uploads(t)
-	e.store.mu.Lock()
-	defer e.store.mu.Unlock()
-	for ref := range e.store.objects {
-		up, ok := ups[ref.FunctionID]
-		require.True(t, ok, "code copy %s has no upload record", ref.FunctionID)
-		require.Equal(t, ref.RevisionID, up.RevisionID)
-	}
-}
-
 type env struct {
 	h     *gwtest.Harness
 	meta  *localClient
@@ -324,15 +275,6 @@ func TestCreateFunction_Success(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotContains(t, string(entry.Value()), "hunter2", "the environment is never stored in plaintext")
 	assert.Contains(t, string(entry.Value()), "envCipher")
-
-	e.requireTracked(t)
-	ups := e.uploads(t)
-	require.Len(t, ups, 1)
-	for _, up := range ups {
-		assert.Equal(t, function.UploadAttached, up.State)
-		assert.Equal(t, entry.Revision(), up.AttachedRevision)
-		assert.Equal(t, "hello", up.FunctionName)
-	}
 }
 
 func TestCreateFunction_RendersTheStoredTimestamps(t *testing.T) {
@@ -365,8 +307,6 @@ func TestCreateFunction_DuplicateNameLeavesFirstUntouched(t *testing.T) {
 	assert.Contains(t, err.Error(), "Function already exist: hello")
 
 	assert.Equal(t, 1, e.store.count(), "the loser wrote no code")
-	assert.Len(t, e.uploads(t), 1, "the refused create leaves no upload record")
-	e.requireTracked(t)
 	after, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.NoError(t, err)
 	assert.Equal(t, first, after)
@@ -381,11 +321,6 @@ func TestCreateFunction_CodePutFailureFreesTheName(t *testing.T) {
 	assert.Equal(t, 1, e.meta.rollbacks)
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound)
-	ups := e.uploads(t)
-	require.Len(t, ups, 1)
-	for _, up := range ups {
-		assert.Equal(t, function.UploadOwed, up.State, "a failed write may still have landed")
-	}
 
 	e.store.putErr = nil
 	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
@@ -402,51 +337,6 @@ func TestCreateFunction_CodeStoredFailureRollsBackOnce(t *testing.T) {
 	assert.Equal(t, 1, e.store.count(), "the code object is left in place, never deleted here")
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound, "the still-storing claim was withdrawn")
-	e.requireTracked(t)
-	for _, up := range e.uploads(t) {
-		assert.Equal(t, function.UploadOwed, up.State, "the unnamed copy stays owed")
-	}
-}
-
-// The first create's write stalls past its claim, a second create takes the
-// name over, then the first write lands and its completion is refused.
-func TestCreateFunction_LateWriterAfterTakeoverStaysTracked(t *testing.T) {
-	e := newEnv(t)
-	key := function.KVFunctionKey(gwtest.Region, "hello")
-	e.store.beforePut = func() {
-		entry, err := e.kv(t).Get(t.Context(), key)
-		require.NoError(t, err)
-		var rec function.Record
-		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
-		rec.ClaimExpiresAt = time.Now().Add(-time.Second)
-		data, err := json.Marshal(rec)
-		require.NoError(t, err)
-		_, err = e.kv(t).Update(t.Context(), key, data, entry.Revision())
-		require.NoError(t, err)
-
-		rec2, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
-			b["Code"] = map[string]any{"ZipFile": mustZip(t, "second")}
-		}))
-		require.NoError(t, err)
-		require.Equal(t, http.StatusCreated, rec2.Code)
-	}
-
-	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.Equal(t, 2, e.store.count(), "both copies landed")
-	e.requireTracked(t)
-
-	winner, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
-	require.NoError(t, err)
-	ups := e.uploads(t)
-	require.Len(t, ups, 2)
-	for id, up := range ups {
-		if id == winner.FunctionID {
-			assert.Equal(t, function.UploadAttached, up.State)
-			continue
-		}
-		assert.Equal(t, function.UploadOwed, up.State, "the late writer's copy is never inferred complete")
-	}
 }
 
 func TestCreateFunction_RefusesUnsupportedInput(t *testing.T) {
@@ -765,7 +655,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "ListVersionsByFunction"}, reg.Inventory.Registered)
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))

spinifex/domains/lambda/awsapi/invoke.go

@@ -1,186 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"log/slog"
-	"net/http"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// maxSyncPayload is the largest synchronous request payload AWS accepts.
-const maxSyncPayload = 6 * 1024 * 1024
-
-// waitOverhead is how much longer than the function timeout a request waits
-// for its result. It bounds the wait only, never the execution.
-const waitOverhead = 60 * time.Second
-
-// SyncInvocation is a RequestResponse request handed to the execution owner,
-// which decides admission.
-type SyncInvocation struct {
-	// Function and Revision are the one resolved configuration the attempt
-	// binds; the environment stays sealed in Function.EnvCipher.
-	Function function.Record
-	Revision uint64
-	// Qualifier is the validated qualifier the request named, or "".
-	Qualifier string
-	Payload   []byte
-	// WaitUntil is when the request stops waiting. Code may still be running
-	// after it; it proves nothing about stop and releases no slot.
-	WaitUntil time.Time
-}
-
-// SyncResult is what the function returned. FunctionError is "" on success;
-// ExecutedVersion is the version that ran.
-type SyncResult struct {
-	Payload         []byte
-	FunctionError   string
-	ExecutedVersion string
-}
-
-// Executor admits and runs one synchronous attempt. Its context ends only at
-// WaitUntil, never on caller disconnect. A *RefusalError means no code was
-// dispatched; any other error means the outcome is unknown.
-type Executor interface {
-	InvokeSync(ctx context.Context, inv SyncInvocation) (SyncResult, error)
-}
-
-// RefusalError is a known pre-dispatch refusal, such as a capacity throttle. Err
-// is the AWS error returned to the caller.
-type RefusalError struct{ Err error }
-
-func (r *RefusalError) Error() string { return "invocation refused before dispatch: " + r.Err.Error() }
-
-func (r *RefusalError) Unwrap() error { return r.Err }
-
-// invokeTarget is the function and qualifier an Invoke request names, once
-// every deterministic check on the request has passed.
-type invokeTarget struct{ Name, Qualifier string }
-
-// validateInvoke runs every check that needs no stored state: options, size,
-// payload JSON, function name and qualifier. Nothing here reads metadata.
-func validateInvoke(c *call) (*invokeTarget, error) {
-	if err := checkInvokeOptions(c.Header); err != nil {
-		return nil, err
-	}
-	if len(c.Body) > maxSyncPayload {
-		return nil, awserrors.Errorf(awserrors.ErrorRequestEntityTooLargeException,
-			"Request must be smaller than %d bytes for the InvokeFunction operation", maxSyncPayload)
-	}
-	// An empty body is dispatched as it is; anything else must be JSON.
-	if len(c.Body) > 0 && !json.Valid(c.Body) {
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidRequestContentException,
-			"Could not parse request body into json")
-	}
-	if len(c.Params[0]) > maxFunctionNameLen {
-		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must have length less than or equal to 256")
-	}
-	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
-	if err != nil {
-		return nil, err
-	}
-	if q != "" && !nameQualifierRE.MatchString(q) {
-		return nil, invalidFunctionName()
-	}
-	if query, ok := c.Query["Qualifier"]; ok {
-		if len(query) != 1 || query[0] == "" {
-			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must have length greater than or equal to 1")
-		}
-		if len(query[0]) > maxQualifierLen {
-			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must have length less than or equal to 128")
-		}
-		if !queryQualifierRE.MatchString(query[0]) {
-			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must satisfy regular expression pattern: \\$(LATEST(\\.PUBLISHED)?)|[a-zA-Z0-9-_$]+")
-		}
-		if q != "" && q != query[0] {
-			return nil, invalid("The derived qualifier from the function name does not match the specified qualifier.")
-		}
-		q = query[0]
-	}
-	return &invokeTarget{Name: name, Qualifier: q}, nil
-}
-
-// invoke implements Invoke for RequestResponse on $LATEST. serve has already
-// validated the request into c.Invoke and authorized it.
-func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
-	if c.Invoke == nil {
-		return reply{}, serviceError("invoke request was not validated")
-	}
-	rec, revision, err := d.lookupTarget(ctx, c, c.Invoke.Name, c.Invoke.Qualifier)
-	if err != nil {
-		return reply{}, err
-	}
-	if rec.State != function.StateActive {
-		return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
-			"The operation cannot be performed at this time. The function is currently in the following state: %s", rec.State)
-	}
-	if d.Exec == nil || rec.Code.State != function.CodeStored {
-		return reply{}, serviceError("function execution is unavailable")
-	}
-
-	if err := ctx.Err(); err != nil {
-		return reply{}, err
-	}
-	// Once accepted, a disconnecting caller does not cancel the attempt; only
-	// the wait is bounded, and a late result is discarded.
-	waitUntil := time.Now().Add(time.Duration(rec.Timeout)*time.Second + waitOverhead)
-	actx, cancel := context.WithDeadline(context.WithoutCancel(ctx), waitUntil)
-	defer cancel()
-	out, err := d.Exec.InvokeSync(actx, SyncInvocation{
-		Function:  rec,
-		Revision:  revision,
-		Qualifier: c.Invoke.Qualifier,
-		Payload:   c.Body,
-		WaitUntil: waitUntil,
-	})
-	if refusal, ok := errors.AsType[*RefusalError](err); ok {
-		if _, coded := awserrors.ResolveErrorCode(refusal.Err); coded {
-			return reply{}, refusal.Err
-		}
-		err = refusal
-	}
-	if err != nil {
-		slog.ErrorContext(ctx, "Lambda: synchronous attempt outcome unknown", "function_id", rec.FunctionID, "err", err)
-		return reply{}, serviceError("the invocation outcome is unknown")
-	}
-	if out.ExecutedVersion == "" || len(out.Payload) > maxSyncPayload {
-		slog.ErrorContext(ctx, "Lambda: executor returned an invalid result", "function_id", rec.FunctionID,
-			"payload_bytes", len(out.Payload))
-		return reply{}, serviceError("the invocation outcome is unknown")
-	}
-	hdr := http.Header{}
-	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
-	if out.FunctionError != "" {
-		hdr.Set("X-Amz-Function-Error", out.FunctionError)
-	}
-	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
-}
-
-// checkInvokeOptions refuses every request option this build does not
-// implement rather than ignoring it.
-func checkInvokeOptions(h http.Header) error {
-	switch t := h.Get("X-Amz-Invocation-Type"); t {
-	case "", "RequestResponse":
-	case "Event", "DryRun":
-		return unsupported("InvocationType " + t)
-	default:
-		return invalid("InvocationType must be RequestResponse, Event or DryRun")
-	}
-	switch l := h.Get("X-Amz-Log-Type"); l {
-	case "", "None":
-	case "Tail":
-		return unsupported("LogType Tail")
-	default:
-		return invalid("LogType must be None or Tail")
-	}
-	for _, name := range []string{"X-Amz-Client-Context", "X-Amz-Tenant-Id", "X-Amz-Durable-Execution-Name"} {
-		if _, ok := h[name]; ok {
-			return unsupported(name)
-		}
-	}
-	return nil
-}

spinifex/domains/lambda/awsapi/invoke_test.go

@@ -1,455 +0,0 @@
-package awsapi_test
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
-	"maps"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
-	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeExec struct {
-	mu     sync.Mutex
-	calls  []awsapi.SyncInvocation
-	ctxs   []context.Context
-	result awsapi.SyncResult
-	err    error
-	// started, when set, is closed once the call begins and the call then
-	// blocks until release is closed.
-	started, release chan struct{}
-}
-
-func (f *fakeExec) InvokeSync(ctx context.Context, inv awsapi.SyncInvocation) (awsapi.SyncResult, error) {
-	f.mu.Lock()
-	f.calls = append(f.calls, inv)
-	f.ctxs = append(f.ctxs, ctx)
-	started, release := f.started, f.release
-	f.mu.Unlock()
-	if started != nil {
-		close(started)
-		<-release
-	}
-	return f.result, f.err
-}
-
-func (f *fakeExec) count() int {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return len(f.calls)
-}
-
-var okResult = awsapi.SyncResult{Payload: []byte(`{"sum":42}`), ExecutedVersion: "$LATEST"}
-
-// newInvokeEnv creates "hello" with its code stored and reports it Active.
-func newInvokeEnv(t *testing.T, statements ...iamdomain.Statement) (*env, *fakeExec) {
-	t.Helper()
-	e := newEnv(t, statements...)
-	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
-		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
-	}))
-	require.NoError(t, err)
-	e.meta.forceActive = true
-	ex := &fakeExec{result: okResult}
-	e.deps.Exec = ex
-	return e, ex
-}
-
-func (e *env) invoke(ctx context.Context, path string, header http.Header, payload []byte) (*httptest.ResponseRecorder, error) {
-	rec := httptest.NewRecorder()
-	req := httptest.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(payload))
-	maps.Copy(req.Header, header)
-	err := e.h.Dispatch(awsapi.NewRegistration(e.deps), awsapi.ServiceName, rec, req)
-	return rec, err
-}
-
-const invokePath = "/2015-03-31/functions/hello/invocations"
-
-func TestInvoke_ReturnsTheExecutorPayloadAndHeaders(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	payload := []byte(`{"a":40,"b":2,"note":"caf\u00e9"}`)
-	rec, err := e.invoke(t.Context(), invokePath, nil, payload)
-	require.NoError(t, err)
-	require.Equal(t, http.StatusOK, rec.Code)
-	assert.Equal(t, `{"sum":42}`, rec.Body.String())
-	assert.Equal(t, "$LATEST", rec.Header().Get("X-Amz-Executed-Version"))
-	assert.NotContains(t, rec.Header(), "X-Amz-Function-Error")
-	assert.NotContains(t, rec.Header(), "X-Amz-Log-Result")
-
-	require.Equal(t, 1, ex.count())
-	got := ex.calls[0]
-	assert.Equal(t, payload, got.Payload)
-	stored, revision, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
-	require.NoError(t, err)
-	assert.Equal(t, revision, got.Revision)
-	assert.Equal(t, stored, got.Function)
-	assert.NotEmpty(t, got.Function.EnvCipher)
-	assert.NotContains(t, got.Function.EnvCipher, "prod")
-}
-
-func TestInvoke_FunctionErrorHeaderComesFromTheExecutor(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	ex.result = awsapi.SyncResult{Payload: []byte(`{"errorType":"x"}`), FunctionError: "Unhandled", ExecutedVersion: "$LATEST"}
-	rec, err := e.invoke(t.Context(), invokePath, nil, nil)
-	require.NoError(t, err)
-	assert.Equal(t, http.StatusOK, rec.Code)
-	assert.Equal(t, "Unhandled", rec.Header().Get("X-Amz-Function-Error"))
-	assert.Equal(t, `{"errorType":"x"}`, rec.Body.String())
-}
-
-func TestInvoke_PendingFunctionIsAConflictAndNeverDispatches(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	e.meta.forceActive = false
-	_, err := e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-	assert.Equal(t, 0, ex.count())
-}
-
-func TestInvoke_FailsClosedWithoutAnExecutor(t *testing.T) {
-	e, _ := newInvokeEnv(t)
-	e.deps.Exec = nil
-	_, err := e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-}
-
-func TestInvoke_DoesNotNeedTheCodeStore(t *testing.T) {
-	e, _ := newInvokeEnv(t)
-	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
-	_, err := e.invoke(t.Context(), invokePath, nil, nil)
-	require.NoError(t, err)
-	e.deps.Codes = nil
-	_, err = e.invoke(t.Context(), invokePath, nil, nil)
-	require.NoError(t, err)
-}
-
-func TestInvoke_RefusalBeforeDispatchIsKeptApartFromAnUnknownOutcome(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-
-	ex.err = &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorThrottling, "Rate exceeded")}
-	_, err := e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorThrottling)
-
-	ex.err = &awsapi.RefusalError{Err: errors.New("uncoded")}
-	_, err = e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-
-	ex.err = errors.New("transport lost")
-	_, err = e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-	assert.Contains(t, err.Error()+errMessage(err), "outcome is unknown")
-
-	ex.err = nil
-	ex.result = awsapi.SyncResult{Payload: []byte(`{}`)}
-	_, err = e.invoke(t.Context(), invokePath, nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-}
-
-func errMessage(err error) string {
-	_, msg, _ := awserrors.ResolveErrorDetail(err)
-	return msg
-}
-
-func TestInvoke_RefusesOptionsItDoesNotImplement(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	for _, h := range []http.Header{
-		{"X-Amz-Invocation-Type": {"Event"}},
-		{"X-Amz-Invocation-Type": {"DryRun"}},
-		{"X-Amz-Invocation-Type": {"Bogus"}},
-		{"X-Amz-Log-Type": {"Tail"}},
-		{"X-Amz-Log-Type": {"Bogus"}},
-		{"X-Amz-Client-Context": {"e30="}},
-		{"X-Amz-Tenant-Id": {"t1"}},
-	} {
-		_, err := e.invoke(t.Context(), invokePath, h, nil)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-	assert.Equal(t, 0, ex.count())
-
-	for _, h := range []http.Header{
-		{"X-Amz-Invocation-Type": {"RequestResponse"}},
-		{"X-Amz-Log-Type": {"None"}},
-	} {
-		_, err := e.invoke(t.Context(), invokePath, h, nil)
-		require.NoError(t, err)
-	}
-}
-
-// jsonOfSize is a JSON string exactly n bytes long.
-func jsonOfSize(n int) []byte {
-	return []byte(`"` + strings.Repeat("a", n-2) + `"`)
-}
-
-func TestInvoke_PayloadLimit(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	_, err := e.invoke(t.Context(), invokePath, nil, jsonOfSize(6*1024*1024))
-	require.NoError(t, err)
-	_, err = e.invoke(t.Context(), invokePath, nil, jsonOfSize(6*1024*1024+1))
-	testutil.RequireAWSCode(t, err, awserrors.ErrorRequestEntityTooLargeException)
-	assert.Equal(t, 1, ex.count())
-}
-
-func TestInvoke_PayloadMustBeJSONUnlessEmpty(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	for _, bad := range []string{"not json", `{"a":`, "{} {}", "\x00"} {
-		_, err := e.invoke(t.Context(), invokePath, nil, []byte(bad))
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidRequestContentException)
-	}
-	assert.Equal(t, 0, ex.count(), "a malformed payload reached the executor")
-
-	for _, good := range []string{"", `{"a":1}`, `[1,2]`, `"s"`, `42`, `null`, " true "} {
-		_, err := e.invoke(t.Context(), invokePath, nil, []byte(good))
-		require.NoError(t, err, "payload %q", good)
-	}
-	require.Equal(t, 7, ex.count())
-	assert.Empty(t, ex.calls[0].Payload, "an empty body is dispatched as it is")
-	assert.Equal(t, []byte(" true "), ex.calls[6].Payload)
-}
-
-func TestInvoke_MissingAndUnsupportedFunctions(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	_, err := e.invoke(t.Context(), "/2015-03-31/functions/ghost/invocations", nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	_, err = e.invoke(t.Context(), invokePath+"?Qualifier=1", nil, nil)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
-	_, err = e.invoke(t.Context(), invokePath+"?Qualifier=%24LATEST", nil, nil)
-	require.NoError(t, err)
-	assert.Equal(t, 1, ex.count())
-}
-
-// Each form follows its own documented Invoke pattern; a documented value this
-// prototype does not serve passes validation and fails closed at lookup.
-func TestInvoke_QualifierFormsAndLengths(t *testing.T) {
-	fn := "/2015-03-31/functions/"
-	long := func(n int) string { return strings.Repeat("a", n) }
-	cases := []struct {
-		name, path, code string
-	}{
-		{"path $LATEST.PUBLISHED", fn + "hello:$LATEST.PUBLISHED/invocations", awserrors.ErrorResourceNotFoundException},
-		{"query $LATEST.PUBLISHED", invokePath + "?Qualifier=%24LATEST.PUBLISHED", awserrors.ErrorResourceNotFoundException},
-		{"query $foo", invokePath + "?Qualifier=%24foo", awserrors.ErrorResourceNotFoundException},
-		{"query of 128 characters", invokePath + "?Qualifier=" + long(128), awserrors.ErrorResourceNotFoundException},
-		{"query of 129 characters", invokePath + "?Qualifier=" + long(129), awserrors.ErrorInvalidParameterValueException},
-		{"empty query", invokePath + "?Qualifier=", awserrors.ErrorInvalidParameterValueException},
-		{"query with a dot", invokePath + "?Qualifier=a.b", awserrors.ErrorInvalidParameterValueException},
-		{"query $LATEST.OTHER", invokePath + "?Qualifier=%24LATEST.OTHER", awserrors.ErrorInvalidParameterValueException},
-		{"path $foo", fn + "hello:$foo/invocations", awserrors.ErrorInvalidParameterValueException},
-		{"path with a dot", fn + "hello:a.b/invocations", awserrors.ErrorInvalidParameterValueException},
-		{"function name of 256 characters", fn + "hello:" + long(250) + "/invocations", awserrors.ErrorResourceNotFoundException},
-		{"function name of 257 characters", fn + "hello:" + long(251) + "/invocations", awserrors.ErrorInvalidParameterValueException},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			e, ex := newInvokeEnv(t)
-			_, err := e.invoke(t.Context(), tc.path, nil, nil)
-			testutil.RequireAWSCode(t, err, tc.code)
-			assert.Equal(t, 0, ex.count())
-		})
-	}
-}
-
-func TestInvoke_CancelledBeforeDispatchNeverReachesTheExecutor(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	ctx, cancel := context.WithCancel(t.Context())
-	cancel()
-	_, err := e.invoke(ctx, invokePath, nil, nil)
-	require.Error(t, err)
-	assert.Equal(t, 0, ex.count())
-}
-
-func TestInvoke_CallerDisconnectDoesNotCancelAnAcceptedAttempt(t *testing.T) {
-	e, ex := newInvokeEnv(t)
-	ex.started, ex.release = make(chan struct{}), make(chan struct{})
-	ctx, cancel := context.WithCancel(t.Context())
-	done := make(chan error, 1)
-	go func() {
-		_, err := e.invoke(ctx, invokePath, nil, nil)
-		done <- err
-	}()
-	<-ex.started
-	cancel()
-
-	actx := ex.ctxs[0]
-	assert.NoError(t, actx.Err(), "the caller's disconnect reached the attempt")
-	deadline, ok := actx.Deadline()
-	require.True(t, ok, "the wait must be bounded")
-	assert.Equal(t, ex.calls[0].WaitUntil, deadline)
-	assert.WithinDuration(t, time.Now().Add(3*time.Second+60*time.Second), deadline, 5*time.Second)
-
-	close(ex.release)
-	select {
-	case <-done:
-	case <-time.After(10 * time.Second):
-		t.Fatal("the handler did not return after the attempt finished")
-	}
-}
-
-func TestInvoke_Authorization(t *testing.T) {
-	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
-	run := func(t *testing.T, statements ...iamdomain.Statement) (*fakeExec, error) {
-		return runPath(t, invokePath, statements...)
-	}
-	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
-		ex, err := run(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))
-		gwtest.AssertDenied(t, err)
-		assert.Equal(t, 0, ex.count())
-	})
-	t.Run("a deny on the function ARN applies", func(t *testing.T) {
-		ex, err := run(t,
-			gwtest.Statement("Allow", "lambda:*", "*"),
-			gwtest.Statement("Deny", "lambda:InvokeFunction", arn))
-		gwtest.AssertDenied(t, err)
-		assert.Equal(t, 0, ex.count())
-	})
-	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
-		_, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"-other"))
-		gwtest.AssertDenied(t, err)
-	})
-	t.Run("an allow of lambda:InvokeFunction on the function ARN passes", func(t *testing.T) {
-		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn))
-		require.NoError(t, err)
-		assert.Equal(t, 1, ex.count())
-	})
-	t.Run("an unqualified call does not match a qualified-only allow", func(t *testing.T) {
-		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"))
-		gwtest.AssertDenied(t, err)
-		assert.Equal(t, 0, ex.count())
-	})
-	t.Run("an unqualified call does not match an allow on any qualified ARN", func(t *testing.T) {
-		_, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":*"))
-		gwtest.AssertDenied(t, err)
-	})
-	t.Run("an unqualified call matches a wildcard allow", func(t *testing.T) {
-		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"*"))
-		require.NoError(t, err)
-		assert.Equal(t, 1, ex.count())
-	})
-
-	// Lambda compares the policy resource with both the name's qualifier and
-	// the Qualifier parameter, so both forms are checked the same way.
-	qualified := map[string]string{
-		"path qualifier":  "/2015-03-31/functions/hello:$LATEST/invocations",
-		"query qualifier": invokePath + "?Qualifier=%24LATEST",
-	}
-	for form, path := range qualified {
-		t.Run(form, func(t *testing.T) {
-			t.Run("a qualified allow passes", func(t *testing.T) {
-				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"))
-				require.NoError(t, err)
-				assert.Equal(t, 1, ex.count())
-			})
-			t.Run("an explicit qualified deny overrides the matching qualified allow", func(t *testing.T) {
-				ex, err := runPath(t, path,
-					gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"),
-					gwtest.Statement("Deny", "lambda:InvokeFunction", arn+":$LATEST"))
-				gwtest.AssertDenied(t, err)
-				assert.Equal(t, 0, ex.count())
-			})
-			t.Run("an unqualified-only allow does not match", func(t *testing.T) {
-				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn))
-				gwtest.AssertDenied(t, err)
-				assert.Equal(t, 0, ex.count())
-			})
-			t.Run("an allow on any qualified ARN passes", func(t *testing.T) {
-				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":*"))
-				require.NoError(t, err)
-				assert.Equal(t, 1, ex.count())
-			})
-			t.Run("a wildcard allow passes", func(t *testing.T) {
-				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"*"))
-				require.NoError(t, err)
-				assert.Equal(t, 1, ex.count())
-			})
-		})
-	}
-}
-
-func runPath(t *testing.T, path string, statements ...iamdomain.Statement) (*fakeExec, error) {
-	t.Helper()
-	e := newEnv(t, statements...)
-	sum := sha256.Sum256(testZip)
-	rev, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
-		FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
-		Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
-	})
-	require.NoError(t, err)
-	_, _, err = e.meta.CodeStored(t.Context(), gwtest.AccountID, gwtest.Region, "hello", rev, hex.EncodeToString(sum[:]), int64(len(testZip)))
-	require.NoError(t, err)
-	e.meta.forceActive = true
-	ex := &fakeExec{result: okResult}
-	e.deps.Exec = ex
-	_, err = e.invoke(t.Context(), path, nil, nil)
-	return ex, err
-}
-
-// countingMeta counts metadata reads, to show a request never got that far.
-type countingMeta struct {
-	function.Client
-
-	gets atomic.Int32
-}
-
-func (m *countingMeta) Get(ctx context.Context, account, region, name string) (function.Record, uint64, error) {
-	m.gets.Add(1)
-	return m.Client.Get(ctx, account, region, name)
-}
-
-// Every deterministic check runs before authorization, so a caller with no
-// Invoke access still gets the validation error, and neither the metadata
-// store nor the executor is reached. Whether AWS orders these the same way
-// is not verified.
-func TestInvoke_ValidationPrecedesAuthorization(t *testing.T) {
-	big := append([]byte(`"`), bytes.Repeat([]byte("a"), 6*1024*1024)...)
-	cases := []struct {
-		name    string
-		path    string
-		header  http.Header
-		payload []byte
-		code    string
-	}{
-		{"unsupported invocation type", invokePath, http.Header{"X-Amz-Invocation-Type": {"Event"}}, nil, awserrors.ErrorInvalidParameterValueException},
-		{"invalid invocation type", invokePath, http.Header{"X-Amz-Invocation-Type": {"Later"}}, nil, awserrors.ErrorInvalidParameterValueException},
-		{"payload too large", invokePath, nil, append(big, '"'), awserrors.ErrorRequestEntityTooLargeException},
-		{"payload not JSON", invokePath, nil, []byte(`{`), awserrors.ErrorInvalidRequestContentException},
-		{"malformed function name", "/2015-03-31/functions/bad%20name/invocations", nil, nil, awserrors.ErrorInvalidParameterValueException},
-		{"malformed path qualifier", "/2015-03-31/functions/hello:a.b/invocations", nil, nil, awserrors.ErrorInvalidParameterValueException},
-		{"malformed query qualifier", invokePath + "?Qualifier=a.b", nil, nil, awserrors.ErrorInvalidParameterValueException},
-		{"empty query qualifier", invokePath + "?Qualifier=", nil, nil, awserrors.ErrorInvalidParameterValueException},
-		{"conflicting qualifiers", "/2015-03-31/functions/hello:$LATEST/invocations?Qualifier=2", nil, nil, awserrors.ErrorInvalidParameterValueException},
-		{"malformed query encoding", invokePath + "?Qualifier=%ZZ", nil, nil, awserrors.ErrorInvalidParameterValueException},
-		{"malformed pair beside a valid qualifier", invokePath + "?Qualifier=%24LATEST&x=%ZZ", nil, nil, awserrors.ErrorInvalidParameterValueException},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			for access, statement := range map[string]iamdomain.Statement{
-				"denied":  gwtest.Statement("Allow", "lambda:GetFunction", "*"),
-				"allowed": gwtest.Statement("Allow", "lambda:InvokeFunction", "*"),
-			} {
-				e := newEnv(t, statement)
-				meta := &countingMeta{Client: e.meta}
-				e.deps.Meta = meta
-				ex := &fakeExec{result: okResult}
-				e.deps.Exec = ex
-				_, err := e.invoke(t.Context(), tc.path, tc.header, tc.payload)
-				testutil.RequireAWSCode(t, err, tc.code)
-				assert.Zero(t, meta.gets.Load(), "%s: metadata was read", access)
-				assert.Equal(t, 0, ex.count(), "%s: the executor ran", access)
-			}
-		})
-	}
-}

spinifex/domains/lambda/awsapi/registration.go

@@ -8,9 +8,7 @@ import (
 	"errors"
 	"io"
 	"log/slog"
-	"maps"
 	"net/http"
-	"net/url"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -52,8 +50,6 @@ type Deps struct {
 	Trusts func(accountID, roleARN string) (bool, error)
 	// AEAD seals function environment variables; it is the master key's cipher.
 	AEAD cipher.AEAD
-	// Exec runs synchronous attempts; nil refuses every Invoke.
-	Exec Executor
 }
 
 // NewRegistration returns the Lambda REST-JSON registration.
@@ -93,17 +89,6 @@ func functionARN(region, accountID, name string) string {
 	return arn.FormatLambdaFunction(region, accountID, name)
 }
 
-// qualifiedFunctionARN names the version or alias a validated request selects,
-// since Lambda matches a qualified request only against a qualified policy
-// resource.
-func qualifiedFunctionARN(region, accountID, name, qualifier string) string {
-	base := functionARN(region, accountID, name)
-	if base == "*" || qualifier == "" {
-		return base
-	}
-	return base + ":" + qualifier
-}
-
 // serve resolves the route, authorizes the request, then runs the action.
 func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
@@ -119,7 +104,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	if err != nil {
 		return err
 	}
-	c := &call{AccountID: inv.AccountID, Region: inv.Region, Params: params, Query: r.URL.Query(), Body: body, Header: r.Header}
+	c := &call{AccountID: inv.AccountID, Region: inv.Region, Params: params, Query: r.URL.Query(), Body: body}
 
 	// The region is the gateway's own, never the caller-supplied credential
 	// scope, so a caller cannot sign for another region to slide out from
@@ -136,19 +121,8 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
-	case "Invoke":
-		// Invoke validates deterministically before authorizing, and its
-		// resource comes from that same validated target. URL.Query drops
-		// malformed pairs, so the raw query is parsed strictly here.
-		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
-			return invalid("The request query string could not be parsed.")
-		}
-		if c.Invoke, err = validateInvoke(c); err != nil {
-			return err
-		}
-		resource = qualifiedFunctionARN(inv.Region, inv.AccountID, c.Invoke.Name, c.Invoke.Qualifier)
 	}
-	if err := inv.Authorize(ServiceName, iamAction(action), []string{resource}, nil); err != nil {
+	if err := inv.Authorize(ServiceName, action, []string{resource}, nil); err != nil {
 		return err
 	}
 	if decodeErr != nil {
@@ -159,9 +133,9 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return err
 		}
 	}
-	// Only CreateFunction and GetFunction touch the code store.
+	// Only GetFunction presigns code; the other reads are metadata only.
 	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
-		action != "ListVersionsByFunction" && action != "Invoke"
+		action != "ListVersionsByFunction"
 	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}
@@ -174,14 +148,6 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	return nil
 }
 
-// iamAction is the IAM action an API operation is authorized as.
-func iamAction(action string) string {
-	if action == "Invoke" {
-		return "InvokeFunction"
-	}
-	return action
-}
-
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	return d.serve(w, inv)
 }
@@ -231,7 +197,6 @@ func writeJSON(w http.ResponseWriter, out reply) {
 		_, _ = w.Write(envelope.JSONBody(awserrors.ErrorServiceException, "failed to marshal response"))
 		return
 	}
-	maps.Copy(w.Header(), out.Header)
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(out.Status)
 	if _, err := w.Write(body); err != nil {

spinifex/domains/lambda/awsapi/routes.go

@@ -2,7 +2,6 @@ package awsapi
 
 import (
 	"context"
-	"net/http"
 	"net/url"
 
 	"github.com/aws/aws-sdk-go/service/lambda"
@@ -16,13 +15,9 @@ type call struct {
 	Params    []string
 	Query     url.Values
 	Body      []byte
-	Header    http.Header
 	// Create is the decoded CreateFunction body, decoded once ahead of the
 	// authorization gate and the PassRole check that both read it.
 	Create *lambda.CreateFunctionInput
-	// Invoke is Invoke's validated target, resolved ahead of the
-	// authorization gate whose resource it names.
-	Invoke *invokeTarget
 }
 
 // rawJSON is a response body written as is, for a shape the SDK encoder
@@ -33,7 +28,6 @@ type rawJSON []byte
 type reply struct {
 	Status int
 	Body   any
-	Header http.Header
 }
 
 type route = rest.Route[routeHandler]
@@ -62,10 +56,6 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunctionCodeSigningConfig(ctx, c)
 		}},
-	{Method: "POST", Pattern: "/2015-03-31/functions/{FunctionName}/invocations", Action: "Invoke",
-		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
-			return d.invoke(ctx, c)
-		}},
 }
 
 var router = rest.NewRouter("lambda", routes)

spinifex/domains/lambda/awsapi/validate.go

@@ -30,20 +30,6 @@ const (
 
 var nameRE = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
 
-// nameQualifierRE is the qualifier suffix of Invoke's documented FunctionName
-// pattern; queryQualifierRE is its Qualifier pattern, bounded by maxQualifierLen.
-var (
-	nameQualifierRE  = regexp.MustCompile(`^(\$LATEST(\.PUBLISHED)?|[a-zA-Z0-9-_]+)$`)
-	queryQualifierRE = regexp.MustCompile(`^(\$LATEST(\.PUBLISHED)?|[a-zA-Z0-9-_$]+)$`)
-)
-
-// maxQualifierLen bounds the Qualifier query; maxFunctionNameLen bounds the
-// whole FunctionName, qualifier suffix included.
-const (
-	maxQualifierLen    = 128
-	maxFunctionNameLen = 256
-)
-
 func invalid(format string, args ...any) error {
 	return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, format, args...)
 }

spinifex/domains/lambda/execution/grant.go

@@ -1,396 +0,0 @@
-// Package execution owns Lambda attempt grants: who may run an attempt and
-// report its result. This is a provisional single-node owner; it is not the
-// ADR-0009 M1 mechanism, which is still open.
-package execution
-
-import (
-	"context"
-	"crypto/rand"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"regexp"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// KVBucket holds one record per node: its grant sequence and current attempt.
-const KVBucket = "lambda-execution"
-
-const formatVersion = 1
-
-// Attempt states. The attempt lives in its node's record, and every change is
-// a conditional write on that one key, so a start, a withdrawal and the next
-// grant can never all succeed.
-const (
-	// StateGranted: recorded; nothing may have started.
-	StateGranted = "granted"
-	// StateRunning: the daemon may have launched the attempt.
-	StateRunning = "running"
-	// StateAccepted: a result was accepted under the grant; stop not yet proven.
-	StateAccepted = "accepted"
-	// StateWithdrawn: the attempt never started, and now never can.
-	StateWithdrawn = "withdrawn"
-	// StateReleased: the VM stop is proven and its resources released, but the
-	// daemon's own record of it is not yet gone; the node stays held.
-	StateReleased = "released"
-	// StateEnded: the stop is proven and nothing of the attempt remains.
-	StateEnded = "ended"
-	// StateAmbiguous: the end is unproven; the attempt keeps the node's slot.
-	StateAmbiguous = "ambiguous"
-)
-
-var (
-	// ErrNodeBusy: the node holds an attempt that has not ended.
-	ErrNodeBusy = errors.New("execution: node holds an unended attempt")
-	// ErrNotCurrent: the grant is not the node's current grant.
-	ErrNotCurrent = errors.New("execution: grant is not current")
-	// ErrWrongState: the attempt is not in a state that allows the change.
-	ErrWrongState = errors.New("execution: attempt is in the wrong state")
-	// ErrExpired: the grant's launch deadline has passed.
-	ErrExpired = errors.New("execution: grant expired before launch")
-	// ErrUnknownRecord: a record this owner cannot read or did not write.
-	ErrUnknownRecord = errors.New("execution: unrecognised record")
-	// ErrMismatch: the request or code differs from what the grant authorises.
-	ErrMismatch = errors.New("execution: request does not match the grant")
-	// ErrOutcomeUnknown: a write's outcome is unknown and may have landed.
-	ErrOutcomeUnknown = errors.New("execution: write outcome unknown")
-)
-
-var segment = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`)
-
-// Grant is the identity that authorises one attempt on one environment
-// generation. Seq orders grants on a node and never changes for the attempt.
-type Grant struct {
-	GrantID       string `json:"grantId"`
-	Seq           uint64 `json:"seq"`
-	AttemptID     string `json:"attemptId"`
-	EnvironmentID string `json:"environmentId"`
-	Generation    uint64 `json:"generation"`
-	Node          string `json:"node"`
-}
-
-// Identity is the exact request and code a grant authorises.
-type Identity struct {
-	RequestID  string `json:"requestId"`
-	AccountID  string `json:"accountId"`
-	Region     string `json:"region"`
-	FunctionID string `json:"functionId"`
-	RevisionID string `json:"revisionId"`
-	CodeSHA256 string `json:"codeSha256"`
-}
-
-// Attempt is the owner's record of one attempt and its grant.
-type Attempt struct {
-	Grant     Grant     `json:"grant"`
-	Identity  Identity  `json:"identity"`
-	State     string    `json:"state"`
-	LaunchBy  time.Time `json:"launchBy"`
-	CreatedAt time.Time `json:"createdAt"`
-	Launch    *Launch   `json:"launch,omitempty"`
-}
-
-// Launch identifies an attempt's VM process, its launcher and what it holds,
-// so a restarted daemon can prove the stop. The pidfs inodes are comparable
-// only within BootID on MachineID; they do not identify the host.
-type Launch struct {
-	MachineID     string `json:"machineId"`
-	BootID        string `json:"bootId"`
-	PIDNamespace  uint64 `json:"pidNamespace"`
-	LauncherPID   int    `json:"launcherPid"`
-	LauncherInode uint64 `json:"launcherInode"`
-	PID           int    `json:"pid"`
-	Inode         uint64 `json:"inode"`
-	TapName       string `json:"tapName"`
-	TapIndex      int    `json:"tapIndex"`
-	TapOwner      string `json:"tapOwner"`
-}
-
-func (l Launch) valid() bool {
-	return l.MachineID != "" && l.BootID != "" && l.PIDNamespace != 0 && l.LauncherPID > 0 &&
-		l.LauncherInode != 0 && l.PID > 0 && l.Inode != 0
-}
-
-// nodeRecord is a node's one slot. Seq survives every attempt so each later
-// grant orders after each earlier one.
-type nodeRecord struct {
-	FormatVersion int      `json:"formatVersion"`
-	Seq           uint64   `json:"seq"`
-	Attempt       *Attempt `json:"attempt,omitempty"`
-}
-
-// Store is the only writer of node records.
-type Store struct {
-	kv  jetstream.KeyValue
-	now func() time.Time
-	// beforeWrite runs between a read and its conditional write; tests use it.
-	beforeWrite func()
-}
-
-// Open opens (or creates) the execution bucket.
-func Open(ctx context.Context, js jetstream.KeyValueManager) (*Store, error) {
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucket, 1)
-	if err != nil {
-		return nil, fmt.Errorf("execution: open bucket: %w", err)
-	}
-	return &Store{kv: kv, now: time.Now}, nil
-}
-
-func nodeKey(node string) string { return "node." + node }
-
-// Issue records a new attempt and its grant before anything is dispatched.
-// It succeeds only if the node's current attempt ended, was withdrawn, or
-// provably never started before its launch deadline.
-func (s *Store) Issue(ctx context.Context, a Attempt) (Attempt, error) {
-	g := a.Grant
-	for _, seg := range []string{g.AttemptID, g.EnvironmentID, g.Node, a.Identity.FunctionID} {
-		if !segment.MatchString(seg) {
-			return Attempt{}, fmt.Errorf("execution: invalid identifier %q", seg)
-		}
-	}
-	id := a.Identity
-	if id.RequestID == "" || id.AccountID == "" || id.Region == "" || id.RevisionID == "" || id.CodeSHA256 == "" {
-		return Attempt{}, errors.New("execution: request and code identity are required")
-	}
-	if g.Generation == 0 {
-		return Attempt{}, errors.New("execution: environment generation is required")
-	}
-	grantID, err := randomID()
-	if err != nil {
-		return Attempt{}, err
-	}
-	a.Grant.GrantID, a.State, a.CreatedAt = grantID, StateGranted, s.now()
-
-	rec, rev, err := s.read(ctx, g.Node)
-	missing := errors.Is(err, jetstream.ErrKeyNotFound)
-	if err != nil && !missing {
-		return Attempt{}, err
-	}
-	if !missing && rec.Attempt != nil {
-		if rec.Attempt.Grant.AttemptID == g.AttemptID {
-			return Attempt{}, fmt.Errorf("%w: attempt %s already granted", ErrWrongState, g.AttemptID)
-		}
-		if !s.supersedable(*rec.Attempt) {
-			return Attempt{}, ErrNodeBusy
-		}
-	}
-	a.Grant.Seq = rec.Seq + 1
-	next := nodeRecord{FormatVersion: formatVersion, Seq: a.Grant.Seq, Attempt: &a}
-	if missing {
-		err = s.create(ctx, g.Node, next)
-	} else {
-		err = s.write(ctx, g.Node, next, rev)
-	}
-	if errors.Is(err, ErrNotCurrent) {
-		return Attempt{}, ErrNodeBusy
-	}
-	if err != nil {
-		return Attempt{}, err
-	}
-	return a, nil
-}
-
-// supersedable reports whether a new grant may replace cur. A granted attempt
-// past its deadline qualifies because Start refuses it then, and both writes
-// are conditional on the same record revision.
-func (s *Store) supersedable(cur Attempt) bool {
-	switch cur.State {
-	case StateEnded, StateWithdrawn:
-		return true
-	case StateGranted:
-		return !s.now().Before(cur.LaunchBy)
-	}
-	return false
-}
-
-// Start moves a current, unexpired grant from granted to running, only for
-// the exact request and code it authorises. The daemon calls it before
-// allocating anything; a repeat, replay or mismatch is refused unchanged.
-func (s *Store) Start(ctx context.Context, g Grant, want Identity) error {
-	return s.transition(ctx, g, func(a *Attempt) error {
-		if a.Identity != want {
-			return ErrMismatch
-		}
-		if a.State != StateGranted {
-			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
-		}
-		if !s.now().Before(a.LaunchBy) {
-			return ErrExpired
-		}
-		a.State = StateRunning
-		return nil
-	})
-}
-
-// Accept records that a result under g is accepted. Only one result is ever
-// accepted, and none once the attempt has ended or been held as ambiguous.
-func (s *Store) Accept(ctx context.Context, g Grant) error {
-	return s.transition(ctx, g, func(a *Attempt) error {
-		if a.State != StateRunning {
-			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
-		}
-		a.State = StateAccepted
-		return nil
-	})
-}
-
-// Launched records the running attempt's launch identity, once, before the
-// daemon sends the guest anything.
-func (s *Store) Launched(ctx context.Context, g Grant, l Launch) error {
-	if !l.valid() {
-		return errors.New("execution: incomplete launch identity")
-	}
-	return s.transition(ctx, g, func(a *Attempt) error {
-		if a.State != StateRunning || a.Launch != nil {
-			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
-		}
-		a.Launch = &l
-		return nil
-	})
-}
-
-// Withdraw ends a grant that never started, for a caller that knows it sent
-// nothing. It fails once the daemon has started the attempt.
-func (s *Store) Withdraw(ctx context.Context, g Grant) error {
-	return s.transition(ctx, g, func(a *Attempt) error {
-		if a.State != StateGranted {
-			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
-		}
-		a.State = StateWithdrawn
-		return nil
-	})
-}
-
-// Released records the caller's proof that the attempt's VM has stopped and
-// everything it held is released. The node stays held until Stopped.
-func (s *Store) Released(ctx context.Context, g Grant) error {
-	return s.transition(ctx, g, func(a *Attempt) error {
-		switch a.State {
-		case StateRunning, StateAccepted, StateAmbiguous:
-			a.State = StateReleased
-			return nil
-		}
-		return fmt.Errorf("%w: %s", ErrWrongState, a.State)
-	})
-}
-
-// Stopped records that nothing of a released attempt remains, which frees the
-// node for the next grant.
-func (s *Store) Stopped(ctx context.Context, g Grant) error {
-	return s.transition(ctx, g, func(a *Attempt) error {
-		if a.State != StateReleased {
-			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
-		}
-		a.State = StateEnded
-		return nil
-	})
-}
-
-// Ambiguous records that the attempt's end is unproven. The node stays held
-// until Stopped; no amount of time releases it.
-func (s *Store) Ambiguous(ctx context.Context, g Grant) error {
-	return s.transition(ctx, g, func(a *Attempt) error {
-		switch a.State {
-		case StateRunning, StateAccepted, StateAmbiguous:
-			a.State = StateAmbiguous
-			return nil
-		}
-		return fmt.Errorf("%w: %s", ErrWrongState, a.State)
-	})
-}
-
-// Current returns the node's current attempt, read-only.
-func (s *Store) Current(ctx context.Context, node string) (Attempt, error) {
-	rec, _, err := s.read(ctx, node)
-	if errors.Is(err, jetstream.ErrKeyNotFound) || err == nil && rec.Attempt == nil {
-		return Attempt{}, ErrNotCurrent
-	}
-	if err != nil {
-		return Attempt{}, err
-	}
-	return *rec.Attempt, nil
-}
-
-// transition applies change to the attempt under g, written only at the
-// revision it was read, so a concurrent change refuses it.
-func (s *Store) transition(ctx context.Context, g Grant, change func(*Attempt) error) error {
-	rec, rev, err := s.read(ctx, g.Node)
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return ErrNotCurrent
-	}
-	if err != nil {
-		return err
-	}
-	if rec.Attempt == nil || rec.Attempt.Grant != g {
-		return ErrNotCurrent
-	}
-	a := *rec.Attempt
-	if err := change(&a); err != nil {
-		return err
-	}
-	rec.Attempt = &a
-	return s.write(ctx, g.Node, rec, rev)
-}
-
-func (s *Store) read(ctx context.Context, node string) (nodeRecord, uint64, error) {
-	e, err := s.kv.Get(ctx, nodeKey(node))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return nodeRecord{}, 0, err
-	}
-	if err != nil {
-		return nodeRecord{}, 0, fmt.Errorf("execution: read node record: %w", err)
-	}
-	var rec nodeRecord
-	if err := json.Unmarshal(e.Value(), &rec); err != nil || rec.FormatVersion != formatVersion || rec.Seq == 0 ||
-		(rec.Attempt != nil && (rec.Attempt.Grant.Node != node || rec.Attempt.Grant.Seq != rec.Seq)) {
-		return nodeRecord{}, 0, ErrUnknownRecord
-	}
-	return rec, e.Revision(), nil
-}
-
-func (s *Store) create(ctx context.Context, node string, rec nodeRecord) error {
-	body, err := json.Marshal(rec)
-	if err != nil {
-		return err
-	}
-	if s.beforeWrite != nil {
-		s.beforeWrite()
-	}
-	if _, err := s.kv.Create(ctx, nodeKey(node), body); err != nil {
-		if errors.Is(err, jetstream.ErrKeyExists) {
-			return ErrNodeBusy
-		}
-		return fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
-	}
-	return nil
-}
-
-// write replaces the node record only at rev. A revision conflict means
-// another change won; any other error may have landed.
-func (s *Store) write(ctx context.Context, node string, rec nodeRecord, rev uint64) error {
-	body, err := json.Marshal(rec)
-	if err != nil {
-		return err
-	}
-	if s.beforeWrite != nil {
-		s.beforeWrite()
-	}
-	if _, err := s.kv.Update(ctx, nodeKey(node), body, rev); err != nil {
-		if errors.Is(err, jetstream.ErrKeyExists) {
-			return fmt.Errorf("%w: record changed concurrently", ErrNotCurrent)
-		}
-		return fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
-	}
-	return nil
-}
-
-func randomID() (string, error) {
-	b := make([]byte, 16)
-	if _, err := rand.Read(b); err != nil {
-		return "", fmt.Errorf("execution: grant ID: %w", err)
-	}
-	return hex.EncodeToString(b), nil
-}

spinifex/domains/lambda/execution/grant_test.go

@@ -1,376 +0,0 @@
-package execution
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"sync"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/require"
-)
-
-type clock struct {
-	mu sync.Mutex
-	t  time.Time
-}
-
-func (c *clock) now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
-
-func (c *clock) add(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() }
-
-func newStore(t *testing.T) (*Store, *clock) {
-	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	s, err := Open(context.Background(), testutil.NewJetStream(t, nc))
-	require.NoError(t, err)
-	c := &clock{t: time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)}
-	s.now = c.now
-	return s, c
-}
-
-func attempt(c *clock, id string) Attempt {
-	return Attempt{
-		Grant:    Grant{AttemptID: id, EnvironmentID: "env-" + id, Generation: 1, Node: "node1"},
-		Identity: identity(id), LaunchBy: c.now().Add(time.Minute),
-	}
-}
-
-func identity(id string) Identity {
-	return Identity{RequestID: "req-" + id, AccountID: "123456789012", Region: "ap-southeast-2",
-		FunctionID: "fn1", RevisionID: "rev1", CodeSHA256: "aa"}
-}
-
-func state(t *testing.T, s *Store, id string) string {
-	t.Helper()
-	rec, _, err := s.read(context.Background(), "node1")
-	require.NoError(t, err)
-	require.NotNil(t, rec.Attempt)
-	require.Equal(t, id, rec.Attempt.Grant.AttemptID)
-	return rec.Attempt.State
-}
-
-func started(t *testing.T, s *Store, c *clock, id string) Attempt {
-	t.Helper()
-	a, err := s.Issue(context.Background(), attempt(c, id))
-	require.NoError(t, err)
-	require.NoError(t, s.Start(context.Background(), a.Grant, a.Identity))
-	return a
-}
-
-func TestGrant_LifecycleFreesTheSlotOnlyOnProvenStop(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a := started(t, s, c, "a1")
-	require.Equal(t, uint64(1), a.Grant.Seq)
-	require.NotEmpty(t, a.Grant.GrantID)
-
-	_, err := s.Issue(ctx, attempt(c, "a2"))
-	require.ErrorIs(t, err, ErrNodeBusy, "no second grant while the first runs")
-
-	require.NoError(t, s.Accept(ctx, a.Grant))
-	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "one result per attempt")
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.ErrorIs(t, err, ErrNodeBusy, "an accepted result does not free the slot")
-
-	require.NoError(t, s.Released(ctx, a.Grant))
-	require.NoError(t, s.Stopped(ctx, a.Grant))
-	require.Equal(t, StateEnded, state(t, s, "a1"))
-	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "no result accepted after the end")
-
-	b, err := s.Issue(ctx, attempt(c, "a2"))
-	require.NoError(t, err)
-	require.Equal(t, uint64(2), b.Grant.Seq, "later grants order after earlier ones")
-}
-
-func TestGrant_AmbiguousHoldsTheSlotUntilStopIsProven(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a := started(t, s, c, "a1")
-	require.NoError(t, s.Ambiguous(ctx, a.Grant))
-	require.NoError(t, s.Ambiguous(ctx, a.Grant), "holding again is a no-op")
-
-	c.add(24 * time.Hour)
-	_, err := s.Issue(ctx, attempt(c, "a2"))
-	require.ErrorIs(t, err, ErrNodeBusy, "time alone never releases an ambiguous attempt")
-	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "no result accepted while ambiguous")
-	require.ErrorIs(t, s.Withdraw(ctx, a.Grant), ErrWrongState, "a started attempt cannot be withdrawn")
-
-	require.NoError(t, s.Released(ctx, a.Grant))
-	require.NoError(t, s.Stopped(ctx, a.Grant))
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.NoError(t, err)
-}
-
-func TestGrant_StartRefusesReplayAndEveryMismatchedIdentity(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-
-	mismatched := map[string]func(g *Grant){
-		"grant ID":    func(g *Grant) { g.GrantID = "other" },
-		"older seq":   func(g *Grant) { g.Seq-- },
-		"environment": func(g *Grant) { g.EnvironmentID = "env-other" },
-		"generation":  func(g *Grant) { g.Generation++ },
-		"attempt":     func(g *Grant) { g.AttemptID = "a9" },
-		"node":        func(g *Grant) { g.Node = "node2" },
-	}
-	for name, mutate := range mismatched {
-		g := a.Grant
-		mutate(&g)
-		require.ErrorIs(t, s.Start(ctx, g, a.Identity), ErrNotCurrent, name)
-		require.ErrorIs(t, s.Stopped(ctx, g), ErrNotCurrent, name)
-	}
-
-	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
-	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrWrongState, "a replayed start is refused")
-}
-
-func TestGrant_ExpiredGrantIsSupersededOnlyIfItNeverStarted(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	old, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-
-	c.add(2 * time.Minute)
-	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrExpired)
-
-	next, err := s.Issue(ctx, attempt(c, "a2"))
-	require.NoError(t, err, "a holder that can never start does not block the node")
-	require.Greater(t, next.Grant.Seq, old.Grant.Seq)
-	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent, "the superseded grant stays refused")
-
-	// A started holder is never superseded, however late.
-	require.NoError(t, s.Start(ctx, next.Grant, next.Identity))
-	c.add(24 * time.Hour)
-	_, err = s.Issue(ctx, attempt(c, "a3"))
-	require.ErrorIs(t, err, ErrNodeBusy)
-	require.Equal(t, StateRunning, state(t, s, "a2"))
-}
-
-func TestGrant_WithdrawFreesOnlyANeverStartedGrant(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-	require.NoError(t, s.Withdraw(ctx, a.Grant))
-	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrWrongState, "a withdrawn grant can never start")
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.NoError(t, err)
-}
-
-func TestGrant_UnexpiredGrantedHolderBlocks(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	_, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.ErrorIs(t, err, ErrNodeBusy)
-}
-
-func TestGrant_ConcurrentIssueGrantsExactlyOne(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	// Seed the slot so racers contend on the conditional update, not Create.
-	seed, err := s.Issue(ctx, attempt(c, "seed"))
-	require.NoError(t, err)
-	require.NoError(t, s.Withdraw(ctx, seed.Grant))
-
-	const racers = 16
-	var wg sync.WaitGroup
-	results := make(chan error, racers)
-	for i := range racers {
-		wg.Go(func() {
-			_, err := s.Issue(ctx, attempt(c, fmt.Sprintf("r%d", i)))
-			results <- err
-		})
-	}
-	wg.Wait()
-	close(results)
-	granted := 0
-	for err := range results {
-		if err == nil {
-			granted++
-		} else {
-			require.ErrorIs(t, err, ErrNodeBusy)
-		}
-	}
-	require.Equal(t, 1, granted)
-}
-
-// TestGrant_StartLosesToASupersedingGrant forces the interleaving where a
-// Start has read an expiring grant and a new grant replaces it before Start
-// writes: Start must fail, leaving only the new grant able to run.
-func TestGrant_StartLosesToASupersedingGrant(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	old, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-
-	// Start reads at a time the old grant is still valid; the new grant is
-	// issued after expiry, between Start's read and its write.
-	var next Attempt
-	s.beforeWrite = func() {
-		s.beforeWrite = nil
-		c.add(2 * time.Minute)
-		next, err = s.Issue(ctx, attempt(c, "a2"))
-		require.NoError(t, err)
-	}
-	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent)
-	c.add(-2 * time.Minute)
-	require.NoError(t, s.Start(ctx, next.Grant, next.Identity))
-	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent)
-	require.Equal(t, StateRunning, state(t, s, "a2"))
-}
-
-func TestGrant_AcceptLosesToAConcurrentHold(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a := started(t, s, c, "a1")
-	s.beforeWrite = func() {
-		s.beforeWrite = nil
-		require.NoError(t, s.Ambiguous(ctx, a.Grant))
-	}
-	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrNotCurrent, "the result was not accepted")
-	require.Equal(t, StateAmbiguous, state(t, s, "a1"))
-}
-
-func TestGrant_DuplicateAttemptIsRefused(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a := started(t, s, c, "a1")
-	require.NoError(t, s.Released(ctx, a.Grant))
-	require.NoError(t, s.Stopped(ctx, a.Grant))
-
-	_, err := s.Issue(ctx, attempt(c, "a1"))
-	require.ErrorIs(t, err, ErrWrongState, "an attempt ID is not granted twice in a row")
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.NoError(t, err, "the refused duplicate does not take the slot")
-}
-
-// lostAck lands attempt Creates and then reports a failure, as a lost
-// acknowledgement would.
-type lostAck struct{ jetstream.KeyValue }
-
-func (l lostAck) Create(ctx context.Context, key string, v []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
-	if _, err := l.KeyValue.Create(ctx, key, v, opts...); err != nil {
-		return 0, err
-	}
-	return 0, errors.New("ack lost")
-}
-
-func TestGrant_AmbiguousRecordWriteKeepsTheSlot(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	inner := s.kv
-	s.kv = lostAck{inner}
-	_, err := s.Issue(ctx, attempt(c, "a1"))
-	require.ErrorIs(t, err, ErrOutcomeUnknown)
-	s.kv = inner
-
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.ErrorIs(t, err, ErrNodeBusy, "a grant that may be recorded keeps the slot")
-	require.Equal(t, StateGranted, state(t, s, "a1"))
-	c.add(2 * time.Minute)
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.NoError(t, err, "past its deadline the landed grant can never start")
-}
-
-func TestGrant_UnrecognisedRecordIsRefusedUnchanged(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-	require.NoError(t, s.Withdraw(ctx, a.Grant))
-
-	e, err := s.kv.Get(ctx, nodeKey("node1"))
-	require.NoError(t, err)
-	var raw map[string]any
-	require.NoError(t, json.Unmarshal(e.Value(), &raw))
-	raw["formatVersion"] = formatVersion + 1
-	body, _ := json.Marshal(raw)
-	rev, err := s.kv.Update(ctx, nodeKey("node1"), body, e.Revision())
-	require.NoError(t, err)
-
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.ErrorIs(t, err, ErrUnknownRecord)
-	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrUnknownRecord)
-	after, err := s.kv.Get(ctx, nodeKey("node1"))
-	require.NoError(t, err)
-	require.Equal(t, rev, after.Revision())
-	require.Equal(t, body, after.Value())
-}
-
-func TestGrant_StartRefusesAMismatchedRequestUnchanged(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-	mismatched := map[string]func(id *Identity){
-		"request":  func(id *Identity) { id.RequestID = "req-other" },
-		"account":  func(id *Identity) { id.AccountID = "210987654321" },
-		"region":   func(id *Identity) { id.Region = "us-east-1" },
-		"function": func(id *Identity) { id.FunctionID = "fn2" },
-		"revision": func(id *Identity) { id.RevisionID = "rev2" },
-		"code":     func(id *Identity) { id.CodeSHA256 = "bb" },
-	}
-	for name, mutate := range mismatched {
-		want := a.Identity
-		mutate(&want)
-		require.ErrorIs(t, s.Start(ctx, a.Grant, want), ErrMismatch, name)
-		require.Equal(t, StateGranted, state(t, s, "a1"), name)
-	}
-	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
-}
-
-// A released attempt still holds the node; only Stopped, and only after
-// Released, frees it for the next grant.
-func TestReleasedHoldsTheNodeUntilStopped(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	a, err := s.Issue(ctx, attempt(c, "a1"))
-	require.NoError(t, err)
-	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
-	require.ErrorIs(t, s.Stopped(ctx, a.Grant), ErrWrongState)
-
-	require.NoError(t, s.Released(ctx, a.Grant))
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.ErrorIs(t, err, ErrNodeBusy)
-
-	require.NoError(t, s.Stopped(ctx, a.Grant))
-	_, err = s.Issue(ctx, attempt(c, "a2"))
-	require.NoError(t, err)
-}
-
-// The launch identity is recorded once, complete, and only while running, so
-// recovery never acts on a partial or replaced identity.
-func TestGrant_LaunchedRecordsACompleteIdentityOnce(t *testing.T) {
-	s, c := newStore(t)
-	ctx := context.Background()
-	l := Launch{MachineID: "m", BootID: "b", PIDNamespace: 4026531836, LauncherPID: 10,
-		LauncherInode: 11, PID: 20, Inode: 21}
-
-	granted, err := s.Issue(ctx, attempt(c, "a0"))
-	require.NoError(t, err)
-	require.ErrorIs(t, s.Launched(ctx, granted.Grant, l), ErrWrongState, "not yet running")
-	require.NoError(t, s.Withdraw(ctx, granted.Grant))
-
-	a := started(t, s, c, "a1")
-	partial := l
-	partial.Inode = 0
-	require.Error(t, s.Launched(ctx, a.Grant, partial))
-	require.NoError(t, s.Launched(ctx, a.Grant, l))
-	other := l
-	other.PID = 99
-	require.ErrorIs(t, s.Launched(ctx, a.Grant, other), ErrWrongState, "recorded once")
-
-	rec, err := s.Current(ctx, "node1")
-	require.NoError(t, err)
-	require.NotNil(t, rec.Launch)
-	require.Equal(t, l, *rec.Launch)
-}

spinifex/domains/lambda/function/meta.go

@@ -26,7 +26,6 @@ var (
 	nameRE   = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
 	regionRE = regexp.MustCompile(`^[a-z0-9-]{1,32}$`)
 	digestRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
-	idRE     = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,64}$`)
 )
 
 // ValidName reports whether name is a legal function name.
@@ -82,15 +81,17 @@ func decode(data []byte) (Record, error) {
 	return rec, nil
 }
 
-// Create records the code upload, then claims the name with kv.Create: Pending,
-// code storing, claim expiring after ClaimTTL. A name held only by an expired,
-// still-storing claim is taken over once; the taken-over upload stays owed.
+// Create binds the name by writing the record with kv.Create. The record starts
+// Pending with its code storing and a claim that expires after ClaimTTL. A name
+// held only by an expired, still-storing claim is taken over once.
+//
+// Orphaned code objects from rolled-back or taken-over claims are not cleaned up yet.
 func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID string) (*CreateResponse, error) {
 	rec := req.Record
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()
 	}
-	if !ValidName(rec.FunctionName) || !idRE.MatchString(rec.FunctionID) || !idRE.MatchString(rec.Code.RevisionID) || accountID == "" {
+	if !ValidName(rec.FunctionName) || rec.FunctionID == "" || rec.Code.RevisionID == "" || accountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "function record is incomplete")
 	}
 	kv, err := s.bucket(ctx, accountID)
@@ -111,21 +112,6 @@ func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID
 	}
 	key := KVFunctionKey(req.Region, rec.FunctionName)
 
-	// The upload is recorded before the claim exists, so no copy can be written
-	// for a claim that has no upload record.
-	upKey := KVUploadKey(req.Region, rec.FunctionID, rec.Code.RevisionID)
-	upData, err := json.Marshal(Upload{
-		FormatVersion: FormatVersion, AccountID: accountID, Region: req.Region, FunctionName: rec.FunctionName,
-		FunctionID: rec.FunctionID, RevisionID: rec.Code.RevisionID, State: UploadOwed, CreatedAt: now,
-	})
-	if err != nil {
-		return nil, err
-	}
-	upRev, err := kv.Create(ctx, upKey, upData)
-	if err != nil {
-		return nil, fmt.Errorf("lambda: record code upload: %w", err)
-	}
-
 	rev, err := kv.Create(ctx, key, data)
 	if err == nil {
 		return &CreateResponse{Revision: rev}, nil
@@ -137,85 +123,27 @@ func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID
 	exists := conflict("Function already exist: %s", rec.FunctionName)
 	entry, err := kv.Get(ctx, key)
 	if err != nil {
-		return nil, s.refused(ctx, kv, upKey, upRev, exists)
+		return nil, exists
 	}
 	old, err := decode(entry.Value())
 	if err != nil || old.Code.State != CodeStoring || !s.now().After(old.ClaimExpiresAt) {
-		return nil, s.refused(ctx, kv, upKey, upRev, exists)
+		return nil, exists
 	}
 	if err := kv.Delete(ctx, key, jetstream.LastRevision(entry.Revision())); err != nil {
-		return nil, s.refused(ctx, kv, upKey, upRev, exists)
+		return nil, exists
 	}
 	slog.WarnContext(ctx, "Lambda: took over an expired function name claim",
 		"function_name", rec.FunctionName, "old_function_id", old.FunctionID)
 	rev, err = kv.Create(ctx, key, data)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, s.refused(ctx, kv, upKey, upRev, exists)
+			return nil, exists
 		}
 		return nil, fmt.Errorf("lambda: create function record: %w", err)
 	}
 	return &CreateResponse{Revision: rev}, nil
 }
 
-// refused drops the upload record of a create that certainly holds no claim:
-// the gateway writes code only after Create succeeds. A failed drop leaves the
-// upload owed, which is safe.
-func (s *MetaService) refused(ctx context.Context, kv jetstream.KeyValue, upKey string, upRev uint64, err error) error {
-	if derr := kv.Delete(ctx, upKey, jetstream.LastRevision(upRev)); derr != nil {
-		slog.WarnContext(ctx, "Lambda: upload record of a refused create was not dropped", "key", upKey, "err", derr)
-	}
-	return err
-}
-
-// checkUpload returns the owed upload record for rec. Any other record, or
-// none, refuses the caller; nothing is rewritten, because the state of a record
-// this owner cannot vouch for is unknown.
-func checkUpload(ctx context.Context, kv jetstream.KeyValue, rec Record) (Upload, uint64, error) {
-	refuse := func(why string, args ...any) (Upload, uint64, error) {
-		slog.WarnContext(ctx, "Lambda: code upload record refused; its state is unknown",
-			"function_id", rec.FunctionID, "revision_id", rec.Code.RevisionID, "reason", fmt.Sprintf(why, args...))
-		return Upload{}, 0, awserrors.Errorf(awserrors.ErrorServiceException, "function code upload is not tracked")
-	}
-	entry, err := kv.Get(ctx, KVUploadKey(rec.Region, rec.FunctionID, rec.Code.RevisionID))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return refuse("missing")
-	}
-	if err != nil {
-		return Upload{}, 0, fmt.Errorf("lambda: get code upload record: %w", err)
-	}
-	var up Upload
-	if err := json.Unmarshal(entry.Value(), &up); err != nil {
-		return refuse("unreadable")
-	}
-	if up.FormatVersion != FormatVersion {
-		return refuse("format version %d", up.FormatVersion)
-	}
-	if up.AccountID != rec.AccountID || up.Region != rec.Region || up.FunctionName != rec.FunctionName ||
-		up.FunctionID != rec.FunctionID || up.RevisionID != rec.Code.RevisionID {
-		return refuse("identity mismatch")
-	}
-	if up.State != UploadOwed {
-		return refuse("state %s", up.State)
-	}
-	return up, entry.Revision(), nil
-}
-
-// attach marks the checked upload as named by the function record at revision.
-// KV has no multi-key write, so a change since the check fails this CAS, and
-// the record is then left as found, its state unknown.
-func attach(ctx context.Context, kv jetstream.KeyValue, up Upload, upRev, revision uint64) {
-	up.State, up.AttachedRevision = UploadAttached, revision
-	data, err := json.Marshal(up)
-	if err == nil {
-		_, err = kv.Update(ctx, KVUploadKey(up.Region, up.FunctionID, up.RevisionID), data, upRev)
-	}
-	if err != nil {
-		slog.WarnContext(ctx, "Lambda: upload record not marked attached; its state is unknown",
-			"function_id", up.FunctionID, "revision_id", up.RevisionID, "err", err)
-	}
-}
-
 // Get returns the record, or Found false when the name is unbound.
 func (s *MetaService) Get(ctx context.Context, req *GetRequest, accountID string) (*GetResponse, error) {
 	if !regionRE.MatchString(req.Region) {
@@ -262,9 +190,8 @@ func (s *MetaService) readStoring(ctx context.Context, kv jetstream.KeyValue, re
 	return rec, nil
 }
 
-// CodeStored marks the code stored. It is refused, leaving both records unchanged,
-// when the revision moved, the code is not storing, the claim has expired, or
-// the upload record is not the owed one this create wrote.
+// CodeStored marks the code stored. It is refused, leaving the record unchanged,
+// when the revision moved, the code is not storing, or the claim has expired.
 func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, accountID string) (*CodeStoredResponse, error) {
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()
@@ -287,10 +214,6 @@ func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, ac
 	if now.After(rec.ClaimExpiresAt) {
 		return nil, conflict("Function creation claim expired: %s", req.FunctionName)
 	}
-	up, upRev, err := checkUpload(ctx, kv, rec)
-	if err != nil {
-		return nil, err
-	}
 	rec.Code.State, rec.Code.SHA256, rec.Code.Size = CodeStored, req.SHA256, req.Size
 	rec.State, rec.StateReasonCode, rec.StateReason = StatePending, ReasonCodeCreating, ReasonCreating
 	rec.LastModified = now.UTC()
@@ -305,13 +228,12 @@ func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, ac
 		}
 		return nil, fmt.Errorf("lambda: update function record: %w", err)
 	}
-	attach(ctx, kv, up, upRev, rev)
 	return &CodeStoredResponse{Record: rec, Revision: rev}, nil
 }
 
 // RollbackCreate frees the name of a create whose own code write failed. It
 // deletes only a record still storing at exactly the revision create returned;
-// an already-absent name is success. The upload stays owed: the write may have landed.
+// an already-absent name is success.
 func (s *MetaService) RollbackCreate(ctx context.Context, req *RollbackCreateRequest, accountID string) (*RollbackCreateResponse, error) {
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()

spinifex/domains/lambda/function/record.go

@@ -62,37 +62,8 @@ const (
 	KVBucketAccountPrefix  = "lambda-account-"
 	KVBucketAccountHistory = 1
 	kvFunctionKeyPrefix    = "fn."
-	kvUploadKeyPrefix      = "upload."
 )
 
-// KVUploadKey returns the key of the record that tracks one code upload. The
-// bucket is shared across Regions, so the Region is part of the key.
-func KVUploadKey(region, functionID, revisionID string) string {
-	return kvUploadKeyPrefix + region + "." + functionID + "." + revisionID
-}
-
-// Upload states. An owed upload may have a code copy that no function record
-// names; it stays owed until the owner records otherwise.
-const (
-	UploadOwed     = "owed"
-	UploadAttached = "attached"
-)
-
-// Upload tracks one code upload. A create begun through this owner writes it
-// before the copy; records and copies from older owners have none.
-type Upload struct {
-	FormatVersion int       `json:"formatVersion"`
-	AccountID     string    `json:"accountId"`
-	Region        string    `json:"region"`
-	FunctionName  string    `json:"functionName"`
-	FunctionID    string    `json:"functionId"`
-	RevisionID    string    `json:"revisionId"`
-	State         string    `json:"state"`
-	CreatedAt     time.Time `json:"createdAt"`
-	// AttachedRevision is the function record revision that names the copy.
-	AttachedRevision uint64 `json:"attachedRevision,omitempty"`
-}
-
 // KVAccountBucket returns the per-account JetStream KV bucket name.
 func KVAccountBucket(accountID string) string { return KVBucketAccountPrefix + accountID }
 

spinifex/domains/lambda/function/upload_test.go

@@ -1,212 +0,0 @@
-package function
-
-import (
-	"encoding/json"
-	"errors"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// upload reads the tracking record for functionID, or reports it absent.
-func upload(t *testing.T, svc *MetaService, functionID string) (Upload, bool) {
-	t.Helper()
-	return uploadIn(t, svc, testRegion, functionID)
-}
-
-func uploadIn(t *testing.T, svc *MetaService, region, functionID string) (Upload, bool) {
-	t.Helper()
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	entry, err := kv.Get(t.Context(), KVUploadKey(region, functionID, "rev-"+functionID))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return Upload{}, false
-	}
-	require.NoError(t, err)
-	var up Upload
-	require.NoError(t, json.Unmarshal(entry.Value(), &up))
-	return up, true
-}
-
-func requireOwed(t *testing.T, svc *MetaService, functionID string) {
-	t.Helper()
-	up, ok := upload(t, svc, functionID)
-	require.True(t, ok, "upload record for %s is missing", functionID)
-	assert.Equal(t, UploadOwed, up.State)
-}
-
-func TestUpload_CreateRecordsItOwedAndCodeStoredAttachesIt(t *testing.T) {
-	svc, c := newTestService(t)
-	rev := mustCreate(t, svc, "hello", "f1")
-	up, ok := upload(t, svc, "f1")
-	require.True(t, ok)
-	assert.Equal(t, Upload{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion, FunctionName: "hello",
-		FunctionID: "f1", RevisionID: "rev-f1", State: UploadOwed, CreatedAt: c.t}, up)
-
-	stored, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-	require.NoError(t, err)
-	up, ok = upload(t, svc, "f1")
-	require.True(t, ok)
-	assert.Equal(t, UploadAttached, up.State)
-	assert.Equal(t, stored.Revision, up.AttachedRevision)
-}
-
-func TestUpload_IsRecordedBeforeTheNameIsClaimed(t *testing.T) {
-	svc, _ := newTestService(t)
-	kv, err := svc.bucket(t.Context(), testAccount)
-	require.NoError(t, err)
-	_, err = kv.Create(t.Context(), KVUploadKey(testRegion, "f1", "rev-f1"), []byte(`{}`))
-	require.NoError(t, err)
-
-	_, err = svc.Create(t.Context(), createReq("hello", "f1"), testAccount)
-	require.Error(t, err)
-	assert.False(t, mustGet(t, svc, "hello").Found, "no claim exists when the upload could not be recorded")
-}
-
-func TestUpload_RefusedDuplicateLeavesNoRecord(t *testing.T) {
-	svc, _ := newTestService(t)
-	mustCreate(t, svc, "hello", "f1")
-	_, err := svc.Create(t.Context(), createReq("hello", "f2"), testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-	_, ok := upload(t, svc, "f2")
-	assert.False(t, ok, "a create refused before any claim can never be written to")
-	requireOwed(t, svc, "f1")
-}
-
-func TestUpload_RollbackKeepsItOwed(t *testing.T) {
-	svc, _ := newTestService(t)
-	rev := mustCreate(t, svc, "hello", "f1")
-	_, err := svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
-	require.NoError(t, err)
-	assert.False(t, mustGet(t, svc, "hello").Found)
-	requireOwed(t, svc, "f1")
-}
-
-func TestUpload_ExpiredClaimIsNotTakenAsCompletion(t *testing.T) {
-	svc, c := newTestService(t)
-	rev := mustCreate(t, svc, "hello", "f1")
-	c.t = c.t.Add(ClaimTTL + time.Second)
-	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-	requireOwed(t, svc, "f1")
-}
-
-// The original writer stalls past its claim, a second create takes the name,
-// and the original then finishes its write and reports it.
-func TestUpload_TakeoverAndLateWriterLeaveTheOriginalOwed(t *testing.T) {
-	svc, c := newTestService(t)
-	oldRev := mustCreate(t, svc, "hello", "f1")
-	c.t = c.t.Add(ClaimTTL + time.Second)
-	newRev := mustCreate(t, svc, "hello", "f2")
-	requireOwed(t, svc, "f1")
-	requireOwed(t, svc, "f2")
-
-	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev, SHA256: testDigest, Size: 9}, testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev}, testAccount)
-	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
-	requireOwed(t, svc, "f1")
-
-	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: newRev, SHA256: testDigest, Size: 9}, testAccount)
-	require.NoError(t, err)
-	up, _ := upload(t, svc, "f2")
-	assert.Equal(t, UploadAttached, up.State)
-	requireOwed(t, svc, "f1")
-}
-
-func TestUpload_RefusesIdentifiersThatCannotBeKeys(t *testing.T) {
-	svc, _ := newTestService(t)
-	for _, id := range []string{"a.b", "a b", "a*"} {
-		_, err := svc.Create(t.Context(), createReq("hello", id), testAccount)
-		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
-	}
-	assert.False(t, mustGet(t, svc, "hello").Found)
-}
-
-func TestUpload_SameIdentifiersInTwoRegionsAreTrackedApart(t *testing.T) {
-	svc, _ := newTestService(t)
-	rev := mustCreate(t, svc, "hello", "f1")
-	other := createReq("hello", "f1")
-	other.Region = "us-east-1"
-	_, err := svc.Create(t.Context(), other, testAccount)
-	require.NoError(t, err)
-
-	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-	require.NoError(t, err)
-	here, ok := uploadIn(t, svc, testRegion, "f1")
-	require.True(t, ok)
-	assert.Equal(t, UploadAttached, here.State)
-	there, ok := uploadIn(t, svc, "us-east-1", "f1")
-	require.True(t, ok)
-	assert.Equal(t, UploadOwed, there.State)
-	assert.Equal(t, "us-east-1", there.Region)
-}
-
-// An upload record the owner cannot vouch for refuses CodeStored, and neither
-// it nor the function record changes.
-func TestUpload_CodeStoredRefusesAnUnrecognisedRecord(t *testing.T) {
-	asJSON := func(mutate func(*Upload)) func([]byte) []byte {
-		return func(b []byte) []byte {
-			var up Upload
-			if err := json.Unmarshal(b, &up); err != nil {
-				panic(err)
-			}
-			mutate(&up)
-			out, err := json.Marshal(up)
-			if err != nil {
-				panic(err)
-			}
-			return out
-		}
-	}
-	for name, rewrite := range map[string]func([]byte) []byte{
-		"missing":          nil,
-		"unreadable":       func([]byte) []byte { return []byte("not json") },
-		"future format":    asJSON(func(u *Upload) { u.FormatVersion = FormatVersion + 1 }),
-		"other function":   asJSON(func(u *Upload) { u.FunctionName = "other" }),
-		"other account":    asJSON(func(u *Upload) { u.AccountID = "000000000002" }),
-		"already attached": asJSON(func(u *Upload) { u.State, u.AttachedRevision = UploadAttached, 7 }),
-	} {
-		t.Run(name, func(t *testing.T) {
-			svc, _ := newTestService(t)
-			rev := mustCreate(t, svc, "hello", "f1")
-			kv, err := svc.bucket(t.Context(), testAccount)
-			require.NoError(t, err)
-			key := KVUploadKey(testRegion, "f1", "rev-f1")
-			entry, err := kv.Get(t.Context(), key)
-			require.NoError(t, err)
-			if rewrite == nil {
-				require.NoError(t, kv.Delete(t.Context(), key))
-			} else {
-				_, err = kv.Update(t.Context(), key, rewrite(entry.Value()), entry.Revision())
-				require.NoError(t, err)
-			}
-			before, err := kv.Get(t.Context(), key)
-			if rewrite == nil {
-				require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
-			} else {
-				require.NoError(t, err)
-			}
-
-			_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
-			testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
-
-			fn := mustGet(t, svc, "hello")
-			assert.Equal(t, rev, fn.Revision, "the function record is unchanged")
-			assert.Equal(t, CodeStoring, fn.Record.Code.State)
-			after, err := kv.Get(t.Context(), key)
-			if rewrite == nil {
-				require.ErrorIs(t, err, jetstream.ErrKeyNotFound, "no record is invented")
-				return
-			}
-			require.NoError(t, err)
-			assert.Equal(t, before.Revision(), after.Revision())
-			assert.Equal(t, before.Value(), after.Value())
-		})
-	}
-}

spinifex/domains/network/external/floating/allocator.go

@@ -2,14 +2,10 @@ package floating
 
 import (
 	"context"
-	"crypto/rand"
-	"encoding/hex"
 	"errors"
 	"fmt"
 	"log/slog"
 	"net/netip"
-	"slices"
-	"sort"
 	"strings"
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
@@ -32,10 +28,6 @@ type Binding struct {
 	// GatewayVPCID names the VPC whose NAT gateway answers on the address. Such
 	// an address has no ENI, so this is what says which node must hold it.
 	GatewayVPCID string `json:"gateway_vpc_id,omitempty"`
-	// Gen is a token drawn at allocation, never reused, so a holder pass that
-	// read the binding before a release and re-allocation can tell it is stale.
-	// Only a fixed set draws one; other providers' records carry none.
-	Gen string `json:"gen,omitempty"`
 	// Holder is the provider host this node last confirmed the address on. It
 	// is observed state, written after the provider agreed, never identity.
 	Holder string `json:"holder,omitempty"`
@@ -61,9 +53,6 @@ type Config struct {
 	Pool external.ExternalPoolConfig
 	// HostID is this node's provider host ID, which the holder pass attaches to.
 	HostID string
-	// KnownHostIDs are the other nodes' provider host IDs. A holder the provider reports
-	// is logged only when it is one of these or HostID.
-	KnownHostIDs []string
 	// LocalPorts lists the OVS ports plugged on this node, keyed by port name.
 	LocalPorts func(ctx context.Context) (map[string]struct{}, error)
 	// LocalGateway reports whether vpcID's gateway chassis is this node.
@@ -75,8 +64,6 @@ type PoolAllocator struct {
 	provider Provider
 	store    Store
 	cfg      Config
-	// fixed is set when the provider hands out a pre-bought set of addresses.
-	fixed bool
 	// suspects holds unbound labelled addresses one reconcile pass has seen,
 	// so the next can collect them. See Reconcile.
 	suspects map[string]struct{}
@@ -88,16 +75,6 @@ var (
 	_ external.AddressClaimer = (*PoolAllocator)(nil)
 )
 
-// holderLabel names a provider-reported holder for a log or a stall reason. For a fixed set
-// the value is shown only if it is a host we configured; anything else, which the provider
-// alone vouches for, is not echoed. Other providers log the holder as they always have.
-func (a *PoolAllocator) holderLabel(id string) string {
-	if !a.fixed || id == "" || id == a.cfg.HostID || slices.Contains(a.cfg.KnownHostIDs, id) {
-		return id
-	}
-	return "unrecognised holder"
-}
-
 // New constructs an allocator bound to a single pool.
 func New(provider Provider, store Store, cfg Config) (*PoolAllocator, error) {
 	if provider == nil {
@@ -109,20 +86,10 @@ func New(provider Provider, store Store, cfg Config) (*PoolAllocator, error) {
 	if cfg.HostID == "" {
 		return nil, fmt.Errorf("floating: pool %q missing provider host id", cfg.Pool.Name)
 	}
-	_, fixed := provider.(FixedSet)
-	if cfg.Pool.LabelPrefix == "" && !fixed {
+	if cfg.Pool.LabelPrefix == "" {
 		return nil, fmt.Errorf("floating: pool %q missing label prefix", cfg.Pool.Name)
 	}
-	return &PoolAllocator{provider: provider, store: store, cfg: cfg, fixed: fixed, suspects: map[string]struct{}{}}, nil
-}
-
-// newGen returns a fresh random token for a binding's Gen.
-func newGen() (string, error) {
-	var b [16]byte
-	if _, err := rand.Read(b[:]); err != nil {
-		return "", fmt.Errorf("floating: draw allocation token: %w", err)
-	}
-	return hex.EncodeToString(b[:]), nil
+	return &PoolAllocator{provider: provider, store: store, cfg: cfg, suspects: map[string]struct{}{}}, nil
 }
 
 // Pool returns the pool this allocator manages.
@@ -138,10 +105,6 @@ func (a *PoolAllocator) Allocate(ctx context.Context, req external.AllocateReque
 		return netip.Addr{}, fmt.Errorf("floating: pool mismatch (got %q, bound to %q)", req.PoolName, a.cfg.Pool.Name)
 	}
 
-	if fs, ok := a.provider.(FixedSet); ok {
-		return a.allocateFixed(ctx, fs, req)
-	}
-
 	addr, err := a.provider.Create(ctx, a.label(req))
 	if err != nil {
 		return netip.Addr{}, capacityErr(req.Purpose, fmt.Errorf("floating: create address: %w", err), err)
@@ -174,53 +137,6 @@ func (a *PoolAllocator) Allocate(ctx context.Context, req external.AllocateReque
 	return addr.IP, nil
 }
 
-// allocateFixed hands out a member of the provider's set that has no binding.
-// The member is chosen inside the record update, which reruns on a conflict,
-// so the choice is made against the record that is actually written.
-func (a *PoolAllocator) allocateFixed(ctx context.Context, fs FixedSet, req external.AllocateRequest) (netip.Addr, error) {
-	members, err := fs.Members(ctx)
-	if err != nil {
-		return netip.Addr{}, capacityErr(req.Purpose, fmt.Errorf("floating: list pool members: %w", err), err)
-	}
-	sort.Slice(members, func(i, j int) bool { return members[i].IP.Less(members[j].IP) })
-
-	gen, err := newGen()
-	if err != nil {
-		return netip.Addr{}, err
-	}
-	var picked Address
-	err = a.store.Mutate(ctx, a.cfg.Pool.Name, func(rec *Record) (bool, error) {
-		picked = Address{}
-		for _, m := range members {
-			if !m.IP.IsValid() || m.Failed {
-				continue
-			}
-			key := m.IP.String()
-			if _, used := rec.Bindings[key]; used {
-				continue
-			}
-			rec.Bindings[key] = Binding{
-				AddrID:       m.ID,
-				Purpose:      req.Purpose,
-				AllocationID: req.AllocationID,
-				ENIID:        req.ENIID,
-				InstanceID:   req.InstanceID,
-				Gen:          gen,
-			}
-			picked = m
-			return true, nil
-		}
-		return false, fmt.Errorf("floating: pool %s has no free address: %w", a.cfg.Pool.Name, ErrQuotaExceeded)
-	})
-	if err != nil {
-		return netip.Addr{}, capacityErr(req.Purpose, fmt.Errorf("floating: allocate from pool: %w", err), err)
-	}
-	slog.InfoContext(ctx, "floating allocated external IP",
-		"pool", a.cfg.Pool.Name, "public_ip", picked.IP.String(), "addr_id", picked.ID,
-		"purpose", req.Purpose, "eni_id", req.ENIID)
-	return picked.IP, nil
-}
-
 // Release drops the binding and deletes the address. ownerENIID scopes the
 // release as StaticPoolAllocator.Release does, so a teardown sweep naming a
 // stale owner is a no-op rather than a double-free.
@@ -326,11 +242,7 @@ func (a *PoolAllocator) undo(ctx context.Context, addr Address) {
 // destroy detaches and then deletes an address; a missing one is already gone.
 // Vultr documents delete as detaching implicitly, but has been measured to
 // refuse it with "Unable to detach subnet" while the address is attached.
-// A fixed set's members belong to whoever bought them, so nothing is destroyed.
 func (a *PoolAllocator) destroy(ctx context.Context, addrID string) error {
-	if a.fixed {
-		return nil
-	}
 	if err := a.provider.Detach(ctx, addrID); err != nil && !errors.Is(err, ErrNotFound) {
 		return fmt.Errorf("detach: %w", err)
 	}

spinifex/domains/network/external/floating/fixedset_test.go

@@ -1,412 +0,0 @@
-package floating
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"maps"
-	"net/netip"
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// fixedFake is a provider whose addresses exist before the pool does: it cannot create, and
-// detach and delete change nothing.
-type fixedFake struct {
-	mu    sync.Mutex
-	addrs map[string]*Address
-	calls []string
-	// onList runs once when the holder pass lists, which is after it read the bindings and before
-	// it acts on them.
-	onList func()
-}
-
-var (
-	_ Provider = (*fixedFake)(nil)
-	_ FixedSet = (*fixedFake)(nil)
-)
-
-func newFixedFake(n int) *fixedFake {
-	f := &fixedFake{addrs: map[string]*Address{}}
-	for i := 1; i <= n; i++ {
-		id := fmt.Sprintf("eip-%d", i)
-		f.addrs[id] = &Address{ID: id, IP: netip.MustParseAddr(fmt.Sprintf("198.51.100.%d", i))}
-	}
-	return f
-}
-
-func (f *fixedFake) record(call string) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.calls = append(f.calls, call)
-}
-
-func (f *fixedFake) callsWith(prefix string) int {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	n := 0
-	for _, c := range f.calls {
-		if len(c) >= len(prefix) && c[:len(prefix)] == prefix {
-			n++
-		}
-	}
-	return n
-}
-
-func (f *fixedFake) holder(id string) string {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return f.addrs[id].HolderID
-}
-
-func (f *fixedFake) snapshot() []Address {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	out := make([]Address, 0, len(f.addrs))
-	for _, a := range f.addrs {
-		out = append(out, *a)
-	}
-	return out
-}
-
-func (f *fixedFake) Create(context.Context, string) (Address, error) {
-	f.record("create")
-	return Address{}, errors.New("a fixed set creates nothing")
-}
-
-func (f *fixedFake) Attach(_ context.Context, id, host string) error {
-	f.record("attach:" + id + ">" + host)
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.addrs[id].HolderID = host
-	return nil
-}
-
-func (f *fixedFake) Detach(context.Context, string) error { f.record("detach"); return nil }
-func (f *fixedFake) Delete(context.Context, string) error { f.record("delete"); return nil }
-
-func (f *fixedFake) Get(_ context.Context, id string) (Address, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	a, ok := f.addrs[id]
-	if !ok {
-		return Address{}, ErrNotFound
-	}
-	return *a, nil
-}
-
-func (f *fixedFake) List(context.Context) ([]Address, error) {
-	f.mu.Lock()
-	hook := f.onList
-	f.onList = nil
-	f.mu.Unlock()
-	if hook != nil {
-		hook()
-	}
-	return f.snapshot(), nil
-}
-
-func (f *fixedFake) Members(context.Context) ([]Address, error) { return f.snapshot(), nil }
-
-func newFixedNode(t *testing.T, fake *fixedFake, store Store, hostID string) *node {
-	t.Helper()
-	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
-	alloc, err := New(fake, store, Config{
-		Pool:   external.ExternalPoolConfig{Name: pool, Source: "latitude"},
-		HostID: hostID,
-		LocalPorts: func(context.Context) (map[string]struct{}, error) {
-			return maps.Clone(n.plugged), nil
-		},
-		LocalGateway: func(_ context.Context, vpcID string) (bool, error) { return n.gateway[vpcID], nil },
-	})
-	require.NoError(t, err)
-	n.alloc = alloc
-	return n
-}
-
-func fixedSetup(t *testing.T, members int) (*fixedFake, *memStore, *node, *node) {
-	t.Helper()
-	fake := newFixedFake(members)
-	store := newMemStore()
-	return fake, store, newFixedNode(t, fake, store, hostA), newFixedNode(t, fake, store, hostB)
-}
-
-func TestFixedSet_AllocatePicksAFreeMemberAndCreatesNothing(t *testing.T) {
-	fake, store, a, _ := fixedSetup(t, 2)
-	ip1 := allocENI(t, a, "eni-1")
-	ip2 := allocENI(t, a, "eni-2")
-
-	assert.NotEqual(t, ip1, ip2)
-	assert.Equal(t, 0, fake.callsWith("create"))
-	b := store.binding(t, ip1)
-	assert.NotEmpty(t, b.Gen)
-	assert.NotEqual(t, b.Gen, store.binding(t, ip2).Gen)
-	assert.Empty(t, fake.holder(b.AddrID), "the owner's node attaches, never the allocating node")
-}
-
-func TestFixedSet_NoFreeMemberIsQuotaForTheCall(t *testing.T) {
-	_, _, a, _ := fixedSetup(t, 1)
-	allocENI(t, a, "eni-1")
-
-	_, err := a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: purposeEIP})
-	assert.ErrorContains(t, err, awserrors.ErrorAddressLimitExceeded)
-	_, err = a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: "eni-public"})
-	assert.ErrorContains(t, err, awserrors.ErrorInsufficientAddressCapacity)
-}
-
-func TestFixedSet_ConcurrentAllocationsNeverShareAMember(t *testing.T) {
-	fake := newFixedFake(3)
-	store := newMemStore()
-	const callers = 8
-	var wg sync.WaitGroup
-	var mu sync.Mutex
-	got := map[netip.Addr]int{}
-	failed := 0
-	for i := range callers {
-		n := newFixedNode(t, fake, store, hostA)
-		wg.Go(func() {
-			ip, err := n.alloc.Allocate(context.Background(), external.AllocateRequest{
-				PoolName: pool, Purpose: "eni-public", ENIID: fmt.Sprintf("eni-%d", i),
-			})
-			mu.Lock()
-			defer mu.Unlock()
-			if err != nil {
-				failed++
-				return
-			}
-			got[ip]++
-		})
-	}
-	wg.Wait()
-
-	assert.Len(t, got, 3)
-	for ip, n := range got {
-		assert.Equal(t, 1, n, "%s was handed out twice", ip)
-	}
-	assert.Equal(t, callers-3, failed)
-}
-
-func TestFixedSet_ReleaseFreesTheMemberAndTouchesNothingAtTheProvider(t *testing.T) {
-	fake, store, a, _ := fixedSetup(t, 1)
-	ip := allocENI(t, a, "eni-1")
-	first := store.binding(t, ip).Gen
-
-	require.NoError(t, a.alloc.Release(context.Background(), pool, ip, "eni-1"))
-	assert.Len(t, fake.snapshot(), 1, "a member is never deleted")
-
-	again := allocENI(t, a, "eni-2")
-	assert.Equal(t, ip, again, "the freed member is handed out again")
-	assert.NotEqual(t, first, store.binding(t, again).Gen, "a reused address is a new allocation")
-}
-
-func TestFixedSet_ClaimMovesTheMemberToTheOwnersNode(t *testing.T) {
-	fake, store, a, b := fixedSetup(t, 1)
-	ip := allocENI(t, a, "eni-1")
-	a.plug("eni-1")
-	_, err := a.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	addrID := store.binding(t, ip).AddrID
-	require.Equal(t, hostA, fake.holder(addrID))
-
-	a.plugged = map[string]struct{}{}
-	b.plug("eni-1")
-	res, err := b.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, []string{ip.String()}, res.Claimed)
-	assert.Equal(t, hostB, fake.holder(addrID))
-	assert.Equal(t, hostB, store.binding(t, ip).Holder)
-}
-
-func TestFixedSet_AnUnownedMemberIsNotSurrendered(t *testing.T) {
-	fake, store, a, _ := fixedSetup(t, 1)
-	ip, err := a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: purposeEIP, AllocationID: "eipalloc-1"})
-	require.NoError(t, err)
-	addrID := store.binding(t, ip).AddrID
-	require.NoError(t, fake.Attach(context.Background(), addrID, hostA))
-	fake.calls = nil
-
-	_, err = a.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, 0, fake.callsWith("detach"), "detach changes nothing at the provider, so the pass must not pretend it did")
-	assert.Equal(t, hostA, fake.holder(addrID))
-}
-
-func TestFixedSet_ReconcileNeverCollectsAMember(t *testing.T) {
-	fake, _, a, _ := fixedSetup(t, 2)
-	for range 2 {
-		res, err := a.alloc.Reconcile(context.Background())
-		require.NoError(t, err)
-		assert.Empty(t, res.Collected)
-		assert.Equal(t, 2, res.Skipped)
-	}
-	assert.Equal(t, 0, fake.callsWith("delete"))
-}
-
-// A holder pass reads the bindings, then acts. Releasing and re-allocating in between puts the
-// same address and provider ID under a different owner, which the pass must not move.
-func TestFixedSet_PassPausedAcrossReallocationDoesNotMove(t *testing.T) {
-	fake, store, a, b := fixedSetup(t, 1)
-	ip := allocENI(t, a, "eni-1")
-	b.plug("eni-1")
-	fake.onList = func() {
-		require.NoError(t, a.alloc.Release(context.Background(), pool, ip, "eni-1"))
-		again := allocENI(t, a, "eni-2")
-		require.Equal(t, ip, again)
-	}
-
-	res, err := b.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, res.Claimed)
-	assert.Empty(t, res.Stalled)
-	assert.Equal(t, 0, fake.callsWith("attach"), "the stale pass must not move the new allocation")
-	assert.Equal(t, "eni-2", store.binding(t, ip).ENIID)
-	assert.Empty(t, store.binding(t, ip).Holder)
-}
-
-func TestFixedSet_PassPausedAcrossAGatewayVPCChangeDoesNotMove(t *testing.T) {
-	fake, store, a, b := fixedSetup(t, 1)
-	ip, err := a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: purposeEIP, AllocationID: "eipalloc-nat"})
-	require.NoError(t, err)
-	binder := NewGatewayBinder(store, []string{pool})
-	require.NoError(t, binder.BindGateway(context.Background(), ip.String(), "vpc-1"))
-	b.gateway["vpc-1"] = true
-	fake.onList = func() {
-		require.NoError(t, binder.BindGateway(context.Background(), ip.String(), "vpc-2"))
-	}
-
-	res, err := b.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, res.Claimed)
-	assert.Equal(t, 0, fake.callsWith("attach"))
-	assert.Equal(t, "vpc-2", store.binding(t, ip).GatewayVPCID)
-}
-
-func TestRecordHolder_IgnoresAStaleAllocation(t *testing.T) {
-	_, store, a, _ := fixedSetup(t, 1)
-	ip := allocENI(t, a, "eni-1")
-	live := store.binding(t, ip)
-
-	stale := live
-	stale.Gen = "an-earlier-allocation"
-	a.alloc.recordHolder(context.Background(), ip.String(), stale, hostA)
-	assert.Empty(t, store.binding(t, ip).Holder)
-
-	a.alloc.recordHolder(context.Background(), ip.String(), live, hostA)
-	assert.Equal(t, hostA, store.binding(t, ip).Holder)
-}
-
-func TestNew_AFixedSetNeedsNoLabelPrefix(t *testing.T) {
-	_, err := New(newFixedFake(1), newMemStore(), Config{Pool: external.ExternalPoolConfig{Name: pool}, HostID: hostA})
-	require.NoError(t, err)
-}
-
-func TestFixedSet_EachAllocationGetsItsOwnGen(t *testing.T) {
-	_, store, a, _ := fixedSetup(t, 2)
-	ip1 := allocENI(t, a, "eni-1")
-	ip2 := allocENI(t, a, "eni-2")
-	assert.NotEmpty(t, store.binding(t, ip1).Gen)
-	assert.NotEqual(t, store.binding(t, ip1).Gen, store.binding(t, ip2).Gen)
-}
-
-// A deployment that does not select the latitude pool keeps the records it always wrote.
-func TestVultrRecordCarriesNoGen(t *testing.T) {
-	_, store, a, _ := setup(t)
-	ip := allocENI(t, a, "eni-1")
-	assert.Empty(t, store.binding(t, ip).Gen)
-	raw, err := json.Marshal(store.binding(t, ip))
-	require.NoError(t, err)
-	assert.NotContains(t, string(raw), "gen")
-}
-
-// failingGets lets the first n reads through, then fails every later one.
-type failingGets struct {
-	Store
-
-	mu   sync.Mutex
-	left int
-}
-
-func (s *failingGets) Get(ctx context.Context, poolName string) (Record, error) {
-	s.mu.Lock()
-	ok := s.left > 0
-	s.left--
-	s.mu.Unlock()
-	if !ok {
-		return Record{}, errors.New("store unavailable")
-	}
-	return s.Store.Get(ctx, poolName)
-}
-
-// A pass that could not re-read the binding moved nothing, so it must say so: the daemon
-// keeps an address's unreachable warning only while it appears in Stalled.
-func TestFixedSet_FailedReReadIsStalledNotSuccess(t *testing.T) {
-	fake, store, a, _ := fixedSetup(t, 1)
-	ip := allocENI(t, a, "eni-1")
-	a.plug("eni-1")
-	_, err := a.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	addrID := store.binding(t, ip).AddrID
-
-	flaky := &failingGets{Store: store, left: 1}
-	b := newFixedNode(t, fake, flaky, hostB)
-	b.plug("eni-1")
-	fake.calls = nil
-
-	res, err := b.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	require.Len(t, res.Stalled, 1)
-	assert.Equal(t, ip.String(), res.Stalled[0].PublicIP)
-	assert.Contains(t, res.Stalled[0].Reason, "re-read")
-	assert.Empty(t, res.Claimed)
-	assert.Equal(t, 0, fake.callsWith("attach"))
-	assert.Equal(t, hostA, fake.holder(addrID), "nothing moved")
-}
-
-func TestClaim_FailedReReadDoesNotStallAVultrClaim(t *testing.T) {
-	fake, store, a, _ := setup(t)
-	ip := allocENI(t, a, "eni-1")
-	a.plug("eni-1")
-	_, err := a.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	flaky := &failingGets{Store: store, left: 1}
-	b := newNode(t, fake, flaky, hostB)
-	b.plug("eni-1")
-
-	res, err := b.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Empty(t, res.Stalled)
-	assert.Equal(t, []string{ip.String()}, res.Claimed)
-}
-
-func TestFixedSet_ReleaseNeverDetachesOrDeletes(t *testing.T) {
-	fake, _, a, _ := fixedSetup(t, 1)
-	ip := allocENI(t, a, "eni-1")
-	require.NoError(t, a.alloc.Release(context.Background(), pool, ip, "eni-1"))
-	assert.Equal(t, 0, fake.callsWith("detach"))
-	assert.Equal(t, 0, fake.callsWith("delete"))
-}
-
-func TestFixedSet_ReconcileNeverDetachesOrDeletesAndStillReportsMissing(t *testing.T) {
-	fake, store, a, _ := fixedSetup(t, 2)
-	// A member the provider no longer lists, bound in the record.
-	ip := allocENI(t, a, "eni-1")
-	delete(fake.addrs, store.binding(t, ip).AddrID)
-	// An unbound member that happens to carry a matching label.
-	fake.addrs["eip-2"].Label = prefix + ":stray"
-
-	for range 3 {
-		res, err := a.alloc.Reconcile(context.Background())
-		require.NoError(t, err)
-		assert.Empty(t, res.Collected)
-		assert.Equal(t, []string{ip.String()}, res.Missing)
-	}
-	assert.Equal(t, 0, fake.callsWith("detach"))
-	assert.Equal(t, 0, fake.callsWith("delete"))
-}

spinifex/domains/network/external/floating/holder.go

@@ -65,7 +65,7 @@ func (a *PoolAllocator) ClaimLocalAddresses(ctx context.Context) (external.Claim
 		case local:
 			res.Local++
 			a.claim(ctx, key, b, addr, &res)
-		case !b.owned() && addr.HolderID == a.cfg.HostID && !a.fixed:
+		case !b.owned() && addr.HolderID == a.cfg.HostID:
 			a.surrender(ctx, key, b)
 		}
 	}
@@ -77,42 +77,19 @@ func (a *PoolAllocator) ClaimLocalAddresses(ctx context.Context) (external.Claim
 // unreachable as this one if the pass gives up here.
 func (a *PoolAllocator) claim(ctx context.Context, key string, b Binding, addr Address, res *external.ClaimResult) {
 	host := a.cfg.HostID
+	if addr.HolderID == host {
+		a.recordHolder(ctx, key, b.AddrID, host)
+		return
+	}
 	stall := func(reason string) {
 		res.Stalled = append(res.Stalled, external.StalledAddress{PublicIP: key, ENIID: b.ENIID, Reason: reason})
 	}
-	if addr.HolderID == host {
-		if addr.Failed {
-			stall(fmt.Sprintf("%s is on %s but the provider reports it in an error state", b.AddrID, host))
-			return
-		}
-		a.recordHolder(ctx, key, b, host)
-		return
-	}
-	// ok is false when the binding changed, which needs no report: the new
-	// allocation is handled on its own. A failed re-read is reported, since the
-	// address was not moved and the pass must not read as complete.
-	stillOurs := func() bool {
-		if !a.fixed {
-			return true
-		}
-		ok, err := a.unchanged(ctx, key, b)
-		if err != nil {
-			stall(fmt.Sprintf("re-read binding for %s before moving it: %v", b.AddrID, err))
-		}
-		return ok
-	}
 	if addr.HolderID != "" {
-		if !stillOurs() {
-			return
-		}
 		if err := a.provider.Detach(ctx, b.AddrID); err != nil && !errors.Is(err, ErrNotFound) {
-			stall(fmt.Sprintf("detach %s from %s: %v", b.AddrID, a.holderLabel(addr.HolderID), err))
+			stall(fmt.Sprintf("detach %s from %s: %v", b.AddrID, addr.HolderID, err))
 			return
 		}
 	}
-	if !stillOurs() {
-		return
-	}
 	if err := a.provider.Attach(ctx, b.AddrID, host); err != nil {
 		stall(fmt.Sprintf("attach %s to %s: %v", b.AddrID, host, err))
 		return
@@ -123,7 +100,7 @@ func (a *PoolAllocator) claim(ctx context.Context, key string, b Binding, addr A
 		return
 	}
 	if got.HolderID != host {
-		stall(fmt.Sprintf("provider accepted the attach of %s to %s and reports holder %q", b.AddrID, host, a.holderLabel(got.HolderID)))
+		stall(fmt.Sprintf("provider accepted the attach of %s to %s and reports holder %q", b.AddrID, host, got.HolderID))
 		return
 	}
 	// A move from another host is worth an operator's attention; a first
@@ -134,8 +111,8 @@ func (a *PoolAllocator) claim(ctx context.Context, key string, b Binding, addr A
 	}
 	slog.Log(ctx, level, "floating claimed an address whose owner runs here",
 		"pool", a.cfg.Pool.Name, "public_ip", key, "addr_id", b.AddrID,
-		"eni_id", b.ENIID, "gateway_vpc_id", b.GatewayVPCID, "from_host", a.holderLabel(addr.HolderID), "to_host", host)
-	a.recordHolder(ctx, key, b, host)
+		"eni_id", b.ENIID, "gateway_vpc_id", b.GatewayVPCID, "from_host", addr.HolderID, "to_host", host)
+	a.recordHolder(ctx, key, b.AddrID, host)
 	res.Claimed = append(res.Claimed, key)
 }
 
@@ -149,7 +126,7 @@ func (a *PoolAllocator) surrender(ctx context.Context, key string, b Binding) {
 	}
 	slog.InfoContext(ctx, "floating detached an address with no owner",
 		"pool", a.cfg.Pool.Name, "public_ip", key, "addr_id", b.AddrID)
-	a.recordHolder(ctx, key, b, "")
+	a.recordHolder(ctx, key, b.AddrID, "")
 }
 
 // ownsBinding answers whether this node is where b's traffic is handled. An
@@ -169,33 +146,13 @@ func (a *PoolAllocator) ownsBinding(ctx context.Context, b Binding, plugged map[
 	return local, nil
 }
 
-// unchanged reads the binding again and reports whether it is still the one
-// this pass snapshotted: same allocation, owner and gateway. A release and
-// re-allocation reuses the address and its provider ID, so the allocation
-// token is what tells them apart. The re-read narrows the window before the
-// provider call but does not close it: a stale move is repaired only when the
-// owning node next runs its claim pass and the provider accepts the move.
-func (a *PoolAllocator) unchanged(ctx context.Context, key string, snap Binding) (bool, error) {
-	rec, err := a.store.Get(ctx, a.cfg.Pool.Name)
-	if err != nil {
-		return false, err
-	}
-	cur, ok := rec.Bindings[key]
-	if ok && cur.Gen == snap.Gen && cur.AddrID == snap.AddrID && cur.ENIID == snap.ENIID && cur.GatewayVPCID == snap.GatewayVPCID {
-		return true, nil
-	}
-	slog.InfoContext(ctx, "floating holder skipped an address whose binding changed during the pass",
-		"pool", a.cfg.Pool.Name, "public_ip", key, "addr_id", snap.AddrID)
-	return false, nil
-}
-
 // recordHolder writes the observed holder under CAS, only while the binding
-// is still the allocation this pass acted on. A failed write is logged: the
+// still names the address this pass acted on. A failed write is logged: the
 // provider already agrees, and the next pass rewrites it.
-func (a *PoolAllocator) recordHolder(ctx context.Context, key string, snap Binding, holder string) {
+func (a *PoolAllocator) recordHolder(ctx context.Context, key, addrID, holder string) {
 	err := a.store.Mutate(ctx, a.cfg.Pool.Name, func(r *Record) (bool, error) {
 		b, ok := r.Bindings[key]
-		if !ok || b.AddrID != snap.AddrID || b.Gen != snap.Gen || b.Holder == holder {
+		if !ok || b.AddrID != addrID || b.Holder == holder {
 			return false, nil
 		}
 		b.Holder = holder

spinifex/domains/network/external/floating/latitude.go

@@ -1,121 +0,0 @@
-package floating
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"slices"
-
-	"github.com/mulgadc/spinifex/spinifex/providers/cloud/latitude"
-)
-
-// latitudeAPI is the slice of the latitude client the adapter calls, so tests can
-// stand the fake in for it.
-type latitudeAPI interface {
-	GetElasticIP(ctx context.Context, id string) (latitude.ElasticIP, error)
-	MoveElasticIP(ctx context.Context, id, serverID string) error
-}
-
-var (
-	_ latitudeAPI = (*latitude.Client)(nil)
-	_ latitudeAPI = (*latitude.Fake)(nil)
-)
-
-// LatitudeProvider adapts a fixed set of latitude.sh Elastic IPs to Provider.
-// The set is named by ID, since the API has no labels to tell ours from the
-// team's others. Addresses are never created, detached or deleted here: the API
-// has no unassign, and the addresses belong to whoever bought them.
-type LatitudeProvider struct {
-	api latitudeAPI
-	ids []string
-}
-
-var (
-	_ Provider = (*LatitudeProvider)(nil)
-	_ FixedSet = (*LatitudeProvider)(nil)
-)
-
-// NewLatitudeProvider wraps a latitude client or fake over the given Elastic IP IDs.
-func NewLatitudeProvider(api latitudeAPI, ids []string) (*LatitudeProvider, error) {
-	if len(ids) == 0 {
-		return nil, errors.New("floating: latitude pool names no Elastic IP IDs")
-	}
-	return &LatitudeProvider{api: api, ids: slices.Clone(ids)}, nil
-}
-
-func fromLatitude(e latitude.ElasticIP) Address {
-	return Address{ID: e.ID, IP: e.Address, HolderID: e.ServerID, Failed: e.Status == latitude.StatusError}
-}
-
-// mapLatitude wraps a latitude error around the engine's sentinel for it.
-func mapLatitude(err error) error {
-	switch {
-	case err == nil:
-		return nil
-	case errors.Is(err, latitude.ErrNotFound):
-		return fmt.Errorf("%w: %w", ErrNotFound, err)
-	}
-	return err
-}
-
-func (p *LatitudeProvider) member(id string) bool { return slices.Contains(p.ids, id) }
-
-// Create is refused: the set is fixed.
-func (p *LatitudeProvider) Create(context.Context, string) (Address, error) {
-	return Address{}, errors.New("floating: a latitude pool cannot create addresses")
-}
-
-// Attach moves the address to hostID and returns once it is active there.
-func (p *LatitudeProvider) Attach(ctx context.Context, addrID, hostID string) error {
-	if !p.member(addrID) {
-		return fmt.Errorf("%w: %s is not in this pool", ErrNotFound, addrID)
-	}
-	return mapLatitude(p.api.MoveElasticIP(ctx, addrID, hostID))
-}
-
-// Detach does nothing: latitude.sh cannot unassign an address, only move it.
-func (p *LatitudeProvider) Detach(context.Context, string) error { return nil }
-
-// Delete does nothing: the addresses are not ours to release.
-func (p *LatitudeProvider) Delete(context.Context, string) error { return nil }
-
-func (p *LatitudeProvider) Get(ctx context.Context, addrID string) (Address, error) {
-	if !p.member(addrID) {
-		return Address{}, fmt.Errorf("%w: %s is not in this pool", ErrNotFound, addrID)
-	}
-	e, err := p.api.GetElasticIP(ctx, addrID)
-	if err != nil {
-		return Address{}, mapLatitude(err)
-	}
-	return fromLatitude(e), nil
-}
-
-// List returns the pool's members.
-func (p *LatitudeProvider) List(ctx context.Context) ([]Address, error) {
-	return p.Members(ctx)
-}
-
-// Members reads each configured address by ID. The project listing is not used: it
-// ignores pagination and is refused when a page is full, so it cannot say who is in the set.
-// An address latitude.sh no longer has is logged and left out, so the rest of the set keeps
-// working; any other failure is returned, since the call did not establish who is there.
-func (p *LatitudeProvider) Members(ctx context.Context) ([]Address, error) {
-	out := make([]Address, 0, len(p.ids))
-	for _, id := range p.ids {
-		e, err := p.api.GetElasticIP(ctx, id)
-		if errors.Is(err, latitude.ErrNotFound) {
-			slog.ErrorContext(ctx, "floating: a configured Elastic IP no longer exists at latitude.sh; the pool runs without it",
-				"pool_member", id)
-			continue
-		}
-		if err != nil {
-			return nil, fmt.Errorf("floating: configured Elastic IP %s: %w", id, mapLatitude(err))
-		}
-		if why := e.Unusable(); why != "" {
-			return nil, fmt.Errorf("floating: configured Elastic IP %s is unusable: %s", id, why)
-		}
-		out = append(out, fromLatitude(e))
-	}
-	return out, nil
-}

spinifex/domains/network/external/floating/latitude_test.go

@@ -1,369 +0,0 @@
-package floating
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"net/http"
-	"net/http/httptest"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
-	"github.com/mulgadc/spinifex/spinifex/providers/cloud/latitude"
-)
-
-func latitudeFake() *latitude.Fake {
-	f := latitude.NewFake("sv_a", "sv_b")
-	f.Seed(latitude.ElasticIP{ID: "eip_1", Address: netip.MustParseAddr("203.0.113.1")})
-	f.Seed(latitude.ElasticIP{ID: "eip_2", Address: netip.MustParseAddr("203.0.113.2"), ServerID: "sv_a"})
-	f.Seed(latitude.ElasticIP{ID: "eip_other", Address: netip.MustParseAddr("203.0.113.9")})
-	return f
-}
-
-func TestLatitudeProvider_MembersAreOnlyTheConfiguredIDs(t *testing.T) {
-	p, err := NewLatitudeProvider(latitudeFake(), []string{"eip_1", "eip_2"})
-	require.NoError(t, err)
-	got, err := p.Members(context.Background())
-	require.NoError(t, err)
-	require.Len(t, got, 2)
-	assert.Equal(t, "eip_1", got[0].ID)
-	assert.Equal(t, "sv_a", got[1].HolderID)
-	assert.Empty(t, got[0].Label)
-}
-
-// The project listing is never read, so a project holding more addresses than a listing
-// page returns does not affect a pool of two.
-func TestLatitudeProvider_MembersIgnoreTheSizeOfTheProject(t *testing.T) {
-	f := latitudeFake()
-	for i := range 150 {
-		f.Seed(latitude.ElasticIP{ID: fmt.Sprintf("eip_x%03d", i), Address: netip.AddrFrom4([4]byte{198, 51, 100, byte(i)})})
-	}
-	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_2"})
-	require.NoError(t, err)
-	got, err := p.Members(context.Background())
-	require.NoError(t, err)
-	assert.Len(t, got, 2)
-	assert.Equal(t, []string{"get", "get"}, f.Calls)
-}
-
-func TestLatitudeProvider_AMissingMemberIsLoggedAndLeftOut(t *testing.T) {
-	buf := captureLog(t)
-	p, err := NewLatitudeProvider(latitudeFake(), []string{"eip_gone", "eip_1"})
-	require.NoError(t, err)
-	got, err := p.Members(context.Background())
-	require.NoError(t, err)
-	require.Len(t, got, 1)
-	assert.Equal(t, "eip_1", got[0].ID)
-	assert.Contains(t, buf.String(), "eip_gone")
-}
-
-func TestLatitudeProvider_AnyOtherReadFailureFailsTheCall(t *testing.T) {
-	f := latitudeFake()
-	f.Fail["get"] = errors.New("timeout")
-	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_2"})
-	require.NoError(t, err)
-	_, err = p.Members(context.Background())
-	assert.ErrorContains(t, err, "eip_1")
-}
-
-// One address gone from latitude.sh must not stop the others from being claimed.
-func TestLatitudeProvider_EngineSurvivesAMissingMember(t *testing.T) {
-	f := latitudeFake()
-	p, err := NewLatitudeProvider(f, []string{"eip_gone", "eip_1"})
-	require.NoError(t, err)
-	n := latitudeNode(t, p, newMemStore(), "sv_b")
-	ip := allocENI(t, n, "eni-1")
-	assert.Equal(t, netip.MustParseAddr("203.0.113.1"), ip)
-	n.plug("eni-1")
-	res, err := n.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, []string{ip.String()}, res.Claimed)
-	assert.Equal(t, "sv_b", f.HolderOf("eip_1"))
-}
-
-// An address the provider reports in error is not handed out, and one already on this node
-// is stalled rather than counted as settled.
-func TestLatitudeProvider_AnErrorStateAddressIsNeitherAllocatedNorSettled(t *testing.T) {
-	f := latitudeFake()
-	f.Seed(latitude.ElasticIP{ID: "eip_1", Address: netip.MustParseAddr("203.0.113.1"), Status: latitude.StatusError})
-	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_2"})
-	require.NoError(t, err)
-	store := newMemStore()
-	n := latitudeNode(t, p, store, "sv_b")
-	ip := allocENI(t, n, "eni-1")
-	assert.Equal(t, netip.MustParseAddr("203.0.113.2"), ip, "eip_1 is skipped")
-
-	n.plug("eni-1")
-	res, err := n.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, []string{ip.String()}, res.Claimed)
-
-	f.Seed(latitude.ElasticIP{ID: "eip_2", Address: ip, ServerID: "sv_b", Status: latitude.StatusError})
-	res, err = n.alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	require.Len(t, res.Stalled, 1)
-	assert.Equal(t, ip.String(), res.Stalled[0].PublicIP)
-	assert.Contains(t, res.Stalled[0].Reason, "error state")
-}
-
-func latitudeNode(t *testing.T, p Provider, store Store, host string) *node {
-	t.Helper()
-	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
-	alloc, err := New(p, store, Config{
-		Pool:       external.ExternalPoolConfig{Name: pool, Source: "latitude"},
-		HostID:     host,
-		LocalPorts: func(context.Context) (map[string]struct{}, error) { return n.plugged, nil },
-	})
-	require.NoError(t, err)
-	n.alloc = alloc
-	return n
-}
-
-func TestLatitudeProvider_AttachMovesAndDetachDeleteChangeNothing(t *testing.T) {
-	f := latitudeFake()
-	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_2"})
-	require.NoError(t, err)
-	ctx := context.Background()
-
-	require.NoError(t, p.Attach(ctx, "eip_2", "sv_b"))
-	assert.Equal(t, "sv_b", f.HolderOf("eip_2"))
-	require.NoError(t, p.Detach(ctx, "eip_2"))
-	require.NoError(t, p.Delete(ctx, "eip_2"))
-	assert.Equal(t, "sv_b", f.HolderOf("eip_2"))
-	got, err := p.Get(ctx, "eip_2")
-	require.NoError(t, err)
-	assert.Equal(t, "sv_b", got.HolderID)
-}
-
-func TestLatitudeProvider_RefusesWhatIsNotInTheSet(t *testing.T) {
-	f := latitudeFake()
-	p, err := NewLatitudeProvider(f, []string{"eip_1"})
-	require.NoError(t, err)
-	ctx := context.Background()
-
-	assert.ErrorIs(t, p.Attach(ctx, "eip_other", "sv_b"), ErrNotFound)
-	assert.Empty(t, f.HolderOf("eip_other"))
-	_, err = p.Get(ctx, "eip_other")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = p.Create(ctx, "x")
-	assert.Error(t, err)
-}
-
-func TestLatitudeProvider_MapsNotFoundAndKeepsRefusals(t *testing.T) {
-	f := latitudeFake()
-	p, err := NewLatitudeProvider(f, []string{"eip_1"})
-	require.NoError(t, err)
-	ctx := context.Background()
-
-	assert.ErrorIs(t, p.Attach(ctx, "eip_1", "sv_unknown"), latitude.ErrMoveRefused)
-	f.Fail["get"] = latitude.ErrNotFound
-	_, err = p.Get(ctx, "eip_1")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
-func TestNewLatitudeProvider_NeedsIDs(t *testing.T) {
-	_, err := NewLatitudeProvider(latitudeFake(), nil)
-	assert.Error(t, err)
-}
-
-// The engine over the adapter: allocation picks, the owner's node claims, and
-// running out is the capacity error.
-func TestLatitudeProvider_EngineEndToEnd(t *testing.T) {
-	f := latitudeFake()
-	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_2"})
-	require.NoError(t, err)
-	store := newMemStore()
-	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
-	alloc, err := New(p, store, Config{
-		Pool:       external.ExternalPoolConfig{Name: pool, Source: "latitude"},
-		HostID:     "sv_b",
-		LocalPorts: func(context.Context) (map[string]struct{}, error) { return n.plugged, nil },
-	})
-	require.NoError(t, err)
-	n.alloc = alloc
-
-	ip := allocENI(t, n, "eni-1")
-	assert.Equal(t, netip.MustParseAddr("203.0.113.1"), ip)
-	n.plug("eni-1")
-	res, err := alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, []string{ip.String()}, res.Claimed)
-	assert.Equal(t, "sv_b", f.HolderOf("eip_1"))
-
-	allocENI(t, n, "eni-2")
-	_, err = alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: "eni-public", ENIID: "eni-3"})
-	assert.ErrorContains(t, err, "InsufficientAddressCapacity")
-}
-
-func TestProviderFor_PicksByPoolSource(t *testing.T) {
-	p, err := providerFor(external.ExternalPoolConfig{
-		Name: pool, Source: external.SourceLatitude, LatitudeProjectID: "proj_1", LatitudeElasticIPIDs: []string{"eip_1"},
-	})
-	require.NoError(t, err)
-	_, isFixed := p.(FixedSet)
-	assert.True(t, isFixed)
-
-	v, err := providerFor(external.ExternalPoolConfig{Name: pool, Source: external.SourceVultr, VultrRegion: "ord"})
-	require.NoError(t, err)
-	_, isFixed = v.(FixedSet)
-	assert.False(t, isFixed)
-
-	_, err = providerFor(external.ExternalPoolConfig{Name: pool, Source: external.SourceStatic})
-	assert.ErrorContains(t, err, "not a floating provider")
-	_, err = providerFor(external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude, LatitudeProjectID: "p"})
-	assert.Error(t, err, "a latitude pool with no IDs is refused")
-}
-
-func captureLog(t *testing.T) *bytes.Buffer {
-	t.Helper()
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil)))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-	return &buf
-}
-
-// deploy-vultr.sh waits for this exact text after a restart.
-func TestReportFirstCall_VultrLogTextIsPinned(t *testing.T) {
-	buf := captureLog(t)
-	reportFirstCall(context.Background(), external.ExternalPoolConfig{Name: pool, Source: external.SourceVultr, VultrRegion: "ord"}, nil)
-	assert.Contains(t, buf.String(), `msg="floating using Vultr reserved IPs"`)
-
-	buf.Reset()
-	reportFirstCall(context.Background(), external.ExternalPoolConfig{Name: pool, Source: external.SourceVultr}, errors.New("refused"))
-	assert.Contains(t, buf.String(), `msg="floating: Vultr refused the pool's first call; public addresses will fail until fixed"`)
-}
-
-func TestReportFirstCall_LatitudeLogsItsOwnLine(t *testing.T) {
-	buf := captureLog(t)
-	reportFirstCall(context.Background(), external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude}, nil)
-	assert.Contains(t, buf.String(), `msg="floating pool ready"`)
-	assert.NotContains(t, buf.String(), "Vultr")
-}
-
-func TestLatitudeProvider_AnUnusableMemberIsAConfigError(t *testing.T) {
-	f := latitudeFake()
-	f.Seed(latitude.ElasticIP{ID: "eip_bgp", Address: netip.MustParseAddr("203.0.113.50"), Mode: "bgp", Family: "IPv4", PrefixLength: 24})
-	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_bgp"})
-	require.NoError(t, err)
-	_, err = p.Members(context.Background())
-	assert.ErrorContains(t, err, "eip_bgp")
-	assert.ErrorContains(t, err, "routed IPv4 /32")
-}
-
-// A response can put anything in server.id. It becomes the holder the engine logs as
-// from_host after a move, and only a host we configured may be named there.
-func TestHolderPass_UnrecognisedHolderIDNeverReachesTheLog(t *testing.T) {
-	const token = "FAKESECRET"
-	var mu sync.Mutex
-	holder := token // what the API reports as the holder
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		mu.Lock()
-		defer mu.Unlock()
-		if r.Method == http.MethodPatch {
-			holder = "sv_bbbbbb"
-			w.WriteHeader(http.StatusAccepted)
-			_, _ = io.WriteString(w, "{}")
-			return
-		}
-		item := `{"id":"eip_aaaaaa","attributes":{"address":"203.0.113.1","status":"active","mode":"routed","family":"IPv4","prefix_length":32,"server":{"id":"` + holder + `"}}}`
-		if strings.HasSuffix(r.URL.Path, "/eip_aaaaaa") {
-			_, _ = io.WriteString(w, `{"data":`+item+`}`)
-			return
-		}
-		_, _ = io.WriteString(w, `{"data":[`+item+`]}`)
-	}))
-	t.Cleanup(srv.Close)
-	keyFile := filepath.Join(t.TempDir(), "token")
-	require.NoError(t, os.WriteFile(keyFile, []byte(token), 0o600))
-	client, err := latitude.New(latitude.Config{BaseURL: srv.URL, KeyFile: keyFile, ProjectID: "p", HTTPClient: srv.Client(), PollInterval: time.Millisecond})
-	require.NoError(t, err)
-	p, err := NewLatitudeProvider(client, []string{"eip_aaaaaa"})
-	require.NoError(t, err)
-
-	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
-	alloc, err := New(p, newMemStore(), Config{
-		Pool:       external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude},
-		HostID:     "sv_bbbbbb",
-		LocalPorts: func(context.Context) (map[string]struct{}, error) { return n.plugged, nil },
-	})
-	require.NoError(t, err)
-	n.alloc = alloc
-	allocENI(t, n, "eni-1")
-	n.plug("eni-1")
-
-	logs := captureLog(t)
-	res, err := alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.Len(t, res.Claimed, 1)
-	assert.Contains(t, logs.String(), "floating claimed an address", "the move is logged")
-	assert.NotContains(t, logs.String(), token)
-	assert.Contains(t, logs.String(), "unrecognised holder")
-}
-
-// A successful response whose id is the token must not reach the bindings or the log.
-func TestMembers_AResponseIDThatIsTheTokenIsRefusedAndNeverLogged(t *testing.T) {
-	const token = "FAKESECRET"
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		_, _ = io.WriteString(w, `{"data":{"id":"`+token+`","attributes":{"address":"203.0.113.1","status":"active","mode":"routed","family":"IPv4","prefix_length":32}}}`)
-	}))
-	t.Cleanup(srv.Close)
-	keyFile := filepath.Join(t.TempDir(), "token")
-	require.NoError(t, os.WriteFile(keyFile, []byte(token), 0o600))
-	client, err := latitude.New(latitude.Config{BaseURL: srv.URL, KeyFile: keyFile, ProjectID: "p", HTTPClient: srv.Client(), PollInterval: time.Millisecond})
-	require.NoError(t, err)
-	p, err := NewLatitudeProvider(client, []string{"eip_aaaaaa"})
-	require.NoError(t, err)
-
-	n := latitudeNode(t, p, newMemStore(), "sv_b")
-	logs := captureLog(t)
-	_, err = n.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: "eni-public", ENIID: "eni-1"})
-	require.Error(t, err)
-	assert.NotContains(t, err.Error(), token)
-	assert.NotContains(t, logs.String(), token)
-}
-
-// A short server ID that is ours is a real holder: the address is settled, with no move.
-func TestHolderPass_ShortConfiguredHolderIDIsSettled(t *testing.T) {
-	f := latitudeFake()
-	f.Seed(latitude.ElasticIP{ID: "eip_1", Address: netip.MustParseAddr("203.0.113.1"), ServerID: "sv_a"})
-	p, err := NewLatitudeProvider(f, []string{"eip_1"})
-	require.NoError(t, err)
-	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
-	alloc, err := New(p, newMemStore(), Config{
-		Pool:       external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude},
-		HostID:     "sv_a",
-		LocalPorts: func(context.Context) (map[string]struct{}, error) { return n.plugged, nil },
-	})
-	require.NoError(t, err)
-	n.alloc = alloc
-	allocENI(t, n, "eni-1")
-	n.plug("eni-1")
-
-	_, err = alloc.ClaimLocalAddresses(context.Background())
-	require.NoError(t, err)
-	assert.NotContains(t, f.Calls, "move")
-	assert.Equal(t, "sv_a", f.HolderOf("eip_1"))
-}
-
-func TestHolderLabel_NamesOnlyConfiguredHosts(t *testing.T) {
-	a := &PoolAllocator{fixed: true, cfg: Config{HostID: "sv_a", KnownHostIDs: []string{"sv_b"}}}
-	assert.Equal(t, "sv_a", a.holderLabel("sv_a"))
-	assert.Equal(t, "sv_b", a.holderLabel("sv_b"))
-	assert.Empty(t, a.holderLabel(""))
-	assert.Equal(t, "unrecognised holder", a.holderLabel("FAKESECRET"))
-	other := &PoolAllocator{cfg: Config{HostID: "sv_a"}}
-	assert.Equal(t, "whatever-the-provider-says", other.holderLabel("whatever-the-provider-says"))
-}

spinifex/domains/network/external/floating/provider.go

@@ -24,9 +24,6 @@ type Address struct {
 	// HolderID is the provider host the address is attached to; empty when
 	// it is attached nowhere.
 	HolderID string
-	// Failed is set when the provider reports the address broken, so it is neither
-	// handed out nor treated as settled where it is. Only a fixed set reports it.
-	Failed bool
 }
 
 // Provider is the vendor half of a floating public address. Errors are wrapped
@@ -46,15 +43,6 @@ type Provider interface {
 	List(ctx context.Context) ([]Address, error)
 }
 
-// FixedSet is implemented by a provider whose addresses are bought ahead of
-// time and handed out, not created per allocation. The engine picks a member
-// that has no binding inside the pool's record update, so two nodes cannot
-// take the same one, and never creates, labels or deletes a member.
-type FixedSet interface {
-	// Members returns every address of the set with its current holder.
-	Members(ctx context.Context) ([]Address, error)
-}
-
 var (
 	// ErrNotFound means the address does not exist at the provider.
 	ErrNotFound = errors.New("floating: address not found")

spinifex/domains/network/external/floating/reconcile.go

@@ -55,7 +55,7 @@ func (a *PoolAllocator) Reconcile(ctx context.Context) (ReconcileResult, error)
 	var errs []error
 	for _, addr := range live {
 		liveIDs[addr.ID] = struct{}{}
-		if a.fixed || !a.ours(addr) {
+		if !a.ours(addr) {
 			res.Skipped++
 			continue
 		}
@@ -73,7 +73,7 @@ func (a *PoolAllocator) Reconcile(ctx context.Context) (ReconcileResult, error)
 		}
 		slog.WarnContext(ctx, "floating reconcile collected a leaked address",
 			"pool", a.cfg.Pool.Name, "addr_id", addr.ID, "public_ip", addr.IP.String(),
-			"label", addr.Label, "holder", a.holderLabel(addr.HolderID))
+			"label", addr.Label, "holder", addr.HolderID)
 		res.Collected = append(res.Collected, addr.ID)
 	}
 	a.suspects = suspects

spinifex/domains/network/external/floating/wiring.go

@@ -10,56 +10,36 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/providers/cloud/latitude"
 	"github.com/mulgadc/spinifex/spinifex/providers/cloud/vultr"
 )
 
 // FromPoolConfig builds a live allocator for one floating pool and proves the
 // credential works before anything depends on it. It does not reconcile; the
 // caller runs that pass. sbAddr is the OVN Southbound address the
-// gateway-chassis question is asked of; empty uses the local socket. peerHostIDs are
-// the other nodes' provider host IDs, the only holders a log may name.
-func FromPoolConfig(ctx context.Context, js jetstream.JetStream, pool external.ExternalPoolConfig, sbAddr string, peerHostIDs []string) (*PoolAllocator, error) {
-	provider, err := providerFor(pool)
+// gateway-chassis question is asked of; empty uses the local socket.
+func FromPoolConfig(ctx context.Context, js jetstream.JetStream, pool external.ExternalPoolConfig, sbAddr string) (*PoolAllocator, error) {
+	if !pool.IsVultr() {
+		return nil, fmt.Errorf("floating: pool %q has source %q, not a floating provider", pool.Name, pool.Source)
+	}
+	client, err := vultr.New(vultr.Config{Region: pool.VultrRegion, KeyFile: pool.VultrKeyFile})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
 	}
-	return fromProvider(ctx, js, pool, sbAddr, provider, peerHostIDs)
-}
+	provider := NewVultrProvider(client)
 
-// providerFor builds the pool's provider from its source.
-func providerFor(pool external.ExternalPoolConfig) (Provider, error) {
-	switch {
-	case pool.IsVultr():
-		client, err := vultr.New(vultr.Config{Region: pool.VultrRegion, KeyFile: pool.VultrKeyFile})
-		if err != nil {
-			return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
-		}
-		return NewVultrProvider(client), nil
-	case pool.IsLatitude():
-		client, err := latitude.New(latitude.Config{ProjectID: pool.LatitudeProjectID, KeyFile: pool.LatitudeKeyFile})
-		if err != nil {
-			return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
-		}
-		p, err := NewLatitudeProvider(client, pool.LatitudeElasticIPIDs)
-		if err != nil {
-			return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
-		}
-		return p, nil
+	// An allowlisted key refused from this node's address looks ready until the
+	// first launch, so the refusal is surfaced at start with Vultr's reason.
+	if _, err := provider.List(ctx); err != nil {
+		slog.ErrorContext(ctx, "floating: Vultr refused the pool's first call; public addresses will fail until fixed",
+			"pool", pool.Name, "region", pool.VultrRegion, "error", err)
+	} else {
+		slog.InfoContext(ctx, "floating using Vultr reserved IPs", "pool", pool.Name,
+			"region", pool.VultrRegion, "provider_host_id", pool.ProviderHostID, "label_prefix", pool.LabelPrefix)
 	}
-	return nil, fmt.Errorf("floating: pool %q has source %q, not a floating provider", pool.Name, pool.Source)
-}
-
-func fromProvider(ctx context.Context, js jetstream.JetStream, pool external.ExternalPoolConfig, sbAddr string, provider Provider, peerHostIDs []string) (*PoolAllocator, error) {
-	// A refused key looks ready until the first launch, so the refusal is
-	// surfaced at start with the provider's reason.
-	_, err := provider.List(ctx)
-	reportFirstCall(ctx, pool, err)
 
 	return New(provider, NewKVStore(js), Config{
-		Pool:         pool,
-		HostID:       pool.ProviderHostID,
-		KnownHostIDs: peerHostIDs,
+		Pool:   pool,
+		HostID: pool.ProviderHostID,
 		LocalPorts: func(ctx context.Context) (map[string]struct{}, error) {
 			return host.ListLocalPorts(ctx, host.NewExecRunner())
 		},
@@ -69,22 +49,3 @@ func fromProvider(ctx context.Context, js jetstream.JetStream, pool external.Ext
 		},
 	})
 }
-
-// reportFirstCall logs the outcome of the pool's first provider call. The Vultr
-// lines are matched by the deploy script, so their text must not change.
-func reportFirstCall(ctx context.Context, pool external.ExternalPoolConfig, err error) {
-	switch {
-	case pool.IsVultr() && err != nil:
-		slog.ErrorContext(ctx, "floating: Vultr refused the pool's first call; public addresses will fail until fixed",
-			"pool", pool.Name, "region", pool.VultrRegion, "error", err)
-	case pool.IsVultr():
-		slog.InfoContext(ctx, "floating using Vultr reserved IPs", "pool", pool.Name,
-			"region", pool.VultrRegion, "provider_host_id", pool.ProviderHostID, "label_prefix", pool.LabelPrefix)
-	case err != nil:
-		slog.ErrorContext(ctx, "floating: the provider refused the pool's first call; public addresses will fail until fixed",
-			"pool", pool.Name, "source", pool.Source, "error", err)
-	default:
-		slog.InfoContext(ctx, "floating pool ready", "pool", pool.Name, "source", pool.Source,
-			"provider_host_id", pool.ProviderHostID)
-	}
-}

spinifex/domains/network/external/types.go

@@ -52,12 +52,6 @@ type ExternalPoolConfig struct {
 	VultrRegion  string
 	VultrKeyFile string
 	LabelPrefix  string
-	// Latitude* configure a source="latitude" pool: the project the Elastic IPs
-	// belong to, the file holding the API token, and the fixed set of Elastic IP
-	// IDs the pool hands out. latitude.sh has no labels, so the set is named.
-	LatitudeProjectID    string
-	LatitudeKeyFile      string
-	LatitudeElasticIPIDs []string
 	// ProviderHostID is this node's instance ID at the provider, which a
 	// floating address is attached to. Per node, so it is filled from the
 	// node's own config rather than the shared pool section.
@@ -90,17 +84,6 @@ func (p *ExternalPoolConfig) IsVultr() bool {
 	return p != nil && p.Source == SourceVultr
 }
 
-// IsLatitude reports whether the pool sources IPs from latitude.sh Elastic IPs.
-func (p *ExternalPoolConfig) IsLatitude() bool {
-	return p != nil && p.Source == SourceLatitude
-}
-
-// IsFloating reports whether the pool's addresses are provider objects moved
-// between nodes by the floating engine.
-func (p *ExternalPoolConfig) IsFloating() bool {
-	return p.IsVultr() || p.IsLatitude()
-}
-
 const (
 	// SourceStatic is the default pool source (inline range math, KV-backed).
 	SourceStatic = "static"
@@ -119,9 +102,6 @@ const (
 	// holder addressed to itself, so like Exoscale there is no private half.
 	// A VM is routed it by MAC; bare metal is ARPed for it on the public NIC.
 	SourceVultr = "vultr"
-	// SourceLatitude hands out a fixed set of latitude.sh Elastic IPs, moved
-	// between servers. A routed Elastic IP arrives addressed to the host's MAC.
-	SourceLatitude = "latitude"
 
 	// OCIAuthConfigFile authenticates from an API-key config file. The default,
 	// because it is what an operator already has from `oci setup config`.

spinifex/domains/network/host/owned_tap.go

@@ -1,128 +0,0 @@
-//go:build lambdatestlink
-
-package host
-
-import (
-	"errors"
-	"fmt"
-	"log/slog"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
-)
-
-// ErrTapExists means a link with the requested name already exists; nothing
-// was changed.
-var ErrTapExists = errors.New("tap name already in use")
-
-// ErrTapNotCreated wraps a CreateOwnedTap failure that left no link behind.
-// Any other failure may have left one.
-var ErrTapNotCreated = errors.New("no tap created")
-
-// ownedTapSysfs is a var so tests can stand in a fake /sys/class/net.
-var ownedTapSysfs = "/sys/class/net"
-
-const ownerExternalID = "spinifex-owner"
-
-// OwnedTap identifies a tap CreateOwnedTap made: a later remove acts only on a
-// link with the same ifindex and alias, and an OVS port with the same owner.
-type OwnedTap struct {
-	Name    string
-	IfIndex int
-	Owner   string
-}
-
-// CreateOwnedTap creates and attaches a new tap, refusing to replace a link or
-// OVS port of the same name. The check-then-create gap remains, so callers use
-// names no other creator would pick.
-func (p *OVSPlumber) CreateOwnedTap(spec vm.TapSpec, owner string) (OwnedTap, error) {
-	if owner == "" || strings.ContainsAny(owner, " \t\n\"'") {
-		return OwnedTap{}, fmt.Errorf("%w: invalid owner", ErrTapNotCreated)
-	}
-	if _, err := os.Lstat(filepath.Join(ownedTapSysfs, spec.Name)); err == nil {
-		return OwnedTap{}, fmt.Errorf("%w: %w: %s", ErrTapNotCreated, ErrTapExists, spec.Name)
-	}
-	addArgs := []string{"tuntap", "add", "dev", spec.Name, "mode", "tap"}
-	if uid := os.Geteuid(); uid != 0 {
-		addArgs = append(addArgs, "user", strconv.Itoa(uid), "group", strconv.Itoa(os.Getegid()))
-	}
-	if out, err := hostcommand.SudoCommand("ip", addArgs...).CombinedOutput(); err != nil {
-		// The add may have taken effect before failing, so this is not ErrTapNotCreated.
-		return OwnedTap{}, fmt.Errorf("create tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err)
-	}
-	idx, err := readIfIndex(spec.Name)
-	if err != nil {
-		// Without its ifindex the new link cannot be told apart, so it stays.
-		return OwnedTap{}, fmt.Errorf("created tap %s but cannot read its ifindex; left in place: %w", spec.Name, err)
-	}
-	tap := OwnedTap{Name: spec.Name, IfIndex: idx, Owner: owner}
-	upArgs := []string{"link", "set", "dev", spec.Name, "alias", owner, "up"}
-	if spec.MTU > 0 {
-		upArgs = append(upArgs, "mtu", strconv.Itoa(spec.MTU))
-	}
-	if out, err := hostcommand.SudoCommand("ip", upArgs...).CombinedOutput(); err != nil {
-		return tap, errors.Join(fmt.Errorf("bring up tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err), p.RemoveOwnedTap(tap))
-	}
-	// No --may-exist: a port record of the same name fails the add.
-	addPort := []string{"add-port", spec.Bridge, spec.Name, "--", "set", "Interface", spec.Name,
-		fmt.Sprintf("external_ids:%s=%s", ownerExternalID, owner)}
-	if out, err := hostcommand.SudoCommand("ovs-vsctl", addPort...).CombinedOutput(); err != nil {
-		return tap, errors.Join(fmt.Errorf("add tap %s to %s: %s: %w", spec.Name, spec.Bridge, strings.TrimSpace(string(out)), err), p.removeOwnedLink(tap))
-	}
-	return tap, nil
-}
-
-// RemoveOwnedTap removes the OVS port and kernel link only where each still
-// carries tap's owner; anything else under that name is left alone. A link
-// already gone is success.
-func (p *OVSPlumber) RemoveOwnedTap(tap OwnedTap) error {
-	out, err := hostcommand.SudoCommand("ovs-vsctl", "--if-exists", "get", "Interface", tap.Name,
-		"external_ids:"+ownerExternalID).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("read OVS owner of %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
-	}
-	switch got := strings.Trim(strings.TrimSpace(string(out)), `"`); got {
-	case tap.Owner:
-		if out, err := hostcommand.SudoCommand("ovs-vsctl", "--if-exists", "del-port", tap.Name).CombinedOutput(); err != nil {
-			return fmt.Errorf("remove OVS port %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
-		}
-	case "":
-	default:
-		return fmt.Errorf("OVS port %s belongs to another owner; left in place", tap.Name)
-	}
-	return p.removeOwnedLink(tap)
-}
-
-func (p *OVSPlumber) removeOwnedLink(tap OwnedTap) error {
-	idx, err := readIfIndex(tap.Name)
-	if errors.Is(err, os.ErrNotExist) {
-		return nil
-	}
-	if err != nil {
-		return fmt.Errorf("read ifindex of %s: %w", tap.Name, err)
-	}
-	alias, err := os.ReadFile(filepath.Join(ownedTapSysfs, tap.Name, "ifalias"))
-	if err != nil {
-		return fmt.Errorf("read alias of %s: %w", tap.Name, err)
-	}
-	if idx != tap.IfIndex || strings.TrimSpace(string(alias)) != tap.Owner {
-		return fmt.Errorf("link %s is not the tap this owner created; left in place", tap.Name)
-	}
-	if out, err := deleteTapLink(tap.Name).CombinedOutput(); err != nil {
-		return fmt.Errorf("delete tap %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
-	}
-	slog.Info("Owned tap removed", "tap", tap.Name)
-	return nil
-}
-
-func readIfIndex(name string) (int, error) {
-	b, err := os.ReadFile(filepath.Join(ownedTapSysfs, name, "ifindex"))
-	if err != nil {
-		return 0, err
-	}
-	return strconv.Atoi(strings.TrimSpace(string(b)))
-}

spinifex/domains/network/host/owned_tap_test.go

@@ -1,164 +0,0 @@
-//go:build lambdatestlink
-
-package host
-
-import (
-	"errors"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"slices"
-	"strings"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
-	"github.com/stretchr/testify/require"
-)
-
-// fakeHost stands in for sysfs and the privileged commands: ip creates and
-// deletes link directories, ovs-vsctl reports a port owner.
-type fakeHost struct {
-	sysfs     string
-	calls     []string
-	ovsOwner  string
-	failCmd   string
-	nextIndex string
-}
-
-func newFakeHost(t *testing.T) *fakeHost {
-	f := &fakeHost{sysfs: t.TempDir(), nextIndex: "41"}
-	orig := ownedTapSysfs
-	ownedTapSysfs = f.sysfs
-	t.Cleanup(func() { ownedTapSysfs = orig })
-	t.Cleanup(hostcommand.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd {
-		call := name + " " + strings.Join(args, " ")
-		f.calls = append(f.calls, call)
-		if f.failCmd != "" && strings.Contains(call, f.failCmd) {
-			return exec.Command("/bin/false")
-		}
-		switch {
-		case name == "ip" && args[0] == "tuntap":
-			f.link(t, args[3], f.nextIndex, "")
-		case name == "ip" && slices.Contains(args, "alias"):
-			i := slices.Index(args, "alias")
-			require.NoError(t, os.WriteFile(filepath.Join(f.sysfs, args[3], "ifalias"), []byte(args[i+1]+"\n"), 0o600))
-		case name == "ip" && args[0] == "link" && args[1] == "del":
-			require.NoError(t, os.RemoveAll(filepath.Join(f.sysfs, args[3])))
-		case name == "ovs-vsctl" && slices.Contains(args, "get"):
-			return exec.Command("/bin/echo", f.ovsOwner)
-		}
-		return exec.Command("/bin/true")
-	}))
-	return f
-}
-
-func (f *fakeHost) link(t *testing.T, name, index, alias string) {
-	dir := filepath.Join(f.sysfs, name)
-	require.NoError(t, os.MkdirAll(dir, 0o700))
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "ifindex"), []byte(index+"\n"), 0o600))
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "ifalias"), []byte(alias+"\n"), 0o600))
-}
-
-func (f *fakeHost) ran(sub string) bool {
-	return slices.ContainsFunc(f.calls, func(c string) bool { return strings.Contains(c, sub) })
-}
-
-func TestCreateOwnedTap_RefusesAnExistingLinkWithoutTouchingIt(t *testing.T) {
-	f := newFakeHost(t)
-	f.link(t, "lm1", "7", "someone-else")
-	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
-	require.ErrorIs(t, err, ErrTapExists)
-	require.ErrorIs(t, err, ErrTapNotCreated)
-	require.Empty(t, f.calls, "an existing link must not be deleted, replaced or detached")
-}
-
-func TestCreateOwnedTap_CreatesWithoutReplacing(t *testing.T) {
-	f := newFakeHost(t)
-	tap, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt", MTU: 1400}, "lambda-a")
-	require.NoError(t, err)
-	require.Equal(t, OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}, tap)
-	require.True(t, f.ran("add-port br-mgmt lm1 -- set Interface lm1 external_ids:spinifex-owner=lambda-a"))
-	require.True(t, f.ran("alias lambda-a up mtu 1400"))
-	require.False(t, f.ran("del-port"), "creation never removes an existing port")
-	require.False(t, f.ran("--may-exist"))
-	require.False(t, f.ran("link del"))
-}
-
-func TestCreateOwnedTap_OnlyARefusalClaimsNothingCreated(t *testing.T) {
-	f := newFakeHost(t)
-	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "bad owner")
-	require.ErrorIs(t, err, ErrTapNotCreated)
-	require.Empty(t, f.calls)
-
-	f = newFakeHost(t)
-	f.failCmd = "tuntap add"
-	_, err = NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
-	require.Error(t, err)
-	require.NotErrorIs(t, err, ErrTapNotCreated, "a failed add may still have created the link")
-
-	f = newFakeHost(t)
-	f.nextIndex = "not-a-number"
-	tap, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm2", Bridge: "br-mgmt"}, "lambda-a")
-	require.Error(t, err)
-	require.NotErrorIs(t, err, ErrTapNotCreated, "a link exists that the caller cannot identify")
-	require.Empty(t, tap.Name)
-	require.False(t, f.ran("link del"))
-}
-
-func TestCreateOwnedTap_FailedAttachRemovesOnlyItsOwnLink(t *testing.T) {
-	f := newFakeHost(t)
-	f.failCmd = "add-port"
-	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
-	require.Error(t, err)
-	require.True(t, f.ran("link del dev lm1"))
-	require.False(t, f.ran("del-port"), "a port this call failed to add is never removed")
-}
-
-func TestRemoveOwnedTap_LeavesAnythingItDidNotCreate(t *testing.T) {
-	cases := map[string]struct {
-		index, alias, ovsOwner string
-		wantDelPort            bool
-	}{
-		"other OVS owner":  {index: "41", alias: "lambda-a", ovsOwner: `"other"`},
-		"replaced link":    {index: "99", alias: "lambda-a", ovsOwner: `"lambda-a"`, wantDelPort: true},
-		"other link alias": {index: "41", alias: "other", ovsOwner: ""},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			f := newFakeHost(t)
-			f.link(t, "lm1", tc.index, tc.alias)
-			f.ovsOwner = tc.ovsOwner
-			err := NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"})
-			require.Error(t, err)
-			require.False(t, f.ran("link del"))
-			require.Equal(t, tc.wantDelPort, f.ran("del-port"))
-		})
-	}
-}
-
-func TestRemoveOwnedTap_RemovesItsOwnTap(t *testing.T) {
-	f := newFakeHost(t)
-	f.link(t, "lm1", "41", "lambda-a")
-	f.ovsOwner = `"lambda-a"`
-	require.NoError(t, NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}))
-	require.True(t, f.ran("del-port lm1"))
-	require.True(t, f.ran("link del dev lm1"))
-
-	// Already gone is success, and touches nothing.
-	f.calls, f.ovsOwner = nil, ""
-	require.NoError(t, NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}))
-	require.False(t, f.ran("del-port"))
-	require.False(t, f.ran("link del"))
-}
-
-func TestRemoveOwnedTap_OVSReadFailureRemovesNothing(t *testing.T) {
-	f := newFakeHost(t)
-	f.link(t, "lm1", "41", "lambda-a")
-	f.failCmd = "get Interface"
-	err := NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"})
-	require.Error(t, err)
-	require.False(t, f.ran("del-port"))
-	require.False(t, f.ran("link del"))
-	require.False(t, errors.Is(err, ErrTapExists))
-}

spinifex/domains/rds/instance/delete.go

@@ -19,10 +19,6 @@ import (
 
 var errFinalSnapshotInProgress = errors.New("rds: final DB snapshot creation is still in progress")
 
-// errModifyInProgress defers a teardown until a modify that already holds the
-// lease, and may be growing the data volume, has let go of it.
-var errModifyInProgress = errors.New("rds: a modification of the DB instance is still in progress")
-
 type finalSnapshotReservation struct {
 	creator  bool
 	revision uint64
@@ -97,10 +93,6 @@ func (s *Service) DeleteDBInstance(ctx context.Context, input *rds.DeleteDBInsta
 	}
 
 	if err := s.teardownDBInstance(ctx, kv, accountID, rec, reservation.creator); err != nil {
-		// The reconciler finishes the delete once the modify lets go.
-		if errors.Is(err, errModifyInProgress) {
-			return &rds.DeleteDBInstanceOutput{DBInstance: s.projectDBInstance(rec)}, nil
-		}
 		return nil, err
 	}
 
@@ -185,29 +177,15 @@ func (s *Service) checkFinalSnapshotAvailable(ctx context.Context, kv *kvstore.B
 // any point it stopped at.
 func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, accountID string,
 	rec *DBInstanceRecord, finalSnapshotCreator bool) error {
-	current, _, err := s.getDBInstance(ctx, kv, rec.DBInstanceIdentifier)
-	if err != nil {
-		return err
-	}
-	if current.ModifyLease.live() {
-		return errModifyInProgress
-	}
-	// A modify that finished since the delete was accepted may have replaced the
-	// VM, so the teardown works from the record as it is now.
-	rec = current
-
 	// Asked to stop cleanly even though the VM is about to go: it is what makes
 	// the final snapshot a clean checkpoint rather than one needing WAL replay.
 	if rec.FinalSnapshotIdentifier != "" {
 		s.stopEngineOrRecordFallback(ctx, accountID, rec, "deleting")
 	}
 
-	if err := s.terminateInstanceVM(ctx, rec.InstanceID, rec.DataVolumeID); err != nil {
+	if err := s.terminateInstanceVM(ctx, rec.InstanceID); err != nil {
 		return fmt.Errorf("rds: terminate the VM behind %s: %w", rec.DBInstanceIdentifier, err)
 	}
-	if err := s.requireDataVolumeDetached(ctx, rec.DataVolumeID, rec.InstanceID); err != nil {
-		return err
-	}
 	if err := s.takeFinalSnapshot(ctx, kv, accountID, rec, finalSnapshotCreator); err != nil {
 		return err
 	}
@@ -246,9 +224,9 @@ func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, ac
 	return nil
 }
 
-// A VM no node owns is either already gone or stopped, and a stopped one is
-// terminated from its shared record once its data volume is out of that record.
-func (s *Service) terminateInstanceVM(ctx context.Context, instanceID, dataVolumeID string) error {
+// A VM no node owns is already terminated, which is the state this is trying to
+// reach.
+func (s *Service) terminateInstanceVM(ctx context.Context, instanceID string) error {
 	if instanceID == "" {
 		return nil
 	}
@@ -256,81 +234,13 @@ func (s *Service) terminateInstanceVM(ctx context.Context, instanceID, dataVolum
 	if launcher == nil {
 		return errors.New("rds: no system-instance launcher configured")
 	}
-	err := launcher.TerminateSystemInstance(instanceID)
-	if err == nil {
-		return nil
-	}
-	if !errors.Is(err, systeminstance.ErrSystemInstanceNotFound) {
+	if err := launcher.TerminateSystemInstance(instanceID); err != nil &&
+		!errors.Is(err, systeminstance.ErrSystemInstanceNotFound) {
 		return err
 	}
-	if s.deps.Instances == nil {
-		return nil
-	}
-	if err := s.releaseDataVolumeFromStopped(ctx, instanceID, dataVolumeID); err != nil {
-		return err
-	}
-	if err := s.deps.Instances.TerminateStoppedInstance(ctx, instanceID); err != nil {
-		return fmt.Errorf("terminate the stopped VM %s: %w", instanceID, err)
-	}
 	return nil
 }
 
-// Anything that later terminates the stopped record, or retries its teardown,
-// detaches every volume the record names, whatever that volume is attached to
-// by then. So the data volume leaves the record before the volume is released.
-func (s *Service) releaseDataVolumeFromStopped(ctx context.Context, instanceID, volumeID string) error {
-	if volumeID == "" {
-		return nil
-	}
-	if err := s.deps.Instances.DetachVolumeFromStopped(ctx, instanceID, volumeID); err != nil {
-		return fmt.Errorf("rds: remove the data volume %s from the stopped VM %s: %w", volumeID, instanceID, err)
-	}
-	attachedTo, err := s.dataVolumeAttachment(ctx, volumeID)
-	if err != nil || attachedTo != instanceID {
-		return err
-	}
-	// The release a customer's detach from a stopped instance makes, which owes
-	// the node still exporting the volume an unpublish.
-	if _, err := s.deps.Storage.ForceDetachVolume(ctx, &ec2.DetachVolumeInput{
-		VolumeId: aws.String(volumeID), Force: aws.Bool(true),
-	}, awsidentifiers.GlobalAccountID); err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidVolumeNotFound) {
-		return fmt.Errorf("rds: release the data volume %s from the stopped VM %s: %w", volumeID, instanceID, err)
-	}
-	return nil
-}
-
-// A volume left attached would be retained as if a snapshot held it, so the
-// teardown stops here and its retry releases the volume.
-func (s *Service) requireDataVolumeDetached(ctx context.Context, volumeID, instanceID string) error {
-	if volumeID == "" || instanceID == "" {
-		return nil
-	}
-	attachedTo, err := s.dataVolumeAttachment(ctx, volumeID)
-	if err != nil {
-		return err
-	}
-	if attachedTo == instanceID {
-		return fmt.Errorf("rds: the data volume %s is still attached to the terminated VM %s", volumeID, instanceID)
-	}
-	return nil
-}
-
-// The instance the data volume is attached to, empty when it is detached, gone
-// or there is no volume service to ask.
-func (s *Service) dataVolumeAttachment(ctx context.Context, volumeID string) (string, error) {
-	if volumeID == "" || s.deps.Storage == nil {
-		return "", nil
-	}
-	_, attachedTo, err := s.dataVolumeState(ctx, volumeID)
-	if err != nil {
-		if awserrors.IsErrorCode(err, awserrors.ErrorInvalidVolumeNotFound) || errors.Is(err, errDataVolumeGone) {
-			return "", nil
-		}
-		return "", fmt.Errorf("rds: read the data volume %s: %w", volumeID, err)
-	}
-	return attachedTo, nil
-}
-
 // Reserves the final snapshot name before deletion becomes destructive. The
 // creator is the only worker allowed to cut new data; retries may adopt data it
 // already cut, but cannot race it into making a duplicate snapshot.

spinifex/domains/rds/instance/delete_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/rds"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -380,145 +379,3 @@ func TestReconciler_ResumesAnInterruptedStop(t *testing.T) {
 	assert.Equal(t, []string{"stop:" + testInstance}, h.cmdr.calls)
 	assert.Equal(t, StatusStopped, h.record(t).Status)
 }
-
-// A stopped DB VM is held by no node, so the system terminate finds nothing. Its
-// data volume leaves the shared record and is released before the record is
-// terminated, so no later terminate of that record can detach the volume again.
-func TestDeleteDBInstance_TerminatesAStoppedVMFromItsRecord(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
-	h.storage.attachedTo = map[string]string{"vol-rdsdata01": testInstance}
-	h.cmdr.onTerminateStopped = func(string) {
-		assert.Empty(t, h.storage.attachedTo, "the data volume is released before the record is terminated")
-	}
-	seedInstance(t, h.svc, availableRecord())
-
-	_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
-	require.NoError(t, err)
-
-	assert.Equal(t, []string{
-		"detach-stopped:" + testInstance + ":vol-rdsdata01",
-		"terminate-stopped:" + testInstance,
-	}, h.cmdr.calls)
-	assert.Equal(t, []string{"vol-rdsdata01"}, h.storage.detached)
-	assert.Equal(t, []string{"vol-rdsdata01"}, h.volumes.deleted)
-	assert.False(t, h.recordExists(t, testDBID))
-}
-
-// A record edit that fails leaves the record naming the volume, so neither the
-// release nor the terminate may run; the teardown stops and is retried.
-func TestDeleteDBInstance_StopsWhileTheStoppedRecordStillNamesTheVolume(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
-	h.storage.attachedTo = map[string]string{"vol-rdsdata01": testInstance}
-	h.cmdr.detachStoppedErr = errors.New(awserrors.ErrorIncorrectInstanceState)
-	seedInstance(t, h.svc, availableRecord())
-
-	_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "remove the data volume")
-
-	assert.NotContains(t, h.cmdr.calls, "terminate-stopped:"+testInstance)
-	assert.Empty(t, h.storage.detached)
-	assert.Empty(t, h.volumes.deleted)
-	assert.True(t, h.recordExists(t, testDBID), "the record stays so the teardown can be resumed")
-}
-
-// A release that keeps failing must not let the volume be read as snapshot-held:
-// the teardown stops with the record kept, and is retried.
-func TestDeleteDBInstance_StopsWhileTheDataVolumeCannotBeReleased(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
-	h.storage.attachedTo = map[string]string{"vol-rdsdata01": testInstance}
-	h.storage.detachErr = errors.New("the volume store is unavailable")
-	seedInstance(t, h.svc, availableRecord())
-
-	_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "release the data volume")
-
-	assert.NotContains(t, h.cmdr.calls, "terminate-stopped:"+testInstance)
-	assert.Empty(t, h.volumes.deleted)
-	_, retained := h.retainedVolume(t, "vol-rdsdata01")
-	assert.False(t, retained, "an attached volume must not be recorded as retained")
-	assert.True(t, h.recordExists(t, testDBID), "the record stays so the teardown can be resumed")
-}
-
-// A terminate that left the data volume attached stops the teardown before the
-// release; the retry finds no VM and releases the volume itself.
-func TestDeleteDBInstance_RetryReleasesWhatATerminateLeftAttached(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	h.storage.attachedTo = map[string]string{"vol-rdsdata01": testInstance}
-	seedInstance(t, h.svc, availableRecord())
-
-	_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "still attached to the terminated VM")
-	assert.Empty(t, h.volumes.deleted)
-	assert.True(t, h.recordExists(t, testDBID))
-
-	h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	rec := h.record(t)
-	require.NoError(t, h.svc.teardownDBInstance(t.Context(), kv, testAccountID, &rec, false))
-	assert.Equal(t, []string{"vol-rdsdata01"}, h.storage.detached)
-	assert.Equal(t, []string{"vol-rdsdata01"}, h.volumes.deleted)
-	assert.False(t, h.recordExists(t, testDBID))
-}
-
-// A stopped VM that refuses the terminate is not gone, so the teardown must not
-// carry on as though it were.
-func TestDeleteDBInstance_ReturnsAStoppedTerminateRefusal(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
-	h.cmdr.terminateStoppedErr = errors.New(awserrors.ErrorIncorrectInstanceState)
-	seedInstance(t, h.svc, availableRecord())
-
-	_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), awserrors.ErrorIncorrectInstanceState)
-	assert.Empty(t, h.volumes.deleted)
-	assert.True(t, h.recordExists(t, testDBID))
-}
-
-// A class change that finished after the delete was accepted replaced the VM;
-// the teardown must take down the replacement, not the VM the delete first saw.
-func TestTeardownDBInstance_UsesTheVMAModifyLeftBehind(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	stale := availableRecord()
-	stale.Status = StatusDeleting
-	replaced := stale
-	replaced.InstanceID = "i-replacement"
-	seedInstance(t, h.svc, replaced)
-
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	require.NoError(t, h.svc.teardownDBInstance(t.Context(), kv, testAccountID, &stale, false))
-
-	assert.Equal(t, []string{"i-replacement"}, h.launcher.terminated)
-	assert.False(t, h.recordExists(t, testDBID))
-}
-
-// A modify that fails after a delete was accepted must not turn the record to
-// failed, or the reconciler would never finish the delete.
-func TestFailTransition_KeepsAnAcceptedDelete(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	rec := availableRecord()
-	rec.Status = StatusDeleting
-	seedInstance(t, h.svc, rec)
-
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	err = h.svc.failTransition(t.Context(), kv, testAccountID, &rec, "the DB instance could not be modified: boom")
-	require.Error(t, err)
-
-	assert.Equal(t, StatusDeleting, h.record(t).Status)
-}

spinifex/domains/rds/instance/lifecycle.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/rds"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
@@ -35,12 +34,6 @@ type instanceCommander interface {
 	// Relaunches a VM whose owning node no longer holds it, from its persisted
 	// stopped-instance record.
 	StartStoppedInstance(ctx context.Context, instanceID string) error
-	// Terminates a VM no node holds from its stopped-instance record. A VM
-	// already gone is not an error.
-	TerminateStoppedInstance(ctx context.Context, instanceID string) error
-	// Removes a volume from a stopped VM's record without changing the volume.
-	// A VM or volume no longer in the record is not an error.
-	DetachVolumeFromStopped(ctx context.Context, instanceID, volumeID string) error
 }
 
 // ErrInstanceNotOnNode means no node subscribes ec2.cmd.{instanceID} for this VM,
@@ -345,15 +338,10 @@ func (s *Service) completeTransition(ctx context.Context, kv *kvstore.Bucket, id
 // Records why an operation failed and returns the error the caller surfaces, so
 // a failed transition cannot be left looking like one still in progress.
 func (s *Service) failTransition(ctx context.Context, kv *kvstore.Bucket, accountID string, rec *DBInstanceRecord, reason string) error {
-	// A delete accepted meanwhile owns the record; failing it would drop the delete.
-	if _, err := s.updateInstanceIf(ctx, kv, rec.DBInstanceIdentifier, func(stored *DBInstanceRecord) bool {
-		if stored.Status == StatusDeleting {
-			return false
-		}
+	if err := s.updateInstance(ctx, kv, rec.DBInstanceIdentifier, func(stored *DBInstanceRecord) {
 		stored.Status = StatusFailed
 		stored.FailureReason = reason
 		stored.TransitionStartedAt = nil
-		return true
 	}); err != nil {
 		slog.ErrorContext(ctx, "rds: marking a failed lifecycle transition failed",
 			"dbInstance", rec.DBInstanceIdentifier, "err", err)
@@ -466,25 +454,3 @@ func (c *natsInstanceCommander) StartStoppedInstance(ctx context.Context, instan
 		ec2instance.StartStoppedInstanceInput{InstanceID: instanceID}, c.timeout, awsidentifiers.GlobalAccountID)
 	return err
 }
-
-func (c *natsInstanceCommander) TerminateStoppedInstance(ctx context.Context, instanceID string) error {
-	_, err := natsmsg.NATSRequest[ec2instance.TerminateStoppedInstanceOutput](ctx, c.nc, "ec2.terminate",
-		ec2instance.TerminateStoppedInstanceInput{InstanceID: instanceID}, c.timeout, awsidentifiers.GlobalAccountID)
-	if err == nil || strings.Contains(err.Error(), awserrors.ErrorInvalidInstanceIDNotFound) {
-		return nil
-	}
-	return err
-}
-
-// The record no longer naming the volume is IncorrectState, which is the state
-// this is trying to reach.
-func (c *natsInstanceCommander) DetachVolumeFromStopped(ctx context.Context, instanceID, volumeID string) error {
-	_, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, c.nc, "ec2.DetachVolumeFromStopped",
-		&ec2.DetachVolumeInput{InstanceId: aws.String(instanceID), VolumeId: aws.String(volumeID)},
-		c.timeout, awsidentifiers.GlobalAccountID)
-	if err == nil || awserrors.IsErrorCode(err, awserrors.ErrorInvalidInstanceIDNotFound) ||
-		awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState) {
-		return nil
-	}
-	return err
-}

spinifex/domains/rds/instance/lifecycle_test.go

@@ -16,10 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/rds"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,12 +52,6 @@ type fakeInstanceCommander struct {
 	// The VM state an accepted stop takes down. Nil models a node that accepts
 	// the command and never lands it.
 	vm *fakeInstanceState
-	// terminateStoppedErr is what terminating a stopped VM returns; onTerminateStopped
-	// runs first, so a test can model what the terminate does to the data volume.
-	terminateStoppedErr error
-	onTerminateStopped  func(instanceID string)
-	// detachStoppedErr is what removing a volume from a stopped VM's record returns.
-	detachStoppedErr error
 }
 
 var _ instanceCommander = (*fakeInstanceCommander)(nil)
@@ -100,19 +91,6 @@ func (f *fakeInstanceCommander) StartStoppedInstance(_ context.Context, instance
 	return f.err
 }
 
-func (f *fakeInstanceCommander) TerminateStoppedInstance(_ context.Context, instanceID string) error {
-	f.calls = append(f.calls, "terminate-stopped:"+instanceID)
-	if f.onTerminateStopped != nil {
-		f.onTerminateStopped(instanceID)
-	}
-	return f.terminateStoppedErr
-}
-
-func (f *fakeInstanceCommander) DetachVolumeFromStopped(_ context.Context, instanceID, volumeID string) error {
-	f.calls = append(f.calls, "detach-stopped:"+instanceID+":"+volumeID)
-	return f.detachStoppedErr
-}
-
 // fakeSnapshots stands in for the EC2 snapshot service. holding is what
 // DescribeSnapshots reports against the data volume, which is what decides
 // between deleting the volume and retaining it.
@@ -287,7 +265,6 @@ type lifecycleHarness struct {
 	sysvpc   *fakeSystemVPC
 	launcher *fakeLauncher
 	volumes  *fakeVolumes
-	storage  *fakeVolumeResizer
 	agent    *stubAgent
 	vmState  *fakeInstanceState
 }
@@ -309,7 +286,6 @@ func newLifecycleHarness(t *testing.T, agentFails bool) *lifecycleHarness {
 		sysvpc:   &fakeSystemVPC{},
 		launcher: &fakeLauncher{},
 		volumes:  &fakeVolumes{},
-		storage:  newFakeVolumeResizer("vol-rdsdata01", 20),
 		vmState:  vmState,
 	}
 	h.agent = newStubAgent(t, nc, testAccountID, testDBID, agentFails)
@@ -318,7 +294,6 @@ func newLifecycleHarness(t *testing.T, agentFails bool) *lifecycleHarness {
 		MasterKey:          testMasterKey,
 		Instances:          h.cmdr,
 		Snapshots:          h.snaps,
-		Storage:            h.storage,
 		InstanceState:      h.vmState,
 		VMStopTimeout:      testVMStopTimeout,
 		ServingCertKeyBits: testServingCertKeyBits,
@@ -881,105 +856,3 @@ func TestUncleanStopMessage_IsEngineAware(t *testing.T) {
 	unknown := uncleanStopMessage(t.Context(), "oracle", "stopping the instance")
 	assert.Equal(t, "The database engine could not be shut down cleanly before stopping the instance.", unknown)
 }
-
-// A stopped DB VM is terminated over ec2.terminate as the system account. A VM
-// already gone is done; a refusal or no terminate service is returned so the
-// teardown retries.
-func TestNATSInstanceCommander_TerminateStoppedInstance(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name    string
-		reply   []byte
-		noReply bool
-		wantErr string
-	}{
-		{name: "terminated", reply: []byte(`{}`)},
-		{name: "already gone", reply: awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound)},
-		{name: "no responder", noReply: true, wantErr: "no responders"},
-		{name: "refused", reply: awserrors.GenerateErrorPayload(awserrors.ErrorIncorrectInstanceState), wantErr: awserrors.ErrorIncorrectInstanceState},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			_, nc := testutil.StartTestNATS(t)
-			var got struct {
-				InstanceID string
-				Account    string
-			}
-			if !tc.noReply {
-				_, err := nc.Subscribe("ec2.terminate", func(msg *nats.Msg) {
-					var in ec2instance.TerminateStoppedInstanceInput
-					_ = json.Unmarshal(msg.Data, &in)
-					got.InstanceID, got.Account = in.InstanceID, msg.Header.Get(natsmsg.AccountIDHeader)
-					_ = msg.Respond(tc.reply)
-				})
-				require.NoError(t, err)
-				require.NoError(t, nc.Flush())
-			}
-
-			err := NewNATSInstanceCommander(nc).TerminateStoppedInstance(t.Context(), testInstance)
-			if tc.wantErr != "" {
-				require.Error(t, err)
-				assert.Contains(t, err.Error(), tc.wantErr)
-			} else {
-				require.NoError(t, err)
-			}
-			if !tc.noReply {
-				assert.Equal(t, testInstance, got.InstanceID)
-				assert.Equal(t, awsidentifiers.GlobalAccountID, got.Account)
-			}
-		})
-	}
-}
-
-// The data volume leaves a stopped DB VM's record over ec2.DetachVolumeFromStopped
-// as the system account. A record that no longer names the volume, or a VM
-// already gone, is done; anything else is returned so the teardown retries.
-func TestNATSInstanceCommander_DetachVolumeFromStopped(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name    string
-		reply   []byte
-		noReply bool
-		wantErr string
-	}{
-		{name: "detached", reply: []byte(`{}`)},
-		{name: "already out of the record", reply: awserrors.GenerateErrorPayload(awserrors.ErrorIncorrectState)},
-		{name: "already gone", reply: awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound)},
-		{name: "no responder", noReply: true, wantErr: "no responders"},
-		{name: "claimed by a start", reply: awserrors.GenerateErrorPayload(awserrors.ErrorIncorrectInstanceState), wantErr: awserrors.ErrorIncorrectInstanceState},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			_, nc := testutil.StartTestNATS(t)
-			var got struct {
-				InstanceID, VolumeID, Account string
-			}
-			if !tc.noReply {
-				_, err := nc.Subscribe("ec2.DetachVolumeFromStopped", func(msg *nats.Msg) {
-					var in ec2.DetachVolumeInput
-					_ = json.Unmarshal(msg.Data, &in)
-					got.InstanceID, got.VolumeID = aws.StringValue(in.InstanceId), aws.StringValue(in.VolumeId)
-					got.Account = msg.Header.Get(natsmsg.AccountIDHeader)
-					_ = msg.Respond(tc.reply)
-				})
-				require.NoError(t, err)
-				require.NoError(t, nc.Flush())
-			}
-
-			err := NewNATSInstanceCommander(nc).DetachVolumeFromStopped(t.Context(), testInstance, testDataVolume)
-			if tc.wantErr != "" {
-				require.Error(t, err)
-				assert.Contains(t, err.Error(), tc.wantErr)
-			} else {
-				require.NoError(t, err)
-			}
-			if !tc.noReply {
-				assert.Equal(t, testInstance, got.InstanceID)
-				assert.Equal(t, testDataVolume, got.VolumeID)
-				assert.Equal(t, awsidentifiers.GlobalAccountID, got.Account)
-			}
-		})
-	}
-}

spinifex/domains/rds/instance/modify_lease.go

@@ -97,9 +97,7 @@ func (s *Service) claimModifyLease(
 ) (bool, time.Time, error) {
 	var expiresAt time.Time
 	claimed, err := s.updateInstanceIf(ctx, kv, id, func(rec *DBInstanceRecord) bool {
-		// A delete moves the record to deleting before its teardown checks for a
-		// live lease, so a modify that has not claimed one by then never does.
-		if rec.Status == StatusDeleting || (rec.ModifyLease.live() && rec.ModifyLease.Holder != holder) {
+		if rec.ModifyLease.live() && rec.ModifyLease.Holder != holder {
 			return false
 		}
 		expiresAt = time.Now().UTC().Add(s.modifyLeaseTTL())

spinifex/domains/rds/instance/modify_lease_test.go

@@ -260,39 +260,3 @@ func TestClaimModifyLease_RefusesAnotherHolderButNotItsOwn(t *testing.T) {
 	require.NoError(t, err)
 	assert.True(t, claimed)
 }
-
-// A delete that lands while a modify holds the lease, possibly mid-grow, must
-// not terminate or detach anything until that modify lets go; the reconciler
-// finishes the delete afterwards.
-func TestDeleteDBInstance_WaitsForAModifyThatHoldsTheLease(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	rec := availableRecord()
-	rec.ModifyLease = heldLease(modifyLeaseTTL)
-	seedInstance(t, h.svc, rec)
-
-	out, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
-	require.NoError(t, err)
-	assert.Equal(t, string(StatusDeleting), aws.StringValue(out.DBInstance.DBInstanceStatus))
-
-	assert.Empty(t, h.launcher.terminated, "nothing may be terminated while a modify holds the lease")
-	assert.Empty(t, h.cmdr.calls)
-	assert.Empty(t, h.volumes.deleted)
-	assert.True(t, h.recordExists(t, testDBID), "the record stays for the reconciler to finish")
-}
-
-// Once the record is deleting, a modify that had not yet claimed the lease can
-// never claim it, so the teardown's lease check cannot be overtaken.
-func TestClaimModifyLease_RefusesADeletingInstance(t *testing.T) {
-	t.Parallel()
-	h := newLifecycleHarness(t, false)
-	rec := availableRecord()
-	rec.Status = StatusDeleting
-	seedInstance(t, h.svc, rec)
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-
-	claimed, _, err := h.svc.claimModifyLease(t.Context(), kv, testDBID, h.svc.newModifyLeaseHolder())
-	require.NoError(t, err)
-	assert.False(t, claimed)
-}

spinifex/domains/rds/instance/reconciler.go

@@ -611,7 +611,7 @@ func (r *Reconciler) reconcileStopping(ctx context.Context, kv *kvstore.Bucket,
 // what lets the customer retry the delete.
 func (r *Reconciler) reconcileDeleting(ctx context.Context, kv *kvstore.Bucket, rev uint64, accountID string, rec *DBInstanceRecord) error {
 	err := r.svc.teardownDBInstance(ctx, kv, accountID, rec, false)
-	if err == nil || errors.Is(err, errFinalSnapshotInProgress) || errors.Is(err, errModifyInProgress) {
+	if err == nil || errors.Is(err, errFinalSnapshotInProgress) {
 		return nil
 	}
 	if time.Since(transitionStarted(rec)) > transitionTimeout {
@@ -861,10 +861,6 @@ func (r *Reconciler) transitionFresh(ctx context.Context, kv *kvstore.Bucket, id
 	if err != nil || !found {
 		return err
 	}
-	// A delete accepted since this pass read the record now owns it.
-	if rec.Status == StatusDeleting {
-		return nil
-	}
 	return r.transition(ctx, kv, rev, &rec, to, reason)
 }
 

spinifex/domains/rds/instance/reconciler_test.go

@@ -418,23 +418,3 @@ func TestReconciler_RelinquishOnlyReleasesItsOwnLease(t *testing.T) {
 
 	assert.False(t, other.lease.TryAcquire(t.Context()), "the original holder still owns the lease")
 }
-
-// An overrun modify failed on a fresh read must not fail a delete accepted since
-// the pass opened; the delete owns the record.
-func TestTransitionFresh_LeavesAnAcceptedDelete(t *testing.T) {
-	t.Parallel()
-	h := newReconcileHarness(t)
-	rec := healthyRecord()
-	rec.Status = StatusDeleting
-	seedInstance(t, h.svc, rec)
-
-	kv, err := h.svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	require.NoError(t, h.rec.transitionFresh(t.Context(), kv, rec.DBInstanceIdentifier, StatusFailed, "overran"))
-
-	var stored DBInstanceRecord
-	found, err := getJSON(t.Context(), kv, DBInstanceKey(rec.DBInstanceIdentifier), &stored)
-	require.NoError(t, err)
-	require.True(t, found)
-	assert.Equal(t, StatusDeleting, stored.Status)
-}

spinifex/domains/rds/instance/replace.go

@@ -86,12 +86,9 @@ func (s *Service) replaceInstanceVM(ctx context.Context, kv *kvstore.Bucket, acc
 	if err := context.Cause(ctx); err != nil {
 		return err
 	}
-	if err := s.terminateInstanceVM(ctx, oldInstanceID, rec.DataVolumeID); err != nil {
+	if err := s.terminateInstanceVM(ctx, oldInstanceID); err != nil {
 		return fmt.Errorf("terminate the VM behind %s: %w", rec.DBInstanceIdentifier, err)
 	}
-	if err := s.requireDataVolumeDetached(ctx, rec.DataVolumeID, oldInstanceID); err != nil {
-		return err
-	}
 	// The system NIC is disposable and the fresh launch mints its own; leaving
 	// this one behind would hold an address in the shared RDS system subnet.
 	if err := context.Cause(ctx); err != nil {
@@ -105,7 +102,7 @@ func (s *Service) replaceInstanceVM(ctx context.Context, kv *kvstore.Bucket, acc
 		return err
 	}
 	if in.GrowStorageToGiB > 0 {
-		if err := s.growDataVolume(ctx, rec.DataVolumeID, "", in.GrowStorageToGiB); err != nil {
+		if err := s.growDataVolume(ctx, rec.DataVolumeID, in.GrowStorageToGiB); err != nil {
 			return err
 		}
 	}

spinifex/domains/rds/instance/replace_test.go

@@ -5,8 +5,6 @@ import (
 	"errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
 	"time"
 
 	"github.com/stretchr/testify/assert"
@@ -201,43 +199,6 @@ func TestReplaceInstanceVM_GrowsTheVolumeWhileNoVMHoldsIt(t *testing.T) {
 	assert.Equal(t, int64(80), h.storage.sizes[testDataVolume])
 }
 
-// A guest poweroff now ends as a stop, so the old VM may be held by no node with
-// the data volume still attached; it is terminated from its record before the grow.
-func TestReplaceInstanceVM_TerminatesAStoppedOldVMBeforeGrowing(t *testing.T) {
-	t.Parallel()
-	h := newModifyHarness(t)
-	rec := seedReplaceable(t, h, modifiableRecord())
-	h.launch.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
-	h.storage.attachedTo = map[string]string{testDataVolume: testInstance}
-
-	require.NoError(t, h.svc.replaceInstanceVM(t.Context(), h.kv(t), testAccountID, &rec, replaceInput{
-		InstanceClass: "db.m5.large", InstanceType: "m5.large", GrowStorageToGiB: 80, Reason: "the instance class changed",
-	}))
-
-	assert.Contains(t, h.cmdr.calls, "detach-stopped:"+testInstance+":"+testDataVolume)
-	assert.Contains(t, h.cmdr.calls, "terminate-stopped:"+testInstance)
-	assert.Equal(t, []string{testDataVolume}, h.storage.detached)
-	assert.Equal(t, int64(80), h.storage.sizes[testDataVolume])
-	assert.NotEqual(t, testInstance, h.record(t).InstanceID)
-}
-
-// A replace that cannot get the data volume off the old VM goes no further: the
-// replacement would boot without its datadir.
-func TestReplaceInstanceVM_StopsWhileTheDataVolumeIsStillOnTheOldVM(t *testing.T) {
-	t.Parallel()
-	h := newModifyHarness(t)
-	rec := seedReplaceable(t, h, modifiableRecord())
-	h.launch.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
-	h.storage.attachedTo = map[string]string{testDataVolume: testInstance}
-	h.storage.detachErr = errors.New("the volume store is unavailable")
-
-	err := h.svc.replaceInstanceVM(t.Context(), h.kv(t), testAccountID, &rec, replaceInput{Reason: "recovery"})
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "release the data volume")
-	assert.Nil(t, h.launch.launcher.input, "no replacement may launch without the data volume")
-	assert.Equal(t, testInstance, h.record(t).InstanceID)
-}
-
 // An unmapped class has no instance type behind it, so the replace is refused
 // before the VM it would land on is torn down.
 func TestReplaceInstanceVM_RefusesAnUnmappedRecordedClass(t *testing.T) {

spinifex/domains/rds/instance/storage.go

@@ -23,8 +23,6 @@ import (
 type volumeResizer interface {
 	DescribeVolumes(ctx context.Context, input *ec2.DescribeVolumesInput, accountID string) (*ec2.DescribeVolumesOutput, error)
 	ModifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID string) (*ec2.ModifyVolumeOutput, error)
-	ExpandAttachedToStopped(ctx context.Context, accountID, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error)
-	ForceDetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error)
 }
 
 // Everything a resize request can be wrong about, checked before the instance
@@ -67,7 +65,7 @@ func (s *Service) growInstanceStorage(ctx context.Context, accountID string, rec
 	if err := context.Cause(ctx); err != nil {
 		return err
 	}
-	if err := s.growDataVolume(ctx, rec.DataVolumeID, rec.InstanceID, targetGiB); err != nil {
+	if err := s.growDataVolume(ctx, rec.DataVolumeID, targetGiB); err != nil {
 		return s.restartAfterStorageGrowFailure(ctx, accountID, rec, err)
 	}
 	if err := context.Cause(ctx); err != nil {
@@ -103,10 +101,10 @@ func (s *Service) restartAfterStorageGrowFailure(
 	return growErr
 }
 
-// Grows the data volume itself, idempotently: a resumed grow already at the
-// target returns without a modify. An attached volume is grown only as
-// stoppedVM's, the VM the caller stopped; an empty stoppedVM requires it detached.
-func (s *Service) growDataVolume(ctx context.Context, volumeID, stoppedVM string, targetGiB int64) error {
+// Grows the data volume itself. Idempotent: a resumed grow re-reads the volume
+// and returns without a second modify once it is already at the target, which
+// ModifyVolume would otherwise reject as a shrink.
+func (s *Service) growDataVolume(ctx context.Context, volumeID string, targetGiB int64) error {
 	if volumeID == "" {
 		return errors.New("rds: the DB instance has no data volume to grow")
 	}
@@ -114,7 +112,7 @@ func (s *Service) growDataVolume(ctx context.Context, volumeID, stoppedVM string
 		return errors.New("rds: no volume resize path configured")
 	}
 
-	current, attachedTo, err := s.dataVolumeState(ctx, volumeID)
+	current, err := s.dataVolumeSize(ctx, volumeID)
 	if err != nil {
 		return err
 	}
@@ -127,45 +125,31 @@ func (s *Service) growDataVolume(ctx context.Context, volumeID, stoppedVM string
 		return err
 	}
 
-	if attachedTo != "" && stoppedVM != "" {
-		_, err = s.deps.Storage.ExpandAttachedToStopped(ctx, awsidentifiers.GlobalAccountID, volumeID, stoppedVM, targetGiB)
-	} else {
-		_, err = s.deps.Storage.ModifyVolume(ctx, &ec2.ModifyVolumeInput{
-			VolumeId: aws.String(volumeID),
-			Size:     aws.Int64(targetGiB),
-		}, awsidentifiers.GlobalAccountID)
-	}
-	if err != nil {
+	if _, err := s.deps.Storage.ModifyVolume(ctx, &ec2.ModifyVolumeInput{
+		VolumeId: aws.String(volumeID),
+		Size:     aws.Int64(targetGiB),
+	}, awsidentifiers.GlobalAccountID); err != nil {
 		return fmt.Errorf("grow the data volume %s to %d GiB: %w", volumeID, targetGiB, err)
 	}
 	slog.InfoContext(ctx, "rds: data volume grown",
-		"volumeId", volumeID, "fromGiB", current, "toGiB", targetGiB, "attachedTo", attachedTo)
+		"volumeId", volumeID, "fromGiB", current, "toGiB", targetGiB)
 	return nil
 }
 
-// The volume's own reported size, which is what makes the grow idempotent, and
-// the instance it is attached to. A volume the store cannot describe is an error
-// rather than a zero, since zero would read as "smaller than the target".
-func (s *Service) dataVolumeState(ctx context.Context, volumeID string) (int64, string, error) {
+// The volume's own reported size, which is what makes the grow idempotent. A
+// volume the store cannot describe is an error rather than a zero, since zero
+// would read as "smaller than the target" and trigger a modify.
+func (s *Service) dataVolumeSize(ctx context.Context, volumeID string) (int64, error) {
 	out, err := s.deps.Storage.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{
 		VolumeIds: aws.StringSlice([]string{volumeID}),
 	}, awsidentifiers.GlobalAccountID)
 	if err != nil {
-		return 0, "", fmt.Errorf("read the data volume %s: %w", volumeID, err)
+		return 0, fmt.Errorf("read the data volume %s: %w", volumeID, err)
 	}
 	for _, volume := range out.Volumes {
-		if aws.StringValue(volume.VolumeId) != volumeID {
-			continue
+		if aws.StringValue(volume.VolumeId) == volumeID {
+			return aws.Int64Value(volume.Size), nil
 		}
-		attachedTo := ""
-		if aws.StringValue(volume.State) == ec2.VolumeStateInUse {
-			for _, a := range volume.Attachments {
-				attachedTo = aws.StringValue(a.InstanceId)
-			}
-		}
-		return aws.Int64Value(volume.Size), attachedTo, nil
 	}
-	return 0, "", fmt.Errorf("%w: %s", errDataVolumeGone, volumeID)
+	return 0, fmt.Errorf("rds: data volume %s no longer exists", volumeID)
 }
-
-var errDataVolumeGone = errors.New("rds: data volume no longer exists")

spinifex/domains/rds/instance/storage_test.go

@@ -20,14 +20,6 @@ type fakeVolumeResizer struct {
 	sizes    map[string]int64
 	modified []*ec2.ModifyVolumeInput
 
-	// attachedTo is the instance each in-use volume is attached to, as a stop now
-	// leaves it; expanded records the grows taken through that attachment.
-	attachedTo map[string]string
-	expanded   []string
-	// detachErr fails the terminate-time detach; detached records the ones made.
-	detachErr error
-	detached  []string
-
 	describeErr error
 	modifyErr   error
 	// missing makes the volume read as gone, which is the state a grow must not
@@ -58,12 +50,7 @@ func (f *fakeVolumeResizer) DescribeVolumes(_ context.Context, in *ec2.DescribeV
 		if !ok {
 			continue
 		}
-		vol := &ec2.Volume{VolumeId: aws.String(id), Size: aws.Int64(size), State: aws.String(ec2.VolumeStateAvailable)}
-		if inst := f.attachedTo[id]; inst != "" {
-			vol.State = aws.String(ec2.VolumeStateInUse)
-			vol.Attachments = []*ec2.VolumeAttachment{{VolumeId: aws.String(id), InstanceId: aws.String(inst)}}
-		}
-		out.Volumes = append(out.Volumes, vol)
+		out.Volumes = append(out.Volumes, &ec2.Volume{VolumeId: aws.String(id), Size: aws.Int64(size)})
 	}
 	return out, nil
 }
@@ -84,33 +71,6 @@ func (f *fakeVolumeResizer) ModifyVolume(ctx context.Context, in *ec2.ModifyVolu
 	return &ec2.ModifyVolumeOutput{}, nil
 }
 
-// Mirrors the store: a grow through an attachment is refused unless the volume
-// is attached to the instance the caller names.
-func (f *fakeVolumeResizer) ExpandAttachedToStopped(ctx context.Context, _, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error) {
-	if f.onModify != nil {
-		f.onModify(ctx)
-	}
-	if f.modifyErr != nil {
-		return nil, f.modifyErr
-	}
-	if instanceID == "" || f.attachedTo[volumeID] != instanceID {
-		return nil, errors.New(awserrors.ErrorIncorrectState)
-	}
-	f.expanded = append(f.expanded, volumeID+":"+instanceID)
-	f.sizes[volumeID] = sizeGiB
-	return &ec2.ModifyVolumeOutput{}, nil
-}
-
-func (f *fakeVolumeResizer) ForceDetachVolume(_ context.Context, in *ec2.DetachVolumeInput, _ string) (*ec2.VolumeAttachment, error) {
-	if f.detachErr != nil {
-		return nil, f.detachErr
-	}
-	volumeID := aws.StringValue(in.VolumeId)
-	delete(f.attachedTo, volumeID)
-	f.detached = append(f.detached, volumeID)
-	return &ec2.VolumeAttachment{VolumeId: in.VolumeId}, nil
-}
-
 // The whole rule set a resize is checked against, ahead of any state change.
 func TestValidateStorageGrow(t *testing.T) {
 	cases := []struct {
@@ -144,7 +104,7 @@ func TestGrowDataVolume_SkipsAVolumeAlreadyAtTheTarget(t *testing.T) {
 	h := newModifyHarness(t)
 	h.storage.sizes[testDataVolume] = 50
 
-	require.NoError(t, h.svc.growDataVolume(t.Context(), testDataVolume, testInstance, 50))
+	require.NoError(t, h.svc.growDataVolume(t.Context(), testDataVolume, 50))
 	assert.Empty(t, h.storage.modified, "a volume already at the target is not modified again")
 }
 
@@ -155,7 +115,7 @@ func TestGrowDataVolume_FailsWhenTheVolumeIsGone(t *testing.T) {
 	h := newModifyHarness(t)
 	h.storage.missing = true
 
-	err := h.svc.growDataVolume(t.Context(), testDataVolume, testInstance, 50)
+	err := h.svc.growDataVolume(t.Context(), testDataVolume, 50)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "no longer exists")
 	assert.Empty(t, h.storage.modified)
@@ -165,7 +125,7 @@ func TestGrowDataVolume_FailsWithoutAVolume(t *testing.T) {
 	t.Parallel()
 	h := newModifyHarness(t)
 
-	require.Error(t, h.svc.growDataVolume(t.Context(), "", testInstance, 50))
+	require.Error(t, h.svc.growDataVolume(t.Context(), "", 50))
 }
 
 // ModifyVolume refuses a volume a running VM holds, so the order is the
@@ -191,35 +151,6 @@ func TestGrowInstanceStorage_StopsTheEngineAndTheVMBeforeGrowing(t *testing.T) {
 	assert.Equal(t, CommandStopEngine, issued[0].Type)
 }
 
-// A stop leaves the data volume attached to the stopped VM, so the grow goes
-// through that attachment rather than the customer path, which refuses it.
-func TestGrowInstanceStorage_GrowsTheVolumeStillAttachedToTheStoppedVM(t *testing.T) {
-	t.Parallel()
-	h := newModifyHarness(t)
-	h.storage.attachedTo = map[string]string{testDataVolume: testInstance}
-	rec := modifiableRecord()
-
-	require.NoError(t, h.svc.growInstanceStorage(t.Context(), testAccountID, &rec, 50))
-
-	assert.Equal(t, []string{testDataVolume + ":" + testInstance}, h.storage.expanded)
-	assert.Empty(t, h.storage.modified, "an attached volume must not go through ModifyVolume")
-	assert.Equal(t, int64(50), h.storage.sizes[testDataVolume])
-	assert.Equal(t, []string{"stop:" + testInstance, "start:" + testInstance}, h.cmdr.calls)
-}
-
-// A data volume attached to some other instance is not this DB VM's to grow.
-func TestGrowDataVolume_RefusesAVolumeAttachedToAnotherInstance(t *testing.T) {
-	t.Parallel()
-	h := newModifyHarness(t)
-	h.storage.attachedTo = map[string]string{testDataVolume: "i-someone-else"}
-
-	err := h.svc.growDataVolume(t.Context(), testDataVolume, testInstance, 50)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), awserrors.ErrorIncorrectState)
-	assert.Empty(t, h.storage.expanded)
-	assert.NotEqual(t, int64(50), h.storage.sizes[testDataVolume])
-}
-
 // The stop command is accepted seconds before the volume detaches, and the
 // store refuses a resize until it has. The grow holds off until the fleet
 // reports the VM down rather than resizing on the accepted command.

spinifex/foundation/aws/errors/awserrors.go

@@ -625,10 +625,8 @@ var (
 	ErrorSerializationException            = "SerializationException"
 	ErrorUnknownOperationException         = "UnknownOperationException"
 	// Lambda wire codes that no other service shares.
-	ErrorResourceConflictException      = "ResourceConflictException"
-	ErrorServiceException               = "ServiceException"
-	ErrorRequestEntityTooLargeException = "RequestEntityTooLargeException"
-	ErrorInvalidRequestContentException = "InvalidRequestContentException"
+	ErrorResourceConflictException = "ResourceConflictException"
+	ErrorServiceException          = "ServiceException"
 )
 
 // ResolveErrorCode returns the first registered AWS error code in err's unwrap tree.
@@ -1428,8 +1426,6 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorUnknownOperationException:         {HTTPCode: 400, Message: "The requested operation is not recognized."},
 	ErrorResourceConflictException:         {HTTPCode: 409, Message: "The resource already exists or is in a state that does not allow this operation."},
 	ErrorServiceException:                  {HTTPCode: 500, Message: "The Lambda service encountered an internal error."},
-	ErrorRequestEntityTooLargeException:    {HTTPCode: 413, Message: "The request payload exceeded the Invoke request body JSON input quota."},
-	ErrorInvalidRequestContentException:    {HTTPCode: 400, Message: "The request body could not be parsed as JSON."},
 
 	// Route 53.
 	ErrorNoSuchHostedZone:          {HTTPCode: 404, Message: "No hosted zone exists with the ID that you specified."},

spinifex/ingress/aws/pipeline/lambda_invoke_wire_test.go

@@ -1,122 +0,0 @@
-package pipeline
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
-	lambdaawsapi "github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
-	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// activeMeta serves one Active function "hello" with its code stored.
-type activeMeta struct{}
-
-var _ function.Client = activeMeta{}
-
-func (activeMeta) Get(_ context.Context, account, region, name string) (function.Record, uint64, error) {
-	if name != "hello" {
-		return function.Record{}, 0, function.ErrNotFound
-	}
-	return function.Record{
-		FunctionID: "f1", AccountID: account, FunctionName: name, Runtime: "provided.al2023",
-		Timeout: 3, MemorySize: 128, State: function.StateActive,
-		Code: function.Code{RevisionID: "r1", State: function.CodeStored},
-	}, 1, nil
-}
-
-func (activeMeta) Create(context.Context, string, string, function.Record) (uint64, error) {
-	return 0, errors.New("unused")
-}
-
-func (activeMeta) CodeStored(context.Context, string, string, string, uint64, string, int64) (function.Record, uint64, error) {
-	return function.Record{}, 0, errors.New("unused")
-}
-
-func (activeMeta) RollbackCreate(context.Context, string, string, string, uint64) error {
-	return errors.New("unused")
-}
-
-// countingExec records how many invocations reached it.
-type countingExec struct{ calls atomic.Int32 }
-
-func (e *countingExec) InvokeSync(context.Context, lambdaawsapi.SyncInvocation) (lambdaawsapi.SyncResult, error) {
-	e.calls.Add(1)
-	return lambdaawsapi.SyncResult{Payload: []byte(`{"ok":true}`), ExecutedVersion: "$LATEST"}, nil
-}
-
-func signedInvoke(t *testing.T, app http.Handler, body []byte) *http.Response {
-	t.Helper()
-	req := httptest.NewRequest(http.MethodPost, "/2015-03-31/functions/hello/invocations", bytes.NewReader(body))
-	req.Host = "localhost:9999"
-	gwfix.SignRequestAs(t, req, body, gwfix.AccessKey, gwfix.SecretKey, lambdaawsapi.ServiceName, time.Now().UTC())
-	return doRequest(app, req)
-}
-
-// paddedJSON is a valid JSON string exactly n bytes long.
-func paddedJSON(n int) []byte {
-	return []byte(`"` + strings.Repeat("a", n-2) + `"`)
-}
-
-// restJSONError decodes the Lambda REST-JSON error body: Type and message,
-// with the code carried only in X-Amzn-Errortype.
-func restJSONError(t *testing.T, resp *http.Response) map[string]string {
-	t.Helper()
-	var out map[string]string
-	require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
-	assert.Equal(t, "User", out["Type"])
-	assert.NotEmpty(t, out["message"])
-	return out
-}
-
-func lambdaInvokeApp(t *testing.T) (http.Handler, *countingExec) {
-	t.Helper()
-	ex := &countingExec{}
-	_, app := dispatchSeamApp(t, allowAllECRDocs(), lambdaawsapi.NewRegistration(lambdaawsapi.Deps{Meta: activeMeta{}, Exec: ex}))
-	return app, ex
-}
-
-func TestLambdaInvokeWire_PayloadSizeLimit(t *testing.T) {
-	app, ex := lambdaInvokeApp(t)
-
-	resp := signedInvoke(t, app, paddedJSON(6291456))
-	require.Equal(t, http.StatusOK, resp.StatusCode)
-	assert.Equal(t, int32(1), ex.calls.Load())
-
-	for _, size := range []int{6291457, 6291556, 7000000} {
-		resp := signedInvoke(t, app, paddedJSON(size))
-		require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode, "size %d", size)
-		assert.Equal(t, "RequestEntityTooLargeException", resp.Header.Get("X-Amzn-Errortype"), "size %d", size)
-		body := restJSONError(t, resp)
-		if size == 7000000 {
-			assert.Equal(t, "Request must be smaller than 6291456 bytes for the InvokeFunction operation", body["message"])
-		}
-	}
-	assert.Equal(t, int32(1), ex.calls.Load(), "an oversized payload reached the executor")
-}
-
-func TestLambdaInvokeWire_MalformedPayload(t *testing.T) {
-	app, ex := lambdaInvokeApp(t)
-
-	resp := signedInvoke(t, app, []byte("not json"))
-	require.Equal(t, http.StatusBadRequest, resp.StatusCode)
-	assert.Equal(t, "InvalidRequestContentException", resp.Header.Get("X-Amzn-Errortype"))
-	restJSONError(t, resp)
-	assert.Equal(t, int32(0), ex.calls.Load(), "a malformed payload reached the executor")
-
-	for _, good := range []string{`{"a":1}`, `[1]`, `"s"`, `7`, `null`} {
-		resp := signedInvoke(t, app, []byte(good))
-		assert.Equal(t, http.StatusOK, resp.StatusCode, "payload %s", good)
-	}
-	assert.Equal(t, int32(5), ex.calls.Load())
-}

spinifex/operator/cli/admin.go

@@ -31,7 +31,6 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
 	ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
@@ -250,18 +249,6 @@ the backend. This is the list of volumes where that would cost something.`,
 	Run: runVolumesUnsealedCmd,
 }
 
-var volumesObligationsCmd = &cobra.Command{
-	Use:   "obligations",
-	Short: "Report volume exports a cleared attachment left on a node",
-	Long: `List the open unpublish obligations: exports a force detach, or a terminate
-whose unmount did not complete, left on the node that held the attachment.
-
-The node named settles each one once it answers. An obligation stays open
-while its node is down; it closes without the node only once the node has
-been removed from the cluster. This command only reports.`,
-	Run: runVolumesObligationsCmd,
-}
-
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -366,7 +353,6 @@ func init() {
 	adminCmd.AddCommand(volumesCmd)
 	volumesCmd.AddCommand(volumesOrphansCmd)
 	volumesCmd.AddCommand(volumesUnsealedCmd)
-	volumesCmd.AddCommand(volumesObligationsCmd)
 
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
@@ -4172,45 +4158,6 @@ func printChecksumError(w io.Writer, imageFile, imageName string, image imagecat
 	fmt.Fprintf(w, "  spx admin images import --name %s --force\n", imageName)
 }
 
-func runVolumesObligationsCmd(_ *cobra.Command, _ []string) {
-	_, nc, err := loadConfigAndConnect()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
-		os.Exit(1)
-	}
-	defer nc.Close()
-
-	open, err := ebsobligation.ListOpen(context.Background(), nc)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "Could not read the unpublish obligations:", err)
-		os.Exit(1)
-	}
-	printUnpublishObligations(os.Stdout, open, time.Now())
-}
-
-// printUnpublishObligations writes the open obligations, oldest first, with
-// the node each waits on and its age.
-func printUnpublishObligations(w io.Writer, open []ebsobligation.UnpublishObligation, now time.Time) {
-	if len(open) == 0 {
-		fmt.Fprintln(w, "No open unpublish obligations: no node holds an export of a cleared attachment.")
-		return
-	}
-	sort.Slice(open, func(i, j int) bool { return open[i].RecordedAt.Before(open[j].RecordedAt) })
-	fmt.Fprintf(w, "%d open unpublish obligation(s):\n\n", len(open))
-	for _, o := range open {
-		fmt.Fprintf(w, "  %s\n", o.VolumeID)
-		fmt.Fprintf(w, "    node:     %s\n", o.NodeID)
-		fmt.Fprintf(w, "    instance: %s\n", o.InstanceID)
-		fmt.Fprintf(w, "    age:      %s\n", now.Sub(o.RecordedAt).Truncate(time.Second))
-		if o.Attempts > 0 {
-			fmt.Fprintf(w, "    attempts: %d\n", o.Attempts)
-		}
-		if o.Error != "" {
-			fmt.Fprintf(w, "    error:    %s\n", o.Error)
-		}
-	}
-}
-
 func runVolumesUnsealedCmd(_ *cobra.Command, _ []string) {
 	_, nc, err := loadConfigAndConnect()
 	if err != nil {

spinifex/operator/cli/admin_volumes_obligations_test.go

@@ -1,37 +0,0 @@
-package cli
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-	"time"
-
-	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestPrintUnpublishObligations_OldestFirstWithFailures(t *testing.T) {
-	now := time.Unix(1_700_010_000, 0)
-	var out bytes.Buffer
-	printUnpublishObligations(&out, []ebsobligation.UnpublishObligation{
-		{VolumeID: "vol-newer", NodeID: "node-b", InstanceID: "i-2", RecordedAt: now.Add(-90 * time.Second)},
-		{VolumeID: "vol-older", NodeID: "node-a", InstanceID: "i-1", RecordedAt: now.Add(-2 * time.Hour),
-			Attempts: 3, Error: "seal refused"},
-	}, now)
-
-	text := out.String()
-	assert.Contains(t, text, "2 open unpublish obligation(s)")
-	assert.Less(t, strings.Index(text, "vol-older"), strings.Index(text, "vol-newer"), "the longest-open obligation is listed first")
-	assert.Contains(t, text, "node:     node-a")
-	assert.Contains(t, text, "instance: i-1")
-	assert.Contains(t, text, "age:      2h0m0s")
-	assert.Contains(t, text, "attempts: 3")
-	assert.Contains(t, text, "error:    seal refused")
-	assert.Equal(t, 1, strings.Count(text, "attempts:"), "an obligation that never failed shows no attempts")
-}
-
-func TestPrintUnpublishObligations_None(t *testing.T) {
-	var out bytes.Buffer
-	printUnpublishObligations(&out, nil, time.Now())
-	assert.Contains(t, out.String(), "No open unpublish obligations")
-}

spinifex/operator/upgrade/VERSIONS.md

@@ -24,7 +24,6 @@ Single source of truth for the current schema version of every config file Spini
 | `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
 | `spinifex-uefi-stores` | `1` | `uefistore.BucketVersion` |
-| `spinifex-ebs-unpublish-obligations` | `1` | `ebsobligation.BucketVersion` |
 
 ## Registered migrations
 

spinifex/providers/cloud/latitude/client.go

@@ -1,442 +0,0 @@
-// Package latitude is the narrow slice of the latitude.sh API that floating
-// public addresses need: reading a project's Elastic IPs and moving one to
-// another server.
-//
-// It is a direct net/http client, like the Vultr one. Request and response
-// shapes follow the published OpenAPI spec and the Terraform provider; no call
-// has been made against the live API, so the tests use bodies written from the
-// spec, not captured responses.
-package latitude
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"math/rand/v2"
-	"net/http"
-	"net/netip"
-	"net/url"
-	"os"
-	"slices"
-	"strings"
-	"time"
-)
-
-// DefaultBaseURL is the public API.
-const DefaultBaseURL = "https://api.latitude.sh"
-
-// DefaultKeyFile holds the node's API token, readable by the daemon's user only.
-const DefaultKeyFile = "/etc/spinifex/latitude/api-token"
-
-// DefaultCallTimeout bounds one HTTP round trip.
-const DefaultCallTimeout = 15 * time.Second
-
-const (
-	defaultAttempts = 3
-	// DefaultMoveTimeout bounds how long a move waits for the address to be active again.
-	DefaultMoveTimeout = 2 * time.Minute
-	// DefaultPollInterval is the gap between status reads while a move settles.
-	DefaultPollInterval = 3 * time.Second
-)
-
-var (
-	// ErrNotFound means the Elastic IP or server does not exist.
-	ErrNotFound = errors.New("latitude: resource not found")
-	// ErrMoveRefused means the API will not move the address to that server:
-	// another project, another site, or an incompatible network. Retrying is pointless.
-	ErrMoveRefused = errors.New("latitude: move refused")
-	// ErrNotActive means the address is still settling from an earlier change, so no move
-	// was attempted; the caller tries again on its next pass.
-	ErrNotActive = errors.New("latitude: Elastic IP not active")
-	// ErrNotSettled means the address did not return to active within the move timeout.
-	ErrNotSettled = errors.New("latitude: Elastic IP did not become active")
-	// ErrUnauthorized means the token was refused.
-	ErrUnauthorized = errors.New("latitude: API token refused")
-	// ErrUnavailable means the API was throttled or failing; retry later.
-	ErrUnavailable = errors.New("latitude: API unavailable")
-)
-
-// APIError is a non-2xx response, carrying latitude.sh's own code and message.
-type APIError struct {
-	Status  int
-	Code    string
-	Message string
-	kind    error
-}
-
-// Error reports the status and, when it is a well-formed code, latitude.sh's error code.
-// The response body is never included: an upstream error can echo a credential.
-func (e *APIError) Error() string {
-	if e.Code != "" {
-		return fmt.Sprintf("latitude: HTTP %d %s", e.Status, e.Code)
-	}
-	return fmt.Sprintf("latitude: HTTP %d %s", e.Status, e.Message)
-}
-
-// Unwrap exposes the classified sentinel, if any, to errors.Is.
-func (e *APIError) Unwrap() error { return e.kind }
-
-// Status values of an Elastic IP.
-const (
-	StatusActive = "active"
-	StatusError  = "error"
-)
-
-// ElasticIP is one Elastic IP.
-type ElasticIP struct {
-	ID           string
-	Address      netip.Addr
-	Status       string
-	Mode         string
-	Family       string
-	PrefixLength int
-	// ServerID is empty when the address is unassigned or its server is not
-	// active, which the API does not distinguish.
-	ServerID string
-}
-
-// Unusable returns why the address cannot be handed out by a floating pool, or
-// empty. Only a routed IPv4 /32 can be moved between servers.
-func (e ElasticIP) Unusable() string {
-	if e.Mode != "routed" || e.Family != "IPv4" || e.PrefixLength != 32 {
-		return fmt.Sprintf("mode %s, family %s, prefix length %d: only a routed IPv4 /32 can be moved",
-			safe(e.Mode, "routed", "bgp"), safe(e.Family, "IPv4", "IPv6"), e.PrefixLength)
-	}
-	return ""
-}
-
-// Config selects the endpoint, credential and project.
-type Config struct {
-	BaseURL     string
-	KeyFile     string
-	ProjectID   string
-	CallTimeout time.Duration
-	HTTPClient  *http.Client
-	// Attempts is the total tries for a retryable failure; zero means the default.
-	Attempts int
-	// Backoff is the first retry delay; it doubles with jitter. Zero means 500ms.
-	Backoff time.Duration
-	// MoveTimeout and PollInterval pace the wait after a move; zero means the defaults.
-	MoveTimeout  time.Duration
-	PollInterval time.Duration
-}
-
-// Client implements the Elastic IP calls against one project.
-type Client struct {
-	cfg  Config
-	http *http.Client
-}
-
-// New builds a client. It makes no call; ListElasticIPs proves the token works.
-func New(cfg Config) (*Client, error) {
-	if cfg.ProjectID == "" {
-		return nil, errors.New("latitude: project id is required")
-	}
-	if cfg.BaseURL == "" {
-		cfg.BaseURL = DefaultBaseURL
-	}
-	cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/")
-	if cfg.KeyFile == "" {
-		cfg.KeyFile = DefaultKeyFile
-	}
-	if cfg.CallTimeout <= 0 {
-		cfg.CallTimeout = DefaultCallTimeout
-	}
-	if cfg.Attempts <= 0 {
-		cfg.Attempts = defaultAttempts
-	}
-	if cfg.Backoff <= 0 {
-		cfg.Backoff = 500 * time.Millisecond
-	}
-	if cfg.MoveTimeout <= 0 {
-		cfg.MoveTimeout = DefaultMoveTimeout
-	}
-	if cfg.PollInterval <= 0 {
-		cfg.PollInterval = DefaultPollInterval
-	}
-	hc := &http.Client{}
-	if cfg.HTTPClient != nil {
-		clone := *cfg.HTTPClient
-		hc = &clone
-	}
-	// The API never redirects a call we make. Following a 307 or 308 would replay a
-	// PATCH body, a second write the caller did not authorise.
-	hc.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
-	return &Client{cfg: cfg, http: hc}, nil
-}
-
-type elasticIPJSON struct {
-	ID         *string `json:"id"`
-	Attributes struct {
-		Address      string `json:"address"`
-		Status       string `json:"status"`
-		Mode         string `json:"mode"`
-		Family       string `json:"family"`
-		PrefixLength int    `json:"prefix_length"`
-		Server       *struct {
-			ID string `json:"id"`
-		} `json:"server"`
-	} `json:"attributes"`
-}
-
-// toElasticIP keeps every ID exactly as the API sent it. IDs are opaque: they are compared, never
-// inspected, and the caller decides which may be logged.
-func (r elasticIPJSON) toElasticIP() (ElasticIP, error) {
-	if r.ID == nil || *r.ID == "" {
-		return ElasticIP{}, errors.New("latitude: Elastic IP with no id in response")
-	}
-	addr, err := netip.ParseAddr(r.Attributes.Address)
-	if err != nil {
-		return ElasticIP{}, errors.New("latitude: Elastic IP response has an unparseable address")
-	}
-	out := ElasticIP{
-		ID: *r.ID, Address: addr, Status: r.Attributes.Status,
-		Mode: r.Attributes.Mode, Family: r.Attributes.Family, PrefixLength: r.Attributes.PrefixLength,
-	}
-	if r.Attributes.Server != nil {
-		if r.Attributes.Server.ID != "" {
-			out.ServerID = r.Attributes.Server.ID
-		}
-	}
-	return out, nil
-}
-
-// GetElasticIP reads one address, including which server holds it.
-func (c *Client) GetElasticIP(ctx context.Context, id string) (ElasticIP, error) {
-	var out struct {
-		Data elasticIPJSON `json:"data"`
-	}
-	if err := c.do(ctx, http.MethodGet, "/elastic_ips/"+url.PathEscape(id), nil, &out); err != nil {
-		return ElasticIP{}, err
-	}
-	got, err := out.Data.toElasticIP()
-	if err != nil {
-		return ElasticIP{}, err
-	}
-	if got.ID != id {
-		return ElasticIP{}, errors.New("latitude: Elastic IP response is for a different id than requested")
-	}
-	return got, nil
-}
-
-// MoveElasticIP points id at serverID and waits until it is active there.
-// It makes at most one write per call: the caller checked that it still owns the
-// address just before calling, and a second write after a wait would outlive that
-// check. An address that is not active is reported as ErrNotActive without a
-// write, and the caller tries again on its next pass. After the write only reads
-// follow, until the move settles or the move timeout passes.
-func (c *Client) MoveElasticIP(ctx context.Context, id, serverID string) error {
-	cur, err := c.GetElasticIP(ctx, id)
-	if err != nil {
-		return err
-	}
-	if why := cur.Unusable(); why != "" {
-		return fmt.Errorf("%w: %s: %s", ErrMoveRefused, id, why)
-	}
-	switch {
-	case cur.Status == StatusError:
-		return fmt.Errorf("%w: %s is in the error state", ErrMoveRefused, id)
-	case cur.Status == StatusActive && cur.ServerID == serverID:
-		return nil
-	case cur.Status != StatusActive:
-		return fmt.Errorf("%w: %s is %s", ErrNotActive, id, safe(cur.Status, "pending", "configuring", "moving", "releasing"))
-	}
-	body := map[string]any{"data": map[string]any{
-		"type":       "elastic_ips",
-		"attributes": map[string]string{"server_id": serverID},
-	}}
-	// Exactly one attempt: a retry after a throttle, a 5xx or a lost connection would be a second
-	// write after the caller's ownership check, and the next engine pass revalidates anyway.
-	payload, err := json.Marshal(body)
-	if err != nil {
-		return fmt.Errorf("latitude: encode move: %w", err)
-	}
-	if err := c.once(ctx, http.MethodPatch, "/elastic_ips/"+url.PathEscape(id), payload, nil); err != nil {
-		return err
-	}
-	return c.awaitActive(ctx, id, serverID)
-}
-
-// awaitActive reads the address until it is active on serverID. It writes nothing.
-func (c *Client) awaitActive(ctx context.Context, id, serverID string) error {
-	ctx, cancel := context.WithTimeout(ctx, c.cfg.MoveTimeout)
-	defer cancel()
-	for {
-		cur, err := c.GetElasticIP(ctx, id)
-		if err != nil {
-			if ctx.Err() != nil {
-				return fmt.Errorf("%w: %s: %w", ErrNotSettled, id, err)
-			}
-			return err
-		}
-		if cur.Status == StatusError {
-			return fmt.Errorf("%w: %s went to the error state", ErrMoveRefused, id)
-		}
-		if cur.Status == StatusActive && cur.ServerID == serverID {
-			return nil
-		}
-		if err := c.pause(ctx); err != nil {
-			return fmt.Errorf("%w: %s: %w", ErrNotSettled, id, err)
-		}
-	}
-}
-
-func (c *Client) pause(ctx context.Context) error {
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-time.After(c.cfg.PollInterval):
-		return nil
-	}
-}
-
-// do runs one call, retrying throttles and server failures. Writes are retried
-// too: a repeated move to the same server is harmless.
-func (c *Client) do(ctx context.Context, method, path string, in, out any) error {
-	var payload []byte
-	if in != nil {
-		var err error
-		if payload, err = json.Marshal(in); err != nil {
-			return fmt.Errorf("latitude: encode %s %s: %w", method, path, err)
-		}
-	}
-	delay := c.cfg.Backoff
-	for attempt := 1; ; attempt++ {
-		err := c.once(ctx, method, path, payload, out)
-		if err == nil || !errors.Is(err, ErrUnavailable) || attempt >= c.cfg.Attempts {
-			return err
-		}
-		wait := delay/2 + rand.N(delay) //#nosec G404 - backoff jitter, not a secret
-		select {
-		case <-ctx.Done():
-			return errors.Join(err, ctx.Err())
-		case <-time.After(wait):
-		}
-		delay *= 2
-	}
-}
-
-// once makes a single round trip. The token is read on every call, so rotating
-// it in place needs no restart.
-func (c *Client) once(ctx context.Context, method, path string, payload []byte, out any) error {
-	key, err := c.key()
-	if err != nil {
-		return err
-	}
-	ctx, cancel := context.WithTimeout(ctx, c.cfg.CallTimeout)
-	defer cancel()
-
-	var body io.Reader
-	if payload != nil {
-		body = bytes.NewReader(payload)
-	}
-	req, err := http.NewRequestWithContext(ctx, method, c.cfg.BaseURL+path, body)
-	if err != nil {
-		return fmt.Errorf("latitude: build %s %s: %w", method, path, err)
-	}
-	req.Header.Set("Authorization", "Bearer "+key)
-	req.Header.Set("Accept", "application/json")
-	if payload != nil {
-		req.Header.Set("Content-Type", "application/json")
-	}
-
-	resp, err := c.http.Do(req)
-	if err != nil {
-		return fmt.Errorf("%w: %s %s: %s", ErrUnavailable, method, path, redact(err.Error(), key))
-	}
-	defer func() { _ = resp.Body.Close() }()
-	raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
-	if err != nil {
-		return fmt.Errorf("%w: read %s %s: %s", ErrUnavailable, method, path, redact(err.Error(), key))
-	}
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return classify(resp.StatusCode, raw)
-	}
-	if out == nil || len(raw) == 0 {
-		return nil
-	}
-	if err := json.Unmarshal(raw, out); err != nil {
-		return fmt.Errorf("latitude: decode %s %s: %s", method, path, redact(err.Error(), key))
-	}
-	return nil
-}
-
-func (c *Client) key() (string, error) {
-	raw, err := os.ReadFile(c.cfg.KeyFile)
-	if err != nil {
-		return "", fmt.Errorf("latitude: read API token: %w", err)
-	}
-	key := strings.TrimSpace(string(raw))
-	if key == "" {
-		return "", fmt.Errorf("latitude: API token file %s is empty", c.cfg.KeyFile)
-	}
-	return key, nil
-}
-
-// knownCodes are the error codes latitude.sh documents for Elastic IPs. Any other
-// code is dropped, so a response cannot put its own text into an error.
-var knownCodes = map[string]struct{}{
-	"ELASTIC_IP_NOT_ACTIVE":        {},
-	"ELASTIC_IP_NOT_RELEASABLE":    {},
-	"ELASTIC_IP_LIMIT_REACHED":     {},
-	"ELASTIC_IP_HAS_BGP_SESSIONS":  {},
-	"IP_ALLOCATION_FAILED":         {},
-	"SERVER_NOT_IN_PROJECT":        {},
-	"SITE_MISMATCH":                {},
-	"SITE_NOT_SUPPORTED":           {},
-	"SERVER_NETWORK_INCOMPATIBLE":  {},
-	"BGP_ELASTIC_IP_NOT_UPDATABLE": {},
-}
-
-// safe returns v when it is one of the documented values, else "unknown". Values
-// read from a response never reach an error otherwise.
-func safe(v string, known ...string) string {
-	if slices.Contains(known, v) {
-		return v
-	}
-	return "unknown"
-}
-
-// classify maps a non-2xx response onto the sentinels callers branch on. Only the
-// status and a well-formed error code are kept; the body itself never reaches an error.
-func classify(status int, raw []byte) error {
-	var body struct {
-		Errors []struct {
-			Code string `json:"code"`
-		} `json:"errors"`
-	}
-	_ = json.Unmarshal(raw, &body)
-	e := &APIError{Status: status, Message: http.StatusText(status)}
-	if len(body.Errors) > 0 {
-		if _, ok := knownCodes[body.Errors[0].Code]; ok {
-			e.Code = body.Errors[0].Code
-		}
-	}
-	switch {
-	case status == http.StatusUnauthorized || status == http.StatusForbidden:
-		e.kind = ErrUnauthorized
-	case status == http.StatusNotFound:
-		e.kind = ErrNotFound
-	case status == http.StatusTooManyRequests || status >= 500:
-		e.kind = ErrUnavailable
-	case e.Code == "ELASTIC_IP_NOT_ACTIVE":
-		e.kind = ErrNotActive
-	case status == http.StatusUnprocessableEntity:
-		switch e.Code {
-		case "SERVER_NOT_IN_PROJECT", "SITE_MISMATCH", "SITE_NOT_SUPPORTED", "SERVER_NETWORK_INCOMPATIBLE", "BGP_ELASTIC_IP_NOT_UPDATABLE":
-			e.kind = ErrMoveRefused
-		}
-	}
-	return e
-}
-
-// redact removes the token from a message that might carry it.
-func redact(msg, token string) string {
-	if token == "" {
-		return msg
-	}
-	return strings.ReplaceAll(msg, token, "[redacted]")
-}