fix: range-check uid, gid and sysfs integer conversions (#1601)

* feat(ec2): add CleanupStorage to delete free volumes and clear instance, volume and spot tag residue

* fix(ec2): read teardown before deleting volumes and report volumes that appear during cleanup

* test: align the fake deleter fields

* feat(ec2): add CleanupStorage to delete free volumes and clear instance, volume and spot tag residue

* fix(ec2): read teardown before deleting volumes and report volumes that appear during cleanup

* test: align the fake deleter fields

.github/CODEOWNERS

@@ -1,15 +0,0 @@
-# CODEOWNERS — GitHub PR review enforcement
-#
-# Format: <path pattern>  <owner> [<owner> …]
-# Order matters — last matching pattern wins.
-#
-# Set the matching branch protection rule on `dev` and `main` to
-# "Require review from Code Owners" or this file is advisory only.
-
-# Network layer is governed by its invariant tests.
-# Changes here must respect those clauses
-# see spinifex/domains/network/invariants/
-# mulga docs/development/proposals/region-local-architecture/0006-spinifex-network-layer-contract.md (proposal, not accepted)
-/spinifex/domains/network/          @joshsiv-mulga
-/spinifex/runtime/roles/vpcd/ @joshsiv-mulga
-/spinifex/domains/network/invariants/ @joshsiv-mulga

.github/pull_request_template.md

@@ -6,20 +6,6 @@
 
 <!-- The notable changes, as bullets. -->
 
-## Account-scoped lifecycle impact
-
-<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
-
-- Lifecycle owner and resource family:
-- Owner-local ordering, recovery, retained-state, or force impact:
-- Cross-domain dependency or ownership impact:
-- Evidence added or rechecked:
-- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
-
-## Mulga design
-
-<!-- One line, so a reviewer knows whether the design records in the private mulga repository need to change. Write either "Updated: <mulga PR or bead id>" or "Not needed: <reason>". This is a reminder, not a gate. -->
-
 ## Community Contribution License
 
 - [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.

.github/scripts/hypervisor-disk-preflight_test.sh

@@ -1,87 +0,0 @@
-#!/bin/sh
-# hypervisor-disk-preflight_test.sh — drives hypervisor-disk-preflight.sh against
-# a stub ssh that answers for each hypervisor from a canned file, so every
-# verdict is checked without a real host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-PREFLIGHT="$SCRIPT_DIR/hypervisor-disk-preflight.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-# The stub answers "<root-kb> <pool-kb> <path>" from $FAKE/<host>, or exits 255
-# when that host has no file, the way ssh does for a host it cannot reach.
-mkdir -p "$TMP/bin" "$TMP/hosts"
-cat > "$TMP/bin/ssh" <<'STUB'
-#!/bin/sh
-for a in "$@"; do case $a in *@*) host=${a#*@} ;; esac; done
-cat >/dev/null
-if [ -f "$FAKE/$host" ]; then cat "$FAKE/$host"; exit 0; fi
-echo "ssh: Could not resolve hostname $host" >&2
-exit 255
-STUB
-chmod +x "$TMP/bin/ssh"
-PATH="$TMP/bin:$PATH"
-FAKE="$TMP/hosts"
-export PATH FAKE
-
-cat > "$TMP/env.tfvars" <<'TFVARS'
-nodes = [
-  { name = "wattle", wan_ip = "192.0.2.1", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  # { name = "retired", wan_ip = "192.0.2.9", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  { name = "banksia", wan_ip = "192.0.2.2", storage_pool = "nvme1-pool", base_image = "/x.qcow2" },
-]
-TFVARS
-
-GiB=1048576
-plenty="$((50 * GiB)) $((200 * GiB)) /mnt/drive1/pool"
-
-# run <label> <want-rc> <want-text>
-run() {
-    rc=0
-    out=$(bash "$PREFLIGHT" "$TMP/env.tfvars" 2>&1) || rc=$?
-    if [ "$rc" -ne "$2" ]; then
-        fail "$1: exit $rc, want $2"
-        echo "$out" | sed 's/^/        /'
-        return
-    fi
-    case $out in
-        *"$3"*) pass "$1" ;;
-        *) fail "$1: output lacks '$3'"; echo "$out" | sed 's/^/        /' ;;
-    esac
-}
-
-echo "-- hypervisor-disk-preflight.sh"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$plenty" > "$FAKE/banksia"
-run "every host has room" 0 "banksia: root fs 51200 MiB free"
-
-echo "0 $((200 * GiB)) /mnt/drive1/pool" > "$FAKE/wattle"
-run "a full root filesystem fails fast" 1 "::error::hypervisor wattle has 0 MiB free on the filesystem holding /etc/libvirt/qemu"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) $((3 * GiB)) /mnt/nvme1/pool" > "$FAKE/banksia"
-run "a nearly full pool fails fast" 1 "::error::hypervisor banksia has 3 GiB free in storage pool nvme1-pool"
-
-echo "$plenty" > "$FAKE/banksia"
-rm -f "$FAKE/wattle"
-run "an unreachable host is reported, not failed" 0 "::warning::hypervisor disk preflight: cannot reach wattle"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) - -" > "$FAKE/banksia"
-run "a pool libvirt does not know is reported, not failed" 0 "pool nvme1-pool (-) not found"
-
-echo "0 0 /x" > "$FAKE/retired"
-echo "$plenty" > "$FAKE/banksia"
-run "a commented-out node is never checked" 0 "wattle: root fs"
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi
-echo "  hypervisor-disk-preflight ok"

.github/scripts/mulga-design-check.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY and
+# never fails the job: the check is a prompt, not a gate.
+set -euo pipefail
+shopt -s nocasematch
+
+section_re='^#{1,6}[[:space:]]*Mulga design[[:space:]]*$'
+line_re='^[[:space:]]*(Updated|Not needed)[[:space:]]*:[[:space:]]*(.*)$'
+placeholder_re='<[^<>]*>'
+
+filled() {
+  local body=$1 line value kept inside=0 pre rest
+  body=${body//$'\r'/}
+  while [[ $body == *'<!--'*'-->'* ]]; do
+    pre=${body%%'<!--'*}
+    rest=${body#*'<!--'}
+    body=$pre${rest#*'-->'}
+  done
+  while IFS= read -r line; do
+    if [[ $line =~ $section_re ]]; then
+      inside=1
+    elif ((inside)) && [[ ${line#"${line%%[![:space:]]*}"} == '#'* ]]; then
+      return 1
+    elif ((inside)) && [[ $line =~ $line_re ]]; then
+      value=${BASH_REMATCH[2]} kept=''
+      while [[ $value =~ $placeholder_re ]]; do
+        kept+=${value%%"${BASH_REMATCH[0]}"*}
+        value=${value#*"${BASH_REMATCH[0]}"}
+      done
+      [[ $kept$value == *[![:space:]]* ]] && return 0
+    fi
+  done <<<"$body"
+  return 1
+}
+
+if filled "${PR_BODY:-}"; then
+  echo "The Mulga design line is filled in."
+else
+  echo '::warning title=Mulga design::Add a "## Mulga design" section with one line: "Updated: <mulga PR>" or "Not needed: <reason>".'
+fi

.github/scripts/mulga_design_check.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-"""Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY; never fails the job."""
-
-import os
-import re
-
-COMMENT_RE = re.compile(r"<!--.*?-->", re.S)
-SECTION_RE = re.compile(r"^#{1,6}\s*Mulga design\s*$", re.I)
-LINE_RE = re.compile(r"^\s*(Updated|Not needed)\s*:\s*(.*?)\s*$", re.I)
-PLACEHOLDER_RE = re.compile(r"<[^<>]*>")
-
-
-def filled(body):
-    inside = False
-    for line in COMMENT_RE.sub("", body or "").splitlines():
-        if SECTION_RE.match(line):
-            inside = True
-        elif inside and line.lstrip().startswith("#"):
-            return False
-        elif inside:
-            m = LINE_RE.match(line)
-            if m and PLACEHOLDER_RE.sub("", m.group(2)).strip():
-                return True
-    return False
-
-
-def main():
-    if filled(os.environ.get("PR_BODY", "")):
-        print("The Mulga design line is filled in.")
-    else:
-        print('::warning title=Mulga design::Add a line under "## Mulga design": "Updated: <mulga PR or bead id>" or "Not needed: <reason>".')
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

.github/scripts/test_mulga_design_check.py

@@ -1,23 +0,0 @@
-import unittest
-
-from mulga_design_check import filled
-
-
-class FilledTest(unittest.TestCase):
-    def test_accepts_either_form(self):
-        self.assertTrue(filled("## Mulga design\nUpdated: mulga#275"))
-        self.assertTrue(filled("## Mulga design\n\nnot needed: internal refactor\n## Next"))
-
-    def test_rejects_unanswered(self):
-        self.assertFalse(filled(""))
-        self.assertFalse(filled("## Mulga design\n<!-- Updated: x -->"))
-        self.assertFalse(filled("## Mulga design\nUpdated: <mulga PR or bead id>"))
-        self.assertFalse(filled("## Mulga design\nUpdated:"))
-
-    def test_scoped_to_the_section(self):
-        self.assertFalse(filled("Updated: mulga#1"))
-        self.assertFalse(filled("## Mulga design\n## Other\nUpdated: mulga#1"))
-
-
-if __name__ == "__main__":
-    unittest.main()

.github/scripts/wait-for-memory_test.sh

@@ -1,47 +0,0 @@
-#!/bin/sh
-# wait-for-memory_test.sh — drives wait-for-memory.sh against a fake meminfo
-# that a background writer changes mid-wait, so each verdict is checked without
-# a loaded host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-WAIT="$SCRIPT_DIR/wait-for-memory.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-meminfo() { printf 'MemTotal:       65536000 kB\nMemAvailable:   %s kB\n' "$(( $1 * 1024 ))" > "$TMP/meminfo"; }
-MEMINFO="$TMP/meminfo"
-MEM_WAIT_POLL_S=1
-export MEMINFO MEM_WAIT_POLL_S
-
-meminfo 30000
-if MEM_WAIT_TIMEOUT_S=5 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  pass "enough memory returns at once"
-else
-  fail "enough memory returns at once: $(cat "$TMP/out")"
-fi
-
-meminfo 21577
-( sleep 2; meminfo 30000 ) &
-if MEM_WAIT_TIMEOUT_S=20 bash "$WAIT" 26624 > "$TMP/out" 2>&1 && grep -q waiting "$TMP/out"; then
-  pass "memory freed mid-wait is admitted"
-else
-  fail "memory freed mid-wait is admitted: $(cat "$TMP/out")"
-fi
-wait
-
-meminfo 21577
-if MEM_WAIT_TIMEOUT_S=2 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  fail "memory that never frees fails: exited 0"
-elif grep -q '::error::need 26624 MiB available, have 21577 MiB' "$TMP/out" && grep -q 'RSS' "$TMP/out"; then
-  pass "memory that never frees fails and names the biggest processes"
-else
-  fail "memory that never frees fails and names the biggest processes: $(cat "$TMP/out")"
-fi
-
-[ "$fails" -eq 0 ] || { echo "wait-for-memory: $fails failed"; exit 1; }
-echo "wait-for-memory: all passed"

.github/workflows/codex-review.yml

@@ -0,0 +1,21 @@
+# Codex review gate: calls bluebottle's reusable workflows from main, so a PR cannot change its own review.
+name: codex-review
+
+on:
+  pull_request_target:
+    branches: [dev]
+    types: [opened, synchronize, reopened, ready_for_review, edited]
+  issue_comment:
+    types: [created]
+
+permissions: {}
+
+jobs:
+  review:
+    if: github.event_name == 'pull_request_target'
+    uses: mulgadc/bluebottle/.github/workflows/codex-pr-review.yml@main
+    secrets: inherit
+  waive:
+    if: github.event_name == 'issue_comment' && github.event.issue.pull_request
+    uses: mulgadc/bluebottle/.github/workflows/codex-waive.yml@main
+    secrets: inherit

.github/workflows/dev-release.yml

@@ -30,6 +30,10 @@ concurrency:
 jobs:
   prerelease:
     runs-on: ubuntu-26.04
+    env:
+      # Dev prereleases, newest published first: GitHub lists releases by tag name
+      # as text, so list order is not build order. The install Worker uses this rule.
+      DEV_RELEASES_JQ: '[.[] | select(.draft == false and .prerelease and .published_at != null and (.tag_name | test("-dev(\\.|$)")))] | sort_by(.published_at) | reverse'
     steps:
       # Full history and tags, because the tag name is git describe and a shallow
       # clone has no tag to describe against.
@@ -164,12 +168,12 @@ jobs:
               exit 1
           fi
           echo "latest is still $latest"
-          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
+          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq "$DEV_RELEASES_JQ | .[0].tag_name")"
           [ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
 
-      # Kept rather than unbounded: the Worker reads the first page of releases to
-      # resolve the channel, so enough prereleases would eventually push the real
-      # ones off it. Five leaves a few recent builds to roll back to.
+      # Kept rather than unbounded: the Worker reads one page of releases to resolve
+      # the channel, so enough prereleases would eventually push the newest off it.
+      # Five leaves a few recent builds to roll back to.
       - name: Prune old dev prereleases
         if: steps.version.outputs.exists != 'true'
         env:
@@ -177,7 +181,7 @@ jobs:
         run: |
           set -euo pipefail
           gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
-              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \
+              --jq "$DEV_RELEASES_JQ | .[5:] | .[].tag_name" \
               > stale.txt
           while read -r tag; do
               [ -n "$tag" ] || continue

.github/workflows/e2e-nightly.yml

@@ -1017,7 +1017,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
       - uses: actions/checkout@v7.0.1
@@ -1275,7 +1275,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
       - uses: actions/checkout@v7.0.1
@@ -1423,7 +1423,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Predastore
@@ -1547,7 +1547,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Predastore

.github/workflows/mulga-design.yml

@@ -2,7 +2,8 @@ name: Mulga design
 
 # Reminds the author to say whether the Mulga design records need to change.
 # It reads only the pull request body, uses no secrets, and warns without ever
-# failing: the check is a prompt, not a gate.
+# failing: the check is a prompt, not a gate. Fork PRs are skipped because their
+# authors cannot see the design records.
 
 on:
   pull_request:
@@ -16,7 +17,7 @@ permissions:
 jobs:
   mulga_design:
     name: Mulga design line
-    if: ${{ !endsWith(github.actor, '[bot]') }}
+    if: ${{ !endsWith(github.actor, '[bot]') && github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: ubuntu-26.04
     steps:
       - name: Checkout Code
@@ -28,6 +29,4 @@ jobs:
         env:
           PR_BODY: ${{ github.event.pull_request.body }}
         working-directory: .github/scripts
-        run: |
-          python3 -m unittest -q test_mulga_design_check
-          python3 mulga_design_check.py
+        run: bash mulga-design-check.sh

.github/workflows/perf-3node.yml

@@ -113,7 +113,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - uses: actions/checkout@v7.0.1

.github/workflows/publish-system-ami.yml

@@ -84,7 +84,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Viperblock
         uses: actions/checkout@v7.0.1

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1
@@ -85,7 +85,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1
@@ -335,7 +335,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1

.github/workflows/segscan-oracle.yml

@@ -34,7 +34,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock

.github/workflows/spinifex.yml

@@ -20,6 +20,12 @@ on:
 permissions:
   contents: read
 
+# A newer push to the same pull request supersedes its run. Pushes and dispatches
+# get a group of their own, so every commit on dev and main keeps its result.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   build_and_test:
     name: Build and Test
@@ -36,7 +42,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock
@@ -166,7 +172,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock
@@ -268,7 +274,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock

.github/workflows/ui.yml

@@ -30,6 +30,12 @@ on:
 permissions:
   contents: read
 
+# A newer push to the same pull request supersedes its run. Pushes and dispatches
+# get a group of their own, so every commit on dev and main keeps its result.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   ui_build_and_test:
     name: UI Build and Test

.gitignore

@@ -27,6 +27,7 @@ build/microvm/sqs-worker/.stage.*/
 build/microvm/lambda-runtime/vmlinuz
 build/microvm/lambda-runtime/initramfs.cpio.gz
 build/microvm/lambda-runtime/.stage.*/
+build/microvm/lambda-testlink-runtime/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -42,11 +42,15 @@ ifdef QUIET
   _COVQ  = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
   _RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
   _SECQ  = >
+  _HDR   = :
+  _RUNSH = sh "$$t" >"$$log" 2>&1 || { echo "-- $$t"; cat "$$log"; exit 1; }
 else
   _Q     =
   _COVQ  =
   _RACEQ =
   _SECQ  = 2>&1 | tee
+  _HDR   = echo -e
+  _RUNSH = echo "-- $$t"; sh "$$t" || exit 1
 endif
 
 build: go_build build-installer build-lb-agent generate-aws-model-coverage
@@ -78,6 +82,35 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
+# Controlled-test only: builds the tagged daemon and its hard-kill initializer
+# together, pinning the initializer's digest into that daemon build. The daemon
+# keeps its symbols so the pin can be read back from the binary.
+# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
+# build-lambda-testlink-image prints; it is read raw and validated before use.
+LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
+LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
+LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
+LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
+build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
+build-lambda-testlink:
+	@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
+	@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
+	@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
+	cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
+	scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
+	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
+	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
+	echo "initializer sha256 $$digest"; \
+	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
+	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
+	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	@mkdir -p $(LAMBDA_TESTLINK_DIR)
+	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
+	@rmdir $(LAMBDA_TESTLINK_DIR).tmp
+
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
 	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -193,6 +226,24 @@ bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
 
 build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
+
+# Controlled-test only: a separately built guest whose agent has the
+# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
+# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
+LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
+build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
+	@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
+	./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
+	install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
+	@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
+	[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
+	echo "LAMBDA_IMAGE_DIGEST=$$digest"
 
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -240,27 +291,29 @@ preflight:
 NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh
 
 test-build-scripts:
-	@echo -e "\n....Running build/scripts/**/*_test.sh...."
-	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running build/scripts/**/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
-	@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running .github/scripts/*_test.sh...."
-	@for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running scripts/terraform/latitude-spx/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find scripts/terraform/latitude-spx -maxdepth 1 -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh \
-		.github/scripts/wait-for-memory.sh .github/scripts/wait-for-memory_test.sh
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
-	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
-	@echo "  test-build-scripts ok"
+	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
+	done
+	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
+		.github/scripts/wait-for-memory.sh \
+		.github/scripts/mulga-design-check.sh
+	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
+	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
+	@$(_HDR) "  test-build-scripts ok"
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
@@ -327,7 +380,10 @@ test-cover:
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
 	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -538,7 +594,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -70,8 +70,8 @@ The platform serves **405 AWS API operations** across the services below. See ex
 | **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
 | **Route 53** | DNS | [Northstar](https://github.com/mulgadc/northstar): authoritative DNS with hosted zones | Available |
 | **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
-| **Lambda** | Serverless functions | | Q1 2027 |
-| **SQS** | Message queues | | Q1 2027 |
+| **Lambda** | Serverless functions | QEMU `microvm` execution environments | Q1 2027 |
+| **SQS** | Message queues | A dedicated NATS JetStream store in a per-Region QEMU `microvm` guest | Q1 2027 |
 
 Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 

cmd/lambda-agent/connect.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 )
 
@@ -29,20 +30,20 @@ func defaultStartup() startup {
 
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
-func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
+func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {
 	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
 	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
-		return nil, exitRefused, errors.Join(loadErr, err)
+		return nil, "", exitRefused, errors.Join(loadErr, err)
 	}
 	if loadErr != nil {
-		return nil, exitRefused, loadErr
+		return nil, "", exitRefused, loadErr
 	}
 	conn, err := hostlink.Dial(ctx, id)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
 		}
-		return nil, exitLink, err
+		return nil, "", exitLink, err
 	}
-	return conn, 0, nil
+	return conn, id.Scope, 0, nil
 }

cmd/lambda-agent/grant_testlink.go

@@ -0,0 +1,66 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+const entryGrant = "test-grant"
+
+// testGrant is the grant the host launched this VM under.
+type testGrant struct {
+	grantID, attemptID, environmentID string
+	generation                        uint64
+}
+
+// readGrant reads the test-grant fw_cfg entry: grant ID, attempt ID,
+// environment ID and generation, one per line.
+func readGrant(fwCfgDir string) (testGrant, error) {
+	b, err := os.ReadFile(filepath.Join(fwCfgDir, entryGrant, "raw"))
+	if err != nil {
+		return testGrant{}, fmt.Errorf("read %s: %w", entryGrant, err)
+	}
+	lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n")
+	if len(lines) != 4 {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	g := testGrant{grantID: lines[0], attemptID: lines[1], environmentID: lines[2]}
+	g.generation, err = strconv.ParseUint(lines[3], 10, 64)
+	if err != nil || g.generation == 0 || g.grantID == "" || g.attemptID == "" || g.environmentID == "" {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	return g, nil
+}
+
+// checkBinding refuses a run unless the grant names the authenticated environment
+// generation, the request carries that grant and its deadline is still ahead. The
+// clock check is a local guard only: it cannot detect skew or a superseded grant.
+func checkBinding(scope lambdav1.Scope, g testGrant, req hostlink.Request, now time.Time) error {
+	parts := strings.Split(string(scope), "/")
+	if len(parts) != 5 {
+		return errors.New("authenticated scope malformed")
+	}
+	if parts[3] != g.environmentID || parts[4] != strconv.FormatUint(g.generation, 10) {
+		return errors.New("grant is not for the authenticated environment generation")
+	}
+	if req.GrantID != g.grantID || req.AttemptID != g.attemptID {
+		return errors.New("request does not carry this VM's grant")
+	}
+	if req.DeadlineMs <= 0 {
+		return errors.New("request has no deadline")
+	}
+	if !now.Before(time.UnixMilli(req.DeadlineMs)) {
+		return errors.New("request deadline already passed")
+	}
+	return nil
+}

cmd/lambda-agent/helpers_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/1"
 
 // guest is a fake fw_cfg tree; unload removes it as unloading the module would.
 type guest struct {

cmd/lambda-agent/package_testlink.go

@@ -18,6 +18,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 
 	"golang.org/x/sys/unix"
 )
@@ -28,6 +29,7 @@ import (
 const (
 	entryPackageSize   = "test-package-size"
 	entryPackageSHA256 = "test-package-sha256"
+	entryHandler       = "test-handler"
 
 	devMaxZipBytes    = 16 << 20
 	devMaxUnzipBytes  = 64 << 20
@@ -37,8 +39,9 @@ const (
 )
 
 type packageSpec struct {
-	size   int64
-	sha256 [sha256.Size]byte
+	size    int64
+	sha256  [sha256.Size]byte
+	handler string
 }
 
 type unpackLimits struct {
@@ -76,6 +79,13 @@ func readPackageSpec(fwCfgDir string) (packageSpec, error) {
 		return spec, errors.New("package sha256 is not 64 hex digits")
 	}
 	copy(spec.sha256[:], b)
+	if spec.handler, err = read(entryHandler); err != nil {
+		return spec, err
+	}
+	if spec.handler == "" || len(spec.handler) > 128 || strings.IndexFunc(spec.handler, unicode.IsSpace) >= 0 ||
+		strings.ContainsRune(spec.handler, 0) {
+		return spec, errors.New("handler must be 1..128 characters with no whitespace")
+	}
 	return spec, nil
 }
 

cmd/lambda-agent/package_testlink_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"syscall"
 	"testing"
 
@@ -165,19 +166,23 @@ func TestUnpackChecksSizeAndHash(t *testing.T) {
 func TestReadPackageSpecBounds(t *testing.T) {
 	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 	cases := map[string]struct {
-		size, hash string
-		want       string
+		size, hash, handler string
+		want                string
 	}{
-		"valid":     {"10", sum, ""},
-		"zero":      {"0", sum, "outside"},
-		"too large": {"16777217", sum, "outside"},
-		"not hex":   {"10", "zz", "64 hex digits"},
-		"short":     {"10", sum[:62], "64 hex digits"},
+		"valid":            {"10", sum, "bootstrap.h", ""},
+		"zero":             {"0", sum, "bootstrap.h", "outside"},
+		"too large":        {"16777217", sum, "bootstrap.h", "outside"},
+		"not hex":          {"10", "zz", "bootstrap.h", "64 hex digits"},
+		"short":            {"10", sum[:62], "bootstrap.h", "64 hex digits"},
+		"empty handler":    {"10", sum, "", "handler"},
+		"handler space":    {"10", sum, "a b", "handler"},
+		"128-char handler": {"10", sum, strings.Repeat("h", 128), ""},
+		"129-char handler": {"10", sum, strings.Repeat("h", 129), "handler"},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			dir := t.TempDir()
-			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash, entryHandler: tc.handler} {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
 				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
 			}
@@ -185,6 +190,7 @@ func TestReadPackageSpecBounds(t *testing.T) {
 			if tc.want == "" {
 				require.NoError(t, err)
 				assert.Equal(t, int64(10), spec.size)
+				assert.Equal(t, tc.handler, spec.handler)
 				return
 			}
 			assert.ErrorContains(t, err, tc.want)

cmd/lambda-agent/start_tcp.go

@@ -25,7 +25,7 @@ func start(ctx context.Context) int {
 
 // run authenticates to the host and holds the connection until it closes.
 func (s startup) run(ctx context.Context) (int, error) {
-	conn, code, err := s.connect(ctx)
+	conn, _, code, err := s.connect(ctx)
 	if err != nil {
 		return code, err
 	}

cmd/lambda-agent/start_testlink.go

@@ -5,8 +5,8 @@ package main
 import (
 	"context"
 	"errors"
-	"flag"
 	"fmt"
+	"io"
 	"log/slog"
 	"net"
 	"net/http"
@@ -24,8 +24,6 @@ const (
 	taskRoot    = "/var/task"
 )
 
-var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
-
 // testlink relays one invocation over the test carrier on the authenticated
 // connection; the carrier is not the guest protocol.
 type testlink struct {
@@ -54,13 +52,14 @@ func start(ctx context.Context) int {
 func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
-	conn, code, err := t.connect(parent)
+	grant, grantErr := readGrant(t.fwCfgDir)
+	conn, scope, code, err := t.connect(parent)
 	if err != nil {
-		return code, errors.Join(err, specErr)
+		return code, errors.Join(err, specErr, grantErr)
 	}
 	defer conn.Close()
-	if specErr != nil {
-		return exitRefused, specErr
+	if err := errors.Join(specErr, grantErr); err != nil {
+		return exitRefused, err
 	}
 	link := hostlink.NewTestStream(conn)
 
@@ -76,6 +75,10 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	// Nothing of the tenant's is installed or run for an unbound request.
+	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
+		return exitRefused, err
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -93,7 +96,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	defer cancel(nil)
 	cmd, err := t.startBootstrap(ctx, []string{
 		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
-		"_HANDLER=" + *handler,
+		"_HANDLER=" + spec.handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
 		"PATH=/bin:/sbin:/usr/bin",
 	})
@@ -104,7 +107,11 @@ func (t testlink) run(parent context.Context) (int, error) {
 	go func() {
 		defer close(exited)
 		err := cmd.Wait()
-		cancel(fmt.Errorf("bootstrap exited: %w", err))
+		// Once a response is being received the result stands, so only an
+		// exit before that cancels the invocation.
+		if srv.BootstrapExited() {
+			cancel(fmt.Errorf("bootstrap exited: %w", err))
+		}
 	}()
 
 	err = agent.Run(ctx, &oneInvocation{Link: link, first: &first}, srv)
@@ -116,11 +123,18 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
+// shutdownGrace is how long past the invocation deadline the agent waits for
+// the host's shutdown frame once its result is written.
+const shutdownGrace = 2 * time.Second
+
 // oneInvocation serves the host's first request and then stops the agent:
-// each test VM runs a single invocation.
+// each test VM runs a single invocation. After the result is written it waits
+// for the host's shutdown frame or EOF, so the result is not cut off by the
+// poweroff that follows the agent's exit.
 type oneInvocation struct {
 	hostlink.Link
-	first *hostlink.Request
+	first      *hostlink.Request
+	deadlineMs int64
 }
 
 var _ hostlink.Link = (*oneInvocation)(nil)
@@ -129,10 +143,36 @@ func (o *oneInvocation) Next(ctx context.Context) (hostlink.Request, error) {
 	if err := ctx.Err(); err != nil {
 		return hostlink.Request{}, err
 	}
-	if o.first == nil {
-		return hostlink.Request{}, hostlink.ErrShutdown
+	if o.first != nil {
+		req := *o.first
+		o.first = nil
+		o.deadlineMs = req.DeadlineMs
+		return req, nil
 	}
-	req := *o.first
-	o.first = nil
-	return req, nil
+	return hostlink.Request{}, o.awaitShutdown(ctx)
+}
+
+// awaitShutdown reads the host's next frame and always ends the agent: with
+// ErrShutdown for a shutdown frame, EOF, a stray frame or an expired wait.
+func (o *oneInvocation) awaitShutdown(ctx context.Context) error {
+	end := time.UnixMilli(o.deadlineMs)
+	if now := time.Now(); end.Before(now) {
+		end = now
+	}
+	wctx, cancel := context.WithDeadline(ctx, end.Add(shutdownGrace))
+	defer cancel()
+	_, err := o.Link.Next(wctx)
+	switch {
+	case errors.Is(err, hostlink.ErrShutdown), errors.Is(err, io.EOF):
+		return hostlink.ErrShutdown
+	case err == nil:
+		slog.Warn("lambda-agent: frame after the invocation was not dispatched")
+		return hostlink.ErrShutdown
+	case ctx.Err() != nil:
+		return err
+	case wctx.Err() != nil:
+		slog.Warn("lambda-agent: no shutdown from the host; exiting")
+		return hostlink.ErrShutdown
+	}
+	return err
 }

cmd/lambda-agent/start_testlink_test.go

@@ -25,12 +25,18 @@ import (
 
 const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"
 
+// helperHandlerOut names the file the helper bootstrap writes its _HANDLER to.
+const helperHandlerOut = "LAMBDA_AGENT_TEST_HANDLER_OUT"
+
 // TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
 // echoes one invocation through the Runtime API, then waits for the next.
 func TestHelperBootstrap(t *testing.T) {
 	if os.Getenv(helperEnv) != "1" {
 		return
 	}
+	if out := os.Getenv(helperHandlerOut); out != "" {
+		require.NoError(t, os.WriteFile(out, []byte(os.Getenv("_HANDLER")), 0o600))
+	}
 	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
 	resp, err := http.Get(base + "next")
 	require.NoError(t, err)
@@ -41,14 +47,28 @@ func TestHelperBootstrap(t *testing.T) {
 	post, err := http.Post(base+id+"/response", "application/json", bytes.NewReader(body))
 	require.NoError(t, err)
 	post.Body.Close()
+	if os.Getenv(helperExitAfterPost) == "1" {
+		return
+	}
 	_, _ = http.Get(base + "next")
 }
 
+// helperExitAfterPost makes the helper bootstrap exit once its response is posted.
+const helperExitAfterPost = "LAMBDA_AGENT_TEST_EXIT_AFTER_POST"
+
+// testGrantEntry grants g-1 for attempt a-1 on the fixture scope's env-1/1.
+const testGrantEntry = "g-1\na-1\nenv-1\n1\n"
+
+// testHandler is the handler the fixture's host configures.
+const testHandler = "bootstrap.fixture"
+
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
 	task    string
 	sealed  bool
+	// handlerOut receives the _HANDLER the bootstrap process itself saw.
+	handlerOut string
 }
 
 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -61,9 +81,16 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	sum := sha256.Sum256(zipBytes)
 	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
 	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	if _, ok := entries[entryHandler]; !ok {
+		entries[entryHandler] = []byte(testHandler)
+	}
+	if _, ok := entries[entryGrant]; !ok {
+		entries[entryGrant] = []byte(testGrantEntry)
+	}
 	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
+	b.handlerOut = filepath.Join(t.TempDir(), "handler")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -77,7 +104,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			b.n++
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
-			cmd.Env = append(env, helperEnv+"=1")
+			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
 			return cmd, cmd.Start()
 		},
 	}
@@ -132,10 +159,105 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	seen, err := os.ReadFile(b.handlerOut)
+	require.NoError(t, err, "bootstrap process did not report its handler")
+	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
 
+// A bootstrap that posts its response and exits at once does not cost the
+// host the result; the agent then waits for the host's shutdown.
+func TestTestlinkBootstrapExitingAfterResponseStillDelivers(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	start := tl.startBootstrap
+	tl.startBootstrap = func(ctx context.Context, env []string) (*exec.Cmd, error) {
+		return start(ctx, append(env, helperExitAfterPost+"=1"))
+	}
+	done := runTestlinkAsync(ctx, tl)
+
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	payload := bytes.Repeat([]byte("0123456789"), 300_000)
+	req := hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: payload}
+	require.NoError(t, hs.HostSend(req))
+	got, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, hostlink.Response{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", Payload: payload}, got)
+
+	require.NoError(t, hs.HostShutdown())
+	res := wait(t, done, "agent to stop after the host's shutdown")
+	assert.Equal(t, exitClosed, res.code)
+	assert.NoError(t, res.err)
+}
+
+// The agent stays up for the host's shutdown rather than exiting as soon as
+// its result is written, and never dispatches a stray frame.
+func TestOneInvocationAfterResult(t *testing.T) {
+	first := hostlink.Request{RequestID: "r-1", DeadlineMs: time.Now().Add(time.Minute).UnixMilli()}
+	cases := map[string]struct {
+		next func(context.Context) (hostlink.Request, error)
+		want error
+	}{
+		"shutdown frame": {func(context.Context) (hostlink.Request, error) { return hostlink.Request{}, hostlink.ErrShutdown }, hostlink.ErrShutdown},
+		"eof":            {func(context.Context) (hostlink.Request, error) { return hostlink.Request{}, io.EOF }, hostlink.ErrShutdown},
+		"a second request is not dispatched": {func(context.Context) (hostlink.Request, error) {
+			return hostlink.Request{RequestID: "r-2"}, nil
+		}, hostlink.ErrShutdown},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			o := &oneInvocation{Link: stubLink{tc.next}, first: &first}
+			_, err := o.Next(context.Background())
+			require.NoError(t, err)
+			req, err := o.Next(context.Background())
+			require.ErrorIs(t, err, tc.want)
+			assert.Empty(t, req.RequestID)
+		})
+	}
+	t.Run("host never sends shutdown", func(t *testing.T) {
+		past := first
+		past.DeadlineMs = time.Now().Add(-time.Second).UnixMilli()
+		o := &oneInvocation{Link: stubLink{func(ctx context.Context) (hostlink.Request, error) {
+			<-ctx.Done()
+			return hostlink.Request{}, ctx.Err()
+		}}, first: &past}
+		_, err := o.Next(context.Background())
+		require.NoError(t, err)
+		start := time.Now()
+		_, err = o.Next(context.Background())
+		require.ErrorIs(t, err, hostlink.ErrShutdown)
+		assert.GreaterOrEqual(t, time.Since(start), shutdownGrace-100*time.Millisecond, "waited the grace")
+	})
+	t.Run("parent cancel is not a shutdown", func(t *testing.T) {
+		o := &oneInvocation{Link: stubLink{func(ctx context.Context) (hostlink.Request, error) {
+			<-ctx.Done()
+			return hostlink.Request{}, ctx.Err()
+		}}, first: &first}
+		_, _ = o.Next(context.Background())
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel()
+		_, err := o.Next(ctx)
+		require.ErrorIs(t, err, context.Canceled)
+	})
+}
+
+type stubLink struct {
+	next func(context.Context) (hostlink.Request, error)
+}
+
+var _ hostlink.Link = stubLink{}
+
+func (l stubLink) Next(ctx context.Context) (hostlink.Request, error) { return l.next(ctx) }
+
+func (stubLink) Complete(context.Context, hostlink.Response) error { return nil }
+
 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -218,7 +340,8 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
 
 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
+		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
 	assert.ErrorContains(t, res.err, "install package")
@@ -244,3 +367,87 @@ func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
 	_, err := os.Stat(tl.fwCfgRoot)
 	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
 }
+
+// A request that does not carry this VM's grant, or a grant for another
+// environment generation than the one authenticated, installs and runs
+// nothing of the tenant's.
+func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
+	cases := map[string]struct {
+		grant            string
+		grantID, attempt string
+		deadlineMs       int64
+		want             string
+	}{
+		"frame with no deadline":      {testGrantEntry, "g-1", "a-1", -1, "no deadline"},
+		"frame with expired deadline": {testGrantEntry, "g-1", "a-1", time.Now().Add(-time.Second).UnixMilli(), "deadline already passed"},
+		"frame for another grant":     {testGrantEntry, "g-2", "a-1", 0, "does not carry this VM's grant"},
+		"frame for another attempt":   {testGrantEntry, "g-1", "a-2", 0, "does not carry this VM's grant"},
+		"frame with no grant":         {testGrantEntry, "", "", 0, "does not carry this VM's grant"},
+		"grant for another env":       {"g-1\na-1\nenv-2\n1\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+		"grant for another gen":       {"g-1\na-1\nenv-1\n2\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			entries[entryGrant] = []byte(tc.grant)
+			done := runTestlinkAsync(ctx, b.testlink(t, entries))
+			conn := h.guest(t)
+			defer conn.Close()
+			deadline := time.Now().Add(5 * time.Second).UnixMilli()
+			switch {
+			case tc.deadlineMs < 0:
+				deadline = 0
+			case tc.deadlineMs > 0:
+				deadline = tc.deadlineMs
+			}
+			req := hostlink.Request{GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
+				DeadlineMs: deadline, Payload: []byte(`{}`)}
+			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+
+			res := wait(t, done, "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, tc.want)
+			assert.Zero(t, b.n)
+			assert.False(t, b.sealed)
+			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
+		})
+	}
+}
+
+// A missing or malformed grant entry refuses before any frame is awaited.
+func TestTestlinkBadGrantEntryRefuses(t *testing.T) {
+	cases := map[string]*string{
+		"missing":       nil,
+		"too few lines": ptr("g-1\na-1\nenv-1\n"),
+		"generation 0":  ptr("g-1\na-1\nenv-1\n0\n"),
+		"empty grant":   ptr("\na-1\nenv-1\n1\n"),
+	}
+	for name, grant := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			if grant != nil {
+				entries[entryGrant] = []byte(*grant)
+			}
+			tl := b.testlink(t, entries)
+			if grant == nil {
+				require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryGrant)))
+			}
+			res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, entryGrant)
+			assert.Zero(t, b.n)
+		})
+	}
+}
+
+func ptr(s string) *string { return &s }

cmd/lambda-image-agent-tag/main.go

@@ -0,0 +1,221 @@
+// Command lambda-image-agent-tag checks whether the lambda-agent archived in a
+// built Lambda guest initramfs carries the lambdatestlink build tag. Build
+// information is recorded by the build; it is not proof of how the agent behaves.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+const (
+	agentPath    = "usr/local/bin/lambda-agent"
+	maxArchive   = 1 << 30
+	maxAgent     = 256 << 20
+	maxName      = 4096
+	newcHeader   = 110
+	trailerName  = "TRAILER!!!"
+	modeTypeMask = 0o170000
+	modeDir      = 0o040000
+	modeRegular  = 0o100000
+)
+
+var agentParents = []string{"usr", "usr/local", "usr/local/bin"}
+
+func main() {
+	if len(os.Args) != 3 || (os.Args[2] != "present" && os.Args[2] != "absent") {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-agent-tag <initramfs.cpio.gz> present|absent")
+		os.Exit(2)
+	}
+	if err := run(os.Args[1], os.Args[2] == "present"); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[1], err)
+		os.Exit(1)
+	}
+}
+
+func run(path string, want bool) error {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+	if !fi.Mode().IsRegular() {
+		return errors.New("not a regular file")
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	agent, err := archivedAgent(f)
+	if err != nil {
+		return err
+	}
+	return checkTag(agent, want)
+}
+
+// archivedAgent returns the single agent entry after checking listed names,
+// entry types, the agent's link count and zero-only padding after the trailer.
+// It is not a full kernel-equivalent parser; 070702 CRCs, for one, are unchecked.
+func archivedAgent(r io.Reader) ([]byte, error) {
+	zr, err := gzip.NewReader(bufio.NewReader(r))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	raw, err := io.ReadAll(io.LimitReader(zr, maxArchive+1))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	if len(raw) > maxArchive {
+		return nil, fmt.Errorf("archive larger than %d bytes", maxArchive)
+	}
+
+	seen := map[string]bool{}
+	var agent []byte
+	off := 0
+	for {
+		e, next, err := parseEntry(raw, off)
+		if err != nil {
+			return nil, fmt.Errorf("entry at offset %d: %w", off, err)
+		}
+		off = next
+		if e.name == trailerName {
+			break
+		}
+		if !canonical(e.name) {
+			return nil, fmt.Errorf("entry name %q is not canonical", e.name)
+		}
+		if seen[e.name] {
+			return nil, fmt.Errorf("entry %q appears more than once", e.name)
+		}
+		seen[e.name] = true
+		for _, p := range agentParents {
+			if e.name == p && e.mode&modeTypeMask != modeDir {
+				return nil, fmt.Errorf("%s is not a directory", p)
+			}
+		}
+		if e.name == agentPath {
+			for _, p := range agentParents {
+				if !seen[p] {
+					return nil, fmt.Errorf("%s has no %s directory entry before it", agentPath, p)
+				}
+			}
+			if e.mode&modeTypeMask != modeRegular {
+				return nil, fmt.Errorf("%s is not a regular file", agentPath)
+			}
+			if e.nlink != 1 {
+				return nil, fmt.Errorf("%s has %d links, want 1", agentPath, e.nlink)
+			}
+			if len(e.data) > maxAgent {
+				return nil, fmt.Errorf("%s is larger than %d bytes", agentPath, maxAgent)
+			}
+			agent = e.data
+		}
+	}
+	if i := bytes.IndexFunc(raw[off:], func(r rune) bool { return r != 0 }); i >= 0 {
+		return nil, fmt.Errorf("data after the archive trailer at offset %d", off+i)
+	}
+	if agent == nil {
+		return nil, fmt.Errorf("no %s entry", agentPath)
+	}
+	return agent, nil
+}
+
+type entry struct {
+	name  string
+	mode  uint64
+	nlink uint64
+	data  []byte
+}
+
+func parseEntry(raw []byte, off int) (entry, int, error) {
+	if len(raw)-off < newcHeader {
+		return entry{}, 0, errors.New("truncated header")
+	}
+	h := raw[off : off+newcHeader]
+	if m := string(h[:6]); m != "070701" && m != "070702" {
+		return entry{}, 0, fmt.Errorf("magic %q is not newc", m)
+	}
+	field := func(i int) (uint64, error) {
+		return strconv.ParseUint(string(h[6+8*i:14+8*i]), 16, 32)
+	}
+	mode, err1 := field(1)
+	nlink, err2 := field(4)
+	size, err3 := field(6)
+	nameSize, err4 := field(11)
+	if err := errors.Join(err1, err2, err3, err4); err != nil {
+		return entry{}, 0, fmt.Errorf("bad header field: %w", err)
+	}
+	if nameSize < 2 || nameSize > maxName {
+		return entry{}, 0, fmt.Errorf("name size %d out of range", nameSize)
+	}
+	nameEnd := off + newcHeader + int(nameSize)
+	dataOff := align4(nameEnd)
+	dataEnd := dataOff + int(size)
+	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
+		return entry{}, 0, errors.New("truncated entry")
+	}
+	name := raw[off+newcHeader : nameEnd]
+	if name[len(name)-1] != 0 || bytes.IndexByte(name[:len(name)-1], 0) >= 0 {
+		return entry{}, 0, errors.New("name is not one NUL-terminated string")
+	}
+	return entry{name: string(name[:len(name)-1]), mode: mode, nlink: nlink, data: raw[dataOff:dataEnd]}, align4(dataEnd), nil
+}
+
+func align4(n int) int { return (n + 3) &^ 3 }
+
+// canonical accepts "." and relative names with no empty, "." or ".." component.
+func canonical(name string) bool {
+	if name == "." {
+		return true
+	}
+	if name == "" || strings.HasPrefix(name, "/") {
+		return false
+	}
+	for c := range strings.SplitSeq(name, "/") {
+		if c == "" || c == "." || c == ".." {
+			return false
+		}
+	}
+	return true
+}
+
+func checkTag(agent []byte, want bool) error {
+	info, err := buildinfo.Read(bytes.NewReader(agent))
+	if err != nil {
+		return fmt.Errorf("no Go build information in %s: %w", agentPath, err)
+	}
+	return decide(info, want)
+}
+
+// decide refuses build information with no recorded settings, since their
+// absence does not show the tag was absent.
+func decide(info *buildinfo.BuildInfo, want bool) error {
+	if len(info.Settings) == 0 {
+		return fmt.Errorf("%s records no build settings", agentPath)
+	}
+	if hasTag(info, "lambdatestlink") != want {
+		return fmt.Errorf("%s: lambdatestlink build tag present=%v, want %v", agentPath, !want, want)
+	}
+	return nil
+}
+
+func hasTag(info *buildinfo.BuildInfo, tag string) bool {
+	for _, s := range info.Settings {
+		if s.Key != "-tags" {
+			continue
+		}
+		if slices.Contains(strings.Split(s.Value, ","), tag) {
+			return true
+		}
+	}
+	return false
+}

cmd/lambda-image-agent-tag/main_test.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type fileEntry struct {
+	name  string
+	mode  uint32
+	nlink uint32
+	data  string
+}
+
+func dir(name string) fileEntry { return fileEntry{name: name, mode: modeDir | 0o755, nlink: 2} }
+func file(name, data string) fileEntry {
+	return fileEntry{name: name, mode: modeRegular | 0o755, nlink: 1, data: data}
+}
+func symlink(name, target string) fileEntry {
+	return fileEntry{name: name, mode: 0o120777, nlink: 1, data: target}
+}
+
+// newc writes entries and a trailer in the format the kernel unpacks.
+func newc(entries ...fileEntry) []byte {
+	var b bytes.Buffer
+	for i, e := range append(entries, fileEntry{name: trailerName, nlink: 1}) {
+		fmt.Fprintf(&b, "070701%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X",
+			i+1, e.mode, 0, 0, e.nlink, 0, len(e.data), 0, 0, 0, 0, len(e.name)+1, 0)
+		b.WriteString(e.name)
+		b.WriteByte(0)
+		pad(&b)
+		b.WriteString(e.data)
+		pad(&b)
+	}
+	return b.Bytes()
+}
+
+func pad(b *bytes.Buffer) {
+	for b.Len()%4 != 0 {
+		b.WriteByte(0)
+	}
+}
+
+func gz(parts ...[]byte) []byte {
+	var b bytes.Buffer
+	for _, p := range parts {
+		w := gzip.NewWriter(&b)
+		_, _ = w.Write(p)
+		_ = w.Close()
+	}
+	return b.Bytes()
+}
+
+func withAgent(agent fileEntry) []byte {
+	return newc(dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent, file("init", "#!/bin/sh"))
+}
+
+func TestArchivedAgent_ReturnsTheOneArchivedAgent(t *testing.T) {
+	got, err := archivedAgent(bytes.NewReader(gz(withAgent(file(agentPath, "agent-bytes")))))
+	require.NoError(t, err)
+	require.Equal(t, "agent-bytes", string(got))
+
+	padded := append(withAgent(file(agentPath, "agent-bytes")), make([]byte, 1024)...)
+	got, err = archivedAgent(bytes.NewReader(gz(padded)))
+	require.NoError(t, err, "zero padding after the trailer")
+	require.Equal(t, "agent-bytes", string(got))
+}
+
+func TestArchivedAgent_Refuses(t *testing.T) {
+	parents := []fileEntry{dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin")}
+	agent := file(agentPath, "untagged")
+	small := newc(append(parents, agent)...)
+	second := newc(append(parents[1:], file(agentPath, "tagged"))...)
+	// The second archive starts inside what would be the first one's final block.
+	sameBlock := append(append([]byte{}, small...), second...)
+	for len(sameBlock)%512 != 0 {
+		sameBlock = append(sameBlock, 0)
+	}
+
+	for name, archive := range map[string][]byte{
+		"aliased duplicate":            gz(newc(append(parents, agent, file("././"+agentPath, "tagged"))...)),
+		"leading slash alias":          gz(newc(append(parents, agent, file("/"+agentPath, "tagged"))...)),
+		"dot-dot component":            gz(newc(append(parents, agent, file("usr/local/../local/bin/lambda-agent", "x"))...)),
+		"empty component":              gz(newc(append(parents, agent, file("usr//local/bin/lambda-agent", "x"))...)),
+		"exact duplicate":              gz(newc(append(parents, agent, file(agentPath, "tagged"))...)),
+		"duplicate directory":          gz(newc(dir("usr"), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent)),
+		"missing directory entry":      gz(newc(dir("usr"), dir("usr/local/bin"), agent)),
+		"directory after the agent":    gz(newc(dir("usr"), dir("usr/local"), agent, dir("usr/local/bin"))),
+		"parent is a symlink":          gz(newc(dir("usr"), dir("usr/local"), symlink("usr/local/bin", "elsewhere"), agent)),
+		"agent is a symlink":           gz(newc(append(parents, symlink(agentPath, "/bin/sh"))...)),
+		"hard-linked agent":            gz(newc(append(parents, fileEntry{name: agentPath, mode: modeRegular | 0o755, nlink: 2, data: "x"})...)),
+		"no agent":                     gz(newc(parents...)),
+		"second archive appended":      gz(small, second),
+		"second archive in same block": gz(sameBlock),
+		"non-zero byte after trailer":  gz(append(append([]byte{}, small...), 0, 0, 1)),
+		"truncated archive":            gz(small[:len(small)-20]),
+		"not newc":                     gz([]byte("not cpio at all, but long enough to fill one newc header: " + string(make([]byte, 120)))),
+		"not gzip":                     small,
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := archivedAgent(bytes.NewReader(archive))
+			require.Error(t, err)
+		})
+	}
+}
+
+func TestRun_RefusesASymlinkedImagePath(t *testing.T) {
+	d := t.TempDir()
+	img := filepath.Join(d, "initramfs.cpio.gz")
+	require.NoError(t, os.WriteFile(img, gz(withAgent(file(agentPath, "x"))), 0o644))
+	require.NoError(t, os.Symlink(img, filepath.Join(d, "link")))
+	require.ErrorContains(t, run(filepath.Join(d, "link"), false), "not a regular file")
+}
+
+func TestHasTag_MatchesWholeTagsOnly(t *testing.T) {
+	info := func(tags string) *buildinfo.BuildInfo {
+		return &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}, {Key: "-tags", Value: tags}}}
+	}
+	require.True(t, hasTag(info("lambdatestlink,fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("lambdatestlinkx,xlambdatestlink"), "lambdatestlink"))
+	require.False(t, hasTag(&buildinfo.BuildInfo{}, "lambdatestlink"))
+}
+
+// The test binary carries real build information, so checkTag reads it and
+// refuses the opposite answer; bytes with no build information are refused.
+func TestCheckTag_ReadsRealBuildInformation(t *testing.T) {
+	exe, err := os.Executable()
+	require.NoError(t, err)
+	raw, err := os.ReadFile(exe)
+	require.NoError(t, err)
+	info, err := buildinfo.Read(bytes.NewReader(raw))
+	require.NoError(t, err)
+	tagged := hasTag(info, "lambdatestlink")
+	require.NoError(t, checkTag(raw, tagged))
+	require.Error(t, checkTag(raw, !tagged))
+	require.Error(t, checkTag([]byte("not a Go binary"), false))
+}
+
+func TestDecide_RefusesMissingBuildSettings(t *testing.T) {
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, false), "no build settings")
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, true), "no build settings")
+	untagged := &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}}}
+	require.NoError(t, decide(untagged, false))
+	require.Error(t, decide(untagged, true))
+}

cmd/lambda-image-digest/main.go

@@ -0,0 +1,26 @@
+//go:build lambdatestlink
+
+// Command lambda-image-digest prints the test-link runtime image digest of one
+// directory, selecting the boot files the same way the daemon does. The digest
+// identifies the bytes; it does not authenticate where they came from.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-digest <image-dir>")
+		os.Exit(2)
+	}
+	digest, err := testlink.DirDigest(os.Args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	fmt.Println(digest)
+}

contracts/cluster/v1/facts.go

@@ -56,6 +56,8 @@ type NodeFacts struct {
 	Memory      MemoryFacts       `json:"memory"`
 	NICs        []NICFacts        `json:"nics"`
 	Planes      []PlaneFacts      `json:"planes"`
+	Disks       []DiskFacts       `json:"disks,omitempty"`
+	Filesystems []FilesystemFacts `json:"filesystems,omitempty"`
 	Sources     map[string]string `json:"sources"`
 }
 
@@ -134,6 +136,42 @@ type MemoryGroup struct {
 	Serials       []string `json:"serials,omitempty"`
 }
 
+// DiskFacts is one physical drive as sysfs reports it. Transport is nvme, sata,
+// usb or virtio, and empty when the device path names none, as a RAID
+// controller's virtual disk does.
+type DiskFacts struct {
+	Name       string `json:"name"`
+	Path       string `json:"path"`
+	SizeBytes  uint64 `json:"size_bytes"`
+	Model      string `json:"model,omitempty"`
+	Transport  string `json:"transport,omitempty"`
+	Rotational bool   `json:"rotational,omitempty"`
+	Removable  bool   `json:"removable,omitempty"`
+}
+
+// FilesystemFacts is one mounted block-device or ZFS filesystem. BindOf names
+// the directory a bind mount shows; Disk names the drives under it. Error says
+// why Usage is missing or stale.
+type FilesystemFacts struct {
+	Mount  string           `json:"mount"`
+	Source string           `json:"source"`
+	Disk   string           `json:"disk,omitempty"`
+	Type   string           `json:"type"`
+	BindOf string           `json:"bind_of,omitempty"`
+	Usage  *FilesystemUsage `json:"usage,omitempty"`
+	Error  string           `json:"error,omitempty"`
+	// ID identifies the mount on its node, so a filesystem replaced at the
+	// same path is read afresh. Empty for a mount the daemon cannot see.
+	ID string `json:"-"`
+}
+
+// FilesystemUsage is a filesystem's last statfs reading, as df counts it.
+type FilesystemUsage struct {
+	SizeBytes  uint64 `json:"size_bytes"`
+	UsedBytes  uint64 `json:"used_bytes"`
+	AvailBytes uint64 `json:"avail_bytes"`
+}
+
 // NICFacts is one physical network interface and, where LLDP reports it, the
 // switch port it is cabled to.
 type NICFacts struct {

contracts/ec2/v1/README.md

@@ -37,6 +37,13 @@ contract or an approved compatibility plan; it is not a package refactor.
 Only the subject is owned here today.
 The request (`account_id`) and reply (`vpc_id`, `error`) bodies are still declared as anonymous structs on each side; moving them into this package is a contract change that needs its own characterization first.
 
+## Volume held elsewhere
+
+A start refused because another node holds one of the instance's volumes is an `IncorrectState` error payload on the `ec2.start`, `ec2.start.<node>` and `ec2.cmd.<instance-id>` replies, and the AWS body cannot say why.
+The owning node adds the `X-Volume-Held-Elsewhere: true` header to that reply, the `ec2.start` forwarder relays it, and `StartInstances` wraps `ErrVolumeHeldElsewhere` around the error it rebuilds, so a caller matches it with `errors.Is` rather than reading the message.
+The owning node sets the header only from the EBS provider's `volume_held_elsewhere` code, which every provider returns for this refusal; no reader matches the message text.
+`VolumeHeldElsewhereText` is the wording viperblockd puts in that refusal's message, not a signal.
+
 ## Platform inventory
 
 `PlatformInventorySubject` (`ec2.v1.PlatformInventory`) is a request/reply route on which a daemon in the `spinifex-workers` queue group reports one account's key pairs, placement groups, launch templates and their `ec2-tags` entries from the EC2 owner stores.

contracts/ec2/v1/volume_held.go

@@ -1,7 +1,13 @@
 package ec2v1
 
-// VolumeHeldElsewhereText is the text of a start refused because another node
-// holds one of the instance's volumes. It crosses NATS as message text, so
-// readers match on it; the provider that refuses and every reader share this
-// string so they cannot drift.
+import "errors"
+
+// VolumeHeldElsewhereText is the message viperblockd gives a lease refusal. It
+// is wording for the operator; callers classify the refusal by its code.
 const VolumeHeldElsewhereText = "volume is leased by another owner"
+
+// ErrVolumeHeldElsewhere marks a start refused because another node still
+// holds one of the instance's volumes. The owning node sets it from the EBS
+// provider's held-elsewhere code, and a start reply carries it across NATS as
+// the X-Volume-Held-Elsewhere header rather than as message text.
+var ErrVolumeHeldElsewhere = errors.New("a volume is leased by another node")

docs/COMMANDS.md

@@ -95,7 +95,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--advertise` (external IP off-host clients dial; auto-detected from the default route), `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--predastore-compaction-interval` (seconds; test clusters set a short one), `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--discard-jetstream` (with `--nodes` >= 2, remove this node's existing JetStream streams once every node has joined; without it a multi-node init over a store that holds streams is refused, and it never proceeds while NATS is running), `--skip-host-dns`, `--email` (operator address for update and security notices), `--no-telemetry`, `--gpu-passthrough`, `--ipsec` (default true; encrypt intra-AZ Geneve via OVN native IPsec cluster-wide — disable only for a trusted single-rack lab; on a host whose IPsec stack is libreswan it defaults to false and `--ipsec=true` is refused), `--host` (leader to join; falls back to multicast discovery), `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--external-iface` (WAN NIC for br-external, auto-detected), `--external-source` (`dhcp` \| `static`), `--external-pool` (`start-end`), `--external-prefix-len` (default 24), `--external-gateway` (auto-detected), `--external-dns` (comma-separated IPv4 addresses, at most 3; the DNS servers written to the external pool's `dns_servers` and handed to guests, verbatim; when omitted they are detected from the host resolver at init time, so set it to give every node of a cluster the same value), `--external-bind-bridge` (default `br-wan` when `--external-source=dhcp`), `--gateway-ip` (OVN gateway router's external IP for SNAT; defaults to the pool's range start, required for `nat` mode without DHCP) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--advertise` (external IP off-host clients dial; auto-detected from the default route), `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--predastore-compaction-interval` (seconds; test clusters set a short one), `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--discard-jetstream` (with `--nodes` >= 2, remove this node's existing JetStream streams once every node has joined; without it a multi-node init over a store that holds streams is refused, and it never proceeds while NATS is running), `--skip-host-dns`, `--email` (operator address for update and security notices), `--no-telemetry`, `--gpu-passthrough`, `--ipsec` (default true; encrypt intra-AZ Geneve via OVN native IPsec cluster-wide — disable only for a trusted single-rack lab; on a host whose IPsec stack is libreswan it defaults to false and `--ipsec=true` is refused), `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--external-iface` (WAN NIC for br-external, auto-detected), `--external-source` (`dhcp` \| `static`), `--external-pool` (`start-end`), `--external-prefix-len` (default 24), `--external-gateway` (auto-detected), `--external-dns` (comma-separated IPv4 addresses, at most 3; the DNS servers written to the external pool's `dns_servers` and handed to guests, verbatim; when omitted they are detected from the host resolver at init time, so set it to give every node of a cluster the same value), `--external-bind-bridge` (default `br-wan` when `--external-source=dhcp`), `--gateway-ip` (OVN gateway router's external IP for SNAT; defaults to the pool's range start, required for `nat` mode without DHCP) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
 | `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--advertise` (defaults to `--bind`, or the auto-detected WAN IP when `--bind` is 0.0.0.0), `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services`, `--skip-host-dns`, `--email`, `--no-telemetry`, `--predastore-compaction-interval`, `--join-timeout` (how long to keep retrying while the formation server is unreachable, default 20m), `--force` (join even though this node is already initialized or holds JetStream streams, **discarding its own CA, master key and JetStream store**; the store is removed only after formation succeeds, and the join is refused while NATS is running even with `--force`) | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations → points the host resolver at this node's local northstar so the Spinifex zones resolve from the node itself (opt out with `--skip-host-dns`) |
 
 ### Version
@@ -135,7 +135,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. |
+| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. It also lists UEFI variable stores nothing owns: a `uefi/i-*/` prefix with no instance record and no store record, a `uefi/ami-*/` prefix with no AMI document, and a store record that is not `deleting` and has no instance record. These are reported and never deleted, because an absent instance record is not proof of a terminate. |
 | `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is refused while that node is alive (its daemon heartbeat is under 60 seconds old, or cannot be read) and is otherwise allowed, in which case it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
 | `spx admin storage status` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Sends the signed `GetStorageStatus` action to the AWS gateway with credentials from the standard AWS chain (e.g. `AWS_PROFILE=spinifex`) and prints what the console's Service Metrics page shows: predastore encoding, meta and blob node counts, cluster health, a meta node table (health, raft state, leader, term, commit and applied index) and the blob nodes. Cluster health is `Degraded` when any meta node is unreachable or the nodes reporting raft state do not agree on one leader. A meta node read only through its host's `/readyz` shows healthy with blank raft fields. |
 

docs/compute/launching-instances/README.md

@@ -23,6 +23,7 @@ resources:
 - [Overview](#overview)
 - [Launch](#launch)
 - [Manage](#manage)
+- [Stopping and Data Volumes](#stopping-and-data-volumes)
 - [Modify Instance Attributes](#modify-instance-attributes)
 - [Instance Metadata Options](#instance-metadata-options)
 - [Spot Instances](#spot-instances)
@@ -73,6 +74,31 @@ aws ec2 terminate-instances --instance-ids $INSTANCE_ID
 aws ec2 reboot-instances --instance-ids $INSTANCE_ID
 ```
 
+## Stopping and Data Volumes
+
+A guest that powers itself off (`poweroff`, `shutdown -h`) goes `stopping` then `stopped`, the same as `stop-instances`.
+It is not restarted automatically.
+`InstanceInitiatedShutdownBehavior=terminate` is not honoured yet: a guest shutdown always stops the instance.
+
+Data volumes stay `in-use` and attached to a stopped instance, and `start-instances` mounts them again.
+While the instance is stopped, `attach-volume` of one of its data volumes to another instance returns `VolumeInUse`, and `delete-volume` is refused.
+
+`detach-volume` works on a stopped instance, with or without `--instance-id`.
+The volume becomes `available` and the next start does not attach it.
+The root volume of a stopped instance can be detached too, as on AWS.
+The internal EFI variable store is not a customer volume: it never appears in `describe-volumes` or `BlockDeviceMappings`, and detaching it is refused with `OperationNotPermitted`.
+The instance then cannot start: `start-instances` returns `InvalidParameterValue` with "Instance does not have a volume attached at root (/dev/sda1)".
+
+`attach-volume` works on a stopped instance too.
+The volume must be `available` and in the instance's availability zone, otherwise the call returns `VolumeInUse` or `InvalidVolume.ZoneMismatch`.
+The attachment is recorded on the stopped instance and the volume becomes `in-use` straight away.
+Attaching at the root device name (for example `/dev/sda1`) makes the volume the root again, and the next `start-instances` boots from it.
+
+An instance stopped by an earlier build had its data volumes released to `available` while its record still listed them.
+On start, a listed data volume that was deleted or is now attached to another instance is dropped with a warning in the daemon log, and the start continues.
+A listed volume that is still `available` and unattached is attached again.
+A missing root volume fails the start with the same `InvalidParameterValue` answer.
+
 ## Modify Instance Attributes
 
 Change instance type, user data, or termination protection. Instance type and user data require the instance to be **stopped** first.
@@ -135,6 +161,30 @@ aws ec2 get-console-output --instance-id $INSTANCE_ID \
   --query 'Output' --output text | base64 -d
 ```
 
+Like AWS, the call returns the most recent 64 KiB of the console.
+While an instance is shutting down after a terminate request, the call reads the renamed file described below.
+Once the instance has left the node's running set, which includes a stopped or terminated instance, the node no longer answers and the call returns `InvalidInstanceID.NotFound`.
+AWS still lists a terminated instance for a while after termination, so a caller may expect this call to work then; it does not here, a known gap.
+Whether the API should answer for a stopped instance is not settled here; only the file on the host is kept.
+
+### Where console logs live and how big they get
+
+The serial console of each instance is written on its host to `/run/spinifex/console-<instance-id>.log`.
+`/run` is memory-backed, so these files are cleared when the host reboots, including any left by older releases.
+The daemon keeps them bounded.
+A file over 4 MiB is cut back in place to its newest 1 MiB, and the guest keeps writing while it happens.
+The console is opened in append mode, so a reboot or auto-restart of the guest adds to the same file instead of starting over.
+A live `console-<instance-id>.log` is deleted only when it has not changed for an hour and no QEMU process on the host has it open, so a stopped instance keeps its console for an hour and a quiet running guest keeps it indefinitely.
+If the daemon cannot read the open files of every QEMU process on the host, it deletes nothing on that pass and logs a warning.
+This also clears the file of an instance that was stopped and then terminated, or that was recovered on another node, within about an hour of its last write.
+When an instance is terminated, including a failed first launch and the system instances behind load balancers, queues and EKS control planes, its file is renamed to `console-<instance-id>.log.terminated`.
+The daemon removes a `.terminated` file an hour after its last write.
+
+Known limits:
+
+- The log shipper follows only `console-*.log`, so it stops following a file when it is renamed on terminate, and the last few seconds of output that it had not yet read can be lost.
+- The instance password retrieval reads only the live file name, so it does not find a console renamed after terminate.
+
 ## Instance Types
 
 List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.

docs/testing/e2e-coverage.md

@@ -110,7 +110,7 @@ Three entry points, in the order they run:
 
 ### `iam` — everything IAM and STS
 
-Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
+Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
 
 ### `cert` — the cluster CA and TLS identity
 

internal/awsmodel/model.go

@@ -92,6 +92,7 @@ type Shape struct {
 	Min             *float64
 	Max             *float64
 	Pattern         string
+	Documentation   string
 	LocationName    string
 	TimestampFormat string
 	XMLNamespace    XMLNamespace

internal/awsmodel/smithy.go

@@ -443,6 +443,9 @@ func (t *smithyTranslator) applyShapeTraits(shape *Shape, traits smithyTraits) e
 	if shape.Pattern, err = traits.string("smithy.api#pattern"); err != nil {
 		return err
 	}
+	if shape.Documentation, err = traits.string("smithy.api#documentation"); err != nil {
+		return err
+	}
 	if shape.LocationName, err = traits.string("smithy.api#xmlName"); err != nil {
 		return err
 	}

internal/ecsgw/client.go

@@ -7,9 +7,10 @@ package ecsgw
 
 import (
 	"bytes"
-	"context"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
@@ -17,10 +18,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
-
 	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 )
 
 // targetPrefix is the X-Amz-Target service prefix the gateway's ECS dispatch
@@ -28,31 +27,26 @@ import (
 // suffix); this one mirrors the AWS JSON 1.1 convention.
 const targetPrefix = "AmazonEC2ContainerServiceV20141113."
 
-// CredentialsFunc yields the current SigV4 credentials. It is called once per
-// request so a provider backed by IMDS can rotate instance-role creds (and their
-// session token) transparently. Must be safe for concurrent use.
-type CredentialsFunc func(ctx context.Context) (accessKey, secretKey, sessionToken string, err error)
-
 // Client posts SigV4-signed AWS JSON 1.1 requests to the gateway for service
 // "ecs". One client is reused for register/heartbeat/state/poll.
 type Client struct {
 	baseURL    string
-	creds      CredentialsFunc
+	signer     *gwsign.Signer
 	region     string
 	httpClient *http.Client
 }
 
 // New builds a client. caPath optionally pins the gateway TLS CA; empty relies on
-// the system trust store. creds supplies the per-call instance-role credentials.
+// the system trust store. signer supplies the per-call instance-role credentials.
 // region defaults to us-east-1 when empty (SigV4 requires a non-empty region).
 // timeout bounds a single call — callers needing a long-poll pass a larger value
 // than the default register/heartbeat timeout.
-func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout time.Duration) (*Client, error) {
+func New(baseURL, caPath string, signer *gwsign.Signer, region string, timeout time.Duration) (*Client, error) {
 	if baseURL == "" {
 		return nil, fmt.Errorf("ecsgw: baseURL is required")
 	}
-	if creds == nil {
-		return nil, fmt.Errorf("ecsgw: credentials func is required")
+	if signer == nil {
+		return nil, fmt.Errorf("ecsgw: signer is required")
 	}
 	if region == "" {
 		region = "us-east-1"
@@ -79,7 +73,7 @@ func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout t
 
 	return &Client{
 		baseURL: strings.TrimRight(baseURL, "/"),
-		creds:   creds,
+		signer:  signer,
 		region:  region,
 		httpClient: &http.Client{
 			Timeout:   timeout,
@@ -90,7 +84,7 @@ func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout t
 
 // Call SigV4-signs and POSTs body under X-Amz-Target "<prefix><action>" to the
 // gateway root, returning the 2xx response body. Credentials are fetched per call
-// and the v4 signer emits X-Amz-Security-Token for instance-role (temporary)
+// and the signer emits X-Amz-Security-Token for instance-role (temporary)
 // creds, so the gateway's ASIA path verifies them. No retry; callers wrap.
 func (c *Client) Call(action string, body []byte) ([]byte, error) {
 	req, err := http.NewRequest(http.MethodPost, c.baseURL+"/", bytes.NewReader(body))
@@ -100,12 +94,8 @@ func (c *Client) Call(action string, body []byte) ([]byte, error) {
 	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
 	req.Header.Set("X-Amz-Target", targetPrefix+action)
 
-	akid, secret, token, err := c.creds(req.Context())
-	if err != nil {
-		return nil, fmt.Errorf("retrieve credentials: %w", err)
-	}
-	signer := v4.NewSigner(credentials.NewStaticCredentials(akid, secret, token))
-	if _, err := signer.Sign(req, bytes.NewReader(body), "ecs", c.region, time.Now()); err != nil {
+	sum := sha256.Sum256(body)
+	if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), "ecs", c.region); err != nil {
 		return nil, fmt.Errorf("sign request: %w", err)
 	}
 

internal/ecsgw/client_test.go

@@ -1,24 +1,29 @@
 package ecsgw
 
 import (
-	"context"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 )
 
-// testCreds returns a fixed temporary credential set (with a session token, as
-// IMDS instance-role creds carry).
-func testCreds(context.Context) (string, string, string, error) {
-	return "ASIATEST", "secret", "session-token-xyz", nil
+const testSecret = "secret"
+
+// testSigner signs with a fixed temporary credential set (with a session token,
+// as IMDS instance-role creds carry).
+func testSigner() *gwsign.Signer {
+	return gwsign.New(credentials.NewStaticCredentialsProvider("ASIATEST", testSecret, "session-token-xyz"))
 }
 
 func newTestClient(t *testing.T, srv *httptest.Server) *Client {
 	t.Helper()
-	c, err := New(srv.URL, "", testCreds, "ap-southeast-2", time.Second)
+	c, err := New(srv.URL, "", testSigner(), "ap-southeast-2", time.Second)
 	if err != nil {
 		t.Fatalf("New: %v", err)
 	}
@@ -30,7 +35,14 @@ func newTestClient(t *testing.T, srv *httptest.Server) *Client {
 // 2xx body.
 func TestClient_CallSignsAndTargets(t *testing.T) {
 	var gotTarget, gotAuth, gotBody, gotToken string
+	var verifyErr error
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// sigv4.Parse reads and hashes the body itself, then restores it.
+		sr, err := sigv4.Parse(r)
+		if err == nil {
+			_, err = sr.Verify(testSecret, "ap-southeast-2", "ecs")
+		}
+		verifyErr = err
 		gotTarget = r.Header.Get("X-Amz-Target")
 		gotAuth = r.Header.Get("Authorization")
 		gotToken = r.Header.Get("X-Amz-Security-Token")
@@ -45,6 +57,9 @@ func TestClient_CallSignsAndTargets(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Call: %v", err)
 	}
+	if verifyErr != nil {
+		t.Fatalf("gateway-side SigV4 verify failed: %v", verifyErr)
+	}
 	if string(out) != `{"ok":true}` {
 		t.Errorf("body = %q", out)
 	}
@@ -78,10 +93,10 @@ func TestClient_CallErrorStatus(t *testing.T) {
 }
 
 func TestNew_Validation(t *testing.T) {
-	if _, err := New("", "", testCreds, "r", 0); err == nil {
+	if _, err := New("", "", testSigner(), "r", 0); err == nil {
 		t.Error("want error for empty baseURL")
 	}
 	if _, err := New("https://gw:9999", "", nil, "r", 0); err == nil {
-		t.Error("want error for nil credentials func")
+		t.Error("want error for nil signer")
 	}
 }

internal/gwsign/signer.go

@@ -15,8 +15,7 @@ import (
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
 // Signer holds a credential provider and signs requests in place.
@@ -24,6 +23,12 @@ type Signer struct {
 	provider aws.CredentialsProvider
 }
 
+// New returns a Signer over provider. Wrap a provider that fetches remotely in
+// aws.NewCredentialsCache, since Sign retrieves on every call.
+func New(provider aws.CredentialsProvider) *Signer {
+	return &Signer{provider: provider}
+}
+
 // NewStatic returns a Signer using fixed long-lived keys (no session token).
 func NewStatic(accessKey, secretKey string) *Signer {
 	return &Signer{provider: credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")}
@@ -46,12 +51,14 @@ func NewIMDS(ctx context.Context, region string) (*Signer, error) {
 
 // NewInstanceRole returns a Signer whose credentials come only from the
 // instance metadata service: no environment, profile or static-key fallback.
+// A failed refresh of expired credentials is an error, not the stale key.
 func NewInstanceRole() *Signer {
-	return newInstanceRole(imds.New(imds.Options{}))
+	return newInstanceRole("")
 }
 
-func newInstanceRole(client *imds.Client) *Signer {
-	return &Signer{provider: aws.NewCredentialsCache(ec2rolecreds.New(func(o *ec2rolecreds.Options) { o.Client = client }))}
+// newInstanceRole reads from the IMDS root base; "" means the default endpoint.
+func newInstanceRole(base string) *Signer {
+	return &Signer{provider: aws.NewCredentialsCache(imdscreds.NewProvider(nil, base))}
 }
 
 // EnsureCredentials retrieves once from the provider, surfacing the error so a

internal/gwsign/signer_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/mulgadc/bluebottle/pkg/sigv4"
 )
 
@@ -145,7 +144,7 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	}))
 	t.Cleanup(srv.Close)
 
-	s := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
+	s := newInstanceRole(srv.URL)
 	req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
 	if err := s.Sign(req, emptySHA, testService, testRegion); err != nil {
 		t.Fatal(err)
@@ -155,10 +154,42 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	}
 
 	srv.Close()
-	cold := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
+	cold := newInstanceRole(srv.URL)
 	if err := cold.EnsureCredentials(context.Background()); err == nil {
 		t.Fatal("environment keys were used when IMDS was unreachable")
 	}
 }
 
+// Once the cached instance-role key has expired, a refresh that cannot reach
+// IMDS must fail the sign rather than reuse the expired key with a new expiry.
+func TestNewInstanceRole_FailedRefreshErrorsInsteadOfStaleKey(t *testing.T) {
+	const staleKey = "ASIASTALEEXAMPLE0001"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			_, _ = w.Write([]byte("token"))
+		case "/latest/meta-data/iam/security-credentials/":
+			_, _ = w.Write([]byte("sqsWorkerRole"))
+		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
+			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + staleKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2000-01-01T00:00:00Z"}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	s := newInstanceRole(srv.URL)
+	if err := s.EnsureCredentials(t.Context()); err != nil {
+		t.Fatalf("first retrieve: %v", err)
+	}
+
+	srv.Close()
+	req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
+	err := s.Sign(req, emptySHA, testService, testRegion)
+	if err == nil {
+		t.Fatalf("Sign after failed refresh = nil, signed with %q", req.Header.Get("Authorization"))
+	}
+	if auth := req.Header.Get("Authorization"); auth != "" {
+		t.Fatalf("request carries Authorization %q after a failed refresh", auth)
+	}
+}
+
 const emptySHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

internal/imdscreds/imdscreds.go

@@ -21,31 +21,18 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 )
 
-// Credentials is a resolved instance-role credential set with parsed expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
 // Fetch resolves the instance role then its credentials via the SDK's IMDSv2
 // client and the ec2rolecreds provider. base is the IMDS root, e.g.
 // http://169.254.169.254/latest.
-func Fetch(client *http.Client, base string) (Credentials, error) {
+func Fetch(client *http.Client, base string) (aws.Credentials, error) {
 	creds, err := NewProvider(client, base).Retrieve(context.Background())
 	if err != nil {
-		return Credentials{}, fmt.Errorf("fetch IMDS credentials: %w", err)
+		return aws.Credentials{}, fmt.Errorf("fetch IMDS credentials: %w", err)
 	}
 	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
-		return Credentials{}, fmt.Errorf("IMDS credentials missing access/secret key")
+		return aws.Credentials{}, fmt.Errorf("IMDS credentials missing access/secret key")
 	}
-	return Credentials{
-		AccessKeyID:     creds.AccessKeyID,
-		SecretAccessKey: creds.SecretAccessKey,
-		SessionToken:    creds.SessionToken,
-		Expiration:      creds.Expires,
-	}, nil
+	return creds, nil
 }
 
 // NewProvider builds an ec2rolecreds provider against base, an IMDS root URL

internal/imdscreds/imdscreds_test.go

@@ -1,9 +1,11 @@
 package imdscreds
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -60,8 +62,8 @@ func TestFetch_OK(t *testing.T) {
 	if got.AccessKeyID != "AKIATEST" || got.SecretAccessKey != "secrettest" || got.SessionToken != "sessiontoken" {
 		t.Errorf("creds mismatch: %+v", got)
 	}
-	if !got.Expiration.Equal(exp) {
-		t.Errorf("Expiration = %v, want %v", got.Expiration, exp)
+	if !got.Expires.Equal(exp) {
+		t.Errorf("Expires = %v, want %v", got.Expires, exp)
 	}
 }
 
@@ -132,3 +134,46 @@ func TestNewProvider_WithholdsExtendOnFailure(t *testing.T) {
 		t.Error("provider dropped AdjustExpiresBy, which a caller using an ExpiryWindow relies on")
 	}
 }
+
+// Behind aws.CredentialsCache, as the ecs-agent uses it, a refresh that fails
+// against an expired credential must surface as an error rather than hand back
+// the dead key with a pushed-out expiry.
+func TestNewProvider_CachedFailedRefreshErrors(t *testing.T) {
+	var hits atomic.Int32
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits.Add(1) > 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"Code":            "Success",
+			"AccessKeyId":     "AKIA",
+			"SecretAccessKey": "secret",
+			"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+		})
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+
+	cache := aws.NewCredentialsCache(NewProvider(srv.Client(), srv.URL+"/latest"))
+	first, err := cache.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("first Retrieve: %v", err)
+	}
+	got, err := cache.Retrieve(context.Background())
+	if err == nil {
+		t.Fatalf("refresh failure was reported as success: %+v", got)
+	}
+	if got.Expires.After(first.Expires) {
+		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expires, got.Expires)
+	}
+}

internal/stsauth/stsauth.go

@@ -8,8 +8,8 @@ package stsauth
 import (
 	"fmt"
 	"net/http"
-	"time"
 
+	awsv2 "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -20,24 +20,15 @@ import (
 // scope and the gateway does not enforce a specific value.
 const defaultRegion = "us-east-1"
 
-// Credentials is the decoded result of AssumeRole: a SigV4 credential set bound
-// to the assumed role, with its expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
 // AssumeRole builds an aws-sdk STS client pointed at the gateway and calls
 // AssumeRole with the supplied static (instance-role) credentials, returning the
 // assumed-role credential set. httpClient must trust the gateway CA.
-func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret, sessionToken, roleARN, sessionName string) (Credentials, error) {
+func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret, sessionToken, roleARN, sessionName string) (awsv2.Credentials, error) {
 	if region == "" {
 		region = defaultRegion
 	}
 	if roleARN == "" || sessionName == "" {
-		return Credentials{}, fmt.Errorf("stsauth: roleARN and sessionName are required")
+		return awsv2.Credentials{}, fmt.Errorf("stsauth: roleARN and sessionName are required")
 	}
 	sess, err := session.NewSession(&aws.Config{
 		Region:      aws.String(region),
@@ -47,7 +38,7 @@ func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret
 		DisableSSL:  aws.Bool(false),
 	})
 	if err != nil {
-		return Credentials{}, fmt.Errorf("new session: %w", err)
+		return awsv2.Credentials{}, fmt.Errorf("new session: %w", err)
 	}
 
 	out, err := sts.New(sess).AssumeRole(&sts.AssumeRoleInput{
@@ -55,16 +46,17 @@ func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret
 		RoleSessionName: aws.String(sessionName),
 	})
 	if err != nil {
-		return Credentials{}, fmt.Errorf("AssumeRole: %w", err)
+		return awsv2.Credentials{}, fmt.Errorf("AssumeRole: %w", err)
 	}
 	if out.Credentials == nil {
-		return Credentials{}, fmt.Errorf("AssumeRole returned no credentials")
+		return awsv2.Credentials{}, fmt.Errorf("AssumeRole returned no credentials")
 	}
 	c := out.Credentials
-	return Credentials{
+	return awsv2.Credentials{
 		AccessKeyID:     aws.StringValue(c.AccessKeyId),
 		SecretAccessKey: aws.StringValue(c.SecretAccessKey),
 		SessionToken:    aws.StringValue(c.SessionToken),
-		Expiration:      aws.TimeValue(c.Expiration),
+		CanExpire:       c.Expiration != nil,
+		Expires:         aws.TimeValue(c.Expiration),
 	}, nil
 }

internal/stsauth/stsauth_test.go

@@ -41,8 +41,8 @@ func TestAssumeRole_DecodesCredentials(t *testing.T) {
 		t.Errorf("bad creds: %+v", creds)
 	}
 	want, _ := time.Parse(time.RFC3339, "2031-01-02T03:04:05Z")
-	if !creds.Expiration.Equal(want) {
-		t.Errorf("Expiration = %v, want %v", creds.Expiration, want)
+	if !creds.Expires.Equal(want) {
+		t.Errorf("Expires = %v, want %v", creds.Expires, want)
 	}
 	// The request must carry the AssumeRole action and the role ARN we asked for.
 	if !strings.Contains(gotBody, "Action=AssumeRole") {

internal/testkit/awserror.go

@@ -0,0 +1,29 @@
+package testutil
+
+import (
+	"testing"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// RequireAWSError asserts err carries the AWS code and exact client-facing
+// message; an empty message asserts the bare code with no message attached.
+func RequireAWSError(t *testing.T, err error, code, message string) {
+	t.Helper()
+	require.Error(t, err)
+	got, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, code, got)
+	assert.Equal(t, message, msg)
+}
+
+// RequireAWSCode asserts err carries the AWS code, whatever its message.
+func RequireAWSCode(t *testing.T, err error, code string) {
+	t.Helper()
+	require.Error(t, err)
+	got, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, code, got)
+}

internal/testkit/nats.go

@@ -167,11 +167,11 @@ func registerVpcdStub(t *testing.T, nc *nats.Conn, defaultPayload []byte) {
 	})
 }
 
-// SeedKV creates a KV bucket and populates it with the given entries.
+// SeedKV creates or updates a KV bucket and populates it with the given entries.
 // Returns the KV handle for further use.
 func SeedKV(t *testing.T, js jetstream.JetStream, bucket string, entries map[string][]byte) jetstream.KeyValue {
 	t.Helper()
-	kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
+	kv, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
 	require.NoError(t, err)
 	for k, v := range entries {
 		_, err := kv.Put(t.Context(), k, v)

internal/testkit/nats_test.go

@@ -0,0 +1,19 @@
+package testutil
+
+import (
+	"testing"
+
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSeedKV_ExistingBucket(t *testing.T) {
+	_, _, js := StartTestJetStream(t)
+	_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "seeded", History: 5})
+	require.NoError(t, err)
+
+	kv := SeedKV(t, js, "seeded", map[string][]byte{"k": []byte("v")})
+	entry, err := kv.Get(t.Context(), "k")
+	require.NoError(t, err)
+	require.Equal(t, []byte("v"), entry.Value())
+}

scripts/build-lambda-runtime-microvm-image.sh

@@ -3,24 +3,28 @@
 # + initramfs: a guest running the Lambda guest agent, with virtio-net for its
 # host control connection. It is not shipped.
 #
+# Usage: build-lambda-runtime-microvm-image.sh [agent-binary [output-dir]]
+#
 # Inputs:
 #   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
-#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
+#   agent binary, default $REPO_ROOT/bin/lambda-agent (make bin/lambda-agent)
 #
-# Outputs:
-#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
-#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
+# Outputs, in output-dir (default $REPO_ROOT/build/microvm/lambda-runtime):
+#   vmlinuz
+#   initramfs.cpio.gz
 #
 # Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
 # shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
-INIT_SH="$BUILD_DIR/init.sh"
-INITTAB="$BUILD_DIR/inittab"
-RUN_AGENT="$BUILD_DIR/run-agent"
-LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
+INPUT_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$INPUT_DIR/init.sh"
+INITTAB="$INPUT_DIR/inittab"
+RUN_AGENT="$INPUT_DIR/run-agent"
+LAMBDA_AGENT_BIN="${1:-$REPO_ROOT/bin/lambda-agent}"
+# The library stages beside BUILD_DIR, so it names the output, not the inputs.
+BUILD_DIR="${2:-$INPUT_DIR}"
 TAG="[build-lambda-runtime-microvm-image]"
 
 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
@@ -38,7 +42,8 @@ MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
 source "$REPO_ROOT/scripts/lib/microvm-image.sh"
 
 echo "$TAG repo root: $REPO_ROOT"
-echo "$TAG build dir: $BUILD_DIR"
+echo "$TAG input dir: $INPUT_DIR"
+echo "$TAG output dir: $BUILD_DIR"
 
 microvm_check_tools
 
@@ -57,6 +62,9 @@ if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
     exit 1
 fi
 
+# Absolute, because packing the initramfs runs from inside the chroot.
+mkdir -p "$BUILD_DIR"
+BUILD_DIR=$(cd "$BUILD_DIR" && pwd)
 microvm_init_dirs
 microvm_make_rootfs
 microvm_empty_dev

scripts/build-lambda-runtime-microvm-image_test.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+# build-lambda-runtime-microvm-image_test.sh — a build into a chosen output
+# directory stages and publishes there, and leaves the default input directory
+# untouched.
+#
+# The script runs in a fixture repo against the real library, with only the
+# rootfs, kernel and packing steps replaced so no image is downloaded.
+set -eu
+
+SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- build-lambda-runtime-microvm-image.sh output directory"
+
+REPO="$TMP/repo"
+IN="$REPO/build/microvm/lambda-runtime"
+mkdir -p "$REPO/scripts/lib" "$IN" "$REPO/bin" "$TMP/bin"
+cp "$SCRIPTS_DIR/build-lambda-runtime-microvm-image.sh" "$REPO/scripts/"
+for f in init.sh inittab run-agent; do printf '%s\n' "$f" > "$IN/$f"; done
+printf 'default image\n' > "$IN/vmlinuz"
+printf '#!/bin/sh\n' > "$REPO/bin/agent-testlink"
+chmod +x "$REPO/bin/agent-testlink"
+printf '#!/bin/sh\necho "ELF 64-bit, statically linked"\n' > "$TMP/bin/file"
+chmod +x "$TMP/bin/file"
+BEFORE=$(cd "$IN" && ls -A | cksum)
+
+# The real library, with the steps that need a rootfs or kernel replaced.
+# Packing runs from inside the chroot as the real step does, and records where
+# the stage is while it still exists.
+cat > "$REPO/scripts/lib/microvm-image.sh" <<EOF
+. "$SCRIPTS_DIR/lib/microvm-image.sh"
+microvm_check_tools() { :; }
+microvm_make_rootfs() { mkdir -p "\$CHROOT_DIR/etc"; }
+microvm_empty_dev() { :; }
+microvm_locate_kernel() { :; }
+microvm_strip_modules() { :; }
+microvm_strip_rootfs() { :; }
+microvm_stage_kernel() { printf 'kernel\n' > "\$STAGE_DIR/vmlinuz"; }
+microvm_pack_initramfs() {
+    (cd "\$CHROOT_DIR" && printf 'initramfs\n' | gzip > "\$STAGE_DIR/initramfs.cpio.gz")
+    dirname "\$STAGE_DIR" > "$TMP/stage-parent"
+    ls -A "$IN" > "$TMP/input-during"
+}
+EOF
+
+# The output is given relative to the repo root, as make passes it.
+OUT="$REPO/build/microvm/lambda-testlink-runtime"
+if ! (cd "$REPO" && PATH="$TMP/bin:$PATH" bash scripts/build-lambda-runtime-microvm-image.sh bin/agent-testlink build/microvm/lambda-testlink-runtime) > "$TMP/log" 2>&1; then
+    fail "build into output dir: $(tr '\n' ' ' < "$TMP/log")"
+else
+    if [ "$(cat "$TMP/stage-parent")" = "$OUT" ]; then
+        pass "stage created in the output directory"
+    else
+        fail "stage created in $(cat "$TMP/stage-parent"), want $OUT"
+    fi
+    if grep -q '^\.stage\.' "$TMP/input-during"; then
+        fail "stage present in the default input directory during the build"
+    else
+        pass "no stage in the default input directory during the build"
+    fi
+    if [ "$(cd "$IN" && ls -A | cksum)" = "$BEFORE" ] && grep -qx 'default image' "$IN/vmlinuz"; then
+        pass "default input directory unchanged"
+    else
+        fail "default input directory changed: $(cd "$IN" && ls -A | tr '\n' ' ')"
+    fi
+    if grep -qx kernel "$OUT/vmlinuz" && gzip -t "$OUT/initramfs.cpio.gz" && [ "$(ls -A "$OUT" | wc -l)" -eq 2 ]; then
+        pass "output directory holds exactly the published pair"
+    else
+        fail "output directory: $(ls -A "$OUT" | tr '\n' ' ')"
+    fi
+fi
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

scripts/images/ecs-agent-gpu/ecs-agent.service

@@ -9,8 +9,7 @@ Wants=network-online.target
 
 [Service]
 Type=simple
-# Static config (gateway URL, CA, region, cluster, seeded IAM creds,
-# containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by
+# Static config (gateway URL, CA, region, cluster, containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by
 # cloud-init at launch. Absent file is tolerated — the agent falls back to
 # its built-in defaults.
 EnvironmentFile=-/etc/spinifex-ecs/agent.env

scripts/images/ecs-agent/ecs-agent.initd

@@ -3,8 +3,7 @@
 # IMDSv2, registers + heartbeats + polls task assignments through the AWS gateway
 # over TLS+SigV4 (no NATS), and pulls task images through containerd.
 #
-# Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
-# socket) comes from /etc/spinifex-ecs/agent.env, dropped by cloud-init at launch.
+# Static config (gateway URL, CA, region, cluster, containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by cloud-init at launch.
 
 description="Spinifex ECS container-instance agent"
 command="/usr/local/bin/ecs-agent"
@@ -28,7 +27,7 @@ start_pre() {
         # shellcheck disable=SC1091
         . /etc/spinifex-ecs/agent.env
         export ECS_GATEWAY_URL ECS_GATEWAY_CA ECS_REGION ECS_CLUSTER \
-            ECS_ACCESS_KEY ECS_SECRET_KEY ECS_IMDS_BASE ECS_CONTAINERD_SOCKET \
+            ECS_IMDS_BASE ECS_CONTAINERD_SOCKET \
             ECS_HEARTBEAT_INTERVAL ECS_POLL_INTERVAL
     fi
 }

scripts/images/eks-node/eks-node-role.sh

@@ -159,6 +159,7 @@ case "${ROLE}" in
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     server-join)
@@ -178,12 +179,14 @@ case "${ROLE}" in
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     agent)
         log "configuring agent role"
         svc_enable k3s-agent
         svc_start k3s-agent
+        svc_optional mulga-eks-log-forward
         ;;
     *)
         die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"

scripts/images/eks-node/eks-node-role_test.sh

@@ -146,6 +146,15 @@ for INIT in openrc systemd; do
     )"
     check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
     check "server(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
@@ -196,6 +205,15 @@ for INIT in openrc systemd; do
     # published the cluster-identical artifacts, and a join re-publish would
     # only race the bootstrap bus. This is deliberate, not an oversight.
     check "server-join(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
+    check "server-join(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server-join(${INIT}): no k3s-first-boot" "yes" "$(absent_all "${CALLS}" "k3s-first-boot")"
     check "server-join(${INIT}): no k3s-agent" "yes" "$(absent_all "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server-join(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
@@ -207,8 +225,39 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+    check "agent(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): log-forward is enabled after k3s-agent" "yes" "$(
+        awk -v k="$(enable_cmd "${INIT}" k3s-agent)" -v f="$(enable_cmd "${INIT}" mulga-eks-log-forward)" '
+            index($0, k) == 1 { kn = NR }
+            index($0, f) == 1 { fn = NR }
+            END { print (kn > 0 && fn > kn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
+
     check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "eks-pod-identity-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
+    # A broken log forwarder must never stop a node joining.
+    for FAILING in enable start; do
+        run_case "${INIT}" agent_logfwd_${FAILING}_fail
+        printf 'SPINIFEX_K3S_ROLE=agent\n' > "${EKS_NODE_ENVFILE}"
+        if [ "${FAILING}" = enable ]; then
+            STUB_FAIL="$(enable_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-update add }"; STUB_FAIL="${STUB_FAIL#systemctl enable }"
+        else
+            STUB_FAIL="$(start_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-service }"; STUB_FAIL="${STUB_FAIL#systemctl start --no-block }"
+        fi
+        invoke
+        unset STUB_FAIL
+        check "agent(${INIT}): log-forward ${FAILING} failure still exits 0" 0 "${rc}"
+        check "agent(${INIT}): log-forward ${FAILING} failure still enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still starts k3s-agent" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
+    done
+
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred
     printf 'K3S_URL=https://nlb:443\nK3S_TOKEN=abc\n' > "${EKS_NODE_AGENT_ENVFILE}"

scripts/images/eks-node/manifest.conf

@@ -45,6 +45,8 @@ scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-s
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
 scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward \
+scripts/images/eks-node/mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-log-forward.initd

@@ -0,0 +1,30 @@
+#!/sbin/openrc-run
+# mulga-eks-log-forward — copies k3s warnings and errors to the serial console
+# so they reach the Elasticsearch sink. Best effort: nothing depends on it and
+# a failure here never affects the node.
+
+description="Forward k3s warnings and errors to the serial console"
+command="/bin/sh"
+command_background="yes"
+supervisor="supervise-daemon"
+respawn_delay=5
+pidfile="/run/mulga-eks-log-forward.pid"
+output_log="/dev/null"
+error_log="/dev/null"
+
+depend() {
+    after k3s k3s-agent
+}
+
+start_pre() {
+    role=$(cat /etc/spinifex-eks/role 2>/dev/null || true)
+    if [ "${role}" = "agent" ]; then
+        log=/var/log/k3s-agent.log
+        FORWARD_PREFIX="[k3s-agent]"
+    else
+        log=/var/log/k3s.log
+        FORWARD_PREFIX="[k3s]"
+    fi
+    export FORWARD_PREFIX
+    command_args="-c 'tail -n +1 -F ${log} 2>/dev/null | /usr/local/sbin/mulga-eks-log-forward'"
+}

scripts/images/eks-node/mulga-eks-log-forward.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+# mulga-eks-log-forward — copy the warnings and errors of a k3s log to the
+# serial console, where the host picks them up for the Elasticsearch sink.
+#
+# Reads the tailed log on stdin. Lines that matter are redacted first, then
+# cut to FORWARD_MAX_BYTES, then rate limited: at most FORWARD_RATE_PER_MIN a
+# minute reach the console and the rest are reported as one "N lines dropped"
+# line. Every console write has a timeout, so a stuck console loses lines
+# instead of stalling the tail or k3s.
+#
+# Env: FORWARD_CONSOLE (default /dev/console), FORWARD_PREFIX, FORWARD_MAX_BYTES
+# (512), FORWARD_RATE_PER_MIN (60), FORWARD_WRITE_TIMEOUT seconds (2) and
+# FORWARD_CLOCK_CMD (default: seconds of uptime, read without forking).
+
+CONSOLE=${FORWARD_CONSOLE:-/dev/console}
+PREFIX=${FORWARD_PREFIX:-[k3s]}
+MAX_BYTES=${FORWARD_MAX_BYTES:-512}
+RATE=${FORWARD_RATE_PER_MIN:-60}
+WRITE_TIMEOUT=${FORWARD_WRITE_TIMEOUT:-2}
+CLOCK_CMD=${FORWARD_CLOCK_CMD:-}
+
+now() {
+    if [ -n "${CLOCK_CMD}" ]; then
+        sh -c "${CLOCK_CMD}"
+    else
+        read -r up _ < /proc/uptime && echo "${up%%.*}"
+    fi
+}
+
+# emit TEXT: one console line, abandoned after WRITE_TIMEOUT. Returns non-zero
+# when the console did not take it.
+emit() {
+    timeout "${WRITE_TIMEOUT}" sh -c 'printf "%s\n" "$1" >> "$2"' sh "$1" "${CONSOLE}" 2>/dev/null
+}
+
+# Filter, redact and truncate in one awk pass so lines that do not matter cost
+# no fork. Redaction runs before the cut so a secret straddling the limit
+# cannot leave a readable prefix behind.
+select_lines() {
+    LC_ALL=C awk -v max="${MAX_BYTES}" '
+    {
+        line = $0
+        gsub(/\r/, "", line)
+        keep = 0
+        if (line ~ /level=(warning|error)/) keep = 1
+        else if (line ~ /^[WE][0-9][0-9][0-9][0-9] /) keep = 1
+        else if (line ~ /NodeNotReady|PLEG|[Ll]ease|[Nn]ode status/) keep = 1
+        if (!keep) next
+
+        gsub(/[Bb]earer +[A-Za-z0-9._~+\/=-]+/, "Bearer [REDACTED]", line)
+        gsub(/K10[0-9a-f]+(::[A-Za-z0-9-]+:[^ "\047]+)?/, "[REDACTED]", line)
+        gsub(/eyJ[A-Za-z0-9_.-]+/, "[REDACTED]", line)
+        gsub(/(AKIA|ASIA)[A-Z0-9]+/, "[REDACTED]", line)
+        gsub(/X-Amz-Signature=[^& "\047]*/, "X-Amz-Signature=[REDACTED]", line)
+        gsub(/X-Amz-Credential=[^& "\047]*/, "X-Amz-Credential=[REDACTED]", line)
+        gsub(/X-Amz-Security-Token=[^& "\047]*/, "X-Amz-Security-Token=[REDACTED]", line)
+        gsub(/[^[:print:]\t]/, " ", line)
+        print substr(line, 1, max)
+    }'
+}
+
+win_start=0
+sent=0
+dropped=0
+
+flush_dropped() {
+    [ "${dropped}" -gt 0 ] || return 0
+    emit "${PREFIX} ${dropped} lines dropped"
+    dropped=0
+}
+
+select_lines | {
+    while IFS= read -r line; do
+        t=$(now) || t=0
+        if [ "${win_start}" -eq 0 ] || [ $((t - win_start)) -ge 60 ]; then
+            flush_dropped
+            win_start=${t}
+            sent=0
+            [ "${win_start}" -eq 0 ] && win_start=1
+        fi
+        if [ "${sent}" -ge "${RATE}" ]; then
+            dropped=$((dropped + 1))
+            continue
+        fi
+        sent=$((sent + 1))
+        emit "${PREFIX} ${line}" || dropped=$((dropped + 1))
+    done
+    flush_dropped
+}
+exit 0

scripts/images/eks-node/mulga-eks-log-forward_test.sh

@@ -0,0 +1,190 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-log-forward.sh: the line filter,
+# redaction, truncation, rate cap and blocked-console behaviour. The script
+# reads the tailed log on stdin and writes to FORWARD_CONSOLE, which the test
+# points at a temp file (or a reader-less FIFO), so no root or tty is needed.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-log-forward_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/mulga-eks-log-forward.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+check() { # check <desc> <expected> <actual>
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        echo "FAIL: $1"
+        echo "  expected: [$2]"
+        echo "  actual:   [$3]"
+    fi
+}
+
+# forward <name>: feeds stdin to the script with a per-case console file and a
+# fixed clock, leaving the forwarded text in ${OUT}.
+forward() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}"
+    OUT="${CASE}/console"
+    : > "${OUT}"
+    FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" \
+        FORWARD_CLOCK_CMD="echo 1000" sh "${SCRIPT}" 2>/dev/null || true
+}
+forwarded() { grep -c . "${OUT}" || true; }
+
+# --- filter: lines that must pass ---
+forward filter_pass <<'EOF'
+time="2026-11-03T03:00:00Z" level=warning msg="slow response from apiserver"
+time="2026-11-03T03:00:01Z" level=error msg="failed to renew"
+W1103 03:00:02.123456    1234 reflector.go:561] watch of *v1.Pod ended
+E1103 03:00:03.123456    1234 kubelet.go:2400] node not found
+I1103 03:00:04.123456    1234 kubelet_node_status.go:73] Node status update failed
+I1103 03:00:05.123456    1234 controller.go:193] Failed to update lease
+I1103 03:00:06.123456    1234 kubelet.go:2200] PLEG is not healthy
+time="2026-11-03T03:00:07Z" level=info msg="Event: NodeNotReady kubelet"
+EOF
+check "filter: all eight interesting lines pass" "8" "$(forwarded)"
+check "filter: lines carry the prefix" "8" "$(grep -c '^\[k3s-agent\] ' "${OUT}" || true)"
+
+# --- filter: lines that must be dropped ---
+forward filter_drop <<'EOF'
+time="2026-11-03T03:00:00Z" level=info msg="Starting k3s agent"
+time="2026-11-03T03:00:01Z" level=debug msg="tick"
+I1103 03:00:02.123456    1234 server.go:100] Version: v1.32.5
+I1103 03:00:03.123456    1234 kubelet.go:1000] SyncLoop (UPDATE, "api")
+EOF
+check "filter: info and debug lines are dropped" "0" "$(forwarded)"
+
+# --- redaction: one case per secret shape ---
+redact() { # redact <name> <line> <secret-that-must-not-appear>
+    forward "redact_$1" <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="$2"
+EOF
+    check "redact(${1}): line still forwarded" "1" "$(forwarded)"
+    if grep -qF "$3" "${OUT}"; then
+        check "redact(${1}): secret removed" "absent" "present"
+    else
+        check "redact(${1}): secret removed" "absent" "absent"
+    fi
+    check "redact(${1}): marker present" "1" "$(grep -c 'REDACTED' "${OUT}" || true)"
+}
+redact bearer "request failed Authorization: Bearer abc123SECRETvalue456" "abc123SECRETvalue456"
+redact k3s_token "join failed token K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" "s3cr3tPassw0rd"
+redact jwt "bad token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.c2lnbmF0dXJlLXNlY3JldA" "eyJhbGciOiJIUzI1NiJ9"
+redact akia "credentials AKIAIOSFODNN7EXAMPLE rejected" "AKIAIOSFODNN7EXAMPLE"
+redact asia "credentials ASIAIOSFODNN7EXAMPLE rejected" "ASIAIOSFODNN7EXAMPLE"
+redact amz_signature "GET /obj?X-Amz-Signature=0123456789abcdef0123456789abcdef&X-Amz-Expires=60 failed" "0123456789abcdef0123456789abcdef"
+redact amz_credential "GET /obj?X-Amz-Credential=ZZZsecretscope%2F20261103%2Fap%2Fs3%2Faws4_request failed" "ZZZsecretscope"
+
+# --- truncation: text over 512 bytes is cut ---
+LONG=$(awk 'BEGIN { for (i = 0; i < 2000; i++) printf "a"; }')
+forward truncate <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="${LONG}"
+EOF
+check "truncate: still forwarded" "1" "$(forwarded)"
+BODY=$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1)
+check "truncate: body is exactly 512 bytes" "512" "$(printf '%s' "${BODY}" | wc -c | tr -d ' ')"
+check "truncate: keeps the start of the line" "time=\"2026-11-03T03:00:00Z\"" "$(printf '%s' "${BODY}" | cut -c1-27)"
+
+# --- redaction runs before the cut: a secret over the limit leaves no prefix ---
+# The pad puts the 512-byte cut right after the bare marker (AKIA, eyJ, K10).
+# A cut fragment that long matches no redaction pattern, so redacting after the
+# cut would leave it in the output.
+HEAD='time="2026-11-03T03:00:00Z" level=error msg="'
+straddle() { # straddle <name> <secret> <bytes of the secret kept by the cut>
+    PAD=$(awk -v n=$((512 - ${#HEAD} - 1 - $3)) 'BEGIN { for (i = 0; i < n; i++) printf "x"; }')
+    forward "straddle_$1" <<EOF
+${HEAD}${PAD} $2 tail"
+EOF
+    check "straddle(${1}): line forwarded" "1" "$(forwarded)"
+    check "straddle(${1}): cut keeps the marker length" "$3" "$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1 | awk -v p="${PAD} " '{ print length($0) - index($0, p) - length(p) + 1 }')"
+    if grep -qF "$(printf '%s' "$2" | cut -c1-"$3")" "${OUT}"; then
+        check "straddle(${1}): no fragment survives the cut" "absent" "present"
+    else
+        check "straddle(${1}): no fragment survives the cut" "absent" "absent"
+    fi
+}
+straddle jwt "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.c2ln" 3
+straddle akia "AKIAIOSFODNN7EXAMPLE" 4
+straddle k3s_token "K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" 3
+
+# --- rate cap: 60 a minute, then one summary of what was dropped ---
+gen() { # gen <count>
+    awk -v n="$1" 'BEGIN { for (i = 0; i < n; i++) printf "time=\"t\" level=error msg=\"loop %d\"\n", i }'
+}
+gen 100 > "${WORK}/burst"
+forward rate_cap < "${WORK}/burst"
+check "rate cap: 60 lines forwarded plus one summary" "61" "$(forwarded)"
+check "rate cap: one dropped summary" "1" "$(grep -c '40 lines dropped' "${OUT}" || true)"
+check "rate cap: first line kept" "1" "$(grep -c 'loop 0"' "${OUT}" || true)"
+check "rate cap: line past the cap is not forwarded" "0" "$(grep -c 'loop 60"' "${OUT}" || true)"
+
+# --- rate cap: a new minute starts a new window and reports the old drops ---
+# The clock command advances 60 s every 70 calls, so lines 0-69 share a window
+# (10 dropped) and lines 70-74 land in the next one.
+CASE="${WORK}/rate_window"
+mkdir -p "${CASE}"
+OUT="${CASE}/console"
+: > "${OUT}"
+echo 0 > "${CASE}/calls"
+cat > "${CASE}/clock" <<EOF
+#!/bin/sh
+n=\$(cat "${CASE}/calls")
+echo \$((n + 1)) > "${CASE}/calls"
+echo \$((1000 + n / 70 * 60))
+EOF
+chmod +x "${CASE}/clock"
+gen 75 | FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="${CASE}/clock" sh "${SCRIPT}" 2>/dev/null || true
+check "window: 60 + 5 forwarded plus one summary" "66" "$(forwarded)"
+check "window: summary names the 10 dropped" "1" "$(grep -c '10 lines dropped' "${OUT}" || true)"
+check "window: new window forwards its lines" "1" "$(grep -c 'loop 74"' "${OUT}" || true)"
+
+# --- blocked console: lines are dropped, the forwarder neither stalls nor fails ---
+CASE="${WORK}/blocked"
+mkdir -p "${CASE}"
+mkfifo "${CASE}/console"
+gen 3 > "${CASE}/in"
+(
+    FORWARD_CONSOLE="${CASE}/console" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="echo 1000" \
+        FORWARD_WRITE_TIMEOUT=1 sh "${SCRIPT}" < "${CASE}/in"
+) &
+PID=$!
+i=0
+while kill -0 "${PID}" 2>/dev/null && [ "${i}" -lt 20 ]; do
+    sleep 1
+    i=$((i + 1))
+done
+if kill -0 "${PID}" 2>/dev/null; then
+    kill "${PID}" 2>/dev/null || true
+    check "blocked console: forwarder finished" "finished" "stalled"
+else
+    check "blocked console: forwarder finished" "finished" "finished"
+fi
+rc=0
+wait "${PID}" || rc=$?
+check "blocked console: exit 0" "0" "${rc}"
+
+# --- the service is wired in, and nothing depends on it ---
+INITD="${SCRIPT_DIR}/mulga-eks-log-forward.initd"
+check "initd ships" "yes" "$([ -f "${INITD}" ] && echo yes || echo no)"
+check "manifest installs the initd" "1" "$(grep -c 'mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+check "manifest installs the script" "1" "$(grep -c 'mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+DEPS=0
+for f in "${SCRIPT_DIR}"/*.initd; do
+    [ "${f}" = "${INITD}" ] && continue
+    if grep -E '^[[:space:]]*(need|use|want)[[:space:]].*mulga-eks-log-forward' "${f}" >/dev/null 2>&1; then
+        DEPS=$((DEPS + 1))
+    fi
+done
+check "nothing depends on the forwarder" "0" "${DEPS}"
+if [ -f "${INITD}" ]; then
+    check "initd does not need anything it could block boot on" "0" "$(grep -cE '^[[:space:]]*need[[:space:]]' "${INITD}" || true)"
+fi
+
+echo "mulga-eks-log-forward_test: ${PASS} passed, ${FAIL} failed"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -45,10 +45,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
     /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server \
-    /etc/init.d/eks-pod-identity-webhook
+    /etc/init.d/eks-pod-identity-webhook /etc/init.d/mulga-eks-log-forward
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
     /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
-    /usr/local/sbin/mulga-eks-node-gc
+    /usr/local/sbin/mulga-eks-node-gc /usr/local/sbin/mulga-eks-log-forward
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

scripts/install-node.sh

@@ -260,6 +260,14 @@ out() {
     ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@" 2>/dev/null || true
 }
 
+# remote_join_cmd prints the command that joins one node. The token is not in it: it arrives on
+# stdin and is read after sudo has logged its command, so sudo's log never holds it. The other
+# values are positional arguments of a literal script, never spliced into it.
+remote_join_cmd() {
+    local script='exec spx admin join --force --node "$1" --bind "$2" --cluster-bind "$2" --advertise "$3" --host "$4" --token "$(cat)" --region "$5" --az "$6"'
+    printf 'sudo bash -c %q _ %q %q %q %q %q %q' "$script" "$@"
+}
+
 SETUP_OVN="/usr/local/share/spinifex/setup-ovn.sh"
 
 # --- Preflight -------------------------------------------------------------
@@ -652,14 +660,18 @@ fi
 [ -n "$EXT_IFACE" ] && init_args+=(--external-iface="$EXT_IFACE")
 [ -n "$EMAIL" ] && init_args+=(--email="$EMAIL")
 
-INIT_LOG=/tmp/spx-init.log
+# init prints the join token, so its log is root-only and outside the world-writable /tmp.
+INIT_LOG=/var/log/spx-init.log
+
+# redact_tokens hides join tokens in text about to be shown to the operator.
+redact_tokens() { sed -E 's/spx_join_[A-Za-z0-9_-]+/spx_join_[redacted]/g'; }
 
 log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s) to join)"
 
 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
     exit 0" || fail "${HOSTS[0]}: could not start spx admin init"
 
 TOKEN=""
@@ -675,7 +687,7 @@ else
         elapsed=$((elapsed + 5))
     done
     if [ -z "$TOKEN" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: spx admin init produced no join token within ${TOKEN_TIMEOUT}s"
     fi
     log "  join token issued"
@@ -691,11 +703,9 @@ fi
 # waiting on a shell that node 2 is still holding.
 JOIN_PIDS=()
 for i in $(seq 1 $((N - 1))); do
-    on "${HOSTS[$i]}" "sudo spx admin join --force \
-        --node ${NODE_NAMES[$i]} \
-        --bind ${LAN_IPS[$i]} --cluster-bind ${LAN_IPS[$i]} --advertise ${WAN_IPS[$i]} \
-        --host ${LAN_IPS[0]}:$PORT --token $TOKEN \
-        --region $REGION --az $AZ" &
+    # The token goes to the node on stdin, so it is in no ssh argument and not in sudo's log.
+    # spx admin join takes it only as --token, so it is still in that process's arguments.
+    on "${HOSTS[$i]}" "$(remote_join_cmd "${NODE_NAMES[$i]}" "${LAN_IPS[$i]}" "${WAN_IPS[$i]}" "${LAN_IPS[0]}:$PORT" "$REGION" "$AZ")" <<<"$TOKEN" &
     JOIN_PIDS+=("$!")
 done
 
@@ -724,7 +734,7 @@ if ! $DRY_RUN; then
         elapsed=$((elapsed + 5))
     done
     if [ "$elapsed" -ge "$FORMATION_TIMEOUT" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: formation did not complete within ${FORMATION_TIMEOUT}s"
     fi
     log "  formation complete"

scripts/install-node_test.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
+# sudo's log (which holds the command it was given) never contains it, and the other values must
+# arrive as plain arguments even when they hold shell syntax.
+# preflight runs this with sh
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+mkdir "$work/bin"
+cd "$work"
+
+cat > "$work/bin/sudo" <<'SH'
+#!/bin/sh
+echo "$*" >> "$FAKE_SUDO_LOG"
+exec "$@"
+SH
+cat > "$work/bin/spx" <<'SH'
+#!/bin/sh
+for a in "$@"; do printf '<%s>' "$a"; done > "$FAKE_SPX_ARGV"
+SH
+chmod +x "$work/bin/sudo" "$work/bin/spx"
+export FAKE_SUDO_LOG="$work/sudo.log" FAKE_SPX_ARGV="$work/spx.argv" PATH="$work/bin:$PATH"
+
+eval "$(sed -n '/^remote_join_cmd()/,/^}/p' "$here/install-node.sh")"
+
+fails=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+
+TOKEN=spx_join_FAKETOKEN-abc_1
+cmd="$(remote_join_cmd node2 10.0.0.2 203.0.113.2 '10.0.0.1:4432;touch PWNED' us-east-1 'az $(touch PWNED2)')"
+bash -c "$cmd" <<<"$TOKEN"
+
+check "the token is not in the command" "$(printf '%s' "$cmd" | grep -c FAKETOKEN || true)" 0
+check "the token is not in sudo's log" "$(grep -c FAKETOKEN "$FAKE_SUDO_LOG" || true)" 0
+check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--token><$TOKEN>"
+check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
+check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
+
+[ "$fails" -eq 0 ]

scripts/lib/lambda-testlink-build_test.sh

@@ -0,0 +1,111 @@
+#!/bin/sh
+# lambda-testlink-build_test.sh — the test-link make targets refuse a malformed
+# image digest and an unsupported host or GOARCH before touching any output.
+#
+# Every case is refused on the recipe's first lines, so nothing is compiled.
+set -eu
+
+LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+REPO_ROOT=$(dirname -- "$(dirname -- "$LIB_DIR")")
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- make build-lambda-testlink / build-lambda-testlink-image guards"
+
+PWN="$TMP/pwn"
+OUT="$TMP/out"
+HEX=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+mkdir -p "$OUT"
+printf 'old daemon\n' > "$OUT/spx"
+printf 'old initializer\n' > "$OUT/hardkill.so"
+PAIR=$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)
+
+# unchanged <name> — the existing pair is intact, no scratch was made and no
+# injected command ran.
+unchanged() {
+    if [ -e "$PWN" ]; then
+        fail "$1: injected command ran"
+        rm -f "$PWN"
+    elif [ -e "$OUT.tmp" ]; then
+        fail "$1: scratch directory created"
+    elif [ "$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)" != "$PAIR" ]; then
+        fail "$1: existing pair replaced"
+    else
+        pass "$1"
+    fi
+}
+
+# refused <name> <want> <log> <status> — make failed with the expected message.
+refused() {
+    if [ "$4" -eq 0 ]; then
+        fail "$1: make succeeded"
+    elif ! grep -q "$2" "$3"; then
+        fail "$1: wrong refusal: $(tr '\n' ' ' < "$3")"
+    else
+        unchanged "$1"
+    fi
+}
+
+testlink() {
+    make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" "$@" > "$TMP/log" 2>&1
+}
+
+DIGEST_MSG="LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"
+NL='
+'
+for v in '' "\$(shell touch $PWN)" "\$\$(touch $PWN)" "\`touch $PWN\`" "$HEX; touch $PWN" \
+    "$HEX\" -X main.x=\"y" "$HEX -X main.x=y" "$HEX${NL}touch $PWN" \
+    0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF "${HEX%?}" "${HEX}0"; do
+    # GNU make itself expands a command-line $(shell ...) while parsing, for
+    # any target, so that one literal is tested only from the environment.
+    case "$v" in
+    "\$(shell"*) ;;
+    *)
+        status=0
+        testlink LAMBDA_IMAGE_DIGEST="$v" || status=$?
+        refused "command-line digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+        ;;
+    esac
+    status=0
+    LAMBDA_IMAGE_DIGEST="$v" testlink || status=$?
+    refused "environment digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+done
+
+status=0
+env -u LAMBDA_IMAGE_DIGEST make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" > "$TMP/log" 2>&1 || status=$?
+refused "digest unset" "$DIGEST_MSG" "$TMP/log" "$status"
+
+ARCH_MSG="supports only an x86_64 host building linux/amd64"
+SHIM="$TMP/shim"
+mkdir -p "$SHIM"
+printf '#!/bin/sh\necho aarch64\n' > "$SHIM/uname"
+chmod +x "$SHIM/uname"
+
+status=0
+GOARCH=arm64 testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "GOARCH=arm64" "$ARCH_MSG" "$TMP/log" "$status"
+status=0
+PATH="$SHIM:$PATH" testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "aarch64 host" "$ARCH_MSG" "$TMP/log" "$status"
+
+# The image target refuses before building qboot or any agent binary.
+for c in "GOARCH=arm64" "PATH=$SHIM:$PATH"; do
+    status=0
+    env "$c" make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink-image > "$TMP/log" 2>&1 || status=$?
+    if [ "$status" -eq 0 ]; then
+        fail "image target [${c%%=*}]: make succeeded"
+    elif ! grep -q "$ARCH_MSG" "$TMP/log" || grep -q "qboot\|go build" "$TMP/log"; then
+        fail "image target [${c%%=*}]: did not refuse first: $(tr '\n' ' ' < "$TMP/log")"
+    else
+        pass "image target [${c%%=*}] refused first"
+    fi
+done
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

scripts/terraform/latitude-spx/.gitignore

@@ -0,0 +1,10 @@
+.terraform/
+terraform.tfstate
+terraform.tfstate.*
+*.tfplan
+*.auto.tfvars
+crash.log
+terraform.tfvars
+*.pem
+*.log
+.terraform.lock.hcl

scripts/terraform/latitude-spx/README.md

@@ -0,0 +1,137 @@
+# latitude-spx
+
+Builds a Spinifex deployment on latitude.sh bare metal from your laptop with `deploy-latitude.sh`.
+The commands, flags, step markers and state layout follow `vultr-spx`.
+
+**Status: not yet run end to end.**
+The script was written from the plan doc findings (`docs/development/feature/latitude-bm.md`, L1 to L26) and from the provider schema.
+No latitude.sh API call or host run was made while building it, so treat the first `up` as the test.
+One node has been the only shape run live, and only in part (see Unverified). The three-node flow, including the floating pool, has not been run: Ben takes that over.
+
+## What it creates
+
+| Resource | Notes |
+|---|---|
+| Project | `spinifex-<name>`, unless `--project` names an existing one (then it is never deleted) |
+| SSH key | `spinifex-<name>`, account scoped |
+| User data | `spinifex-<name>`, host preparation only (no `{{ }}`, latitude.sh substitutes them) |
+| Servers | `m4-metal-medium`, site `DAL`, Ubuntu 26.04, hourly billing, hostnames `spx-<name>-nodeN` |
+| Elastic IP | one by default (`elastic_ip_count` in `variables.tf`, 1 to 5; set `TF_VAR_elastic_ip_count` to change it). With one node it is a static pool address and the count must stay 1, because a second pool is never tried when the first is full; with three nodes the addresses form the `latitude` floating pool, which Spinifex moves between servers. `nginx-webserver` needs one address and runs on one node; `nginx-alb` needs two public addresses and is a three-node workbook |
+| Virtual network | three nodes only: `spinifex-<name>`, one VLAN assignment per server |
+
+## Requirements
+
+OpenTofu (or Terraform) 1.5 or later, `aws` CLI, `python3`, `curl`, `ssh`, an SSH key pair.
+
+| Variable | When | Meaning |
+|---|---|---|
+| `LATITUDESH_AUTH_TOKEN` | always | the account API token; used by the provider and this script only, never sent to a node |
+| `LATITUDESH_NODE_TOKEN` | `--nodes 3` | a second, different, writable API key, copied to each node (`/etc/spinifex/latitude/api-token`, owned by `spinifex-daemon` with mode 0400, in a root-owned 0711 directory; the other services share the `spinifex` group, so no group may read it; ownership and mode are set on every `up`; a symlink or any other owner there is refused) so the daemon can move Elastic IPs |
+
+The script refuses a node key it can see is read-only (it lists `/auth/api_keys` and matches the key's last five characters). If the key is not among the account's keys, write authority is first proven by the first Elastic IP move.
+
+Exposure: a latitude.sh API key is team-wide. It is not scoped to Elastic IPs, so anyone who reads it from a node can do whatever the key's owner can in the team (the API exposes `read_only` and `allowed_ips` on a key, and no narrower scope). Create the node key with `allowed_ips` set to the nodes' public addresses, and revoke it after `destroy`.
+
+Nodes are reached as `ubuntu` with the deployment's SSH key, and remote commands run through passwordless `sudo`: latitude.sh puts the key on the image's login user and root login is not enabled.
+
+## Usage
+
+```
+export LATITUDESH_AUTH_TOKEN=...        # from the latitude.sh dashboard
+./deploy-latitude.sh up --name demo     # one node
+
+export LATITUDESH_NODE_TOKEN=...        # a second, writable key
+./deploy-latitude.sh up --name demo3 --nodes 3
+./deploy-latitude.sh status --name demo
+./deploy-latitude.sh ssh --name demo node1 'systemctl status spinifex.target'
+./deploy-latitude.sh credentials --name demo
+./deploy-latitude.sh stop node1 --name demo
+./deploy-latitude.sh start node1 --name demo
+./deploy-latitude.sh destroy --name demo
+```
+
+Options: `--name`, `--site` (alias `--region`), `--plan`, `--os`, `--project ID|SLUG`, `--distro TARBALL`, `--channel`, `--spx-region`, `--spx-az`, `--ssh-key`, `--image`, `--data-disk-id`, `--vpc-subnet` (the VLAN's /24, three nodes), `--nodes 1|3`, `--yes`.
+`./deploy-latitude.sh --help` lists them with defaults.
+
+Three nodes need `--distro` with a tarball built from this branch (`make distro-amd64`) until a release carries the latitude provider.
+A released spx rejects `source="latitude"` and its daemon restarts in a loop, so `up --nodes 3` without `--distro` stops in preflight.
+Before the pool is written, each node's `/usr/local/bin/spx` is checked for the `latitude_elastic_ip_ids` key name; spx has no validate subcommand, so this is a string search of the binary, not a run of its config check.
+
+## What `up` does
+
+`terraform` then, per node: `host` (wait for SSH as `ubuntu` and cloud-init, `/dev/kvm`), `disk`, `vlan` (three nodes), `install`, `form`, `start`; then `pool` (one node) or `cluster` (three nodes: `install-node.sh` forms the cluster); three nodes then install the node token on every `up` and add the `latitude` pool (`pool`); then `restart` (three nodes wait for the current daemon to log "floating pool ready" for the pool), `public`, `image` and the AWS profile.
+Each step writes `step.<name>[.nodeN]` in the state directory, and a rerun resumes at the first step without one.
+
+- **Data disk.** `host-data-disk.sh` picks the blank NVMe by content and by-id EUI, never by `nvmeXn1` (L1, L11).
+A candidate is at least 100 GB with no signature, partitions, holders or mounts, and not in the root filesystem's ancestry; with none it stops and lists the disks.
+With several (LON2 has two 1.7 TB and one spare 447 GB blank NVMe next to the root disk) it takes the largest, the lowest by-id path among equals, prints what it chose and the blank disks it left, and never touches the others (`--data-disk-id` chooses another).
+It mounts by UUID with `nofail` and the script records the UUID in `data_uuid.<node>`.
+- **Mount check.** Before anything writes under `/var/lib/spinifex`, and on every run, the mounted filesystem UUID must equal the recorded one.
+The units that write there also get a `RequiresMountsFor=/var/lib/spinifex` drop-in.
+The mount is by UUID in fstab, so renamed disks do not change it, and there is no check at service start.
+- **Firewall.** `setup.sh --firewall on` runs before Spinifex first starts.
+- **One node.** The Elastic IP is a static pool (`gateway_ip = "192.0.2.1"`, the node's next hop as `gateway`, no `prefix_len`) with the IMDS addresses moved off the ones latitude.sh routes.
+- **Three nodes.** The `vlan` step makes `eno2.<vid>` with `ip link` and persists it as systemd-networkd files for that link plus a drop-in on the `eno2` unit.
+It never runs `netplan apply`, refuses a subnet that overlaps the SSH peer or a route, and removes the VLAN again if the public interface's address, default route or the route to the SSH peer differ afterwards.
+`form_cluster` then puts the cluster and Geneve planes on that VLAN with `install-node.sh` (`--user ubuntu`); its output is not kept, as it can hold the join token.
+- **Floating pool (three nodes).** The `latitude` pool names the Elastic IP IDs and the project ID from the Terraform outputs, and each node's `provider_host_id` is its server ID. `spinifex.toml` is edited idempotently and read back. Failover moves an address with one `PATCH /elastic_ips/{id}` and waits for it to be active; latitude.sh has no unassign, so a node that loses an address is not told to release it.
+- **State** is in `~/.spinifex/latitude/<name>/`: Terraform state, `ids` (every `latitudesh_*` object in the state, merged on each apply and before each destroy), `step.*` markers, logs and the data disk UUIDs.
+- **AWS profile** `spinifex-latitude-<name>` is replaced on each `up`, with the CA at `~/.spinifex/latitude-<name>-ca.pem`.
+The secret key is written to `~/.aws/credentials` (0600, other profiles kept, the old file saved beside it) and never appears in a process argument.
+
+## Token handling
+
+`LATITUDESH_AUTH_TOKEN` is read from the environment by the provider and by the script's API calls.
+It is not a Terraform variable, is on no command line (curl reads it from stdin), is not sent to a node, and is written to no file.
+`LATITUDESH_NODE_TOKEN` is checked with curl reading it from stdin and reaches each node on stdin of the install script, so it is in no argument on either side.
+`TF_LOG` is unset by the script.
+The join token that `install-node.sh` creates is shared with Vultr and OCI: its init log is root-only, and `spx admin join` still takes it as `--token` on the joining node (tracked separately).
+
+## Recovery
+
+If SSH is lost once the firewall is armed, use the serial console.
+`up` enables the server's out-of-band connection with the deployment's SSH key and prints the `ssh` command it returns before it arms the firewall, falling back to the Out-of-Band tab in the dashboard.
+Logging in there needs an OS user password, which you set yourself while SSH still works.
+A fresh deployment that cannot be recovered is cheaper to `destroy` and `up` again.
+The VLAN step prints the console command first.
+It refuses a subnet that holds the SSH peer or overlaps an existing route, and removes the VLAN again if the route to the SSH peer or the public interface changes.
+
+## destroy
+
+`destroy` asks you to type the deployment name (`--yes` skips only that), then records every ID in the Terraform state and stops without destroying anything if it cannot.
+It runs `terraform destroy`, fetches every recorded ID by itself (2xx present, 404 gone, anything else fails and keeps the records), and deletes what remains of those.
+Servers, keys, user data, virtual networks and projects whose names match the deployment but that are not recorded are listed by ID and deleted only after you type `delete-untracked` for exactly those IDs, which `--yes` never supplies; an ID that shows up later needs its own confirmation, and without a terminal they are kept and `destroy` exits 1.
+Elastic IPs are touched only by recorded ID.
+Elastic IP release is asynchronous and only works once the address is active or in error, so the sweep retries.
+Because the Elastic IP list ignores pagination, `destroy` ends by asking you to check the dashboard for leftovers and type `checked`.
+An `up` that failed while creating can leave an object nothing recorded, such as an Elastic IP; that dashboard check is the only thing that finds it.
+
+## Unverified
+
+Nothing in the three-node flow has run against real servers. For one node, only the parts the person ran have been exercised, and the script has changed since.
+
+- The OS slug `ubuntu_26_04_x64_lts` (the `latitudesh_operating_system` data source fails the plan if it is wrong).
+- That the deploy key is on `ubuntu` and `sudo -n` works for it, and that `sudo` resets nothing the script needs (the SSH peer address is read before `sudo` for that reason).
+- The power action request (`POST /servers/{id}/actions` with `power_off` and `power_on`).
+- Resolving `--project` as a slug through `GET /projects/{slug}`, and the `in_stock` shape of `GET /plans` used by the stock check, which only warns when it cannot read it.
+- The `GET` and `DELETE` paths for `/user_data/{id}` and `/virtual_networks/{id}`.
+- That the Spinifex API certificate names the node's public address, which the profile check needs.
+- That the private NIC is `eno2`, its tagging and MTU (the script uses 1500), and that the networkd files persist the VLAN across a reboot.
+- `GET /auth/api_keys` and its `token_last_slice` and `read_only` fields, which the node key check relies on; and that a key with `allowed_ips` set to the nodes' addresses still works from them.
+- The Elastic IP move itself: `PATCH /elastic_ips/{id}` with `server_id`, its 202 then `active`, the error codes, and how long a move takes; the client waits up to its move timeout and makes one write per call.
+- That the daemon reads the node token and logs "floating pool ready", then fails over when a node stops (`stop node1`) and the address lands on another node.
+- `install-node.sh` as `--user ubuntu` on latitude.sh servers.
+
+## Not exercised
+
+Any real API call from the three-node flow, `destroy`, `host-data-disk.sh` on a host and the VLAN step on a server.
+Guest launch, public reachability of the Elastic IP, the Terraform workbooks and failover are not part of `up`; prove them by hand.
+`e2e-latitude.sh` and removing the old `install-node.sh` in this directory are not done.
+
+## Tests
+
+`sh node-scripts_test.sh` checks the node token install (install, unchanged, rotation, modes, an older install converging, symlinks and owners refused; run it as root too, with `sudo sh node-scripts_test.sh`, to check that only the daemon's user can read the file) and the readiness check (old success then current failure) with stand-in tools.
+`bash deploy-argv_test.sh` checks that neither API token, nor the five characters used to find the node key, is an argument of any external command in the node token paths, and that the SSH peer address comes from the same connection after sudo's environment reset.
+`scripts/install-node_test.sh` (part of `make preflight`) checks that the join token reaches `spx admin join` only on stdin, so sudo's log never holds it.
+The first two are not part of `make preflight`.
+The Go side: `cd spinifex && GOWORK=off go test ./providers/cloud/latitude/ ./domains/network/external/floating/ ./bootstrap/config/ ./runtime/daemon/`.

scripts/terraform/latitude-spx/cloud-init/user-data.yaml

@@ -0,0 +1,23 @@
+#cloud-config
+# Host preparation only. Spinifex is installed and formed over SSH by deploy-latitude.sh.
+# latitude.sh substitutes double-brace variables in user data, so this file holds none.
+write_files:
+  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
+    content: |
+      network: {config: disabled}
+  # The image's netplan lists eno2 with no address and without optional, so boot waits two minutes
+  # for it. --any returns once one link, eno1, is online; it changes the wait, not the network.
+  # UNVERIFIED: the binary path is the Ubuntu one and the drop-in first applies on the next boot.
+  - path: /etc/systemd/system/systemd-networkd-wait-online.service.d/10-spinifex-any.conf
+    content: |
+      [Service]
+      ExecStart=
+      ExecStart=/usr/lib/systemd/systemd-networkd-wait-online --any
+# runcmd carries on past a failing command, so each line stands alone.
+runcmd:
+  - [sh, -c, "command -v ufw >/dev/null && ufw --force disable || true"]
+  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
+  # The image ships empty apt lists and no git, which the workbooks need.
+  - [sh, -c, "apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git || true"]
+  - [sh, -c, "systemctl daemon-reload || true"]
+  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]

scripts/terraform/latitude-spx/compute.tf

@@ -0,0 +1,70 @@
+locals {
+  hostname_prefix = "spx-${var.deployment_name}-node"
+  project         = var.project != "" ? var.project : latitudesh_project.deploy[0].slug
+  project_id      = var.project != "" ? var.project_id : latitudesh_project.deploy[0].id
+}
+
+# Both lookups fail the plan when a slug does not exist, which is the only check of the OS slug.
+data "latitudesh_operating_system" "os" {
+  slug = var.operating_system
+}
+
+data "latitudesh_plan" "plan" {
+  slug = var.plan
+}
+
+resource "latitudesh_project" "deploy" {
+  count             = var.project == "" ? 1 : 0
+  name              = "spinifex-${var.deployment_name}"
+  description       = "Spinifex deployment ${var.deployment_name}"
+  environment       = "Development"
+  provisioning_type = "on_demand"
+}
+
+# The key is team scoped. The server login user is ubuntu, not root.
+resource "latitudesh_ssh_key" "deploy" {
+  name       = "spinifex-${var.deployment_name}"
+  public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
+}
+
+# Host preparation only. Latitude substitutes double-brace variables inside user data, so the file
+# must hold none. Spinifex is installed over SSH by deploy-latitude.sh.
+resource "latitudesh_user_data" "host" {
+  description = "spinifex-${var.deployment_name}"
+  content     = filebase64("${path.module}/cloud-init/user-data.yaml")
+}
+
+# billing must be hourly: the default is monthly, which bills upfront with no refund.
+resource "latitudesh_server" "node" {
+  count            = var.node_count
+  project          = local.project
+  hostname         = "${local.hostname_prefix}${count.index + 1}"
+  site             = var.site
+  plan             = data.latitudesh_plan.plan.slug
+  operating_system = data.latitudesh_operating_system.os.slug
+  billing          = "hourly"
+  ssh_keys         = [latitudesh_ssh_key.deploy.id]
+  user_data        = latitudesh_user_data.host.id
+
+  timeouts = {
+    create = "60m"
+  }
+}
+
+# Elastic IPs are bought here and never created or deleted by Spinifex. Each starts on a server, spread
+# over the nodes, and server_id is ignored afterwards so a move by Spinifex is not undone by a later apply.
+resource "latitudesh_elastic_ip" "pool" {
+  count = var.elastic_ip_count
+  # The provider documents ID or slug but posts the value as project_id unresolved: a slug gave 404.
+  project   = local.project_id
+  server_id = latitudesh_server.node[count.index % var.node_count].id
+
+  lifecycle {
+    ignore_changes = [server_id]
+  }
+}
+
+moved {
+  from = latitudesh_elastic_ip.node1
+  to   = latitudesh_elastic_ip.pool[0]
+}

scripts/terraform/latitude-spx/deploy-argv_test.sh

@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+# Checks that neither API token is ever an argument of an external command in the node token
+# paths of deploy-latitude.sh. External commands are stand-ins that record their arguments and
+# their standard input; the test fails if a token is in the first and proves it is in the second.
+# preflight runs this with sh
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+set -euo pipefail
+exec </dev/null
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+mkdir "$work/bin" "$work/home"
+real_python="$(command -v python3)"
+
+AUTH=FAKEAUTHTOKEN9
+NODE=FAKENODETOKEN7
+
+# Every command the script runs outside bash records "name args" and, for the first line of
+# standard input, "name stdin: ...".
+for c in curl ssh scp aws tofu ssh-keygen; do
+    cat > "$work/bin/$c" <<'SH'
+#!/bin/sh
+echo "$(basename "$0") $*" >> "$FAKE_ARGV"
+if [ ! -t 0 ]; then
+    in="$(cat)"
+    [ -z "$in" ] || echo "$(basename "$0") stdin: $in" >> "$FAKE_STDIN"
+fi
+case "$(basename "$0")" in
+    curl) printf '{"data":[{"attributes":{"token_last_slice":"%s","read_only":false}}]}\n200' "$(printf '%s' "$FAKE_NODE" | tail -c 5)" ;;
+    ssh)
+        if [ -n "${FAKE_SSH_EVAL:-}" ]; then
+            # Behave like a login on a node: skip the options and host, run the rest with the
+            # connection variable of this connection, and let sudo reset the environment.
+            while [ "$#" -gt 0 ]; do case "$1" in ubuntu@*) shift; break ;; *) shift ;; esac; done
+            SSH_CONNECTION="$FAKE_SSH_CONNECTION" bash -c "$*"
+        else
+            echo installed
+        fi ;;
+esac
+SH
+    chmod +x "$work/bin/$c"
+done
+cat > "$work/bin/sudo" <<'SH'
+#!/bin/sh
+while [ "$#" -gt 0 ]; do case "$1" in -*) shift ;; *) break ;; esac; done
+exec env -i PATH="$PATH" "$@"
+SH
+chmod +x "$work/bin/sudo"
+cat > "$work/bin/python3" <<SH
+#!/bin/sh
+echo "python3 \$*" >> "\$FAKE_ARGV"
+exec "$real_python" "\$@"
+SH
+chmod +x "$work/bin/python3"
+
+export FAKE_ARGV="$work/argv" FAKE_STDIN="$work/stdin" FAKE_NODE="$NODE" PATH="$work/bin:$PATH" HOME="$work/home"
+: > "$FAKE_ARGV"
+: > "$FAKE_STDIN"
+export LATITUDESH_AUTH_TOKEN="$AUTH" LATITUDESH_NODE_TOKEN="$NODE"
+
+# shellcheck source=deploy-latitude.sh
+source "$here/deploy-latitude.sh"
+
+# shellcheck disable=SC2034 # read by the sourced functions
+NODES=3 IP=192.0.2.10 NODE_NAME=node1
+check_node_token >/dev/null
+install_key >/dev/null 2>&1
+
+fails=0
+for tok in "$AUTH" "$NODE" "${NODE: -5}"; do
+    if grep -q "$tok" "$FAKE_ARGV"; then
+        echo "FAIL token $tok is in a command's arguments:"
+        grep "$tok" "$FAKE_ARGV"
+        fails=$((fails + 1))
+    else
+        echo "ok   $tok is in no argument"
+    fi
+done
+for what in "curl stdin:.*Bearer $NODE" "ssh stdin: $NODE"; do
+    if grep -q "$what" "$FAKE_STDIN"; then
+        echo "ok   the check can see the token where it is sent ($what)"
+    else
+        echo "FAIL the token never reached stdin ($what): the test would pass without checking anything"
+        fails=$((fails + 1))
+    fi
+done
+
+# The peer address comes from the same connection that runs the command, whichever source
+# address that connection has, and survives sudo's environment reset.
+export FAKE_SSH_EVAL=1
+for src in 198.51.100.7 203.0.113.99; do
+    export FAKE_SSH_CONNECTION="$src 51000 192.0.2.10 22"
+    got="$(on 'printf %s "$SSH_PEER"')"
+    if [ "$got" = "$src" ]; then echo "ok   peer $src reaches the command"; else echo "FAIL peer for $src was '$got'"; fails=$((fails + 1)); fi
+done
+
+# A three-node run needs a build that knows the latitude source: a release would be refused by its
+# own config check. The guard needs --distro, and the node is checked before it gets the pool.
+check_result() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+check_result "three nodes without --distro stop" "$( (NODES=3 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
+check_result "three nodes with --distro go on" "$( (NODES=3 DISTRO=/tmp/x.tar.gz require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node without --distro goes on" "$( (NODES=1 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node with one address goes on" "$( (NODES=1 TF_VAR_elastic_ip_count=1 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node with no setting goes on" "$( (NODES=1 TF_VAR_elastic_ip_count="" require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+check_result "one node with two addresses stops" "$( (NODES=1 TF_VAR_elastic_ip_count=2 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
+check_result "three nodes with three addresses go on" "$( (NODES=3 TF_VAR_elastic_ip_count=3 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+printf 'binary with latitude_elastic_ip_ids inside\n' > "$work/spx-new"
+printf 'binary of a release\n' > "$work/spx-old"
+SPX_BIN="$work/spx-new"
+check_result "a build that knows latitude is accepted" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" ran
+# shellcheck disable=SC2034 # read by the sourced functions
+SPX_BIN="$work/spx-old"
+check_result "a release is refused" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
+# The profile spx writes under sudo is in the ubuntu user's home, so node-side reads name that
+# directory instead of relying on root's HOME.
+mkdir "$work/awsdir"
+printf '[other]\naws_access_key_id = X\n[spinifex]\naws_access_key_id = AK1\naws_secret_access_key = SK1\n[z]\nq = 1\n' > "$work/awsdir/credentials"
+node_ip() { printf 192.0.2.10; }
+# shellcheck disable=SC2034 # read by the sourced functions
+NODE_AWS_DIR="$work/awsdir"
+check_result "credentials come from the node's AWS dir" "$(cmd_credentials | tr '\n' ' ')" "[spinifex] aws_access_key_id = AK1 aws_secret_access_key = SK1 "
+
+# A failed API call logs its status and a documented code, never the response text, which can
+# echo the request's headers.
+mkdir "$work/bin2"
+cat > "$work/bin2/curl" <<'SH'
+#!/bin/sh
+cat > /dev/null
+printf '{"errors":[{"code":"%s","detail":"Authorization: Bearer %s"}]}\n401' "$FAKE_CODE" "$FAKE_LEAK"
+SH
+chmod +x "$work/bin2/curl"
+logged="$(FAKE_CODE=UNAUTHORIZED FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
+check_result "a failed call names its status and code" "$(grep -c 'returned 401 (UNAUTHORIZED)' <<<"$logged" || true)" 1
+check_result "a failed call never logs the token" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
+logged="$(FAKE_CODE="$AUTH" FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
+check_result "an undocumented code is not logged" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
+
+# A recorded ID that now names another deployment's object is not ours to delete.
+# shellcheck disable=SC2034 # read by the sourced functions
+NAME=demo
+latitude() { printf '%s' "$FAKE_LAT"; }
+orig_expected_project_id="$(declare -f expected_project_id)"
+expected_project_id() { echo proj_1; }
+srv() {
+    local frag=',"project":{"id":"proj_1"}'
+    [ "$#" -lt 2 ] || frag="$2"
+    printf '{"data":{"id":"sv_1","attributes":{"hostname":"%s"%s}}}' "$1" "$frag"
+}
+FAKE_LAT="$(srv spx-demo-node2)"
+check_result "a recorded server with our hostname is ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" ours
+FAKE_LAT="$(srv spx-other-node1)"
+check_result "a recorded server with another hostname is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
+FAKE_LAT="$(srv spx-demo2-node1)"
+check_result "a hostname that only starts like ours is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
+FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
+check_result "our hostname in another project is not ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
+FAKE_LAT="$(srv spx-demo-node1 '')"
+check_result "a server with no project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+FAKE_LAT="$(srv spx-demo-node1 ',"project":{}')"
+check_result "a project with no id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+FAKE_LAT="$(srv spx-demo-node1 ',"project":null')"
+check_result "a null project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+expected_project_id() { return 1; }
+FAKE_LAT="$(srv spx-demo-node1)"
+check_result "no known project id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+expected_project_id() { echo proj_1; }
+FAKE_LAT='not json'
+check_result "an unreadable object is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
+KNOWN_IDS="$work/ids"
+printf 'server sv_1\n' > "$KNOWN_IDS"
+lat_exists() { return 0; }
+FAKE_LAT="$(srv spx-other-node1)"
+check_result "destroy stops before Terraform when a recorded ID was reused" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
+FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
+check_result "destroy stops when the same hostname is in another project" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
+FAKE_LAT="$(srv spx-demo-node1 '')"
+check_result "destroy stops when the project cannot be read" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
+FAKE_LAT="$(srv spx-demo-node1)"
+check_result "destroy goes on when the recorded server is ours" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" went
+leftovers_out="$( (lat_match() { return 0; }; FAKE_LAT="$(srv spx-other-node1)" leftovers) 2>/dev/null || true)"
+check_result "a reused ID is not listed as a leftover to delete" "$(grep -c sv_1 <<<"$leftovers_out" || true)" 0
+
+# After terraform destroy the state's project output is gone; the remembered ID still lets the
+# sweep delete a matching leftover and keep a foreign one.
+eval "$orig_expected_project_id"
+TF_BIN="$work/tofu"
+printf '#!/bin/sh\nif [ -f "%s/gone" ]; then exit 1; fi\nprintf proj_1\n' "$work" > "$TF_BIN"
+chmod +x "$TF_BIN"
+EXPECTED_PROJECT=""
+remember_project_id
+check_result "the project ID is remembered while the state holds it" "$EXPECTED_PROJECT" proj_1
+: > "$work/gone"
+check_result "the state no longer answers after destroy" "$("$TF_BIN" output >/dev/null 2>&1 && echo up || echo gone)" gone
+printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
+lat_match() { return 1; }
+latitude() { if [ "$2" = /servers/sv_1 ]; then srv spx-demo-node1; else srv spx-other-node1 ',"project":{"id":"proj_other"}'; fi; }
+swept="$(leftovers 2>/dev/null || true)"
+check_result "after destroy a matching leftover is still listed" "$(grep -c 'server sv_1 ' <<<"$swept" || true)" 1
+check_result "after destroy a foreign one is kept out" "$(grep -c 'server sv_2 ' <<<"$swept" || true)" 0
+EXPECTED_PROJECT=""
+swept="$(leftovers 2>/dev/null || true)"
+check_result "without the remembered ID a leftover is unknown, not matched" "$(grep -c 'server sv_1 sv_1 (unknown)' <<<"$swept" || true)" 1
+expected_project_id() { echo proj_1; }
+
+# Cleanup never deletes an object whose ownership could not be read, recorded or not.
+deleted="$work/deleted"
+: > "$deleted"
+latitude() { printf '%s %s\n' "$1" "$2" >> "$deleted"; }
+printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
+delete_left $'server sv_1 sv_1 (unknown)\nserver sv_2 spx-demo-node2\n' "" >/dev/null 2>&1
+check_result "an unreadable recorded object is not deleted" "$(grep -c sv_1 "$deleted" || true)" 0
+check_result "a verified recorded object is deleted" "$(grep -c 'DELETE .*sv_2' "$deleted" || true)" 1
+
+# Both deployment sizes write the metadata remap.
+cfg="$work/spinifex.toml"
+extract() { sed -n "/^$1()/,/^}/p" "$here/deploy-latitude.sh" | sed -n "/<<'PY'/,/^PY\$/p" | sed '1d;$d' | sed "s|/etc/spinifex/spinifex.toml|$cfg|"; }
+printf '[network]\nexternal_mode = "nat"\n\n[bootstrap]\ny = 1\n' > "$cfg"
+extract configure_pool | python3 -I - 203.0.113.1 203.0.113.254
+check_result "one node gets the metadata remap" "$(grep -c 'imds_host_meta_ip = "169.254.42.254"' "$cfg")" 1
+printf '[network]\nexternal_mode = "nat"\n\n[nodes.node1]\nx = 1\n\n[nodes.node2]\nx = 1\n\n[nodes.node3]\nx = 1\n\n[bootstrap]\ny = 1\n' > "$cfg"
+extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
+check_result "three nodes get the metadata remap" "$(grep -c 'imds_host_dns_ip  = "169.254.42.253"' "$cfg")" 1
+check_result "three nodes get the latitude pool" "$(grep -c 'source                  = "latitude"' "$cfg")" 1
+extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
+check_result "writing the config again changes nothing" "$(grep -c 'imds_host_meta_ip' "$cfg")" 1
+unset FAKE_SSH_EVAL
+
+[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/host-data-disk.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# host-data-disk.sh — format and mount the blank data disk at /var/lib/spinifex. Run as root ON the host.
+#
+#   host-data-disk.sh [/dev/disk/by-id/nvme-eui.<id>]
+#
+# The disk is chosen by content, never by its nvmeXn1 name, which swaps between boots. A candidate is a
+# whole-disk by-id EUI link whose device is not in the root filesystem's ancestry, has no partitions,
+# holders, swap or mounts, no filesystem signature and no partition table. With none the script stops
+# and lists the disks. With several it takes the largest, and the lowest by-id path among equals, and
+# prints what it chose and the blank disks it left alone; pass the by-id path to choose another.
+# It never reformats: a rerun on a host already done prints the UUID and exits. The last line of stdout
+# is DATA_UUID=<uuid>, which the later install phases need.
+
+set -euo pipefail
+
+MP=/var/lib/spinifex
+LABEL=spinifex-data
+FSTAB_BAK=/etc/fstab.pre-spinifex
+MIN_BYTES="${MIN_DATA_BYTES:-100000000000}"
+WANT="${1:-}"
+
+die() { echo "STOP: $*" >&2; exit 1; }
+[ "$(id -u)" = 0 ] || die "run as root"
+
+# Resume: the mount is already there and is ours.
+if mp_uuid="$(findmnt --mountpoint "$MP" -no UUID 2>/dev/null)" && [ -n "$mp_uuid" ]; then
+    src="$(findmnt --mountpoint "$MP" -no SOURCE)"
+    [ "$(blkid -s LABEL -o value "$src")" = "$LABEL" ] || die "$MP is mounted from $src, which is not labelled $LABEL; not touching it"
+    grep -q "UUID=${mp_uuid}[[:space:]]" /etc/fstab || die "$MP is mounted but /etc/fstab has no UUID=$mp_uuid entry"
+    echo "already done: $MP is mounted from $src (UUID $mp_uuid)"
+    echo "DATA_UUID=$mp_uuid"
+    exit 0
+fi
+if grep -q "$MP" /etc/fstab; then die "/etc/fstab mentions $MP but nothing is mounted there; fix that by hand first"; fi
+[ ! -e "$MP" ] || [ -z "$(ls -A "$MP")" ] || die "$MP exists and is not empty"
+
+root_src="$(findmnt -no SOURCE /)" || die "cannot find the root source"
+root_anc="$(lsblk -s -nro NAME "$root_src")" || die "cannot read the root ancestry"
+[ -n "$root_anc" ] || die "empty root ancestry"
+
+# blank_reason ID prints why the disk is not a candidate and returns 1, or returns 0.
+blank_reason() {
+    local id="$1" dev name size kids hold sw mnt rc w
+    dev="$(readlink -e "$id")" || { echo "link does not resolve"; return 1; }
+    [ -b "$dev" ] || { echo "not a block device"; return 1; }
+    name="$(basename "$dev")"
+    if grep -qx "$name" <<<"$root_anc"; then echo "in the root ancestry"; return 1; fi
+    size="$(blockdev --getsize64 "$id")" || { echo "size unreadable"; return 1; }
+    [ "$size" -ge "$MIN_BYTES" ] || { echo "only $size bytes"; return 1; }
+    kids="$(lsblk -nro NAME "$id")" || { echo "children unreadable"; return 1; }
+    [ "$kids" = "$name" ] || { echo "has children"; return 1; }
+    hold="$(ls -A "/sys/block/$name/holders/")" || { echo "holders unreadable"; return 1; }
+    [ -z "$hold" ] || { echo "has holders"; return 1; }
+    sw="$(swapon --show=NAME --noheadings)" || { echo "swapon failed"; return 1; }
+    if grep -q "$name" <<<"$sw"; then echo "is swap"; return 1; fi
+    mnt="$(lsblk -nro MOUNTPOINTS "$id")" || { echo "mounts unreadable"; return 1; }
+    [ -z "${mnt//[[:space:]]/}" ] || { echo "is mounted"; return 1; }
+    set +e; blkid -p "$id" >/dev/null 2>&1; rc=$?; set -e
+    [ "$rc" -eq 2 ] || { echo "blkid -p says it has a signature (rc $rc)"; return 1; }
+    w="$(wipefs -n "$id")" || { echo "wipefs failed"; return 1; }
+    [ -z "$w" ] || { echo "wipefs finds signatures"; return 1; }
+    return 0
+}
+
+# choose_data_disk ID... prints the largest disk's ID, the lowest path among equal sizes, and says on
+# stderr which it left. The choice depends only on size and path, so every node of one plan agrees.
+choose_data_disk() {
+    local chosen rest
+    chosen="$(for id in "$@"; do printf '%s %s\n' "$(blockdev --getsize64 "$id")" "$id"; done |
+        sort -k1,1nr -k2,2 | head -n1)"
+    chosen="${chosen#* }"
+    if [ "$#" -gt 1 ]; then
+        rest="$(printf '%s\n' "$@" | grep -vxF "$chosen" | sort | tr '\n' ' ')"
+        echo "chose $chosen ($(blockdev --getsize64 "$chosen") bytes), the largest of $# blank disks; left untouched: ${rest% }" >&2
+    fi
+    printf '%s\n' "$chosen"
+}
+
+candidates=()
+shopt -s nullglob
+for id in /dev/disk/by-id/nvme-eui.*; do
+    case "$id" in *-part*) continue ;; esac
+    if [ -n "$WANT" ] && [ "$id" != "$WANT" ]; then continue; fi
+    if why="$(blank_reason "$id")"; then
+        candidates+=("$id")
+    else
+        echo "skip $id: $why"
+    fi
+done
+shopt -u nullglob
+
+if [ "${#candidates[@]}" -eq 0 ]; then
+    lsblk -o NAME,SIZE,TYPE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS >&2 || true
+    die "no blank data disk among the by-id EUI disks"
+fi
+ID="$(choose_data_disk "${candidates[@]}")"
+DEV="$(readlink -e "$ID")"
+echo "CHECKS PASSED: $ID -> $DEV"
+
+mkdir -p "$MP"
+mkfs.ext4 -q -L "$LABEL" "$ID"
+U="$(blkid -p -o value -s UUID "$ID")" || die "blkid UUID failed"
+[ -n "$U" ] || die "no UUID after mkfs"
+[ "$(blkid -p -o value -s TYPE "$ID")" = ext4 ] || die "not ext4 after mkfs"
+
+NEW="$(mktemp /etc/fstab.spinifex.XXXXXX)"
+cp /etc/fstab "$NEW"
+echo "UUID=$U $MP ext4 defaults,nofail,x-systemd.device-timeout=30s 0 2" >> "$NEW"
+findmnt --verify --tab-file "$NEW" || { rm -f "$NEW"; die "staged fstab failed verify, /etc/fstab untouched"; }
+BAK="$FSTAB_BAK.$(date +%s)"
+cp -a /etc/fstab "$BAK"
+trap 'cp -a "$BAK" /etc/fstab; systemctl daemon-reload; echo "ROLLED BACK /etc/fstab from $BAK" >&2' EXIT
+chmod 644 "$NEW"
+mv "$NEW" /etc/fstab
+systemctl daemon-reload
+mount "$MP"
+[ "$(findmnt --mountpoint "$MP" -no UUID)" = "$U" ] || die "wrong UUID at $MP"
+[ "$(findmnt --mountpoint "$MP" -no FSTYPE)" = ext4 ] || die "wrong fstype at $MP"
+[ "$(findmnt --mountpoint "$MP" -no MAJ:MIN | tr -d ' ')" = "$(lsblk -dno MAJ:MIN "$ID" | tr -d ' ')" ] || die "wrong device at $MP"
+trap - EXIT
+findmnt --mountpoint "$MP" -o TARGET,SOURCE,MAJ:MIN,UUID,FSTYPE,OPTIONS
+echo "STORAGE OK"
+echo "DATA_UUID=$U"

scripts/terraform/latitude-spx/network.tf

@@ -0,0 +1,14 @@
+# A private network needs two or more servers, so a single node has none. The VLAN carries the cluster
+# and Geneve traffic; its address is set over SSH, since the VID is only known after apply.
+resource "latitudesh_virtual_network" "lan" {
+  count       = var.node_count > 1 ? 1 : 0
+  project     = local.project_id
+  site        = var.site
+  description = "spinifex-${var.deployment_name}"
+}
+
+resource "latitudesh_vlan_assignment" "node" {
+  count              = var.node_count > 1 ? var.node_count : 0
+  server_id          = latitudesh_server.node[count.index].id
+  virtual_network_id = latitudesh_virtual_network.lan[0].id
+}

scripts/terraform/latitude-spx/node-install-api-token.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+# Runs as root on a node. Reads the API token on stdin and installs it for the daemon.
+# The directory belongs to root and is not writable by anyone else, so no service can create or
+# swap an entry in it. The file belongs to the daemon's user alone (mode 0400): the other
+# services share the spinifex group, so group access would hand them the team-wide token. The
+# temporary file has an unpredictable name and is created exclusively. A symlink, or an owner
+# other than root or the daemon's user, where the directory or the file should be is refused.
+# Ownership and mode are set on every run, so an older install converges. The file's content is
+# replaced only when the token differs. Prints installed or unchanged.
+set -eu
+dir="${API_TOKEN_DIR:-/etc/spinifex/latitude}"
+root="${API_TOKEN_ROOT-root}"
+rootgroup="${API_TOKEN_ROOT_GROUP-root}"
+user="${API_TOKEN_USER-spinifex-daemon}"
+umask 077
+
+fail() { echo "$*" >&2; exit 1; }
+owner_of() { stat -c %U "$1"; }
+
+[ ! -L "$dir" ] || fail "$dir is a symlink"
+if [ ! -e "$dir" ]; then mkdir -p "$dir"; fi
+[ -d "$dir" ] || fail "$dir is not a directory"
+[ "$(owner_of "$dir")" = "$root" ] || fail "$dir is not owned by $root"
+chown "$root:$rootgroup" "$dir"
+chmod 0711 "$dir"
+
+target="$dir/api-token"
+[ ! -L "$target" ] || fail "$target is a symlink"
+if [ -e "$target" ]; then
+    [ -f "$target" ] || fail "$target is not a regular file"
+    holder="$(owner_of "$target")"
+    [ "$holder" = "$root" ] || [ "$holder" = "$user" ] || fail "$target is owned by $holder"
+fi
+
+tmp="$(mktemp "$dir/.api-token.XXXXXX")"
+trap 'rm -f "$tmp"' EXIT
+cat > "$tmp"
+[ -s "$tmp" ] || fail "empty token on stdin"
+if [ -f "$target" ] && cmp -s "$tmp" "$target"; then
+    chown "$user" "$target"
+    chmod 0400 "$target"
+    echo unchanged
+    exit 0
+fi
+chown "$user" "$tmp"
+chmod 0400 "$tmp"
+mv -f "$tmp" "$target"
+echo installed

scripts/terraform/latitude-spx/node-pool-state.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# Runs as root on a node with the pool name as $1. Reads only the journal of the daemon's
+# current invocation, so a success logged before a restart cannot hide a failure after it.
+# Prints ready, refused or waiting.
+set -eu
+pool="${1:?pool name}"
+id="$(systemctl show -p InvocationID --value spinifex-daemon)"
+[ -n "$id" ] || { echo waiting; exit 0; }
+log="$(journalctl "_SYSTEMD_INVOCATION_ID=$id" -u spinifex-daemon --no-pager -o cat 2>/dev/null || true)"
+named() { grep -F -e "pool=$pool" -e "\"pool\":\"$pool\""; }
+if printf '%s\n' "$log" | grep -F "refused the pool's first call" | named >/dev/null; then
+    echo refused
+elif printf '%s\n' "$log" | grep -F "floating pool ready" | named >/dev/null; then
+    echo ready
+else
+    echo waiting
+fi

scripts/terraform/latitude-spx/node-scripts_test.sh

@@ -0,0 +1,119 @@
+#!/bin/sh
+# Tests node-install-api-token.sh and node-pool-state.sh with stand-in tools.
+set -eu
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+fails=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+
+# token install: A, A again, then B across a rotation; the file ends as B, owned by the daemon
+# user alone (0400) in a 0711 directory that only root may change
+me="$(id -un)"
+export API_TOKEN_DIR="$work/tok" API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
+install_token() { sh "$here/node-install-api-token.sh"; }
+check "first install" "$(printf 'TOKENA' | install_token)" installed
+check "same token" "$(printf 'TOKENA' | install_token)" unchanged
+check "rotation" "$(printf 'TOKENB' | install_token)" installed
+check "file holds B" "$(cat "$work/tok/api-token")" TOKENB
+check "file mode and owner" "$(stat -c '%U %a' "$work/tok/api-token")" "$me 400"
+check "dir mode and owner" "$(stat -c '%U %a' "$work/tok")" "$me 711"
+check "no temp left after install" "$(ls -A "$work/tok")" api-token
+check "empty refused" "$(printf '' | install_token 2>/dev/null || echo failed)" failed
+check "file kept after refusal" "$(cat "$work/tok/api-token")" TOKENB
+
+# an older install converges on the unchanged path too
+chmod 0640 "$work/tok/api-token"
+chmod 0755 "$work/tok"
+check "unchanged path repairs" "$(printf 'TOKENB' | install_token)" unchanged
+check "mode repaired on unchanged" "$(stat -c %a "$work/tok/api-token")" 400
+check "dir mode repaired on unchanged" "$(stat -c %a "$work/tok")" 711
+
+# who can read it: only the daemon's identity. This needs two real identities, so it runs as root only.
+if [ "$(id -u)" = 0 ]; then
+    chmod 0711 "$work"
+    export API_TOKEN_ROOT=root API_TOKEN_ROOT_GROUP=root API_TOKEN_USER=daemon
+    for mode in installed unchanged; do
+        [ "$mode" = installed ] && rm -f "$work/tok/api-token"
+        check "$mode path" "$(printf 'TOKENB' | install_token)" "$mode"
+        check "daemon reads ($mode)" "$(setpriv --reuid=daemon --regid=daemon --clear-groups cat "$work/tok/api-token")" TOKENB
+        check "another identity in the shared group cannot read ($mode)" \
+            "$(setpriv --reuid=nobody --regid=daemon --clear-groups cat "$work/tok/api-token" 2>/dev/null || echo denied)" denied
+    done
+    export API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
+else
+    echo "skip reading as the daemon and as another service: needs root"
+fi
+
+# a link planted where the file or the directory belongs is refused, and nothing it points at is touched
+echo SENTINEL > "$work/sentinel"
+rm -f "$work/tok/api-token"
+ln -s "$work/sentinel" "$work/tok/api-token"
+check "symlinked file refused" "$(printf 'TOKENC' | sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
+check "symlink target untouched" "$(cat "$work/sentinel")" SENTINEL
+ln -s "$work/tok" "$work/tok-link"
+check "symlinked dir refused" "$(printf 'TOKENC' | API_TOKEN_DIR="$work/tok-link" sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
+check "dir owned by someone else refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=nobody-such install_token 2>/dev/null || echo refused)" refused
+rm -f "$work/tok/api-token"
+printf 'TOKENB' | install_token >/dev/null
+check "file owned by a stranger refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=$me API_TOKEN_USER=nobody-such install_token 2>/dev/null || echo refused)" refused
+check "no temp left" "$(ls -A "$work/tok")" api-token
+
+# data disk choice: the largest blank disk, the lowest path among equals, the rest left alone
+choose_dir="$work/choose"
+mkdir -p "$choose_dir/bin"
+cat > "$choose_dir/bin/blockdev" <<'SH'
+#!/bin/sh
+# blockdev --getsize64 <id>: the fixture's size for that disk
+cat "$CHOOSE_DIR/size.$(basename "$2")"
+SH
+chmod +x "$choose_dir/bin/blockdev"
+eval "$(sed -n '/^choose_data_disk()/,/^}/p' "$here/host-data-disk.sh")"
+size_of() { printf '%s' "$2" > "$choose_dir/size.$1"; }
+pick() { CHOOSE_DIR="$choose_dir" PATH="$choose_dir/bin:$PATH" choose_data_disk "$@" 2>"$choose_dir/err"; }
+size_of nvme-eui.0b 1700000000000
+size_of nvme-eui.0a 1700000000000
+size_of nvme-eui.09 447000000000
+size_of nvme-eui.0c 1900000000000
+check "largest of three wins" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0c)" /dev/disk/by-id/nvme-eui.0c
+check "equal sizes: the lowest path" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a)" /dev/disk/by-id/nvme-eui.0a
+check "order of the list does not matter" "$(pick /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.0a
+check "what was left is said" "$(grep -c 'left untouched: /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b' "$choose_dir/err")" 1
+check "one disk is just chosen" "$(pick /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.09
+check "one disk says nothing" "$(wc -c < "$choose_dir/err" | tr -d ' ')" 0
+
+# pool state: the stand-in journal serves only the lines of the invocation asked for
+mkdir "$work/bin"
+cat > "$work/bin/systemctl" <<'SH'
+#!/bin/sh
+cat "$FAKE_DIR/invocation"
+SH
+cat > "$work/bin/journalctl" <<'SH'
+#!/bin/sh
+for a in "$@"; do
+    case "$a" in _SYSTEMD_INVOCATION_ID=*) id="${a#_SYSTEMD_INVOCATION_ID=}" ;; esac
+done
+cat "$FAKE_DIR/journal.$id" 2>/dev/null
+SH
+chmod +x "$work/bin/systemctl" "$work/bin/journalctl"
+export FAKE_DIR="$work" PATH="$work/bin:$PATH"
+state() { sh "$here/node-pool-state.sh" latitude; }
+
+echo old > "$work/invocation"
+echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.old"
+check "current success" "$(state)" ready
+
+echo new > "$work/invocation"
+echo 'ERROR floating: the provider refused the pool'"'"'s first call; public addresses will fail until fixed pool=latitude' > "$work/journal.new"
+check "old success then current failure" "$(state)" refused
+
+echo 'INFO starting' > "$work/journal.new"
+check "current invocation not yet ready" "$(state)" waiting
+
+echo 'INFO floating pool ready pool=other source=latitude' > "$work/journal.new"
+check "another pool does not count" "$(state)" waiting
+
+echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.new"
+check "current success" "$(state)" ready
+
+[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/outputs.tf

@@ -0,0 +1,56 @@
+output "deployment_name" {
+  value = var.deployment_name
+}
+
+output "site" {
+  value = var.site
+}
+
+output "project" {
+  value = local.project
+}
+
+# provider_host_id is the server ID. public_ip is the server's own /31 address, which SSH and the
+# Spinifex API use; the Elastic IP is separate.
+output "nodes" {
+  value = [for n in latitudesh_server.node : {
+    name             = trimprefix(n.hostname, "spx-${var.deployment_name}-")
+    provider_host_id = n.id
+    public_ip        = n.primary_ipv4
+  }]
+}
+
+# The first address, for a single node's static pool.
+output "elastic_ip" {
+  value = latitudesh_elastic_ip.pool[0].address
+}
+
+# IDs as well as addresses: destroy deletes only these IDs, and the floating pool lists them.
+output "elastic_ips" {
+  value = [for e in latitudesh_elastic_ip.pool : { id = e.id, address = e.address }]
+}
+
+# The VLAN ID of the cluster network; null for one node, which has no private network.
+output "vid" {
+  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].vid
+}
+
+output "ssh_key_id" {
+  value = latitudesh_ssh_key.deploy.id
+}
+
+output "user_data_id" {
+  value = latitudesh_user_data.host.id
+}
+
+output "project_id" {
+  value = local.project_id
+}
+
+output "project_created" {
+  value = length(latitudesh_project.deploy) > 0
+}
+
+output "virtual_network_id" {
+  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].id
+}

scripts/terraform/latitude-spx/provider.tf

@@ -0,0 +1,4 @@
+# The token comes from LATITUDESH_AUTH_TOKEN in the environment, which the provider reads
+# itself (its auth_token argument documents that variable). It is never a variable here, so it
+# is not written to the state or to a tfvars file. The default project is set per resource.
+provider "latitudesh" {}

scripts/terraform/latitude-spx/terraform.tfvars.example

@@ -0,0 +1,5 @@
+deployment_name     = "l1"
+site                = "DAL"
+plan                = "m4-metal-medium"
+node_count          = 1
+ssh_public_key_path = "~/.ssh/id_ed25519.pub"

scripts/terraform/latitude-spx/variables.tf

@@ -0,0 +1,63 @@
+variable "deployment_name" {
+  type        = string
+  description = "Names every resource and the server hostnames, so two deployments in one account never collide."
+  validation {
+    # A hostname is at most 32 characters: "spx-" + name + "-node1".
+    condition     = can(regex("^[a-z0-9][a-z0-9-]{0,15}$", var.deployment_name))
+    error_message = "deployment_name must be 1-16 lowercase letters, digits or hyphens."
+  }
+}
+
+variable "project" {
+  type        = string
+  default     = ""
+  description = "ID or slug of an existing latitude.sh project to deploy into. Empty creates a project spinifex-<deployment_name> and removes it on destroy."
+}
+
+variable "project_id" {
+  type        = string
+  default     = ""
+  description = "ID of the project named by project. The Elastic IP create sends it as project_id without resolving a slug (provider 4.8.0), so it must be an ID. Unused when project is empty."
+}
+
+variable "site" {
+  type        = string
+  default     = "DAL"
+  description = "latitude.sh site slug. Pick one with stock for the plan and Elastic IP support; not SAO."
+}
+
+variable "plan" {
+  type        = string
+  default     = "m4-metal-medium"
+  description = "Server plan slug (the pricing page calls it m4.metal.medium)."
+}
+
+variable "operating_system" {
+  type        = string
+  default     = "ubuntu_26_04_x64_lts"
+  description = "OS slug. UNVERIFIED: the 26.04 slug follows the 24.04 pattern (ubuntu_24_04_x64_lts) and is checked by a data source at plan time."
+}
+
+variable "node_count" {
+  type    = number
+  default = 1
+  validation {
+    condition     = contains([1, 3], var.node_count)
+    error_message = "node_count must be 1 or 3."
+  }
+}
+
+variable "elastic_ip_count" {
+  type        = number
+  default     = 1
+  description = "Elastic IPs to buy, each 4 USD a month. Every one starts on a server and keeps it unless Spinifex moves it. latitude.sh allows 5 per team."
+  validation {
+    condition     = var.elastic_ip_count >= 1 && var.elastic_ip_count <= 5 && floor(var.elastic_ip_count) == var.elastic_ip_count
+    error_message = "elastic_ip_count must be a whole number from 1 to 5."
+  }
+}
+
+variable "ssh_public_key_path" {
+  type    = string
+  default = "~/.ssh/id_ed25519.pub"
+}

scripts/terraform/latitude-spx/versions.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_version = ">= 1.5.0"
+  required_providers {
+    # Pinned exactly: 4.8.0 is the version whose schema these files were written against.
+    latitudesh = { source = "latitudesh/latitudesh", version = "= 4.8.0" }
+  }
+}

spinifex/agents/ecs/agent/agent.go

@@ -12,8 +12,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 	ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
 )
 
@@ -102,7 +103,9 @@ func New(cfg Config) (*Agent, error) {
 		AgentVersion: cfg.AgentVersion,
 	}
 
-	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
+	// The cache refetches only once the real expiry passes, and a failed
+	// refresh is an error rather than an extended expiry.
+	creds := aws.NewCredentialsCache(imdscreds.NewProvider(imdsClient, cfg.IMDSBase))
 	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
 
 	// A dial that fails here is not fatal and not final: the node starts the

spinifex/agents/ecs/agent/config.go

@@ -21,17 +21,14 @@ const (
 
 // Config holds the static settings the agent reads at boot. Cluster identity
 // (account ID, instance ID) is discovered at runtime from IMDS, not configured;
-// only the cluster *name* the instance was launched into is static. AccessKey /
-// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
-// gateway calls with instance-role credentials from IMDS; the agent never holds
-// a NATS token.
+// only the cluster *name* the instance was launched into is static. The agent
+// SigV4-signs gateway calls with instance-role credentials from IMDS and never
+// holds a NATS token.
 type Config struct {
 	GatewayURL       string
 	GatewayCA        string
 	Region           string
 	ClusterName      string
-	AccessKey        string
-	SecretKey        string
 	IMDSBase         string
 	ContainerdSocket string
 	Heartbeat        time.Duration
@@ -53,8 +50,6 @@ func LoadConfig(envFile string) Config {
 		GatewayCA:        get("ECS_GATEWAY_CA"),
 		Region:           get("ECS_REGION"),
 		ClusterName:      get("ECS_CLUSTER"),
-		AccessKey:        get("ECS_ACCESS_KEY"),
-		SecretKey:        get("ECS_SECRET_KEY"),
 		IMDSBase:         get("ECS_IMDS_BASE"),
 		ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
 	}

spinifex/agents/ecs/agent/config_test.go

@@ -10,7 +10,7 @@ import (
 func TestLoadConfig_Defaults(t *testing.T) {
 	// Ensure env vars don't leak in from the host.
 	for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
-		"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
+		"ECS_CLUSTER", "ECS_IMDS_BASE",
 		"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
 		t.Setenv(k, "")
 	}
@@ -43,7 +43,6 @@ func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
 		t.Fatal(err)
 	}
 	t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
-	t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
 
 	cfg := LoadConfig(envFile)
 	if cfg.GatewayURL != "https://gw.env" {
@@ -55,9 +54,6 @@ func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
 	if cfg.ClusterName != "prod" {
 		t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
 	}
-	if cfg.AccessKey != "AKIAENV" {
-		t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
-	}
 }
 
 func TestLoadConfig_HeartbeatOverride(t *testing.T) {

spinifex/agents/ecs/agent/controlplane.go

@@ -1,17 +1,17 @@
 package agent
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"time"
 
+	awsv2 "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/ecsgw"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 )
 
@@ -47,26 +47,14 @@ const pollTimeout = 30 * time.Second
 
 // newGatewayControlPlane builds the SigV4 client signing with the instance-role
 // credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
-func newGatewayControlPlane(cfg Config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
-	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
+func newGatewayControlPlane(cfg Config, creds awsv2.CredentialsProvider) (*gatewayControlPlane, error) {
+	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, gwsign.New(creds), cfg.Region, pollTimeout)
 	if err != nil {
 		return nil, err
 	}
 	return &gatewayControlPlane{client: client}, nil
 }
 
-// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
-// per-call credentials hook.
-func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
-	return func(ctx context.Context) (string, string, string, error) {
-		c, err := p.Retrieve(ctx)
-		if err != nil {
-			return "", "", "", err
-		}
-		return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
-	}
-}
-
 func (g *gatewayControlPlane) Register(id identity) error {
 	in := &ecs.RegisterContainerInstanceInput{
 		Cluster:                  aws.String(id.ClusterName),

spinifex/agents/ecs/agent/controlplane_test.go

@@ -8,17 +8,17 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 )
 
-// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// fixedCredsProvider is a static aws.CredentialsProvider for tests
 // that only need a valid SigV4 signature, not real IMDS rotation.
 type fixedCredsProvider struct{}
 
-func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
-	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+func (fixedCredsProvider) Retrieve(context.Context) (aws.Credentials, error) {
+	return aws.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
 }
 
 func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {

spinifex/agents/ecs/agent/credendpoint.go

@@ -13,9 +13,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/ecrauth"
 	"github.com/mulgadc/spinifex/internal/stsauth"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 )
 
 const (
@@ -56,7 +56,7 @@ type credRole struct {
 
 type credEntry struct {
 	roleARN string
-	creds   stsauth.Credentials
+	creds   aws.Credentials
 }
 
 // credEndpoint is the host-netns HTTP server that hands a task its IAM role
@@ -64,7 +64,7 @@ type credEntry struct {
 // mints/caches credentials on demand by assuming the role over the gateway with
 // the agent's instance credentials.
 type credEndpoint struct {
-	provider   credentials.CredentialsProvider
+	provider   aws.CredentialsProvider
 	region     string
 	gatewayURL string
 	caPath     string
@@ -75,7 +75,7 @@ type credEndpoint struct {
 
 	// assume mints fresh credentials for a role; defaults to AssumeRole over the
 	// gateway and is overridable in tests.
-	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
+	assume func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error)
 
 	mu         sync.Mutex
 	roles      map[string]credRole  // credID -> task role
@@ -89,7 +89,7 @@ type credEndpoint struct {
 // newCredEndpoint builds the endpoint. An empty ip/port falls back to the
 // AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
 // adds the link-local address to the dummy interface (real or fake).
-func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
+func newCredEndpoint(provider aws.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
 	if ip == "" {
 		ip = defaultCredEndpointIP
 	}
@@ -165,7 +165,7 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		AccessKeyID:     creds.AccessKeyID,
 		SecretAccessKey: creds.SecretAccessKey,
 		Token:           creds.SessionToken,
-		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
+		Expiration:      creds.Expires.UTC().Format(time.RFC3339),
 	}); err != nil {
 		slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
 	}
@@ -176,12 +176,12 @@ var errUnknownCredID = fmt.Errorf("unknown credId")
 
 // fetch returns valid credentials for credID, reusing the cache until within the
 // refresh margin and otherwise assuming the task role over the gateway.
-func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, credRole, error) {
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (aws.Credentials, credRole, error) {
 	c.mu.Lock()
 	role, known := c.roles[credID]
 	if !known {
 		c.mu.Unlock()
-		return stsauth.Credentials{}, credRole{}, errUnknownCredID
+		return aws.Credentials{}, credRole{}, errUnknownCredID
 	}
 	if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
 		c.mu.Unlock()
@@ -191,7 +191,7 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden
 
 	creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
 	if err != nil {
-		return stsauth.Credentials{}, role, err
+		return aws.Credentials{}, role, err
 	}
 
 	c.mu.Lock()
@@ -204,7 +204,7 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden
 // roleARN over the gateway — the execution-role path for ECR image pulls. It
 // caches the assumed set and re-assumes within the refresh margin. session scopes
 // the STS session name.
-func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+func (c *credEndpoint) AssumeProvider(roleARN, session string) aws.CredentialsProvider {
 	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
 }
 
@@ -216,39 +216,34 @@ type assumeProvider struct {
 	session string
 
 	mu     sync.Mutex
-	cached stsauth.Credentials
+	cached aws.Credentials
 }
 
-var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+var _ aws.CredentialsProvider = (*assumeProvider)(nil)
 
-func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+func (p *assumeProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	if !credValid(p.cached, credRefreshMargin) {
 		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
 		if err != nil {
-			return credentials.Credentials{}, err
+			return aws.Credentials{}, err
 		}
 		p.cached = creds
 	}
-	return credentials.Credentials{
-		AccessKeyID:     p.cached.AccessKeyID,
-		SecretAccessKey: p.cached.SecretAccessKey,
-		SessionToken:    p.cached.SessionToken,
-		Expiration:      p.cached.Expiration,
-	}, nil
+	return p.cached, nil
 }
 
 // assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
 // against the gateway with the agent's instance credentials.
-func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
 	instCreds, err := c.provider.Retrieve(ctx)
 	if err != nil {
-		return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
+		return aws.Credentials{}, fmt.Errorf("instance credentials: %w", err)
 	}
 	client, err := c.client()
 	if err != nil {
-		return stsauth.Credentials{}, err
+		return aws.Credentials{}, err
 	}
 	return stsauth.AssumeRole(c.region, c.gatewayURL, client,
 		instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
@@ -387,14 +382,14 @@ func (c *credEndpoint) delEndpointAddr() {
 }
 
 // credValid reports whether creds are usable now with margin to spare.
-func credValid(c stsauth.Credentials, margin time.Duration) bool {
+func credValid(c aws.Credentials, margin time.Duration) bool {
 	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
 		return false
 	}
-	if c.Expiration.IsZero() {
+	if c.Expires.IsZero() {
 		return true
 	}
-	return time.Now().Add(margin).Before(c.Expiration)
+	return time.Now().Add(margin).Before(c.Expires)
 }
 
 // sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the

spinifex/agents/ecs/agent/credendpoint_test.go

@@ -9,15 +9,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/mulgadc/spinifex/internal/stsauth"
+	"github.com/aws/aws-sdk-go-v2/aws"
 )
 
 // newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
 // counting calls so cache behaviour is observable. run is nil (no iface plumbing).
-func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
+func newTestCredEndpoint(creds aws.Credentials) (*credEndpoint, *int32) {
 	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 	var calls int32
-	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
 		atomic.AddInt32(&calls, 1)
 		return creds, nil
 	}
@@ -26,8 +26,8 @@ func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
 
 func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
-	c, _ := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
+	c, _ := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expires: exp,
 	})
 	c.Register("cred-1", "task-1", "arn:aws:iam::111122223333:role/task")
 
@@ -54,11 +54,11 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 // The STS session name lands in the assumed-role ARN and in STS logs, so it is
 // built from the public task ID and never from the secret credID.
 func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	c, _ := newTestCredEndpoint(aws.Credentials{})
 	var gotSession string
-	c.assume = func(ctx context.Context, roleARN, session string) (stsauth.Credentials, error) {
+	c.assume = func(ctx context.Context, roleARN, session string) (aws.Credentials, error) {
 		gotSession = session
-		return stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
+		return aws.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
 	}
 	c.Register("secret-cred", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
@@ -72,7 +72,7 @@ func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
 }
 
 func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
-	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, calls := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
 	if rr.Code != http.StatusNotFound {
@@ -84,7 +84,7 @@ func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 }
 
 func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
@@ -94,8 +94,8 @@ func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
 }
 
 func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
-	c, calls := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	c, calls := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Hour),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	for hit := range 3 {
@@ -112,8 +112,8 @@ func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
 
 func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 	// Expiry inside the refresh margin -> every hit re-assumes.
-	c, calls := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
+	c, calls := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Minute),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	for range 2 {
@@ -126,7 +126,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 }
 
 func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	c.Register("cred-1", "task-1", "arn:role")
 	c.Deregister("cred-1")
 	rr := httptest.NewRecorder()
@@ -137,9 +137,9 @@ func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
 }
 
 func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{})
-	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
-		return stsauth.Credentials{}, context.DeadlineExceeded
+	c, _ := newTestCredEndpoint(aws.Credentials{})
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
+		return aws.Credentials{}, context.DeadlineExceeded
 	}
 	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
@@ -150,8 +150,8 @@ func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
 }
 
 func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	c, _ := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Hour),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	c.port = 0 // bind an ephemeral port instead of privileged 80

spinifex/agents/ecs/agent/credinject_test.go

@@ -5,7 +5,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/mulgadc/spinifex/internal/stsauth"
+	"github.com/aws/aws-sdk-go-v2/aws"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 )
@@ -30,7 +30,7 @@ func TestTaskCredID(t *testing.T) {
 // The task ID is public (it is the task ARN suffix), so naming another task's ID
 // at the credential endpoint must not yield that task's role.
 func TestRegisterTaskCreds_TaskIDDoesNotServeCredentials(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
 	a := Agent{cred: c}
 	get := func(id string) int {
 		rr := httptest.NewRecorder()

spinifex/agents/ecs/agent/doc.go

@@ -5,8 +5,7 @@
 // task assignments, runs them through containerd, and reports state back over
 // the gateway.
 //
-// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
-// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
+// Static config (gateway URL, CA, region, cluster, containerd socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
 // (KEY=value); real env vars override it. Host identity (account, instance, AZ)
 // comes from IMDS at boot.
 package agent

spinifex/agents/ecs/agent/ecrresolver.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/ecrauth"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 	"github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
 )
 
@@ -22,7 +22,7 @@ import (
 // pulled, so the agent can still register + heartbeat on a host with absent or
 // wrong ECR config.
 type ecrResolver struct {
-	creds      credentials.CredentialsProvider
+	creds      aws.CredentialsProvider
 	region     string
 	gatewayURL string
 	caPath     string
@@ -35,13 +35,13 @@ var _ runtime.Resolver = (*ecrResolver)(nil)
 
 // newECRResolver builds a resolver with an injected HTTP client (tests). The
 // client is used as-is and never rebuilt from caPath.
-func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
+func newECRResolver(creds aws.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
 	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
 }
 
 // newLazyECRResolver builds a resolver that constructs its gateway HTTP client
 // from caPath on first use (production path).
-func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
+func newLazyECRResolver(creds aws.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
 	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
 }
 

spinifex/agents/ecs/agent/ecrresolver_test.go

@@ -14,16 +14,16 @@ import (
 	"testing"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/aws/aws-sdk-go-v2/aws"
 )
 
-// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
+// stubCreds is a aws.CredentialsProvider returning fixed creds or error.
 type stubCreds struct {
-	c   credentials.Credentials
+	c   aws.Credentials
 	err error
 }
 
-func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
+func (s stubCreds) Retrieve(context.Context) (aws.Credentials, error) {
 	return s.c, s.err
 }
 
@@ -58,9 +58,9 @@ func trusting(srv *httptest.Server) *http.Client {
 
 func TestECRResolver_Authorize(t *testing.T) {
 	srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
+	creds := stubCreds{c: aws.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
+		Expires: time.Now().Add(time.Hour),
 	}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
 
@@ -85,7 +85,7 @@ func TestECRResolver_CredsError(t *testing.T) {
 
 func TestECRResolver_GatewayError(t *testing.T) {
 	srv := gatewayStub(t, "", "", http.StatusInternalServerError)
-	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
 	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
 		t.Fatal("expected gateway error")
@@ -93,7 +93,7 @@ func TestECRResolver_GatewayError(t *testing.T) {
 }
 
 func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
-	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
 	// Construction with a bogus CA path must not fail — only Authorize does.
 	r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
 	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
@@ -141,9 +141,9 @@ func TestIsECRRef(t *testing.T) {
 // registry, even though it lacks a ".dkr.ecr." name.
 func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
 	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
+	creds := stubCreds{c: aws.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
+		Expires: time.Now().Add(time.Hour),
 	}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
 

spinifex/agents/ecs/credentials/imds.go

@@ -1,41 +0,0 @@
-package credentials
-
-import (
-	"context"
-	"net/http"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/mulgadc/spinifex/internal/imdscreds"
-)
-
-// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
-// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
-// which refetches once the real expiry passes. A failed refresh is an error
-// here, not an extended expiry. Safe for concurrent use.
-type IMDSProvider struct {
-	cache *aws.CredentialsCache
-}
-
-var _ CredentialsProvider = (*IMDSProvider)(nil)
-
-// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
-// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
-// default HTTP client.
-func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
-	provider := imdscreds.NewProvider(client, base)
-	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
-}
-
-// Retrieve returns cached credentials when still valid, else fetches fresh ones.
-func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
-	creds, err := p.cache.Retrieve(ctx)
-	if err != nil {
-		return Credentials{}, err
-	}
-	return Credentials{
-		AccessKeyID:     creds.AccessKeyID,
-		SecretAccessKey: creds.SecretAccessKey,
-		SessionToken:    creds.SessionToken,
-		Expiration:      creds.Expires,
-	}, nil
-}

spinifex/agents/ecs/credentials/imds_test.go

@@ -1,164 +0,0 @@
-package credentials
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
-// real /latest prefix the SDK IMDS client always requests.
-func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
-	t.Helper()
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		// Real IMDS echoes the requested TTL back as a response header; the SDK
-		// client parses it from there, not the body.
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits != nil {
-			atomic.AddInt32(hits, 1)
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-func TestRetrieve_FetchesAndCaches(t *testing.T) {
-	var hits int32
-	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
-	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
-	srv := imdsStub(t, &hits, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		"Expiration":      exp.Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	got, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("Retrieve: %v", err)
-	}
-	if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
-		t.Errorf("creds mismatch: %+v", got)
-	}
-
-	// Second call well within validity must not re-hit IMDS.
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatalf("Retrieve 2: %v", err)
-	}
-	if hits != 1 {
-		t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
-	}
-}
-
-func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
-	var hits int32
-	// Already expired: aws.CredentialsCache refetches once the stored
-	// credentials' real expiry has passed, regardless of ExpiryWindow.
-	srv := imdsStub(t, &hits, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatal(err)
-	}
-	if hits != 2 {
-		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
-	}
-}
-
-// imdsStubFailAfterFirst serves creds on the first credential request and 500s
-// on every one after it, so a test can drive a refresh that fails against a
-// credential the cache has already stored.
-func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
-	t.Helper()
-	var hits atomic.Int32
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits.Add(1) > 1 {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-// A refresh that fails must surface as an error. Left to itself the SDK's
-// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
-// key with Expires pushed 5-15 minutes out, so the agent would sign requests
-// with a credential the control plane has already rejected — indefinitely,
-// since every subsequent refresh fails the same way.
-func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
-	srv := imdsStubFailAfterFirst(t, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		// Already expired, so the next Retrieve must go back to IMDS.
-		"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	first, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("first Retrieve: %v", err)
-	}
-
-	got, err := p.Retrieve(context.Background())
-	if err == nil {
-		t.Fatalf("refresh failure was reported as success: %+v", got)
-	}
-	if got.Expiration.After(first.Expiration) {
-		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
-	}
-}
-
-func TestRetrieve_CancelledContext(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	if _, err := p.Retrieve(ctx); err == nil {
-		t.Fatal("expected error from cancelled context")
-	}
-}
-
-func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	if _, err := p.Retrieve(context.Background()); err == nil {
-		t.Fatal("expected error when Code != Success")
-	}
-}

spinifex/agents/ecs/credentials/provider.go

@@ -1,24 +0,0 @@
-// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
-// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
-// with the host's instance-role credentials, fetched from IMDS and cached until
-// near expiry.
-package credentials
-
-import (
-	"context"
-	"time"
-)
-
-// Credentials is a SigV4 credential set with an expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
-// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
-// must be safe for concurrent use.
-type CredentialsProvider interface {
-	Retrieve(ctx context.Context) (Credentials, error)
-}

spinifex/agents/eks/konnectivitycert/cert.go

@@ -80,7 +80,17 @@ func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.I
 	if err != nil {
 		return "", "", err
 	}
-	certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
+	now := time.Now().UTC()
+	certPEM, keyPEM, err := Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              dns,
+		IPAddresses:           ips,
+	}, caCert, caKey)
 	if err != nil {
 		return "", "", err
 	}
@@ -181,8 +191,9 @@ func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
 	return ec, nil
 }
 
-// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
-func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
+// Mint generates an ECDSA-P256 key, issues tmpl for it with a random serial and
+// signs it with parentKey, or self-signs when parent is nil. tmpl is not modified.
+func Mint(tmpl, parent *x509.Certificate, parentKey *ecdsa.PrivateKey) (certPEM, keyPEM []byte, err error) {
 	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("generate key: %w", err)
@@ -191,19 +202,12 @@ func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns
 	if err != nil {
 		return nil, nil, fmt.Errorf("serial: %w", err)
 	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: cn},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		DNSNames:              dns,
-		IPAddresses:           ips,
+	t := *tmpl
+	t.SerialNumber = serial
+	if parent == nil {
+		parent, parentKey = &t, priv
 	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
+	der, err := x509.CreateCertificate(rand.Reader, &t, parent, &priv.PublicKey, parentKey)
 	if err != nil {
 		return nil, nil, fmt.Errorf("create cert: %w", err)
 	}

spinifex/agents/eks/podidentity/certs.go

@@ -1,15 +1,11 @@
 package podidentity
 
 import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"fmt"
-	"math/big"
 	"net"
 	"os"
 	"path/filepath"
@@ -83,17 +79,8 @@ func ensureLeaf(certPath, keyPath, caCertPath, caKeyPath string, spec leafSpec)
 	if leafCurrent(certPath, keyPath, ca, spec) {
 		return nil
 	}
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return fmt.Errorf("serial: %w", err)
-	}
 	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
+	certPEM, keyPEM, err := konnectivitycert.Mint(&x509.Certificate{
 		Subject:               pkix.Name{CommonName: spec.cn},
 		NotBefore:             now.Add(-time.Hour),
 		NotAfter:              now.Add(certValidity),
@@ -101,20 +88,15 @@ func ensureLeaf(certPath, keyPath, caCertPath, caKeyPath string, spec leafSpec)
 		ExtKeyUsage:           []x509.ExtKeyUsage{spec.usage},
 		BasicConstraintsValid: true,
 		IPAddresses:           spec.ips,
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, ca, &priv.PublicKey, caKey)
+	}, ca, caKey)
 	if err != nil {
-		return fmt.Errorf("create certificate: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return fmt.Errorf("marshal key: %w", err)
-	}
-	// Key first: a cert without its key is never published.
-	if err := writeAtomic(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
 		return err
 	}
-	return writeAtomic(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)
+	// Key first: a cert without its key is never published.
+	if err := writeAtomic(keyPath, keyPEM, 0o600); err != nil {
+		return err
+	}
+	return writeAtomic(certPath, certPEM, 0o644)
 }
 
 // leafCurrent reports whether the cached pair loads, matches, was signed by ca,

spinifex/agents/eks/tokenwebhook/cert.go

@@ -1,21 +1,17 @@
 package tokenwebhook
 
 import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/base64"
-	"encoding/pem"
 	"fmt"
-	"math/big"
 	"net"
 	"os"
 	"time"
 
 	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
 )
 
 // serverTLSConfig is the webhook's loopback serving TLS config.
@@ -41,7 +37,18 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 		}
 	}
 
-	certPEM, keyPEM, err := generateSelfSigned()
+	now := time.Now().UTC()
+	certPEM, keyPEM, err := konnectivitycert.Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: "eks-token-webhook"},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
+		DNSNames:              []string{"localhost"},
+	}, nil, nil)
 	if err != nil {
 		return tls.Certificate{}, nil, err
 	}
@@ -58,41 +65,6 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 	return cert, certPEM, nil
 }
 
-func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return nil, nil, fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return nil, nil, fmt.Errorf("serial: %w", err)
-	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: "eks-token-webhook"},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		IsCA:                  true,
-		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
-		DNSNames:              []string{"localhost"},
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("create cert: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("marshal key: %w", err)
-	}
-	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
-	return certPEM, keyPEM, nil
-}
-
 // writeAPIServerKubeconfig writes the kubeconfig kube-apiserver loads via
 // --authentication-token-webhook-config-file. It points the apiserver at the
 // loopback webhook and trusts its self-signed serving cert as the CA bundle.