fix: release only system-tagged addresses in the terminate backstop

An internet-facing ALB's address is now tagged managed-by at allocation, and
ReleaseAddressByInstanceID leaves any untagged record alone with a warning, so
a wrong instance ID can no longer release a customer's own address.

.github/scripts/e2e-select.sh

@@ -1,148 +0,0 @@
-#!/usr/bin/env bash
-# e2e-select.sh push <before> <sha> | suites "<e2e-x ...>"
-# Prints the e2e gate's single_legs, multi_suites and multi_timeout outputs.
-# Exits non-zero on anything it cannot read, which fails the e2e status.
-set -euo pipefail
-
-SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-# shellcheck source=suite-sets.sh
-. "${SCRIPTS_DIR}/suite-sets.sh"
-
-# Provision, install and teardown around the suites; the floor is what the
-# jobs have always had, so a budget can only be raised from it.
-SETUP_ALLOWANCE_MIN=15
-FLOOR_MIN=60
-
-BASE_RUNNER=ci-single
-SERVICE_RUNNER=ci-pseudo
-
-log() { printf '[e2e-select] %s\n' "$*" >&2; }
-die() { log "error: $*"; exit 1; }
-
-# in_list <word> <list>
-in_list() {
-  local w
-  for w in $2; do [ "$w" = "$1" ] && return 0; done
-  return 1
-}
-
-# keep <list> <eligible>: the words of <list> that are in <eligible>, deduplicated.
-keep() {
-  local out="" s
-  for s in $1; do
-    in_list "$s" "$2" || continue
-    in_list "$s" "$out" || out="${out:+$out }$s"
-  done
-  printf '%s' "$out"
-}
-
-# budget <list>: max over the suites' own timeouts plus the setup allowance.
-# A sum would demand hours for a set that shares one job and never takes them.
-budget() {
-  local want="${FLOOR_MIN}" s t
-  for s in $1; do
-    t="$(e2e_suite_timeout "$s")"
-    t=$((${t%m} + SETUP_ALLOWANCE_MIN))
-    [ "$t" -gt "$want" ] && want="$t"
-  done
-  printf '%s' "$want"
-}
-
-prefixed() {
-  local out="" s
-  for s in $1; do out="${out:+$out }e2e-$s"; done
-  printf '%s' "$out"
-}
-
-# leg <name> <runner> <suites>: one matrix entry, or nothing for no suites.
-leg() {
-  [ -n "$3" ] || return 0
-  jq -nc --arg leg "$1" --arg runner "$2" --arg suites "$(prefixed "$3")" \
-    --argjson timeout "$(budget "$3")" \
-    '{leg: $leg, runner: $runner, suites: $suites, timeout: $timeout}'
-}
-
-# emit <base-single> <service-single> <multi>
-emit() {
-  local legs
-  legs=$( { leg base "${BASE_RUNNER}" "$1"; leg service "${SERVICE_RUNNER}" "$2"; } | jq -sc .)
-  log "base leg: ${1:-none}"
-  log "service leg: ${2:-none}"
-  log "multi: ${3:-none}"
-  printf 'single_legs=%s\n' "$legs"
-  printf 'multi_suites=%s\n' "$(prefixed "$3")"
-  printf 'multi_timeout=%s\n' "$(budget "$3")"
-}
-
-# select_for_paths <newline-separated paths> [all]: base set plus matches.
-# With "all", every mapped suite is selected, for a push with no baseline.
-select_for_paths() {
-  local matched="" p s
-  if [ "${2:-}" = all ]; then
-    matched=$(e2e_path_suites_all | tr '\n' ' ')
-  else
-    while IFS= read -r p; do
-      [ -n "$p" ] || continue
-      for s in $(e2e_suites_for_path "$p"); do
-        in_list "$s" "$matched" || { matched="${matched:+$matched }$s"; log "${p} -> ${s}"; }
-      done
-    done <<<"$1"
-  fi
-  # A matched suite runs where it is eligible, preferring one node, so route53
-  # goes to the service leg and lb to the multi job.
-  local extra_single="" extra_multi=""
-  for s in $matched; do
-    if in_list "$s" "${E2E_SUITES_SINGLE}"; then
-      extra_single="${extra_single:+$extra_single }$s"
-    elif in_list "$s" "${E2E_SUITES_MULTI}"; then
-      extra_multi="${extra_multi:+$extra_multi }$s"
-    else
-      die "E2E_PATH_SUITES names '${s}', which no topology can run"
-    fi
-  done
-  local base_single base_multi
-  base_single=$(keep "${E2E_SUITES_PUSH_BASE}" "${E2E_SUITES_SINGLE}")
-  base_multi=$(keep "${E2E_SUITES_PUSH_BASE}" "${E2E_SUITES_MULTI}")
-  # A matched suite already in the base leg runs there, not twice.
-  extra_single=$(for s in $extra_single; do in_list "$s" "$base_single" || printf '%s ' "$s"; done)
-  emit "$base_single" "$(keep "$extra_single" "${E2E_SUITES_SINGLE}")" \
-    "$(keep "$base_multi $extra_multi" "${E2E_SUITES_MULTI}")"
-}
-
-mode="${1:-}"
-case "$mode" in
-  push)
-    [ $# -eq 3 ] || die "usage: $0 push <before> <sha>"
-    before="$2" sha="$3"
-    git cat-file -e "${sha}^{commit}" 2>/dev/null || die "commit ${sha} is not in this checkout"
-    if [ -z "${before//0/}" ]; then
-      log "no previous commit for this push; selecting every mapped suite"
-      select_for_paths "" all
-      exit 0
-    fi
-    if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then
-      git fetch -q --filter=blob:none origin "${before}" 2>/dev/null || true
-    fi
-    if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then
-      log "previous commit ${before} is gone; selecting every mapped suite"
-      select_for_paths "" all
-      exit 0
-    fi
-    # A rename counts as both paths, so moving code out of a service selects it.
-    changed=$(git diff --no-renames --name-only "${before}" "${sha}") || die "git diff ${before} ${sha} failed"
-    log "$(printf '%s\n' "$changed" | grep -c . || true) paths changed ${before:0:9}..${sha:0:9}"
-    select_for_paths "$changed"
-    ;;
-  suites)
-    [ $# -eq 2 ] || die "usage: $0 suites \"<e2e-x ...>\""
-    if [ -z "${2// /}" ]; then
-      select_for_paths ""
-      exit 0
-    fi
-    requested=$(printf '%s' "$2" | sed 's/e2e-//g')
-    emit "$(keep "$requested" "${E2E_SUITES_SINGLE}")" "" "$(keep "$requested" "${E2E_SUITES_MULTI}")"
-    ;;
-  *)
-    die "usage: $0 push <before> <sha> | suites \"<list>\""
-    ;;
-esac

.github/scripts/go-build-cache.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-# Trims GOCACHE to the entries one CI run used. Go refreshes an entry's mtime
-# on use only when it is over an hour old, so "mark" backdates every restored
-# entry first and "trim" drops whatever this run neither read nor wrote.
-set -euo pipefail
-
-cache="$(go env GOCACHE)"
-mark="${RUNNER_TEMP:?}/go-build-cache.mark"
-
-# stale prints each entry this run did not touch and no touched action entry
-# names. Go skips rewriting an output it already holds, so a fresh action
-# entry can point at a backdated output.
-stale() {
-  local keep="${RUNNER_TEMP}/go-build-cache.keep"
-  find "$cache" -mindepth 2 -maxdepth 2 -name '*-a' -newer "$mark" -exec cat {} + |
-    awk '{ print "/" $3 "-d" }' >"$keep"
-  find "$cache" -mindepth 2 -maxdepth 2 ! -newer "$mark" | { grep -vFf "$keep" || true; }
-}
-
-case "${1:-}" in
-mark)
-  mkdir -p "$cache"
-  find "$cache" -mindepth 2 -maxdepth 2 -exec touch -m -d '2 hours ago' {} +
-  touch "$mark"
-  ;;
-stale | trim)
-  if [[ ! -f "$mark" ]]; then
-    echo "go-build-cache: no mark, leaving the cache untrimmed" >&2
-    exit 0
-  fi
-  if [[ "$1" == stale ]]; then
-    stale
-    exit 0
-  fi
-  before=$(du -sm "$cache" | cut -f1)
-  # An entry is a file, or a directory holding one cached executable whose
-  # mtime Go refreshes in place of the binary's, so it goes as a unit.
-  stale | xargs -r rm -rf --
-  echo "trimmed ${before}MB -> $(du -sm "$cache" | cut -f1)MB"
-  ;;
-*)
-  echo "usage: $0 mark|stale|trim" >&2
-  exit 2
-  ;;
-esac

.github/scripts/mulga-design-check.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY and
+# never fails the job: the check is a prompt, not a gate.
+set -euo pipefail
+shopt -s nocasematch
+
+section_re='^#{1,6}[[:space:]]*Mulga design[[:space:]]*$'
+line_re='^[[:space:]]*(Updated|Not needed)[[:space:]]*:[[:space:]]*(.*)$'
+placeholder_re='<[^<>]*>'
+
+filled() {
+  local body=$1 line value kept inside=0 pre rest
+  body=${body//$'\r'/}
+  while [[ $body == *'<!--'*'-->'* ]]; do
+    pre=${body%%'<!--'*}
+    rest=${body#*'<!--'}
+    body=$pre${rest#*'-->'}
+  done
+  while IFS= read -r line; do
+    if [[ $line =~ $section_re ]]; then
+      inside=1
+    elif ((inside)) && [[ ${line#"${line%%[![:space:]]*}"} == '#'* ]]; then
+      return 1
+    elif ((inside)) && [[ $line =~ $line_re ]]; then
+      value=${BASH_REMATCH[2]} kept=''
+      while [[ $value =~ $placeholder_re ]]; do
+        kept+=${value%%"${BASH_REMATCH[0]}"*}
+        value=${value#*"${BASH_REMATCH[0]}"}
+      done
+      [[ $kept$value == *[![:space:]]* ]] && return 0
+    fi
+  done <<<"$body"
+  return 1
+}
+
+if filled "${PR_BODY:-}"; then
+  echo "The Mulga design line is filled in."
+else
+  echo '::warning title=Mulga design::Add a "## Mulga design" section with one line: "Updated: <mulga PR>" or "Not needed: <reason>".'
+fi

.github/scripts/suite-sets.sh

@@ -66,67 +66,6 @@ E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancere
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
 E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 
-# The push gate always runs this set. A push adds to it the suites whose code
-# it changed, by E2E_PATH_SUITES below; a dispatch names its own set instead.
-E2E_SUITES_PUSH_BASE="single multinode cert iam partialblock"
-
-# "suite prefix" pairs: a push changing a path under prefix adds suite. The
-# recovery and fault suites stay nightly, and shared code adds nothing because
-# the base set covers it. Image directories carry each guest's own scripts.
-E2E_PATH_SUITES="
-rds cmd/rds-agent/
-rds internal/rdsgw/
-rds spinifex/agents/rds/
-rds spinifex/domains/rds/
-rds tests/e2e/rds/
-rds scripts/images/rds-mariadb/
-rds scripts/images/rds-postgres/
-ecs cmd/ecs-agent/
-ecs contracts/ecs/
-ecs internal/ecsgw/
-ecs spinifex/agents/ecs/
-ecs spinifex/domains/ecs/
-ecs tests/e2e/ecs/
-ecs scripts/images/ecs-agent/
-eks cmd/eks-
-eks contracts/eks/
-eks internal/eksgw/
-eks spinifex/agents/eks/
-eks spinifex/domains/eks/
-eks tests/e2e/eks/
-eks scripts/images/eks-node/
-rds scripts/images/common/
-ecs scripts/images/common/
-eks scripts/images/common/
-rds scripts/build-system-image.sh
-ecs scripts/build-system-image.sh
-eks scripts/build-system-image.sh
-route53 spinifex/domains/route53/
-route53 tests/e2e/route53/
-lb cmd/lb-agent/
-lb spinifex/agents/elbv2/
-lb spinifex/domains/elbv2/
-lb tests/e2e/lb/
-"
-
-# Every suite E2E_PATH_SUITES can name, once each, in table order.
-#   e2e_path_suites_all
-e2e_path_suites_all() {
-  printf '%s\n' "${E2E_PATH_SUITES}" | awk 'NF == 2 && !seen[$1]++ { print $1 }'
-}
-
-# The suites a change to <path> triggers, one per line.
-#   e2e_suites_for_path <path>
-e2e_suites_for_path() {
-  local path="$1" suite prefix
-  while read -r suite prefix; do
-    [ -n "${prefix}" ] || continue
-    case "${path}" in
-      "${prefix}"*) printf '%s\n' "${suite}" ;;
-    esac
-  done <<<"${E2E_PATH_SUITES}"
-}
-
 # What every e2e-nightly permutation cell must prove, deliberately narrower
 # than the two sets above: those cells have a 35-minute budget and exist to show
 # that every install / network / host-OS combination boots and serves. The

.github/workflows/actions.lock

@@ -50,6 +50,8 @@ workflows:
     '.github/workflows/image-scripts.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-go@v7.0.0'
+    '.github/workflows/mulga-design.yml':
+        - 'actions/checkout@v7.0.1'
     '.github/workflows/perf-3node.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/create-github-app-token@v3.3.0'

.github/workflows/dev-release.yml

@@ -30,10 +30,6 @@ concurrency:
 jobs:
   prerelease:
     runs-on: ubuntu-26.04
-    env:
-      # Dev prereleases, newest published first: GitHub lists releases by tag name
-      # as text, so list order is not build order. The install Worker uses this rule.
-      DEV_RELEASES_JQ: '[.[] | select(.draft == false and .prerelease and .published_at != null and (.tag_name | test("-dev(\\.|$)")))] | sort_by(.published_at) | reverse'
     steps:
       # Full history and tags, because the tag name is git describe and a shallow
       # clone has no tag to describe against.
@@ -168,12 +164,12 @@ jobs:
               exit 1
           fi
           echo "latest is still $latest"
-          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq "$DEV_RELEASES_JQ | .[0].tag_name")"
+          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
           [ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
 
-      # Kept rather than unbounded: the Worker reads one page of releases to resolve
-      # the channel, so enough prereleases would eventually push the newest off it.
-      # Five leaves a few recent builds to roll back to.
+      # Kept rather than unbounded: the Worker reads the first page of releases to
+      # resolve the channel, so enough prereleases would eventually push the real
+      # ones off it. Five leaves a few recent builds to roll back to.
       - name: Prune old dev prereleases
         if: steps.version.outputs.exists != 'true'
         env:
@@ -181,7 +177,7 @@ jobs:
         run: |
           set -euo pipefail
           gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
-              --jq "$DEV_RELEASES_JQ | .[5:] | .[].tag_name" \
+              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \
               > stale.txt
           while read -r tag; do
               [ -n "$tag" ] || continue

.github/workflows/e2e-nightly.yml

@@ -6,32 +6,11 @@ name: E2E Nightly
 # `main` and `dev` as separate nightlies, and it would always execute `main`'s
 # copy of this file — leaving a fix to the workflow itself untestable until it
 # had already shipped. A dispatch takes both the file and the source from its ref.
-#
-# Cells (matrix cells are defined in .github/e2e-cells.json):
-#
-#   Single node                       Three nodes
-#    1  Debian 13, DHCP                 8  Debian 13, DHCP
-#    2  Ubuntu 26.04, static            9  Ubuntu 26.04, static
-#    3  Ubuntu 26.04, DHCP             10  Ubuntu 26.04, DHCP
-#   17  tofu examples                  29  storage fault injection
-#   18  reboot                         30  routed NAT
-#   19  routed NAT                     31  instance recovery
-#   25  eks                            32  load balancer recovery
-#   26  ecs                            33  KV bucket quorum under node loss
-#   27  rds                            37  Oracle Linux 10, DHCP
-#   34  Oracle Linux 10, DHCP          38  Oracle Linux 10, routed NAT
-#   35  Oracle Linux 10, reboot
-#   36  Oracle Linux 10, routed NAT
-#
-#   Standalone jobs
-#   20  bare metal (PXE)               23  predastore throughput
-#   21  ISO, single node               24  predastore fault injection
-#   22  ISO, three nodes               28  guest disk performance (bare metal)
 on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers, no spaces, e.g. 1,25,27 (blank = all). The cell list is at the top of .github/workflows/e2e-nightly.yml."
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 34-36 Oracle Linux single node (basic, reboot, routed NAT), 37-38 Oracle Linux on three nodes (multi-node subset with IPsec, routed NAT), 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
         required: false
 
 permissions:

.github/workflows/e2e.yml

@@ -12,15 +12,13 @@ on:
       - go.sum
       - Makefile
       - tests/e2e/**
-      - scripts/images/**
-      - scripts/build-system-image.sh
       - .github/workflows/e2e.yml
   workflow_dispatch:
     inputs:
       suites:
         description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam e2e-rds')"
         required: false
-        default: "e2e-single e2e-multinode e2e-cert e2e-iam"
+        default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
       test_run:
         description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
         required: false
@@ -33,75 +31,86 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # A push runs the base set plus the suites its changed paths map to in
-  # suite-sets.sh; a dispatch runs its suites input, or the base set if empty.
-  # Budgets come from the suites' own timeouts there.
+  # The inner timeout must fire before the outer one. `go test` reports where it
+  # stopped and the suite's own failure; a job cancelled on timeout reports only
+  # "The operation was canceled" and takes the artifacts with it.
+  #
+  # A job budget cannot be a literal here because the suite set is a dispatch
+  # input: storagefault asks for 90m and diskperf for 180m, both eligible on one
+  # node, so any fixed number is either wrong for them or wasteful for everyone
+  # else. Resolving it from suite-sets.sh keeps the one source of truth and
+  # leaves the default push set at exactly the 60m it has always had.
   plan:
-    name: Select Suites
+    name: Resolve Job Budgets
     runs-on: ubuntu-26.04
     timeout-minutes: 5
     outputs:
-      single_legs: ${{ steps.select.outputs.single_legs }}
-      multi_suites: ${{ steps.select.outputs.multi_suites }}
-      multi_timeout: ${{ steps.select.outputs.multi_timeout }}
+      single_timeout: ${{ steps.budgets.outputs.single_timeout }}
+      multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
     steps:
-      # Full history without file contents: the diff needs both commits' trees.
       - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
-          fetch-depth: 0
-          filter: blob:none
           sparse-checkout: .github/scripts
 
-      - name: Select suites
-        id: select
-        working-directory: spinifex
+      - name: Resolve Budgets
+        id: budgets
         env:
-          EVENT: ${{ github.event_name }}
-          BEFORE: ${{ github.event.before }}
-          IN_SUITES: ${{ inputs.suites }}
+          # Provision, install and teardown around the suites, plus the headroom
+          # a multi-suite job needs for whatever ran before the one that hangs.
+          SETUP_ALLOWANCE_MIN: 15
+          # What the job has always had. Kept as a floor so resolving the budget
+          # can only ever raise it, never quietly shorten an existing run.
+          FLOOR_MIN: 60
         run: |
-          set -euo pipefail
-          SELECT=.github/scripts/e2e-select.sh
-          if [ "${EVENT}" = push ]; then
-            bash "${SELECT}" push "${BEFORE}" "${GITHUB_SHA}" | tee -a "$GITHUB_OUTPUT"
-          else
-            bash "${SELECT}" suites "${IN_SUITES}" | tee -a "$GITHUB_OUTPUT"
-          fi
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          # Only max() is a sound bound: the suites share one job, so a sum would
+          # demand hours for a set that has never taken them. Max is what makes
+          # the hanging suite panic on its own timeout first, which is the point.
+          budget() {
+            local want="${FLOOR_MIN}" s t
+            for s in $(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "$1"); do
+              t="$(e2e_suite_timeout "${s}")"
+              t=$(( ${t%m} + SETUP_ALLOWANCE_MIN ))
+              [ "${t}" -gt "${want}" ] && want="${t}"
+            done
+            echo "${want}"
+          }
+          single=$(budget "${E2E_SUITES_SINGLE_RE}")
+          multi=$(budget "${E2E_SUITES_MULTI_RE}")
+          echo "single_timeout=${single}" >> "$GITHUB_OUTPUT"
+          echo "multi_timeout=${multi}" >> "$GITHUB_OUTPUT"
+          echo "suites='${SUITES}' -> single ${single}m, multi ${multi}m"
 
   e2e_single:
-    # The base leg keeps the job and artifact names it always had: mulga's
-    # registry intake finds a run's results by them.
-    name: ${{ matrix.leg == 'base' && 'Single-Node E2E' || format('Single-Node E2E ({0})', matrix.leg) }}
-    # `base` runs on ci-single; `service`, on ci-pseudo, carries the matched
-    # single-node suites so a long one stays off the base leg's critical path.
-    if: needs.plan.outputs.single_legs != '[]'
+    name: Single-Node E2E
+    # The dispatch check gates on exactly the suites this job's Run step
+    # allow-lists, i.e. E2E_SUITES_SINGLE in .github/scripts/suite-sets.sh —
+    # an `if:` expression cannot source that file, so this one list is
+    # hand-mirrored and must be updated with it. Gating on a suite the job
+    # won't run — lb, which only executes in the multinode job — would
+    # provision a tofu VM and pay the Spinifex install only to run nothing and
+    # tear down. Push triggers keep running via the null-check on
+    # inputs.suites.
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     needs: plan
-    runs-on: [self-hosted, "${{ matrix.runner }}"]
-    timeout-minutes: ${{ matrix.timeout }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include: ${{ fromJSON(needs.plan.outputs.single_legs) }}
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: ${{ fromJSON(needs.plan.outputs.single_timeout) }}
     env:
-      SUITES: ${{ matrix.suites }}
-      # Both legs can run on banksia at once, so every host path carries the leg.
-      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}-${{ matrix.leg }}.log
-      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}-${{ matrix.leg }}
-      # The base leg keeps the cell name it always had, so its history in
-      # Elastic and telemetry continues; any other leg appends its name.
-      E2E_CELL: ${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
+      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
       # Job-level env wins over the workflow-level default. This job and
       # e2e_multi share one run id and both name their hosts node1/node2/node3,
       # so without ci.cell their telemetry collapses into one bucket. This job
       # is where "Bootstrap via Binary Install" bakes SPINIFEX_OTEL_ATTRS into
       # the VM's telemetry.env, so the label has to be set here.
-      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -171,7 +180,7 @@ jobs:
       - name: Resolve CI job id
         uses: $/.github/actions/otel-job-id
         with:
-          job-name: ${{ matrix.leg == 'base' && 'Single-Node E2E' || format('Single-Node E2E ({0})', matrix.leg) }}
+          job-name: Single-Node E2E
 
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
@@ -215,12 +224,6 @@ jobs:
           echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
           echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
 
-      # build-system-image.sh reuses a raw image under ten minutes old from
-      # its build directory, and every runner on banksia shares the default
-      # one in /tmp, so a job could test another commit's guest agent.
-      - name: Use a per-job image build directory
-        run: echo "SYSTEM_IMAGE_BUILD_DIR=${RUNNER_TEMP}/system-image-build" >> "$GITHUB_ENV"
-
       # The eks suite's CreateCluster boots a control-plane VM from the
       # eks-node AMI, so that AMI must exist in the VM's image store before the
       # suite runs. build-system-image.sh customizes the image with libguestfs
@@ -244,7 +247,7 @@ jobs:
           group_open "make build-eks-node-image"
           make build-eks-node-image
           group_close
-          ok "raw: $(du -h "${SYSTEM_IMAGE_BUILD_DIR}/eks-node-image-build/eks-node-alpine.raw" | cut -f1)"
+          ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
 
       - name: Import eks-node AMI on VM
         if: contains(env.SUITES, 'e2e-eks')
@@ -256,16 +259,13 @@ jobs:
           SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
           VM=tf-user@${{ steps.vm.outputs.wan_ip }}
           banner "import eks-node AMI to VM image store"
-          # The VM's /tmp is a tmpfs and scp writes each raw at its full size,
-          # so every import removes its file before the next image arrives.
           scp "${SSH_OPTS[@]}" \
-            "${SYSTEM_IMAGE_BUILD_DIR}/eks-node-image-build/eks-node-alpine.raw" \
+            /tmp/eks-node-image-build/eks-node-alpine.raw \
             "$VM":/tmp/eks-node-alpine.raw
           ssh "${SSH_OPTS[@]}" "$VM" \
             "spx admin images import --file /tmp/eks-node-alpine.raw \
                --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}; \
-               rc=\$?; rm -f /tmp/eks-node-alpine.raw; exit \$rc" >/dev/null 2>&1
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
           ok "eks-node AMI imported"
 
       # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
@@ -299,40 +299,6 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
-      # The ecs suite boots its container instance from the ecs-node AMI, found
-      # by its managed-by tag, so the tag matters. The build directory is named
-      # for the manifest's IMAGE_NAME (ecs-agent), not for the make target.
-      - name: Build ecs-node AMI (libguestfs, runner)
-        if: contains(env.SUITES, 'e2e-ecs')
-        working-directory: spinifex
-        run: |
-          set -euo pipefail
-          # direct backend = no libvirt. Set here rather than in the script:
-          # it has to be in the environment the build itself runs in.
-          export LIBGUESTFS_BACKEND=direct
-          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
-          make build-ecs-node-image
-
-      - name: Import ecs-node AMI on VM
-        if: contains(env.SUITES, 'e2e-ecs')
-        run: |
-          set -euo pipefail
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
-            spinifex/scripts/images/ecs-agent/manifest.conf)"
-          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
-          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          banner "import ecs-node AMI to VM image store"
-          scp "${SSH_OPTS[@]}" \
-            "${SYSTEM_IMAGE_BUILD_DIR}/ecs-agent-image-build/ecs-agent-alpine.raw" \
-            "$VM":/tmp/ecs-agent-alpine.raw
-          ssh "${SSH_OPTS[@]}" "$VM" \
-            "spx admin images import --file /tmp/ecs-agent-alpine.raw \
-               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}; \
-               rc=\$?; rm -f /tmp/ecs-agent-alpine.raw; exit \$rc" >/dev/null 2>&1
-          ok "ecs-node AMI imported"
-
       # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
       # must be in the VM's image store before the suite runs — same split as the
       # eks-node pair above: build on the runner (needs source + network), import
@@ -355,7 +321,7 @@ jobs:
             group_open "make build-${IMAGE}-image"
             make "build-${IMAGE}-image"
             group_close
-            ok "raw: $(du -h "${SYSTEM_IMAGE_BUILD_DIR}/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
           done
 
       - name: Import RDS AMIs on VM
@@ -377,13 +343,12 @@ jobs:
             for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
             banner "import ${IMAGE} AMI to VM image store"
             scp "${SSH_OPTS[@]}" \
-              "${SYSTEM_IMAGE_BUILD_DIR}/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
               "$VM":"/tmp/${IMAGE}-alpine.raw"
             ssh "${SSH_OPTS[@]}" "$VM" \
               "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
                  --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}; \
-                 rc=\$?; rm -f /tmp/${IMAGE}-alpine.raw; exit \$rc" >/dev/null 2>&1
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
             ok "${IMAGE} AMI imported"
           done
 
@@ -561,7 +526,7 @@ jobs:
         uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: ${{ env.E2E_CELL }}
+          cell: single
 
       - name: Analyze e2e failures
         if: always()
@@ -581,7 +546,7 @@ jobs:
       - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
-          name: ${{ matrix.leg == 'base' && format('e2e-artifacts-{0}', github.run_id) || format('e2e-artifacts-{0}-{1}', github.run_id, matrix.leg) }}
+          name: e2e-artifacts-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14
@@ -624,7 +589,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v7.0.2
         with:
-          name: ${{ matrix.leg == 'base' && format('tofu-log-single-{0}', github.run_id) || format('tofu-log-single-{0}-{1}', github.run_id, matrix.leg) }}
+          name: tofu-log-single-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
           if-no-files-found: ignore
           retention-days: 14
@@ -640,13 +605,15 @@ jobs:
 
   e2e_multi:
     name: Multi-Node E2E
-    # Runs when the plan job selected any multi-node suite, and only then.
-    if: needs.plan.outputs.multi_suites != ''
+    # The dispatch check gates on exactly the suites this job's Run step
+    # allow-lists (multinode|lb|cert) so lb — which only runs here — still
+    # provisions the 3-node env when requested on its own, and the env stays
+    # unprovisioned when none of them are asked for.
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
     needs: plan
     runs-on: [self-hosted, ci-multi]
     timeout-minutes: ${{ fromJSON(needs.plan.outputs.multi_timeout) }}
     env:
-      SUITES: ${{ needs.plan.outputs.multi_suites }}
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
       # See the matching note on e2e_single: ci.cell is what separates this
@@ -1103,10 +1070,7 @@ jobs:
   e2e_status:
     name: Report E2E Status
     if: always()
-    # plan is a dependency because a failed plan skips both e2e jobs, and a
-    # skipped job counts as passed below: without it a broken selection would
-    # report green having run nothing.
-    needs: [plan, e2e_single, e2e_multi]
+    needs: [e2e_single, e2e_multi]
     runs-on: ubuntu-26.04
     timeout-minutes: 5
     steps:
@@ -1115,16 +1079,15 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           REPORT_SHA: ${{ github.sha }}
         run: |
-          plan_ok=true; single_ok=true; multi_ok=true
-          [ "${{ needs.plan.result }}" = success ] || plan_ok=false
+          single_ok=true; multi_ok=true
           case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
           case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
-          if $plan_ok && $single_ok && $multi_ok; then
+          if $single_ok && $multi_ok; then
             STATE="success"
             DESC="E2E suites passed"
           else
             STATE="failure"
-            DESC="E2E failed: plan=${{ needs.plan.result }} single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
           fi
           gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
             -f state="$STATE" \
@@ -1141,7 +1104,7 @@ jobs:
   # status-report check.
   e2e_es_summary:
     name: ES CI Run Summary
-    needs: [plan, e2e_single, e2e_multi]
+    needs: [e2e_single, e2e_multi]
     if: always()
     runs-on: [self-hosted, ci-single]
     timeout-minutes: 5
@@ -1153,11 +1116,6 @@ jobs:
       # this host, so no credentials are needed for the POST.
       MULGA_ES_SINK: ${{ vars.MULGA_ES_SINK || '192.168.1.13:9200' }}
     steps:
-      # The workspace outlives the run on a self-hosted runner, and the status
-      # files are collected recursively, so an earlier run's would count here.
-      - name: Clear previous summary downloads
-        run: rm -rf "${GITHUB_WORKSPACE}/es-summary-single" "${GITHUB_WORKSPACE}/es-summary-multi"
-
       # Either leg may not have run (e.g. a workflow_dispatch that only
       # requested single-node suites), so a missing artifact is tolerated —
       # the aggregation script below treats an absent suite-status.txt as
@@ -1166,7 +1124,7 @@ jobs:
         uses: actions/download-artifact@v8.0.2
         continue-on-error: true
         with:
-          pattern: e2e-artifacts-${{ github.run_id }}*
+          name: e2e-artifacts-${{ github.run_id }}
           path: ${{ github.workspace }}/es-summary-single
 
       - name: Download multi-node artifacts
@@ -1186,7 +1144,6 @@ jobs:
           BRANCH: ${{ github.head_ref || github.ref_name }}
           SHA: ${{ github.sha }}
           SERVER_URL: ${{ github.server_url }}
-          PLAN_RESULT: ${{ needs.plan.result }}
           SINGLE_RESULT: ${{ needs.e2e_single.result }}
           MULTI_RESULT: ${{ needs.e2e_multi.result }}
           SINGLE_DIR: ${{ github.workspace }}/es-summary-single
@@ -1194,8 +1151,7 @@ jobs:
         run: |
           set -euo pipefail
 
-          # One subdirectory per single-node leg's artifact.
-          SINGLE_STATUS=$(find "${SINGLE_DIR}" -name suite-status.txt 2>/dev/null | sort || true)
+          SINGLE_STATUS="${SINGLE_DIR}/suite-status.txt"
           MULTI_STATUS="${MULTI_DIR}/suite-status.txt"
 
           # Collect "<suite>: FAILED" lines from both legs' suite-status.txt
@@ -1204,9 +1160,7 @@ jobs:
           # rather than a bash array so this stays correct under `set -u`
           # even when nothing failed.
           FAILED_SUITES=""
-          # SINGLE_STATUS is a newline-separated list, so it is split here.
-          # shellcheck disable=SC2086
-          for f in ${SINGLE_STATUS} "$MULTI_STATUS"; do
+          for f in "$SINGLE_STATUS" "$MULTI_STATUS"; do
             [ -f "$f" ] || continue
             while IFS= read -r line; do
               case "$line" in
@@ -1226,7 +1180,7 @@ jobs:
           case "$MULTI_RESULT" in success | skipped) ;; *) multi_ok=false ;; esac
 
           STATE=success
-          if [ "$PLAN_RESULT" != success ] || [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
+          if [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
             STATE=failure
           fi
 

.github/workflows/mulga-design.yml

@@ -0,0 +1,32 @@
+name: Mulga design
+
+# Reminds the author to say whether the Mulga design records need to change.
+# It reads only the pull request body, uses no secrets, and warns without ever
+# failing: the check is a prompt, not a gate. Fork PRs are skipped because their
+# authors cannot see the design records.
+
+on:
+  pull_request:
+    branches:
+      - dev
+    types: [opened, edited, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  mulga_design:
+    name: Mulga design line
+    if: ${{ !endsWith(github.actor, '[bot]') && github.event.pull_request.head.repo.full_name == github.repository }}
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Check the Mulga design line
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        working-directory: .github/scripts
+        run: bash mulga-design-check.sh

.github/workflows/spinifex.yml

@@ -20,12 +20,6 @@ on:
 permissions:
   contents: read
 
-# A newer push to the same pull request supersedes its run. Pushes and dispatches
-# get a group of their own, so every commit on dev and main keeps its result.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
-  cancel-in-progress: true
-
 jobs:
   build_and_test:
     name: Build and Test
@@ -105,13 +99,6 @@ jobs:
           key: gobuild-${{ github.job }}-${{ github.run_id }}
           restore-keys: gobuild-${{ github.job }}-
 
-      # Go refreshes an entry's mtime on use only when it is over an hour old, so
-      # restored entries are backdated past that before any Go command runs and
-      # the trim below can tell this run's working set from the last run's.
-      - name: Mark Go build cache
-        if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
-        run: spinifex/.github/scripts/go-build-cache.sh mark
-
       - name: Setup Go Workspace
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
@@ -144,12 +131,19 @@ jobs:
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
-      # Keeps what this run read or wrote and drops the rest, so the saved entry
-      # is one run's working set however often dev is pushed. A red run may have
-      # stopped early and used only part of the set, so it saves untrimmed.
+      # Compiled output stays valid whatever the tests said, and skipping the
+      # save on a red run would leave the next one rebuilding it, so this is
+      # gated on the branch rather than on success.
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
       - name: Trim Go build cache
-        if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: spinifex/.github/scripts/go-build-cache.sh trim
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
@@ -160,7 +154,6 @@ jobs:
 
   race_detection:
     name: Race Detection
-    if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-26.04
 
     steps:
@@ -236,25 +229,22 @@ jobs:
           key: gobuild-${{ github.job }}-${{ github.run_id }}
           restore-keys: gobuild-${{ github.job }}-
 
-      # Go refreshes an entry's mtime on use only when it is over an hour old, so
-      # restored entries are backdated past that before any Go command runs and
-      # the trim below can tell this run's working set from the last run's.
-      - name: Mark Go build cache
-        if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
-        run: spinifex/.github/scripts/go-build-cache.sh mark
-
       - name: Setup Go Workspace
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Tests with Race Detector
         run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
-      # Keeps what this run read or wrote and drops the rest, so the saved entry
-      # is one run's working set however often dev is pushed. A red run may have
-      # stopped early and used only part of the set, so it saves untrimmed.
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
       - name: Trim Go build cache
-        if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: spinifex/.github/scripts/go-build-cache.sh trim
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
@@ -341,13 +331,6 @@ jobs:
           key: gobuild-${{ github.job }}-${{ github.run_id }}
           restore-keys: gobuild-${{ github.job }}-
 
-      # Go refreshes an entry's mtime on use only when it is over an hour old, so
-      # restored entries are backdated past that before any Go command runs and
-      # the trim below can tell this run's working set from the last run's.
-      - name: Mark Go build cache
-        if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
-        run: spinifex/.github/scripts/go-build-cache.sh mark
-
       - name: Setup Go Workspace
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
@@ -364,16 +347,16 @@ jobs:
         if: ${{ !cancelled() }}
         run: make -C spinifex test-integration
 
-      - name: Run Gated Suites
-        if: ${{ !cancelled() }}
-        run: make -C spinifex test-gated GOTESTFLAGS=-count=1
-
-      # Keeps what this run read or wrote and drops the rest, so the saved entry
-      # is one run's working set however often dev is pushed. A red run may have
-      # stopped early and used only part of the set, so it saves untrimmed.
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
       - name: Trim Go build cache
-        if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: spinifex/.github/scripts/go-build-cache.sh trim
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}

.github/workflows/ui.yml

@@ -30,12 +30,6 @@ on:
 permissions:
   contents: read
 
-# A newer push to the same pull request supersedes its run. Pushes and dispatches
-# get a group of their own, so every commit on dev and main keeps its result.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
-  cancel-in-progress: true
-
 jobs:
   ui_build_and_test:
     name: UI Build and Test

.gitignore

@@ -27,7 +27,6 @@ build/microvm/sqs-worker/.stage.*/
 build/microvm/lambda-runtime/vmlinuz
 build/microvm/lambda-runtime/initramfs.cpio.gz
 build/microvm/lambda-runtime/.stage.*/
-build/microvm/lambda-testlink-runtime/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -82,36 +82,6 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
-# Controlled-test only: builds the tagged daemon and its hard-kill initializer
-# together, pinning the initializer's digest into that daemon build. The daemon
-# keeps its symbols so the pin can be read back from the binary.
-# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
-# build-lambda-testlink-image prints; it is read raw and validated before use.
-LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
-LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
-LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
-LAMBDA_TESTLINK_GW := github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw
-LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
-build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
-build-lambda-testlink:
-	@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
-	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
-	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
-	@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
-	@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
-	@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
-	cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
-	scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
-	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
-	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
-	echo "initializer sha256 $$digest"; \
-	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST -X $(LAMBDA_TESTLINK_GW).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
-	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ ./spinifex/runtime/roles/awsgw/ && \
-	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
-	@mkdir -p $(LAMBDA_TESTLINK_DIR)
-	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
-	@rmdir $(LAMBDA_TESTLINK_DIR).tmp
-
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
 	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -227,24 +197,6 @@ bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
 
 build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
-	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
-
-# Controlled-test only: a separately built guest whose agent has the
-# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
-# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
-LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
-build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
-	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
-	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
-	$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
-	@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
-	GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
-	./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
-	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
-	install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
-	@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
-	[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
-	echo "LAMBDA_IMAGE_DIGEST=$$digest"
 
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -300,18 +252,13 @@ test-build-scripts:
 	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
 		$(_RUNSH); \
 	done
-	@$(_HDR) "\n....Running scripts/terraform/latitude-spx/*_test.sh...."
-	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find scripts/terraform/latitude-spx -maxdepth 1 -name '*_test.sh' | sort); do \
-		$(_RUNSH); \
-	done
 	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
 	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
 		$(_RUNSH); \
 	done
 	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
 		.github/scripts/wait-for-memory.sh \
-		.github/scripts/go-build-cache.sh \
-		.github/scripts/e2e-select.sh
+		.github/scripts/mulga-design-check.sh
 	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
@@ -346,16 +293,6 @@ test-integration:
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
 	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
 
-# Suites the unit tiers skip: the three-node SQS R3 scenarios behind
-# SPINIFEX_SQS_R3 and the lambdatestlink-tagged tests. CI runs these on every PR.
-test-gated:
-	@echo -e "\n....Running env- and tag-gated suites...."
-	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -timeout 180s -run '^TestR3$$' ./spinifex/domains/sqs/message/jetstream/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/guestconn/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
-
 generate-aws-model-coverage:
 	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR) -json $(AWS_MODEL_OPERATION_COVERAGE_DIR)/coverage.json
 
@@ -391,7 +328,8 @@ test-cover:
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -602,7 +540,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration test-gated generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

architecture/layering/allowlist_test.go

@@ -17,6 +17,7 @@ var allowlist = []allowance{
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
+	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 
 	{From: "agents/ecs/agent", To: "domains/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},

build/microvm/init.sh

@@ -32,8 +32,6 @@ load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
 load_mod failover || true
 load_mod net_failover || true
 load_mod virtio_net || true
-# lb-agent probes each data NIC's gateway on an AF_PACKET socket.
-load_mod af_packet || true
 
 NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 i=0

build/microvm/lambda-runtime/init.sh

@@ -16,8 +16,8 @@ load_mod() {
     done
     return 1
 }
-# lambda-agent reads its identity from fw_cfg, or the configuration disk, and
-# seals it before anything else runs. virtio_net needs net_failover and failover, and the
+# lambda-agent reads its identity from fw_cfg and unloads qemu_fw_cfg before
+# anything else runs. virtio_net needs net_failover and failover, and the
 # insmod fallback resolves no dependencies, so load the chain bottom-up.
 load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
 load_mod failover || true
@@ -26,30 +26,8 @@ load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
 
 ip link set lo up
 
-# Without fw_cfg data the executor attaches the same entries as a tar archive
-# on the second disk; lambda-agent reads them here and erases the disk before
-# the function starts. qemu_fw_cfg loads without a device, so test the entry.
-CFG_ROOT=/spinifex-cfg
-FWCFG_ROOT=/sys/firmware/qemu_fw_cfg
-if [ ! -f "$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw" ]; then
-    load_mod virtio_blk || echo "[init] virtio_blk did not load" >/dev/console
-    i=0
-    while [ "$i" -lt 25 ] && [ ! -b /dev/vdb ]; do
-        sleep 0.04
-        i=$((i + 1))
-    done
-    if [ -b /dev/vdb ]; then
-        mkdir -m 0700 "$CFG_ROOT"
-        if ! mount -t tmpfs -o mode=0700,size=1m tmpfs "$CFG_ROOT" ||
-            ! tar -xf /dev/vdb -C "$CFG_ROOT"; then
-            echo "[init] configuration disk not readable" >/dev/console
-        fi
-        FWCFG_ROOT=$CFG_ROOT
-    fi
-fi
-
 # The guest's addresses come from the same netcfg entry as other system VMs.
-NETCFG=$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw
+NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 i=0
 while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
     sleep 0.04

cmd/lambda-agent/connect.go

@@ -5,9 +5,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"os"
 
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 )
 
@@ -26,38 +24,25 @@ type startup struct {
 }
 
 func defaultStartup() startup {
-	if _, err := os.Stat(hostlink.ConfigDiskRoot); err == nil {
-		return configDiskStartup(hostlink.ConfigDiskRoot, hostlink.ConfigDiskDevice)
-	}
 	return startup{fwCfgDir: hostlink.FwCfgDir, fwCfgRoot: hostlink.FwCfgRoot, unload: hostlink.UnloadFwCfg}
 }
 
-// configDiskStartup reads the entries init extracted from the configuration
-// disk, and seals by erasing the disk and removing the extracted tree.
-func configDiskStartup(root, device string) startup {
-	return startup{
-		fwCfgDir:  root + "/by_name/" + lambdav1.FwCfgPrefix,
-		fwCfgRoot: root,
-		unload:    func() error { return hostlink.SealConfigDisk(device, root) },
-	}
-}
-
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
-func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {
+func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
 	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
 	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
-		return nil, "", exitRefused, errors.Join(loadErr, err)
+		return nil, exitRefused, errors.Join(loadErr, err)
 	}
 	if loadErr != nil {
-		return nil, "", exitRefused, loadErr
+		return nil, exitRefused, loadErr
 	}
 	conn, err := hostlink.Dial(ctx, id)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
 		}
-		return nil, "", exitLink, err
+		return nil, exitLink, err
 	}
-	return conn, id.Scope, 0, nil
+	return conn, 0, nil
 }

cmd/lambda-agent/connect_configdisk_test.go

@@ -1,43 +0,0 @@
-//test:in-package — package main cannot be imported by an external test package.
-
-package main
-
-import (
-	"bytes"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-// An identity that does not load still seals the configuration disk: the
-// guest key on it is erased before the agent refuses to start.
-func TestConfigDiskStartup_RefusedLoadErasesDisk(t *testing.T) {
-	dir := t.TempDir()
-	root := filepath.Join(dir, "cfg")
-	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
-	device := filepath.Join(dir, "vdb")
-	require.NoError(t, os.WriteFile(device, bytes.Repeat([]byte("guest-key"), 512), 0o600))
-
-	_, _, code, err := configDiskStartup(root, device).connect(t.Context())
-	require.Error(t, err)
-	require.Equal(t, exitRefused, code)
-	got, rerr := os.ReadFile(device)
-	require.NoError(t, rerr)
-	require.Len(t, got, 9*512)
-	require.Equal(t, make([]byte, len(got)), got, "configuration disk erased")
-}
-
-// A disk that cannot be erased fails closed: the agent refuses, and the
-// extracted tree is kept so the seal check sees it.
-func TestConfigDiskStartup_UnerasableDiskRefuses(t *testing.T) {
-	dir := t.TempDir()
-	root := filepath.Join(dir, "cfg")
-	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
-
-	_, _, code, err := configDiskStartup(root, filepath.Join(dir, "missing")).connect(t.Context())
-	require.ErrorContains(t, err, "erase configuration disk")
-	require.Equal(t, exitRefused, code)
-	require.DirExists(t, root)
-}

cmd/lambda-agent/grant_testlink.go

@@ -1,66 +0,0 @@
-//go:build lambdatestlink
-
-package main
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
-)
-
-const entryGrant = "test-grant"
-
-// testGrant is the grant the host launched this VM under.
-type testGrant struct {
-	grantID, attemptID, environmentID string
-	generation                        uint64
-}
-
-// readGrant reads the test-grant fw_cfg entry: grant ID, attempt ID,
-// environment ID and generation, one per line.
-func readGrant(fwCfgDir string) (testGrant, error) {
-	b, err := os.ReadFile(filepath.Join(fwCfgDir, entryGrant, "raw"))
-	if err != nil {
-		return testGrant{}, fmt.Errorf("read %s: %w", entryGrant, err)
-	}
-	lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n")
-	if len(lines) != 4 {
-		return testGrant{}, errors.New(entryGrant + " entry malformed")
-	}
-	g := testGrant{grantID: lines[0], attemptID: lines[1], environmentID: lines[2]}
-	g.generation, err = strconv.ParseUint(lines[3], 10, 64)
-	if err != nil || g.generation == 0 || g.grantID == "" || g.attemptID == "" || g.environmentID == "" {
-		return testGrant{}, errors.New(entryGrant + " entry malformed")
-	}
-	return g, nil
-}
-
-// checkBinding refuses a run unless the grant names the authenticated environment
-// generation, the request carries that grant and its deadline is still ahead. The
-// clock check is a local guard only: it cannot detect skew or a superseded grant.
-func checkBinding(scope lambdav1.Scope, g testGrant, req hostlink.Request, now time.Time) error {
-	parts := strings.Split(string(scope), "/")
-	if len(parts) != 5 {
-		return errors.New("authenticated scope malformed")
-	}
-	if parts[3] != g.environmentID || parts[4] != strconv.FormatUint(g.generation, 10) {
-		return errors.New("grant is not for the authenticated environment generation")
-	}
-	if req.GrantID != g.grantID || req.AttemptID != g.attemptID {
-		return errors.New("request does not carry this VM's grant")
-	}
-	if req.DeadlineMs <= 0 {
-		return errors.New("request has no deadline")
-	}
-	if !now.Before(time.UnixMilli(req.DeadlineMs)) {
-		return errors.New("request deadline already passed")
-	}
-	return nil
-}

cmd/lambda-agent/helpers_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/1"
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
 
 // guest is a fake fw_cfg tree; unload removes it as unloading the module would.
 type guest struct {

cmd/lambda-agent/package_testlink.go

@@ -18,7 +18,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-	"unicode"
 
 	"golang.org/x/sys/unix"
 )
@@ -29,7 +28,6 @@ import (
 const (
 	entryPackageSize   = "test-package-size"
 	entryPackageSHA256 = "test-package-sha256"
-	entryHandler       = "test-handler"
 
 	devMaxZipBytes    = 16 << 20
 	devMaxUnzipBytes  = 64 << 20
@@ -39,9 +37,8 @@ const (
 )
 
 type packageSpec struct {
-	size    int64
-	sha256  [sha256.Size]byte
-	handler string
+	size   int64
+	sha256 [sha256.Size]byte
 }
 
 type unpackLimits struct {
@@ -79,13 +76,6 @@ func readPackageSpec(fwCfgDir string) (packageSpec, error) {
 		return spec, errors.New("package sha256 is not 64 hex digits")
 	}
 	copy(spec.sha256[:], b)
-	if spec.handler, err = read(entryHandler); err != nil {
-		return spec, err
-	}
-	if spec.handler == "" || len(spec.handler) > 128 || strings.IndexFunc(spec.handler, unicode.IsSpace) >= 0 ||
-		strings.ContainsRune(spec.handler, 0) {
-		return spec, errors.New("handler must be 1..128 characters with no whitespace")
-	}
 	return spec, nil
 }
 

cmd/lambda-agent/package_testlink_test.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 	"syscall"
 	"testing"
 
@@ -166,23 +165,19 @@ func TestUnpackChecksSizeAndHash(t *testing.T) {
 func TestReadPackageSpecBounds(t *testing.T) {
 	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 	cases := map[string]struct {
-		size, hash, handler string
-		want                string
+		size, hash string
+		want       string
 	}{
-		"valid":            {"10", sum, "bootstrap.h", ""},
-		"zero":             {"0", sum, "bootstrap.h", "outside"},
-		"too large":        {"16777217", sum, "bootstrap.h", "outside"},
-		"not hex":          {"10", "zz", "bootstrap.h", "64 hex digits"},
-		"short":            {"10", sum[:62], "bootstrap.h", "64 hex digits"},
-		"empty handler":    {"10", sum, "", "handler"},
-		"handler space":    {"10", sum, "a b", "handler"},
-		"128-char handler": {"10", sum, strings.Repeat("h", 128), ""},
-		"129-char handler": {"10", sum, strings.Repeat("h", 129), "handler"},
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			dir := t.TempDir()
-			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash, entryHandler: tc.handler} {
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
 				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
 			}
@@ -190,7 +185,6 @@ func TestReadPackageSpecBounds(t *testing.T) {
 			if tc.want == "" {
 				require.NoError(t, err)
 				assert.Equal(t, int64(10), spec.size)
-				assert.Equal(t, tc.handler, spec.handler)
 				return
 			}
 			assert.ErrorContains(t, err, tc.want)

cmd/lambda-agent/sdk_config_testlink.go

@@ -1,65 +0,0 @@
-//go:build lambdatestlink
-
-package main
-
-import (
-	"crypto/x509"
-	"errors"
-	"fmt"
-	"net"
-	"net/url"
-	"os"
-	"path/filepath"
-)
-
-const (
-	sdkEndpointEntry = "test-sdk-endpoint"
-	sdkCAEntry       = "test-sdk-ca"
-	sdkBundlePath    = "/run/lambda-sdk/ca-bundle.pem"
-	systemBundlePath = "/etc/ssl/certs/ca-certificates.crt"
-)
-
-type sdkConfig struct {
-	endpoint string
-	ca       []byte
-}
-
-// readSDKConfig reads optional public settings before the boot config is sealed.
-func readSDKConfig(dir string) (*sdkConfig, error) {
-	endpoint, endpointErr := os.ReadFile(filepath.Join(dir, sdkEndpointEntry, "raw"))
-	ca, caErr := os.ReadFile(filepath.Join(dir, sdkCAEntry, "raw"))
-	if errors.Is(endpointErr, os.ErrNotExist) && errors.Is(caErr, os.ErrNotExist) {
-		return nil, nil
-	}
-	if endpointErr != nil || caErr != nil {
-		return nil, errors.New("incomplete SDK gateway settings")
-	}
-	u, err := url.Parse(string(endpoint))
-	if err != nil || u.Scheme != "https" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" ||
-		net.ParseIP(u.Hostname()) == nil || u.Port() == "" {
-		return nil, errors.New("invalid SDK gateway endpoint")
-	}
-	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
-		return nil, errors.New("invalid SDK gateway CA")
-	}
-	return &sdkConfig{endpoint: string(endpoint), ca: ca}, nil
-}
-
-// writeBundle combines the system roots with the cluster's public root.
-func (c *sdkConfig) writeBundle(path, systemPath string) error {
-	system, err := os.ReadFile(systemPath)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return fmt.Errorf("read system CA bundle: %w", err)
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
-		return fmt.Errorf("create SDK CA directory: %w", err)
-	}
-	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
-		return fmt.Errorf("make SDK CA directory readable: %w", err)
-	}
-	bundle := append(append(system, '\n'), c.ca...)
-	if err := os.WriteFile(path, bundle, 0o644); err != nil {
-		return fmt.Errorf("write SDK CA bundle: %w", err)
-	}
-	return os.Chmod(path, 0o644)
-}

cmd/lambda-agent/sdk_config_testlink_test.go

@@ -1,106 +0,0 @@
-//go:build lambdatestlink
-
-//test:in-package exercises the unexported boot configuration reader
-package main
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"math/big"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSDKConfig_OptionalAndInvalid(t *testing.T) {
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
-	defer server.Close()
-	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
-	for _, tc := range []struct {
-		name     string
-		endpoint []byte
-		ca       []byte
-		wantErr  bool
-		want     bool
-	}{
-		{name: "absent"},
-		{name: "valid", endpoint: []byte(server.URL), ca: ca, want: true},
-		{name: "missing ca", endpoint: []byte(server.URL), wantErr: true},
-		{name: "missing endpoint", ca: ca, wantErr: true},
-		{name: "invalid endpoint", endpoint: []byte("http://127.0.0.1:1234"), ca: ca, wantErr: true},
-		{name: "invalid ca", endpoint: []byte(server.URL), ca: []byte("not a certificate"), wantErr: true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			dir := t.TempDir()
-			for name, value := range map[string][]byte{sdkEndpointEntry: tc.endpoint, sdkCAEntry: tc.ca} {
-				if value != nil {
-					require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
-					require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), value, 0o600))
-				}
-			}
-			cfg, err := readSDKConfig(dir)
-			if tc.wantErr {
-				require.Error(t, err)
-				require.Nil(t, cfg)
-				return
-			}
-			require.NoError(t, err)
-			if !tc.want {
-				require.Nil(t, cfg)
-				return
-			}
-			bundle := filepath.Join(t.TempDir(), "sdk", "ca.pem")
-			require.NoError(t, cfg.writeBundle(bundle, filepath.Join(t.TempDir(), "absent")))
-			data, err := os.ReadFile(bundle)
-			require.NoError(t, err)
-			roots := x509.NewCertPool()
-			require.True(t, roots.AppendCertsFromPEM(data))
-			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
-			resp, err := client.Get(cfg.endpoint)
-			require.NoError(t, err)
-			defer resp.Body.Close()
-			assert.Equal(t, http.StatusOK, resp.StatusCode)
-		})
-	}
-}
-
-func TestSDKConfig_BundleKeepsSystemRoots(t *testing.T) {
-	cluster := httptest.NewTLSServer(nil)
-	defer cluster.Close()
-	clusterCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Certificate().Raw})
-	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err)
-	template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test system root"},
-		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
-		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
-	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
-	require.NoError(t, err)
-	systemCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	dir := t.TempDir()
-	systemPath, bundlePath := filepath.Join(dir, "system.pem"), filepath.Join(dir, "sdk", "ca.pem")
-	require.NoError(t, os.WriteFile(systemPath, systemCA, 0o600))
-	require.NoError(t, (&sdkConfig{ca: clusterCA}).writeBundle(bundlePath, systemPath))
-	bundle, err := os.ReadFile(bundlePath)
-	require.NoError(t, err)
-	roots := x509.NewCertPool()
-	require.True(t, roots.AppendCertsFromPEM(bundle))
-	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
-	resp, err := client.Get(cluster.URL)
-	require.NoError(t, err)
-	require.NoError(t, resp.Body.Close())
-	systemCert, err := x509.ParseCertificate(der)
-	require.NoError(t, err)
-	_, err = systemCert.Verify(x509.VerifyOptions{Roots: roots})
-	require.NoError(t, err)
-}

cmd/lambda-agent/start_tcp.go

@@ -25,7 +25,7 @@ func start(ctx context.Context) int {
 
 // run authenticates to the host and holds the connection until it closes.
 func (s startup) run(ctx context.Context) (int, error) {
-	conn, _, code, err := s.connect(ctx)
+	conn, code, err := s.connect(ctx)
 	if err != nil {
 		return code, err
 	}

cmd/lambda-agent/start_testlink.go

@@ -5,21 +5,18 @@ package main
 import (
 	"context"
 	"errors"
+	"flag"
 	"fmt"
-	"io"
 	"log/slog"
 	"net"
 	"net/http"
 	"os"
 	"os/exec"
-	"sort"
-	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )
 
 const (
@@ -27,6 +24,8 @@ const (
 	taskRoot    = "/var/task"
 )
 
+var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
+
 // testlink relays one invocation over the test carrier on the authenticated
 // connection; the carrier is not the guest protocol.
 type testlink struct {
@@ -34,17 +33,13 @@ type testlink struct {
 	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
-	caBundlePath   string
-	systemCAPath   string
 }
 
 func start(ctx context.Context) int {
 	t := testlink{
-		startup:      defaultStartup(),
-		pkg:          devTaskPackage(),
-		runtimeAddr:  runtimeAddr,
-		caBundlePath: sdkBundlePath,
-		systemCAPath: systemBundlePath,
+		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
+		runtimeAddr: runtimeAddr,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
 		},
@@ -59,15 +54,13 @@ func start(ctx context.Context) int {
 func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
-	grant, grantErr := readGrant(t.fwCfgDir)
-	sdk, sdkErr := readSDKConfig(t.fwCfgDir)
-	conn, scope, code, err := t.connect(parent)
+	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, errors.Join(err, specErr, grantErr)
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
-	if err := errors.Join(specErr, grantErr); err != nil {
-		return exitRefused, err
+	if specErr != nil {
+		return exitRefused, specErr
 	}
 	link := hostlink.NewTestStream(conn)
 
@@ -83,28 +76,6 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
-	// Nothing of the tenant's is installed or run for an unbound request.
-	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
-		return exitRefused, err
-	}
-	// A host that sent no credential is refused before anything is installed.
-	if first.Credential == nil || !first.Credential.Complete() {
-		return exitRefused, errors.New("first frame carries no execution credential")
-	}
-	cred := *first.Credential
-	first.Credential = nil
-	if err := lambdaenv.Validate(cred.Environment); err != nil {
-		return exitRefused, errors.New("first frame carries an invalid function environment")
-	}
-	if sdkErr != nil {
-		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
-	}
-	if sdk != nil {
-		if err := sdk.writeBundle(t.caBundlePath, t.systemCAPath); err != nil {
-			slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", err)
-			sdk = nil
-		}
-	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -120,8 +91,12 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	region, _, _ := strings.Cut(string(scope), "/")
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred, region, sdk, t.caBundlePath))
+	cmd, err := t.startBootstrap(ctx, []string{
+		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
+		"_HANDLER=" + *handler,
+		"LAMBDA_TASK_ROOT=" + taskRoot,
+		"PATH=/bin:/sbin:/usr/bin",
+	})
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -129,11 +104,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	go func() {
 		defer close(exited)
 		err := cmd.Wait()
-		// Once a response is being received the result stands, so only an
-		// exit before that cancels the invocation.
-		if srv.BootstrapExited() {
-			cancel(fmt.Errorf("bootstrap exited: %w", err))
-		}
+		cancel(fmt.Errorf("bootstrap exited: %w", err))
 	}()
 
 	err = agent.Run(ctx, &oneInvocation{Link: link, first: &first}, srv)
@@ -145,49 +116,11 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// bootstrapEnv supplies the runtime, execution-role and SDK settings.
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
-	vars := map[string]string{
-		"AWS_LAMBDA_RUNTIME_API": runtimeAPI,
-		"_HANDLER":               handler,
-		"LAMBDA_TASK_ROOT":       taskRoot,
-		"PATH":                   "/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID":      cred.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY":  cred.SecretAccessKey,
-		"AWS_SESSION_TOKEN":      cred.SessionToken,
-		"AWS_REGION":             region,
-		"AWS_DEFAULT_REGION":     region,
-	}
-	if sdk != nil {
-		vars["AWS_ENDPOINT_URL"], vars["AWS_CA_BUNDLE"] = sdk.endpoint, caPath
-	}
-	for k, v := range cred.Environment {
-		vars[k] = v
-	}
-	keys := make([]string, 0, len(vars))
-	for k := range vars {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	env := make([]string, 0, len(keys))
-	for _, k := range keys {
-		env = append(env, k+"="+vars[k])
-	}
-	return env
-}
-
-// shutdownGrace is how long past the invocation deadline the agent waits for
-// the host's shutdown frame once its result is written.
-const shutdownGrace = 2 * time.Second
-
 // oneInvocation serves the host's first request and then stops the agent:
-// each test VM runs a single invocation. After the result is written it waits
-// for the host's shutdown frame or EOF, so the result is not cut off by the
-// poweroff that follows the agent's exit.
+// each test VM runs a single invocation.
 type oneInvocation struct {
 	hostlink.Link
-	first      *hostlink.Request
-	deadlineMs int64
+	first *hostlink.Request
 }
 
 var _ hostlink.Link = (*oneInvocation)(nil)
@@ -196,36 +129,10 @@ func (o *oneInvocation) Next(ctx context.Context) (hostlink.Request, error) {
 	if err := ctx.Err(); err != nil {
 		return hostlink.Request{}, err
 	}
-	if o.first != nil {
-		req := *o.first
-		o.first = nil
-		o.deadlineMs = req.DeadlineMs
-		return req, nil
+	if o.first == nil {
+		return hostlink.Request{}, hostlink.ErrShutdown
 	}
-	return hostlink.Request{}, o.awaitShutdown(ctx)
-}
-
-// awaitShutdown reads the host's next frame and always ends the agent: with
-// ErrShutdown for a shutdown frame, EOF, a stray frame or an expired wait.
-func (o *oneInvocation) awaitShutdown(ctx context.Context) error {
-	end := time.UnixMilli(o.deadlineMs)
-	if now := time.Now(); end.Before(now) {
-		end = now
-	}
-	wctx, cancel := context.WithDeadline(ctx, end.Add(shutdownGrace))
-	defer cancel()
-	_, err := o.Link.Next(wctx)
-	switch {
-	case errors.Is(err, hostlink.ErrShutdown), errors.Is(err, io.EOF):
-		return hostlink.ErrShutdown
-	case err == nil:
-		slog.Warn("lambda-agent: frame after the invocation was not dispatched")
-		return hostlink.ErrShutdown
-	case ctx.Err() != nil:
-		return err
-	case wctx.Err() != nil:
-		slog.Warn("lambda-agent: no shutdown from the host; exiting")
-		return hostlink.ErrShutdown
-	}
-	return err
+	req := *o.first
+	o.first = nil
+	return req, nil
 }

cmd/lambda-agent/start_testlink_test.go

@@ -7,15 +7,12 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
-	"encoding/pem"
 	"io"
 	"net/http"
-	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
-	"strings"
 	"testing"
 	"time"
 
@@ -28,18 +25,12 @@ import (
 
 const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"
 
-// helperHandlerOut names the file the helper bootstrap writes its _HANDLER to.
-const helperHandlerOut = "LAMBDA_AGENT_TEST_HANDLER_OUT"
-
 // TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
 // echoes one invocation through the Runtime API, then waits for the next.
 func TestHelperBootstrap(t *testing.T) {
 	if os.Getenv(helperEnv) != "1" {
 		return
 	}
-	if out := os.Getenv(helperHandlerOut); out != "" {
-		require.NoError(t, os.WriteFile(out, []byte(os.Getenv("_HANDLER")), 0o600))
-	}
 	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
 	resp, err := http.Get(base + "next")
 	require.NoError(t, err)
@@ -50,31 +41,14 @@ func TestHelperBootstrap(t *testing.T) {
 	post, err := http.Post(base+id+"/response", "application/json", bytes.NewReader(body))
 	require.NoError(t, err)
 	post.Body.Close()
-	if os.Getenv(helperExitAfterPost) == "1" {
-		return
-	}
 	_, _ = http.Get(base + "next")
 }
 
-// helperExitAfterPost makes the helper bootstrap exit once its response is posted.
-const helperExitAfterPost = "LAMBDA_AGENT_TEST_EXIT_AFTER_POST"
-
-// testGrantEntry grants g-1 for attempt a-1 on the fixture scope's env-1/1.
-const testGrantEntry = "g-1\na-1\nenv-1\n1\n"
-
-// testHandler is the handler the fixture's host configures.
-const testHandler = "bootstrap.fixture"
-
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
 	task    string
 	sealed  bool
-	// handlerOut receives the _HANDLER the bootstrap process itself saw.
-	handlerOut string
-	caBundle   string
-	// env is the environment the agent gave the bootstrap.
-	env []string
 }
 
 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -87,17 +61,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	sum := sha256.Sum256(zipBytes)
 	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
 	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
-	if _, ok := entries[entryHandler]; !ok {
-		entries[entryHandler] = []byte(testHandler)
-	}
-	if _, ok := entries[entryGrant]; !ok {
-		entries[entryGrant] = []byte(testGrantEntry)
-	}
 	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
-	b.handlerOut = filepath.Join(t.TempDir(), "handler")
-	b.caBundle = filepath.Join(t.TempDir(), "sdk", "ca-bundle.pem")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -106,15 +72,12 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			mountDir:   func(string) error { return nil },
 			sealDir:    func(string) error { b.sealed = true; return nil },
 		},
-		runtimeAddr:  "127.0.0.1:0",
-		caBundlePath: b.caBundle,
-		systemCAPath: filepath.Join(t.TempDir(), "system-ca.pem"),
+		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
-			b.env = env
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
-			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
+			cmd.Env = append(env, helperEnv+"=1")
 			return cmd, cmd.Start()
 		},
 	}
@@ -150,7 +113,7 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	case <-time.After(100 * time.Millisecond):
 	}
 	hs := hostlink.NewTestStream(conn)
-	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{"a":40,"b":2}`)}
+	req := hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{"a":40,"b":2}`)}
 	require.NoError(t, hs.HostSend(req))
 	got, err := hs.HostResult(ctx)
 	require.NoError(t, err)
@@ -169,231 +132,10 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
-	seen, err := os.ReadFile(b.handlerOut)
-	require.NoError(t, err, "bootstrap process did not report its handler")
-	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
-	assert.ElementsMatch(t, []string{
-		"AWS_LAMBDA_RUNTIME_API=" + envValue(b.env, "AWS_LAMBDA_RUNTIME_API"),
-		"_HANDLER=" + testHandler,
-		"LAMBDA_TASK_ROOT=" + taskRoot,
-		"PATH=/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
-		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
-		"AWS_REGION=ap-southeast-2",
-		"AWS_DEFAULT_REGION=ap-southeast-2",
-	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
-	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
 }
 
-func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
-	defer server.Close()
-	h := newHost(t)
-	h.accept(ctx)
-	entries := h.entries()
-	entries[sdkEndpointEntry] = []byte(server.URL)
-	entries[sdkCAEntry] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
-	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(t, entries))
-	conn := h.guest(t)
-	defer conn.Close()
-	hs := hostlink.NewTestStream(conn)
-	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
-	require.NoError(t, hs.HostSend(req))
-	_, err := hs.HostResult(ctx)
-	require.NoError(t, err)
-	res := wait(t, done, "agent to stop")
-	require.NoError(t, res.err)
-	assert.Equal(t, server.URL, envValue(b.env, "AWS_ENDPOINT_URL"))
-	assert.Equal(t, b.caBundle, envValue(b.env, "AWS_CA_BUNDLE"))
-	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_REGION"))
-	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_DEFAULT_REGION"))
-	ca, err := os.ReadFile(b.caBundle)
-	require.NoError(t, err)
-	assert.Contains(t, string(ca), string(entries[sdkCAEntry]))
-	info, err := os.Stat(b.caBundle)
-	require.NoError(t, err)
-	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
-}
-
-func TestTestlinkProvidesFunctionEnvironment(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	h := newHost(t)
-	h.accept(ctx)
-	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
-	conn := h.guest(t)
-	defer conn.Close()
-	hs := hostlink.NewTestStream(conn)
-	cred := testCredential
-	cred.Environment = map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin"}
-	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
-	require.NoError(t, hs.HostSend(req))
-	_, err := hs.HostResult(ctx)
-	require.NoError(t, err)
-	require.NoError(t, wait(t, done, "agent to stop").err)
-	assert.Equal(t, "hello $WORLD", envValue(b.env, "GREETING"))
-	assert.Equal(t, "/my/bin", envValue(b.env, "PATH"))
-	assert.Equal(t, testCredential.SecretAccessKey, envValue(b.env, "AWS_SECRET_ACCESS_KEY"))
-}
-
-func TestTestlinkRejectsReservedFunctionEnvironment(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	h := newHost(t)
-	h.accept(ctx)
-	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
-	conn := h.guest(t)
-	defer conn.Close()
-	cred := testCredential
-	cred.Environment = map[string]string{"AWS_SECRET_ACCESS_KEY": "forged"}
-	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
-	result := wait(t, done, "agent to refuse the environment")
-	assert.Equal(t, exitRefused, result.code)
-	assert.Error(t, result.err)
-	assert.NotContains(t, result.err.Error(), "forged")
-	assert.Zero(t, b.n)
-}
-
-var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
-
-func envValue(env []string, name string) string {
-	for _, kv := range env {
-		if v, ok := strings.CutPrefix(kv, name+"="); ok {
-			return v
-		}
-	}
-	return ""
-}
-
-// A first frame without a complete credential installs and runs nothing.
-func TestTestlinkFrameWithoutCredentialNeverStartsBootstrap(t *testing.T) {
-	for name, cred := range map[string]*hostlink.Credential{
-		"none":       nil,
-		"incomplete": {AccessKeyID: "ASIAGUESTKEY"},
-	} {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
-			conn := h.guest(t)
-			defer conn.Close()
-			require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{Credential: cred, GrantID: "g-1", AttemptID: "a-1",
-				RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}))
-
-			res := wait(t, done, "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, "no execution credential")
-			assert.Zero(t, b.n)
-			assert.False(t, b.sealed)
-			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
-		})
-	}
-}
-
-// A bootstrap that posts its response and exits at once does not cost the
-// host the result; the agent then waits for the host's shutdown.
-func TestTestlinkBootstrapExitingAfterResponseStillDelivers(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	h := newHost(t)
-	h.accept(ctx)
-	var b bootstrapStarts
-	tl := b.testlink(t, h.entries())
-	start := tl.startBootstrap
-	tl.startBootstrap = func(ctx context.Context, env []string) (*exec.Cmd, error) {
-		return start(ctx, append(env, helperExitAfterPost+"=1"))
-	}
-	done := runTestlinkAsync(ctx, tl)
-
-	conn := h.guest(t)
-	defer conn.Close()
-	hs := hostlink.NewTestStream(conn)
-	payload := bytes.Repeat([]byte("0123456789"), 300_000)
-	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: payload}
-	require.NoError(t, hs.HostSend(req))
-	got, err := hs.HostResult(ctx)
-	require.NoError(t, err)
-	assert.Equal(t, hostlink.Response{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", Payload: payload}, got)
-
-	require.NoError(t, hs.HostShutdown())
-	res := wait(t, done, "agent to stop after the host's shutdown")
-	assert.Equal(t, exitClosed, res.code)
-	assert.NoError(t, res.err)
-}
-
-// The agent stays up for the host's shutdown rather than exiting as soon as
-// its result is written, and never dispatches a stray frame.
-func TestOneInvocationAfterResult(t *testing.T) {
-	first := hostlink.Request{RequestID: "r-1", DeadlineMs: time.Now().Add(time.Minute).UnixMilli()}
-	cases := map[string]struct {
-		next func(context.Context) (hostlink.Request, error)
-		want error
-	}{
-		"shutdown frame": {func(context.Context) (hostlink.Request, error) { return hostlink.Request{}, hostlink.ErrShutdown }, hostlink.ErrShutdown},
-		"eof":            {func(context.Context) (hostlink.Request, error) { return hostlink.Request{}, io.EOF }, hostlink.ErrShutdown},
-		"a second request is not dispatched": {func(context.Context) (hostlink.Request, error) {
-			return hostlink.Request{RequestID: "r-2"}, nil
-		}, hostlink.ErrShutdown},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			o := &oneInvocation{Link: stubLink{tc.next}, first: &first}
-			_, err := o.Next(context.Background())
-			require.NoError(t, err)
-			req, err := o.Next(context.Background())
-			require.ErrorIs(t, err, tc.want)
-			assert.Empty(t, req.RequestID)
-		})
-	}
-	t.Run("host never sends shutdown", func(t *testing.T) {
-		past := first
-		past.DeadlineMs = time.Now().Add(-time.Second).UnixMilli()
-		o := &oneInvocation{Link: stubLink{func(ctx context.Context) (hostlink.Request, error) {
-			<-ctx.Done()
-			return hostlink.Request{}, ctx.Err()
-		}}, first: &past}
-		_, err := o.Next(context.Background())
-		require.NoError(t, err)
-		start := time.Now()
-		_, err = o.Next(context.Background())
-		require.ErrorIs(t, err, hostlink.ErrShutdown)
-		assert.GreaterOrEqual(t, time.Since(start), shutdownGrace-100*time.Millisecond, "waited the grace")
-	})
-	t.Run("parent cancel is not a shutdown", func(t *testing.T) {
-		o := &oneInvocation{Link: stubLink{func(ctx context.Context) (hostlink.Request, error) {
-			<-ctx.Done()
-			return hostlink.Request{}, ctx.Err()
-		}}, first: &first}
-		_, _ = o.Next(context.Background())
-		ctx, cancel := context.WithCancel(context.Background())
-		cancel()
-		_, err := o.Next(ctx)
-		require.ErrorIs(t, err, context.Canceled)
-	})
-}
-
-type stubLink struct {
-	next func(context.Context) (hostlink.Request, error)
-}
-
-var _ hostlink.Link = stubLink{}
-
-func (l stubLink) Next(ctx context.Context) (hostlink.Request, error) { return l.next(ctx) }
-
-func (stubLink) Complete(context.Context, hostlink.Response) error { return nil }
-
 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -476,8 +218,7 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
 
 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
-		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
 	assert.ErrorContains(t, res.err, "install package")
@@ -503,87 +244,3 @@ func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
 	_, err := os.Stat(tl.fwCfgRoot)
 	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
 }
-
-// A request that does not carry this VM's grant, or a grant for another
-// environment generation than the one authenticated, installs and runs
-// nothing of the tenant's.
-func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
-	cases := map[string]struct {
-		grant            string
-		grantID, attempt string
-		deadlineMs       int64
-		want             string
-	}{
-		"frame with no deadline":      {testGrantEntry, "g-1", "a-1", -1, "no deadline"},
-		"frame with expired deadline": {testGrantEntry, "g-1", "a-1", time.Now().Add(-time.Second).UnixMilli(), "deadline already passed"},
-		"frame for another grant":     {testGrantEntry, "g-2", "a-1", 0, "does not carry this VM's grant"},
-		"frame for another attempt":   {testGrantEntry, "g-1", "a-2", 0, "does not carry this VM's grant"},
-		"frame with no grant":         {testGrantEntry, "", "", 0, "does not carry this VM's grant"},
-		"grant for another env":       {"g-1\na-1\nenv-2\n1\n", "g-1", "a-1", 0, "not for the authenticated environment"},
-		"grant for another gen":       {"g-1\na-1\nenv-1\n2\n", "g-1", "a-1", 0, "not for the authenticated environment"},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			entries := h.entries()
-			entries[entryGrant] = []byte(tc.grant)
-			done := runTestlinkAsync(ctx, b.testlink(t, entries))
-			conn := h.guest(t)
-			defer conn.Close()
-			deadline := time.Now().Add(5 * time.Second).UnixMilli()
-			switch {
-			case tc.deadlineMs < 0:
-				deadline = 0
-			case tc.deadlineMs > 0:
-				deadline = tc.deadlineMs
-			}
-			req := hostlink.Request{Credential: &testCredential, GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
-				DeadlineMs: deadline, Payload: []byte(`{}`)}
-			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
-
-			res := wait(t, done, "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, tc.want)
-			assert.Zero(t, b.n)
-			assert.False(t, b.sealed)
-			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
-		})
-	}
-}
-
-// A missing or malformed grant entry refuses before any frame is awaited.
-func TestTestlinkBadGrantEntryRefuses(t *testing.T) {
-	cases := map[string]*string{
-		"missing":       nil,
-		"too few lines": ptr("g-1\na-1\nenv-1\n"),
-		"generation 0":  ptr("g-1\na-1\nenv-1\n0\n"),
-		"empty grant":   ptr("\na-1\nenv-1\n1\n"),
-	}
-	for name, grant := range cases {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			entries := h.entries()
-			if grant != nil {
-				entries[entryGrant] = []byte(*grant)
-			}
-			tl := b.testlink(t, entries)
-			if grant == nil {
-				require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryGrant)))
-			}
-			res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, entryGrant)
-			assert.Zero(t, b.n)
-		})
-	}
-}
-
-func ptr(s string) *string { return &s }

cmd/lambda-image-agent-tag/main.go

@@ -1,224 +0,0 @@
-// Command lambda-image-agent-tag checks whether the lambda-agent archived in a
-// built Lambda guest initramfs carries the lambdatestlink build tag. Build
-// information is recorded by the build; it is not proof of how the agent behaves.
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"compress/gzip"
-	"debug/buildinfo"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"slices"
-	"strconv"
-	"strings"
-)
-
-const (
-	agentPath    = "usr/local/bin/lambda-agent"
-	maxArchive   = 1 << 30
-	maxAgent     = 256 << 20
-	maxName      = 4096
-	newcHeader   = 110
-	trailerName  = "TRAILER!!!"
-	modeTypeMask = 0o170000
-	modeDir      = 0o040000
-	modeRegular  = 0o100000
-)
-
-var agentParents = []string{"usr", "usr/local", "usr/local/bin"}
-
-func main() {
-	if len(os.Args) != 3 || (os.Args[2] != "present" && os.Args[2] != "absent") {
-		fmt.Fprintln(os.Stderr, "usage: lambda-image-agent-tag <initramfs.cpio.gz> present|absent")
-		os.Exit(2)
-	}
-	if err := run(os.Args[1], os.Args[2] == "present"); err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[1], err)
-		os.Exit(1)
-	}
-}
-
-func run(path string, want bool) error {
-	fi, err := os.Lstat(path)
-	if err != nil {
-		return err
-	}
-	if !fi.Mode().IsRegular() {
-		return errors.New("not a regular file")
-	}
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	agent, err := archivedAgent(f)
-	if err != nil {
-		return err
-	}
-	return checkTag(agent, want)
-}
-
-// archivedAgent returns the single agent entry after checking listed names,
-// entry types, the agent's link count and zero-only padding after the trailer.
-// It is not a full kernel-equivalent parser; 070702 CRCs, for one, are unchecked.
-func archivedAgent(r io.Reader) ([]byte, error) {
-	zr, err := gzip.NewReader(bufio.NewReader(r))
-	if err != nil {
-		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
-	}
-	raw, err := io.ReadAll(io.LimitReader(zr, maxArchive+1))
-	if err != nil {
-		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
-	}
-	if len(raw) > maxArchive {
-		return nil, fmt.Errorf("archive larger than %d bytes", maxArchive)
-	}
-
-	seen := map[string]bool{}
-	var agent []byte
-	off := 0
-	for {
-		e, next, err := parseEntry(raw, off)
-		if err != nil {
-			return nil, fmt.Errorf("entry at offset %d: %w", off, err)
-		}
-		off = next
-		if e.name == trailerName {
-			break
-		}
-		if !canonical(e.name) {
-			return nil, fmt.Errorf("entry name %q is not canonical", e.name)
-		}
-		if seen[e.name] {
-			return nil, fmt.Errorf("entry %q appears more than once", e.name)
-		}
-		seen[e.name] = true
-		for _, p := range agentParents {
-			if e.name == p && e.mode&modeTypeMask != modeDir {
-				return nil, fmt.Errorf("%s is not a directory", p)
-			}
-		}
-		if e.name == agentPath {
-			for _, p := range agentParents {
-				if !seen[p] {
-					return nil, fmt.Errorf("%s has no %s directory entry before it", agentPath, p)
-				}
-			}
-			if e.mode&modeTypeMask != modeRegular {
-				return nil, fmt.Errorf("%s is not a regular file", agentPath)
-			}
-			if e.nlink != 1 {
-				return nil, fmt.Errorf("%s has %d links, want 1", agentPath, e.nlink)
-			}
-			if len(e.data) > maxAgent {
-				return nil, fmt.Errorf("%s is larger than %d bytes", agentPath, maxAgent)
-			}
-			agent = e.data
-		}
-	}
-	if i := bytes.IndexFunc(raw[off:], func(r rune) bool { return r != 0 }); i >= 0 {
-		return nil, fmt.Errorf("data after the archive trailer at offset %d", off+i)
-	}
-	if agent == nil {
-		return nil, fmt.Errorf("no %s entry", agentPath)
-	}
-	return agent, nil
-}
-
-type entry struct {
-	name  string
-	mode  uint64
-	nlink uint64
-	data  []byte
-}
-
-func parseEntry(raw []byte, off int) (entry, int, error) {
-	if len(raw)-off < newcHeader {
-		return entry{}, 0, errors.New("truncated header")
-	}
-	h := raw[off : off+newcHeader]
-	if m := string(h[:6]); m != "070701" && m != "070702" {
-		return entry{}, 0, fmt.Errorf("magic %q is not newc", m)
-	}
-	field := func(i int) (uint64, error) {
-		return strconv.ParseUint(string(h[6+8*i:14+8*i]), 16, 32)
-	}
-	mode, err1 := field(1)
-	nlink, err2 := field(4)
-	size, err3 := field(6)
-	nameSize, err4 := field(11)
-	if err := errors.Join(err1, err2, err3, err4); err != nil {
-		return entry{}, 0, fmt.Errorf("bad header field: %w", err)
-	}
-	if nameSize < 2 || nameSize > maxName {
-		return entry{}, 0, fmt.Errorf("name size %d out of range", nameSize)
-	}
-	nameEnd := off + newcHeader + int(nameSize)
-	dataOff := align4(nameEnd)
-	if dataOff > len(raw) || size > uint64(len(raw)-dataOff) {
-		return entry{}, 0, errors.New("truncated entry")
-	}
-	dataEnd := dataOff + int(size)
-	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
-		return entry{}, 0, errors.New("truncated entry")
-	}
-	name := raw[off+newcHeader : nameEnd]
-	if name[len(name)-1] != 0 || bytes.IndexByte(name[:len(name)-1], 0) >= 0 {
-		return entry{}, 0, errors.New("name is not one NUL-terminated string")
-	}
-	return entry{name: string(name[:len(name)-1]), mode: mode, nlink: nlink, data: raw[dataOff:dataEnd]}, align4(dataEnd), nil
-}
-
-func align4(n int) int { return (n + 3) &^ 3 }
-
-// canonical accepts "." and relative names with no empty, "." or ".." component.
-func canonical(name string) bool {
-	if name == "." {
-		return true
-	}
-	if name == "" || strings.HasPrefix(name, "/") {
-		return false
-	}
-	for c := range strings.SplitSeq(name, "/") {
-		if c == "" || c == "." || c == ".." {
-			return false
-		}
-	}
-	return true
-}
-
-func checkTag(agent []byte, want bool) error {
-	info, err := buildinfo.Read(bytes.NewReader(agent))
-	if err != nil {
-		return fmt.Errorf("no Go build information in %s: %w", agentPath, err)
-	}
-	return decide(info, want)
-}
-
-// decide refuses build information with no recorded settings, since their
-// absence does not show the tag was absent.
-func decide(info *buildinfo.BuildInfo, want bool) error {
-	if len(info.Settings) == 0 {
-		return fmt.Errorf("%s records no build settings", agentPath)
-	}
-	if hasTag(info, "lambdatestlink") != want {
-		return fmt.Errorf("%s: lambdatestlink build tag present=%v, want %v", agentPath, !want, want)
-	}
-	return nil
-}
-
-func hasTag(info *buildinfo.BuildInfo, tag string) bool {
-	for _, s := range info.Settings {
-		if s.Key != "-tags" {
-			continue
-		}
-		if slices.Contains(strings.Split(s.Value, ","), tag) {
-			return true
-		}
-	}
-	return false
-}

cmd/lambda-image-agent-tag/main_test.go

@@ -1,154 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"debug/buildinfo"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime/debug"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-type fileEntry struct {
-	name  string
-	mode  uint32
-	nlink uint32
-	data  string
-}
-
-func dir(name string) fileEntry { return fileEntry{name: name, mode: modeDir | 0o755, nlink: 2} }
-func file(name, data string) fileEntry {
-	return fileEntry{name: name, mode: modeRegular | 0o755, nlink: 1, data: data}
-}
-func symlink(name, target string) fileEntry {
-	return fileEntry{name: name, mode: 0o120777, nlink: 1, data: target}
-}
-
-// newc writes entries and a trailer in the format the kernel unpacks.
-func newc(entries ...fileEntry) []byte {
-	var b bytes.Buffer
-	for i, e := range append(entries, fileEntry{name: trailerName, nlink: 1}) {
-		fmt.Fprintf(&b, "070701%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X",
-			i+1, e.mode, 0, 0, e.nlink, 0, len(e.data), 0, 0, 0, 0, len(e.name)+1, 0)
-		b.WriteString(e.name)
-		b.WriteByte(0)
-		pad(&b)
-		b.WriteString(e.data)
-		pad(&b)
-	}
-	return b.Bytes()
-}
-
-func pad(b *bytes.Buffer) {
-	for b.Len()%4 != 0 {
-		b.WriteByte(0)
-	}
-}
-
-func gz(parts ...[]byte) []byte {
-	var b bytes.Buffer
-	for _, p := range parts {
-		w := gzip.NewWriter(&b)
-		_, _ = w.Write(p)
-		_ = w.Close()
-	}
-	return b.Bytes()
-}
-
-func withAgent(agent fileEntry) []byte {
-	return newc(dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent, file("init", "#!/bin/sh"))
-}
-
-func TestArchivedAgent_ReturnsTheOneArchivedAgent(t *testing.T) {
-	got, err := archivedAgent(bytes.NewReader(gz(withAgent(file(agentPath, "agent-bytes")))))
-	require.NoError(t, err)
-	require.Equal(t, "agent-bytes", string(got))
-
-	padded := append(withAgent(file(agentPath, "agent-bytes")), make([]byte, 1024)...)
-	got, err = archivedAgent(bytes.NewReader(gz(padded)))
-	require.NoError(t, err, "zero padding after the trailer")
-	require.Equal(t, "agent-bytes", string(got))
-}
-
-func TestArchivedAgent_Refuses(t *testing.T) {
-	parents := []fileEntry{dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin")}
-	agent := file(agentPath, "untagged")
-	small := newc(append(parents, agent)...)
-	second := newc(append(parents[1:], file(agentPath, "tagged"))...)
-	// The second archive starts inside what would be the first one's final block.
-	sameBlock := append(append([]byte{}, small...), second...)
-	for len(sameBlock)%512 != 0 {
-		sameBlock = append(sameBlock, 0)
-	}
-
-	for name, archive := range map[string][]byte{
-		"aliased duplicate":            gz(newc(append(parents, agent, file("././"+agentPath, "tagged"))...)),
-		"leading slash alias":          gz(newc(append(parents, agent, file("/"+agentPath, "tagged"))...)),
-		"dot-dot component":            gz(newc(append(parents, agent, file("usr/local/../local/bin/lambda-agent", "x"))...)),
-		"empty component":              gz(newc(append(parents, agent, file("usr//local/bin/lambda-agent", "x"))...)),
-		"exact duplicate":              gz(newc(append(parents, agent, file(agentPath, "tagged"))...)),
-		"duplicate directory":          gz(newc(dir("usr"), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent)),
-		"missing directory entry":      gz(newc(dir("usr"), dir("usr/local/bin"), agent)),
-		"directory after the agent":    gz(newc(dir("usr"), dir("usr/local"), agent, dir("usr/local/bin"))),
-		"parent is a symlink":          gz(newc(dir("usr"), dir("usr/local"), symlink("usr/local/bin", "elsewhere"), agent)),
-		"agent is a symlink":           gz(newc(append(parents, symlink(agentPath, "/bin/sh"))...)),
-		"hard-linked agent":            gz(newc(append(parents, fileEntry{name: agentPath, mode: modeRegular | 0o755, nlink: 2, data: "x"})...)),
-		"no agent":                     gz(newc(parents...)),
-		"second archive appended":      gz(small, second),
-		"second archive in same block": gz(sameBlock),
-		"non-zero byte after trailer":  gz(append(append([]byte{}, small...), 0, 0, 1)),
-		"truncated archive":            gz(small[:len(small)-20]),
-		"not newc":                     gz([]byte("not cpio at all, but long enough to fill one newc header: " + string(make([]byte, 120)))),
-		"not gzip":                     small,
-	} {
-		t.Run(name, func(t *testing.T) {
-			_, err := archivedAgent(bytes.NewReader(archive))
-			require.Error(t, err)
-		})
-	}
-}
-
-func TestRun_RefusesASymlinkedImagePath(t *testing.T) {
-	d := t.TempDir()
-	img := filepath.Join(d, "initramfs.cpio.gz")
-	require.NoError(t, os.WriteFile(img, gz(withAgent(file(agentPath, "x"))), 0o644))
-	require.NoError(t, os.Symlink(img, filepath.Join(d, "link")))
-	require.ErrorContains(t, run(filepath.Join(d, "link"), false), "not a regular file")
-}
-
-func TestHasTag_MatchesWholeTagsOnly(t *testing.T) {
-	info := func(tags string) *buildinfo.BuildInfo {
-		return &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}, {Key: "-tags", Value: tags}}}
-	}
-	require.True(t, hasTag(info("lambdatestlink,fips140v1.0"), "lambdatestlink"))
-	require.False(t, hasTag(info("fips140v1.0"), "lambdatestlink"))
-	require.False(t, hasTag(info("lambdatestlinkx,xlambdatestlink"), "lambdatestlink"))
-	require.False(t, hasTag(&buildinfo.BuildInfo{}, "lambdatestlink"))
-}
-
-// The test binary carries real build information, so checkTag reads it and
-// refuses the opposite answer; bytes with no build information are refused.
-func TestCheckTag_ReadsRealBuildInformation(t *testing.T) {
-	exe, err := os.Executable()
-	require.NoError(t, err)
-	raw, err := os.ReadFile(exe)
-	require.NoError(t, err)
-	info, err := buildinfo.Read(bytes.NewReader(raw))
-	require.NoError(t, err)
-	tagged := hasTag(info, "lambdatestlink")
-	require.NoError(t, checkTag(raw, tagged))
-	require.Error(t, checkTag(raw, !tagged))
-	require.Error(t, checkTag([]byte("not a Go binary"), false))
-}
-
-func TestDecide_RefusesMissingBuildSettings(t *testing.T) {
-	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, false), "no build settings")
-	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, true), "no build settings")
-	untagged := &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}}}
-	require.NoError(t, decide(untagged, false))
-	require.Error(t, decide(untagged, true))
-}

cmd/lambda-image-digest/main.go

@@ -1,26 +0,0 @@
-//go:build lambdatestlink
-
-// Command lambda-image-digest prints the test-link runtime image digest of one
-// directory, selecting the boot files the same way the daemon does. The digest
-// identifies the bytes; it does not authenticate where they came from.
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
-)
-
-func main() {
-	if len(os.Args) != 2 {
-		fmt.Fprintln(os.Stderr, "usage: lambda-image-digest <image-dir>")
-		os.Exit(2)
-	}
-	digest, err := testlink.DirDigest(os.Args[1])
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-	fmt.Println(digest)
-}

contracts/cluster/v1/facts.go

@@ -56,8 +56,6 @@ type NodeFacts struct {
 	Memory      MemoryFacts       `json:"memory"`
 	NICs        []NICFacts        `json:"nics"`
 	Planes      []PlaneFacts      `json:"planes"`
-	Disks       []DiskFacts       `json:"disks,omitempty"`
-	Filesystems []FilesystemFacts `json:"filesystems,omitempty"`
 	Sources     map[string]string `json:"sources"`
 }
 
@@ -136,42 +134,6 @@ type MemoryGroup struct {
 	Serials       []string `json:"serials,omitempty"`
 }
 
-// DiskFacts is one physical drive as sysfs reports it. Transport is nvme, sata,
-// usb or virtio, and empty when the device path names none, as a RAID
-// controller's virtual disk does.
-type DiskFacts struct {
-	Name       string `json:"name"`
-	Path       string `json:"path"`
-	SizeBytes  uint64 `json:"size_bytes"`
-	Model      string `json:"model,omitempty"`
-	Transport  string `json:"transport,omitempty"`
-	Rotational bool   `json:"rotational,omitempty"`
-	Removable  bool   `json:"removable,omitempty"`
-}
-
-// FilesystemFacts is one mounted block-device or ZFS filesystem. BindOf names
-// the directory a bind mount shows; Disk names the drives under it. Error says
-// why Usage is missing or stale.
-type FilesystemFacts struct {
-	Mount  string           `json:"mount"`
-	Source string           `json:"source"`
-	Disk   string           `json:"disk,omitempty"`
-	Type   string           `json:"type"`
-	BindOf string           `json:"bind_of,omitempty"`
-	Usage  *FilesystemUsage `json:"usage,omitempty"`
-	Error  string           `json:"error,omitempty"`
-	// ID identifies the mount on its node, so a filesystem replaced at the
-	// same path is read afresh. Empty for a mount the daemon cannot see.
-	ID string `json:"-"`
-}
-
-// FilesystemUsage is a filesystem's last statfs reading, as df counts it.
-type FilesystemUsage struct {
-	SizeBytes  uint64 `json:"size_bytes"`
-	UsedBytes  uint64 `json:"used_bytes"`
-	AvailBytes uint64 `json:"avail_bytes"`
-}
-
 // NICFacts is one physical network interface and, where LLDP reports it, the
 // switch port it is cabled to.
 type NICFacts struct {

contracts/ec2/v1/commands.go

@@ -12,11 +12,6 @@ const (
 	// this contract family. It is the NATS permission/subscription pattern for
 	// the per-instance command route, rather than the broader descendant match.
 	InstanceCommandSubjectWildcard = InstanceCommandSubjectPrefix + "*"
-
-	// HandledHeader is set to "1" on every reply an owner sends to a
-	// modify-instance-metadata-options or set-disable-api-termination command.
-	// An owner that predates those commands cannot set it.
-	HandledHeader = "Spinifex-Ec2-Cmd-Handled"
 )
 
 // InstanceCommandSubject returns the targeted command subject for instanceID.
@@ -28,9 +23,7 @@ func InstanceCommandSubject(instanceID string) string {
 }
 
 // EC2InstanceCommand is the NATS wire format for EC2 instance commands
-// (stop, terminate, start, reboot, attach/detach-volume, attach/detach-eni,
-// IAM profile, spot lineage, tags, monitoring, metadata options and
-// termination protection).
+// (stop, terminate, start, attach/detach-volume, attach/detach-eni).
 type EC2InstanceCommand struct {
 	ID                        string                     `json:"id"`
 	Attributes                EC2CommandAttributes       `json:"attributes"`
@@ -42,9 +35,6 @@ type EC2InstanceCommand struct {
 	SpotLineageData           *SpotLineageData           `json:"spot_lineage_data,omitempty"`
 	InstanceTagsData          *InstanceTagsData          `json:"instance_tags_data,omitempty"`
 	InstanceMonitoringData    *InstanceMonitoringData    `json:"instance_monitoring_data,omitempty"`
-
-	InstanceMetadataOptionsData *InstanceMetadataOptionsData `json:"instance_metadata_options_data,omitempty"`
-	DisableApiTerminationData   *DisableApiTerminationData   `json:"disable_api_termination_data,omitempty"`
 }
 
 // EC2CommandAttributes indicates which action the daemon should perform.
@@ -62,9 +52,6 @@ type EC2CommandAttributes struct {
 	SetInstanceTags             bool `json:"set_instance_tags,omitempty"`
 	RemoveInstanceTags          bool `json:"remove_instance_tags,omitempty"`
 	SetInstanceMonitoring       bool `json:"set_instance_monitoring,omitempty"`
-
-	ModifyInstanceMetadataOptions bool `json:"modify_instance_metadata_options,omitempty"`
-	SetDisableApiTermination      bool `json:"set_disable_api_termination,omitempty"`
 }
 
 // AttachVolumeData carries parameters for an attach-volume command.
@@ -116,23 +103,6 @@ type InstanceMonitoringData struct {
 	Enabled bool `json:"enabled"`
 }
 
-// InstanceMetadataOptionsData carries the requested metadata options for a
-// modify-instance-metadata-options command. Unset fields are left unchanged.
-type InstanceMetadataOptionsData struct {
-	HttpTokens              string `json:"http_tokens,omitempty"`
-	HttpEndpoint            string `json:"http_endpoint,omitempty"`
-	HttpProtocolIpv6        string `json:"http_protocol_ipv6,omitempty"`
-	InstanceMetadataTags    string `json:"instance_metadata_tags,omitempty"`
-	HttpPutResponseHopLimit *int64 `json:"http_put_response_hop_limit,omitempty"`
-}
-
-// DisableApiTerminationData carries the desired termination-protection value
-// for a set-disable-api-termination command. Carried as data so "false" is an
-// explicit value, not the absence of one.
-type DisableApiTerminationData struct {
-	Value bool `json:"value"`
-}
-
 // SpotLineageData carries the SIR id stamped onto a spot-launched instance.
 // Lifecycle is always "spot" for this command, so only the SIR id travels.
 type SpotLineageData struct {

contracts/ec2/v1/commands_test.go

@@ -24,9 +24,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			SetInstanceTags:             true,
 			RemoveInstanceTags:          true,
 			SetInstanceMonitoring:       true,
-
-			ModifyInstanceMetadataOptions: true,
-			SetDisableApiTermination:      true,
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
@@ -36,15 +33,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		SpotLineageData:           &SpotLineageData{SpotInstanceRequestId: "sir-1"},
 		InstanceTagsData:          &InstanceTagsData{Tags: map[string]string{"Name": "example"}, TagKeys: []string{"old"}},
 		InstanceMonitoringData:    &InstanceMonitoringData{Enabled: true},
-
-		InstanceMetadataOptionsData: &InstanceMetadataOptionsData{
-			HttpTokens:              "required",
-			HttpEndpoint:            "enabled",
-			HttpProtocolIpv6:        "disabled",
-			InstanceMetadataTags:    "enabled",
-			HttpPutResponseHopLimit: int64Ptr(3),
-		},
-		DisableApiTerminationData: &DisableApiTerminationData{Value: false},
 	}
 
 	data, err := json.Marshal(command)
@@ -64,9 +52,7 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"set_spot_lineage": true,
 			"set_instance_tags": true,
 			"remove_instance_tags": true,
-			"set_instance_monitoring": true,
-			"modify_instance_metadata_options": true,
-			"set_disable_api_termination": true
+			"set_instance_monitoring": true
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
@@ -75,62 +61,8 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
 		"spot_lineage_data": {"spot_instance_request_id": "sir-1"},
 		"instance_tags_data": {"tags": {"Name": "example"}, "tag_keys": ["old"]},
-		"instance_monitoring_data": {"enabled": true},
-		"instance_metadata_options_data": {
-			"http_tokens": "required",
-			"http_endpoint": "enabled",
-			"http_protocol_ipv6": "disabled",
-			"instance_metadata_tags": "enabled",
-			"http_put_response_hop_limit": 3
-		},
-		"disable_api_termination_data": {"value": false}
-	}`, string(data))
-}
-
-func int64Ptr(v int64) *int64 { return &v }
-
-// An owner that predates the metadata and termination-protection commands
-// sends and receives payloads without the new fields.
-func TestEC2InstanceCommandNewFieldsAreOptionalOnTheWire(t *testing.T) {
-	var n1 EC2InstanceCommand
-	require.NoError(t, json.Unmarshal([]byte(`{
-		"id": "i-0123456789abcdef0",
-		"attributes": {"stop_instance": true},
 		"instance_monitoring_data": {"enabled": true}
-	}`), &n1))
-	require.Equal(t, "i-0123456789abcdef0", n1.ID)
-	require.True(t, n1.Attributes.StopInstance)
-	require.False(t, n1.Attributes.ModifyInstanceMetadataOptions)
-	require.False(t, n1.Attributes.SetDisableApiTermination)
-	require.Nil(t, n1.InstanceMetadataOptionsData)
-	require.Nil(t, n1.DisableApiTerminationData)
-
-	unset, err := json.Marshal(EC2InstanceCommand{ID: "i-1", Attributes: EC2CommandAttributes{StopInstance: true}})
-	require.NoError(t, err)
-	require.NotContains(t, string(unset), "modify_instance_metadata_options")
-	require.NotContains(t, string(unset), "set_disable_api_termination")
-	require.NotContains(t, string(unset), "instance_metadata_options_data")
-	require.NotContains(t, string(unset), "disable_api_termination_data")
-}
-
-func TestInstanceMetadataOptionsDataOmitsUnsetFields(t *testing.T) {
-	data, err := json.Marshal(InstanceMetadataOptionsData{HttpPutResponseHopLimit: int64Ptr(1)})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"http_put_response_hop_limit": 1}`, string(data))
-
-	data, err = json.Marshal(InstanceMetadataOptionsData{})
-	require.NoError(t, err)
-	require.JSONEq(t, `{}`, string(data))
-}
-
-func TestDisableApiTerminationDataKeepsExplicitFalse(t *testing.T) {
-	data, err := json.Marshal(DisableApiTerminationData{Value: false})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"value": false}`, string(data))
-}
-
-func TestHandledHeaderName(t *testing.T) {
-	require.Equal(t, "Spinifex-Ec2-Cmd-Handled", HandledHeader)
+	}`, string(data))
 }
 
 func TestInstanceCommandSubjects(t *testing.T) {

contracts/lambda/v1/README.md

@@ -21,16 +21,8 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 **Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
-ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/3` on both host and guest, whose first frame carries the execution-role credential and function environment, so a test-link end and a release end fail the handshake; release builds are unchanged.
-In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/4`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
-On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
-Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
-The credential and environment travel as their own frame of at most 40 KiB, and the reply has the same payload and frame bounds as the request.
-The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
-In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
-Release builds never leave `Pending`.
-The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=true`; this profile does not meet Q-199 fail-closed revocation, and its guests take management addresses whose record can expire (mulga-0hsub).
 
 **Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
 
@@ -44,8 +36,6 @@ The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=tr
 
 The guest's own interface settings come from `opt/spinifex/netcfg`, the entry other system VMs use.
 The agent unloads `qemu_fw_cfg` even when an entry is missing, and refuses to start if the fw_cfg tree is still readable afterwards.
-On a VMM without fw_cfg (the test-link Firecracker executor), the executor attaches the same entries as a tar archive on the second, writable virtio-blk disk, laid out as `by_name/<entry>/raw`.
-The runtime image's init extracts it to a root-only tmpfs at `/spinifex-cfg`, and the agent reads the entries there, then overwrites the whole disk with zeros and removes the tree; it refuses to start if the tree is still present.
 Until the message format exists, the agent authenticates and holds the connection but accepts no host message and starts no user code.
 
 **Authority:** one environment's attempt only.

docs/COMMANDS.md

@@ -19,7 +19,7 @@ Every service is `PartOf=spinifex.target` and ordered by its own `After=`/`Wants
 | `spinifex-predastore.service` | `spx service predastore start` | S3-compatible object storage, serving that host's nodes and the S3 gate among them |
 | `spinifex-northstar.service` | `spx service northstar start` | Authoritative DNS for `*.spx3.net`, recursive via upstream forwarders, syncing zones from its S3 bucket |
 | `spinifex-viperblock.service` | `spx service viperblock start` | Block storage with the NBD plugin. `KillMode=process`: nbdkit serving a guest survives a restart of this unit |
-| `spinifex-daemon.service` | `spx service spinifex start` | VM orchestration, NATS subscriptions, health endpoint. Two `ExecStartPre` steps run first: `spx admin hostnet check`, and `spx admin node-facts collect`, which runs as root and whose failure does not stop the daemon. `KillMode=process`: guest QEMU survives a restart and the new daemon reattaches. `SIGHUP` reloads config |
+| `spinifex-daemon.service` | `spx service spinifex start` | VM orchestration, NATS subscriptions, health endpoint. `KillMode=process`: guest QEMU survives a restart and the new daemon reattaches. `SIGHUP` reloads config |
 | `spinifex-awsgw.service` | `spx service awsgw start` | AWS-compatible gateway: SigV4 auth, IAM policy enforcement, TLS. Also serves the Private Admin API |
 | `spinifex-vpcd.service` | `spx service vpcd start` | VPC daemon: translates `vpc.*` NATS events to OVN logical switches, ports and routers |
 | `spinifex-ui.service` | `spx service spinifex-ui start` | Web UI serving the React frontend |
@@ -129,7 +129,6 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. On x86_64 it also checks that the QEMU binary the daemon will run lists the `microvm` machine type and that the microvm firmware `/usr/share/spinifex/microvm/qboot.rom` is installed, since load balancers boot on both. It runs the host network check `spinifex-daemon` and `spinifex-vpcd` run before they start, and fails while firewalld or ufw is active, since the Spinifex nftables policy is the only supported node firewall. It reports a Warning, without failing, when systemd-resolved is active but `/etc/resolv.conf` is not its stub, since lookups then bypass the route that sends Spinifex names to northstar. Exits non-zero if any asset is Missing, Stale, Ungranted, Unsupported or Failed. |
 | `spx admin node js-probe` | `--timeout` (default: 10s) | Round-trips a canary key through the cluster-state KV bucket and exits non-zero if the write or the read-back fails. JetStream can latch a write error per message block and keep serving reads — nats-server's filestore records the failure on the block and returns it on every later write, and nothing clears it short of a restart. The HTTP health endpoint reports the server-wide JetStream state, which that latch never touches, so only an actual write observes it. |
-| `spx admin node-facts collect` | — | Runs `dmidecode`, `ethtool`, `udevadm`, `networkctl lldp` and `systemd-detect-virt`, and writes their raw output to `/etc/spinifex/node-facts/`, readable by the spinifex group. `spinifex-daemon.service` runs it as a root `ExecStartPre` because the daemon itself is unprivileged; the daemon parses the files when the operator console asks. A probe that fails writes no new file, so a file from an earlier successful run can remain and the console reads it as available; a source with no file is shown as unavailable. |
 | `spx admin banner` | `--boot-check` | Writes the node information banner to `/etc/issue` (shown before login on the physical/serial console) and `/etc/motd` (shown after SSH login). With `--boot-check` it also detects a management IP that has changed since last boot and updates `/etc/spinifex/node.conf` accordingly. |
 
 ### Storage Inspection
@@ -137,7 +136,6 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Flags | Description |
 |---------|-------|-------------|
 | `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. It also lists UEFI variable stores nothing owns: a `uefi/i-*/` prefix with no instance record and no store record, a `uefi/ami-*/` prefix with no AMI document, and a store record that is not `deleting` and has no instance record. These are reported and never deleted, because an absent instance record is not proof of a terminate. |
-| `spx admin volumes obligations` | — | Lists the open unpublish obligations, oldest first: a volume export that a force detach, or a terminate whose unmount did not complete, left on the node that held the attachment. Each row names the volume, the node, the instance and the age, plus attempts and the last error once the node has tried. The node settles each one once it answers; an obligation stays open while its node is down and closes without the node only once the node has been removed from the cluster. **Reports only.** |
 | `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is refused while that node is alive (its daemon heartbeat is under 60 seconds old, or cannot be read) and is otherwise allowed, in which case it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
 | `spx admin storage status` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Sends the signed `GetStorageStatus` action to the AWS gateway with credentials from the standard AWS chain (e.g. `AWS_PROFILE=spinifex`) and prints what the console's Service Metrics page shows: predastore encoding, meta and blob node counts, cluster health, a meta node table (health, raft state, leader, term, commit and applied index) and the blob nodes. Cluster health is `Degraded` when any meta node is unreachable or the nodes reporting raft state do not agree on one leader. A meta node read only through its host's `/readyz` shows healthy with blank raft fields. |
 
@@ -157,9 +155,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS credential chain: on a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command fails to find credentials), or equivalent credentials; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
 | `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
 | `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
-| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--non-finalizing` (hidden), `--yes` (requires `--name`), `--name` (account name, must match the stored record), `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Always runs through `POST /admin/DeleteAccount` over SigV4 and follows `DescribeAccountDeletion` to completion; the CLI never tears an account down in its own process, so a CLI delete and an API delete cannot race. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
+| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record), `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Always runs through `POST /admin/DeleteAccount` over SigV4 and follows `DescribeAccountDeletion` to completion; the CLI never tears an account down in its own process, so a CLI delete and an API delete cannot race. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
 | `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
-| `spx admin account delete --non-finalizing` | as above | An exercise run, not a finalising delete, but the cleanup is real. Marks the account `TERMINATING`, deletes the resources of each EC2 platform kind the evaluation found quiesced (for example key pairs, empty placement groups and orphaned tags), re-reads the owner inventory against that cleanup, and ends `EXERCISED` with the account still `TERMINATING`. No other stage runs and the account record, name reservation and quota are left in place. It refuses `--force` and `--dry-run`. |
 | `spx admin account delete --remote` | as above | Accepted for existing scripts and ignored: every delete already goes through the admin API. |
 | `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method — `CreateAccount`, `DeleteAccount`, `DescribeAccountDeletion`, `GetAccountQuota`, `ListAccounts`, `PutAccountQuota`) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
 | `spx admin principal list` | — | Lists every principal in the super-admin account with its access-key count and what it may call. An attached policy is reported by name (`AdministratorAccess (attached)`) rather than expanded, which is how an unscoped principal is spotted. |

docs/DESIGN.md

@@ -112,8 +112,6 @@ When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
 2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-   A UEFI instance whose record carries a variable store pointer has no EFI volume: its `vars.fd` is fetched from the object under `uefi/` in the configured Predastore bucket that the pointer names, recorded in the `spinifex-uefi-stores` JetStream bucket (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a record with both or neither is refused.
-   RunInstances still creates the EFI volume, nothing writes `vars.fd` back to the object store yet, and a terminated instance's store is removed by a cluster-wide reaper.
 3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
@@ -227,9 +225,6 @@ It may keep writing for 40 seconds after the last write the store acknowledged,
 A slow acknowledgement therefore shortens the remaining validity rather than extending it, and an acquisition acknowledged after its validity has already run out is refused as retryable.
 
 For a holder that cannot renew, a timer is armed for the validity deadline, leaving a nominal 5 seconds before the entry can expire; when it fires, the export's nbdkit is fenced with SIGKILL (`volume_fence.go`).
-The kill comes first: the read that names the winner (bounded at 2 seconds) and the notice to the daemon follow it, so a hung lease store cannot hold the writer alive.
-The notice is a request that viperblockd repeats every 2 seconds until the local daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting.
-On the notice the daemon moves the guest using the volume to `error` with state reason `Server.VolumeFenced` and SIGKILLs its QEMU immediately rather than shutting it down gracefully, because there is no disk left to flush to; it only stops the guest process and removes its taps, and never terminates the instance or deletes a volume.
 The fence runs on its own goroutine, so Go scheduling, that goroutine and process termination all delay the kill, and enough delay can use up the whole margin; a kill held in uninterruptible sleep, which the fence waits up to 30 seconds for, is the extreme case.
 Once a lease is lost or has lapsed it stays lost: a renewal that succeeds afterwards is ignored, and another open on the same node cannot share it while the fence runs.
 

docs/compute/vpc-networking/README.md

@@ -315,7 +315,7 @@ prefix_len = 24
 
 ### Host access to instances (jumpbox pattern)
 
-The spinifex host automatically reaches every instance's **private IP**: each node converges a host route into OVN (`<vpc-cidr> via <gateway-transit-ip> dev spx-nat-host`) for every attached VPC, and the IGW attach exempts the transit net from SNAT, so replies to host-initiated connections come back un-NATted. No per-instance setup. This only exists in routed NAT mode; other modes install no host routes. Every node runs this loop itself, on each intent change, on a nudge from the local IGW hooks and every 30 seconds, so a node that missed the attach event still gets the route. A route is removed on every node when no attached VPC holds its CIDR any more, such as after a detach. When several attached VPCs share one CIDR, the bootstrap VPC wins, then the lowest VPC ID, so detaching one gateway repoints the route to another attached VPC instead of removing it.
+The spinifex host automatically reaches every instance's **private IP**: IGW attach installs a host route into OVN (`<vpc-cidr> via <gateway-transit-ip> dev spx-nat-host`) and exempts the transit net from SNAT, so replies to host-initiated connections come back un-NATted. No per-instance setup.
 
 Security groups still apply and the default SG is closed to the host, same as AWS — open SSH/ICMP from the transit net first:
 

docs/coverage/ec2/README.md

@@ -43,21 +43,6 @@ A missing `Name`, `BlockDeviceMappings` or root snapshot answers `MissingParamet
 Registering with `--ena-support`, `--no-ena-support` and neither, copying the image, and three refusals were run on a three-node cluster with the AWS CLI.
 Launching an instance from an `EnaSupport` image was not run, so the launch path is unchanged by construction and not by test.
 
-### CreateSnapshot Error Mapping
-
-`CreateSnapshot` drains and snapshots the volume inside the EBS provider.
-A provider that cannot snapshot a volume because it is in use, as the `qemunbd` provider cannot for an attached volume, answers `ErrVolumeInUse`, which becomes `IncorrectState` with the message `The volume 'vol-...' is in use. Detach it before creating a snapshot.`
-The `viperblockd` provider drains a mounted export and snapshots it, so an attached volume succeeds there.
-A provider that cannot answer now, for example because another node holds the volume, answers `ServiceUnavailable` so the caller retries.
-Any other provider failure is `ServerInternal`.
-The snapshot taken for `CreateImage` maps these errors the same way.
-
-### EBS Encryption by Default Follows the Platform
-
-Whether new volumes are encrypted is decided by the platform's encryption key, not by the account.
-`GetEbsEncryptionByDefault` and `EnableEbsEncryptionByDefault` report `true` when the platform has an encryption key configured and `false` otherwise, and `EnableEbsEncryptionByDefault` does not change that.
-`DisableEbsEncryptionByDefault` is refused with `OperationNotPermitted` and the message `EBS encryption by default cannot be disabled: this platform encrypts every volume` while the platform encrypts, and answers `false` where it does not.
-
 ### Operations
 
 | Operation |

docs/security/network-connections/README.md

@@ -99,7 +99,7 @@ behavior in other words.
 | 4432 | Formation server | HTTPS | Cluster (bootstrap only) | Cluster join coordination; active only while a join token is valid. Binds the node's `--bind` (lan) address. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
 | 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
 | 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
-| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client is refused unless the target address is the one the named target node reports on that plane |
+| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client may only target its own plane subnet |
 | 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
 | 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
 | 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |

docs/testing/e2e-coverage.md

@@ -108,8 +108,6 @@ Three entry points, in the order they run:
 
 **Its own suite and its own cell, for instancerecovery's reasons plus one.** It takes a node away, so anything sharing the environment reports that outage as its own failure; it needs its own VPC, public subnet and a backend placement no shared fixture describes; and a failure here has to read as "the load balancer did not survive its host" rather than as one more red subtest in a suite about load balancer features.
 
-**A dead data NIC is the other half, and there the state does change.** `LBSelfHealsADeadDataNIC` disables one data NIC of a running load balancer guest through a second QMP client and runs for an internet-facing NLB and an internal ALB. The backends and an unrelated customer instance in the same subnet are launched more than 21 minutes before the fault, so a repair that reached a customer instance would find one the platform has long stopped expecting to change. A state watch samples `DescribeLoadBalancers` every two seconds and requires the sequence `active`, `active_impaired`, `active` with nothing in between, while the internet-facing run also watches the name continuously and serves traffic through it within five minutes. Afterwards the guest, its interfaces and its address are unchanged but its process is not, the backends and the bystander keep their instance ID, launch time and process (ADR-0003:S7), and the daemon journals exactly one `ELBv2 data-path repair started` line carrying the fields an operator needs. The internal ALB has no client inside the VPC, so it asserts the state sequence and target health rather than traffic through its name.
-
 ### `iam` — everything IAM and STS
 
 Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
@@ -131,7 +129,7 @@ The client reports on its serial console because the runner has no route into th
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. The private-endpoint NIC of the cluster's endpoint NLB is disabled through QMP and every worker must be Ready again within ten minutes while the cluster stays `ACTIVE` and the NLB keeps its guest instance. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 

internal/awsmodel/ec2error_test.go

@@ -77,8 +77,8 @@ func TestEC2ErrorCatalogMetadataAndClasses(t *testing.T) {
 	if catalog.metadata.VerifiedOn == "" {
 		t.Fatal("verifiedOn is empty")
 	}
-	if len(catalog.codes) != 116 {
-		t.Fatalf("catalog code count = %d, want 116", len(catalog.codes))
+	if len(catalog.codes) != 115 {
+		t.Fatalf("catalog code count = %d, want 115", len(catalog.codes))
 	}
 	if catalog.codes["InvalidParameterValue"] != ec2ErrorClassClient {
 		t.Fatal("InvalidParameterValue is not classified as client")

internal/awsmodel/models/ec2/error-codes.json

@@ -1,6 +1,6 @@
 {
   "source": "https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html",
-  "verifiedOn": "2026-11-04",
+  "verifiedOn": "2026-08-05",
   "commonClient": [
     "AuthFailure",
     "Blocked",
@@ -95,7 +95,6 @@
     "InvalidVolumeID.Malformed",
     "InvalidVpcID.NotFound",
     "InvalidVpcRange",
-    "NatGatewayNotFound",
     "OperationNotPermitted",
     "ReservationCapacityExceeded",
     "Resource.AlreadyAssociated",

internal/gwsign/signer.go

@@ -15,7 +15,6 @@ import (
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
@@ -58,9 +57,8 @@ func NewInstanceRole() *Signer {
 }
 
 // newInstanceRole reads from the IMDS root base; "" means the default endpoint.
-// optFns adjust the SDK IMDS client, e.g. its retryer; none keeps the defaults.
-func newInstanceRole(base string, optFns ...func(*imds.Options)) *Signer {
-	return &Signer{provider: aws.NewCredentialsCache(imdscreds.NewProvider(nil, base, optFns...))}
+func newInstanceRole(base string) *Signer {
+	return &Signer{provider: aws.NewCredentialsCache(imdscreds.NewProvider(nil, base))}
 }
 
 // EnsureCredentials retrieves once from the provider, surfacing the error so a

internal/gwsign/signer_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/mulgadc/bluebottle/pkg/sigv4"
 )
 
@@ -125,41 +124,27 @@ func TestNewIMDS_BuildsProvider(t *testing.T) {
 	}
 }
 
-// imdsStub serves one instance-role credential over IMDSv2. It echoes the token
-// TTL header as real IMDS does, so the SDK client caches the token rather than
-// re-fetching it, through the retry backoff, on every later call.
-func imdsStub(t *testing.T, accessKey, expiration string) *httptest.Server {
-	t.Helper()
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.URL.Path {
-		case "/latest/api/token":
-			w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-			_, _ = w.Write([]byte("token"))
-		case "/latest/meta-data/iam/security-credentials/":
-			_, _ = w.Write([]byte("sqsWorkerRole"))
-		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
-			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + accessKey + `","SecretAccessKey":"s","Token":"t","Expiration":"` + expiration + `"}`))
-		default:
-			http.NotFound(w, r)
-		}
-	}))
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-// singleAttempt disables IMDS client retries so an unreachable endpoint fails
-// on the first attempt instead of after the SDK's 1s-per-retry backoff.
-func singleAttempt(o *imds.Options) { o.Retryer = aws.NopRetryer{} }
-
 // The instance-role signer ignores environment keys and signs only with the
 // credentials the metadata service issues.
 func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	t.Setenv("AWS_ACCESS_KEY_ID", testAccessKey)
 	t.Setenv("AWS_SECRET_ACCESS_KEY", testSecretKey)
 	const imdsKey = "ASIAIMDSONLYEXAMPLE1"
-	srv := imdsStub(t, imdsKey, "2099-01-01T00:00:00Z")
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			_, _ = w.Write([]byte("token"))
+		case "/latest/meta-data/iam/security-credentials/":
+			_, _ = w.Write([]byte("sqsWorkerRole"))
+		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
+			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + imdsKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2099-01-01T00:00:00Z"}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(srv.Close)
 
-	s := newInstanceRole(srv.URL, singleAttempt)
+	s := newInstanceRole(srv.URL)
 	req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
 	if err := s.Sign(req, emptySHA, testService, testRegion); err != nil {
 		t.Fatal(err)
@@ -169,7 +154,7 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	}
 
 	srv.Close()
-	cold := newInstanceRole(srv.URL, singleAttempt)
+	cold := newInstanceRole(srv.URL)
 	if err := cold.EnsureCredentials(context.Background()); err == nil {
 		t.Fatal("environment keys were used when IMDS was unreachable")
 	}
@@ -179,8 +164,19 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 // IMDS must fail the sign rather than reuse the expired key with a new expiry.
 func TestNewInstanceRole_FailedRefreshErrorsInsteadOfStaleKey(t *testing.T) {
 	const staleKey = "ASIASTALEEXAMPLE0001"
-	srv := imdsStub(t, staleKey, "2000-01-01T00:00:00Z")
-	s := newInstanceRole(srv.URL, singleAttempt)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			_, _ = w.Write([]byte("token"))
+		case "/latest/meta-data/iam/security-credentials/":
+			_, _ = w.Write([]byte("sqsWorkerRole"))
+		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
+			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + staleKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2000-01-01T00:00:00Z"}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	s := newInstanceRole(srv.URL)
 	if err := s.EnsureCredentials(t.Context()); err != nil {
 		t.Fatalf("first retrieve: %v", err)
 	}

internal/imdscreds/imdscreds.go

@@ -36,10 +36,10 @@ func Fetch(client *http.Client, base string) (aws.Credentials, error) {
 }
 
 // NewProvider builds an ec2rolecreds provider against base, an IMDS root URL
-// such as http://169.254.169.254/latest. client and optFns are passed to
-// NewClient.
-func NewProvider(client *http.Client, base string, optFns ...func(*imds.Options)) aws.CredentialsProvider {
-	imdsClient := NewClient(client, base, optFns...)
+// such as http://169.254.169.254/latest. A nil client uses the SDK IMDS
+// client's default HTTP client.
+func NewProvider(client *http.Client, base string) aws.CredentialsProvider {
+	imdsClient := NewClient(client, base)
 	return noExtendOnFailure{provider: ec2rolecreds.New(func(o *ec2rolecreds.Options) {
 		o.Client = imdsClient
 	})}
@@ -71,15 +71,16 @@ func (p noExtendOnFailure) AdjustExpiresBy(creds aws.Credentials, dur time.Durat
 	return p.provider.AdjustExpiresBy(creds, dur)
 }
 
-// NewClient builds an SDK IMDS client against base, an IMDS root URL with or
-// without a trailing /latest. A nil client uses the SDK's default HTTP client;
-// optFns adjust the options after the defaults here, as imds.New applies them.
-func NewClient(client *http.Client, base string, optFns ...func(*imds.Options)) *imds.Client {
+// NewClient builds an SDK IMDS client against base, an IMDS root URL such as
+// http://169.254.169.254/latest. A trailing /latest is stripped since the
+// client appends its own; both forms of base keep working. A nil client uses
+// the SDK IMDS client's default HTTP client.
+func NewClient(client *http.Client, base string) *imds.Client {
 	opts := imds.Options{Endpoint: rootEndpoint(base)}
 	if client != nil {
 		opts.HTTPClient = client
 	}
-	return imds.New(opts, optFns...)
+	return imds.New(opts)
 }
 
 // rootEndpoint strips a trailing /latest from base so it can be used as the

internal/testkit/nats.go

@@ -27,7 +27,8 @@ func StartTestNATS(t testing.TB) (*server.Server, *nats.Conn) {
 	}
 	ns, err := server.NewServer(opts)
 	require.NoError(t, err)
-	startServer(t, ns)
+	go ns.Start()
+	waitReadyForConnections(t, ns)
 	t.Cleanup(func() { ns.Shutdown() })
 
 	nc, err := nats.Connect(ns.ClientURL())
@@ -36,14 +37,17 @@ func StartTestNATS(t testing.TB) (*server.Server, *nats.Conn) {
 	return ns, nc
 }
 
-// startServer runs Start in this goroutine: it returns once JetStream is
-// enabled and the client listener is bound, so the readiness check passes on
-// its first look instead of spinning or sleeping while the server comes up.
-func startServer(t testing.TB, ns *server.Server) {
+// waitReadyForConnections spins until the server accepts connections. Calling
+// ReadyForConnections with a longer timeout makes it sleep 25ms per attempt,
+// which every embedded server in the suite would otherwise pay.
+func waitReadyForConnections(t testing.TB, ns *server.Server) {
 	t.Helper()
-	ns.Start()
-	if !ns.ReadyForConnections(5 * time.Second) {
-		t.Fatal("embedded NATS server not ready for connections within 5s")
+	deadline := time.Now().Add(5 * time.Second)
+	for !ns.ReadyForConnections(time.Millisecond) {
+		if time.Now().After(deadline) {
+			t.Fatal("embedded NATS server not ready for connections within 5s")
+		}
+		time.Sleep(200 * time.Microsecond)
 	}
 }
 
@@ -62,7 +66,8 @@ func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, jetstream.Jet
 	}
 	ns, err := server.NewServer(opts)
 	require.NoError(t, err)
-	startServer(t, ns)
+	go ns.Start()
+	waitReadyForConnections(t, ns)
 	t.Cleanup(func() { ns.Shutdown() })
 
 	nc, err := nats.Connect(ns.ClientURL())

scripts/README.md

@@ -98,10 +98,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
 | `build-system-image.sh` | Builds a minimal system AMI (Alpine or Ubuntu) from a manifest. All customisation happens inside the libguestfs appliance, so the build touches no host block device, no host mount, and needs no sudo. `--import` also registers it as an AMI. |
 | `publish-system-image.sh` | Converts a built raw image to compressed qcow2, writes the `.sha256` sidecar the catalog verifier expects, and uploads both to the R2 bucket behind `iso.mulgadc.com/system-ami/`. |
 | `build-microvm-image.sh` | Builds the minimal Alpine kernel + initramfs used by the QEMU microvm machine type and the lb-agent, carrying haproxy (ALB) and nginx-stream (NLB) data planes. |
-| `build-lambda-runtime-microvm-image.sh` | Builds the Lambda runtime microVM kernel and initramfs: a guest running the Lambda guest agent (`bin/lambda-agent` by default) with virtio-net for its host control connection. Takes an optional agent binary and output directory (default `build/microvm/lambda-runtime`). Not shipped. |
-| `build-sqs-store-microvm-image.sh` | Builds the SQS microVM kernel and initramfs: one guest running the dedicated SQS `nats-server` and the SQS worker as separate processes, with ext4 support and a clean power-down. The guest never formats a disk. Output goes to `build/microvm/sqs-worker`. |
-| `lib/microvm-image.sh` | Shared steps for the two builders above, sourced rather than run: tool checks, the Alpine rootfs, and module stripping that keeps `virtio*`, `*fw_cfg*` and the modules the caller names with their dependencies. |
-| `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. A host whose `/usr/share/qemu/qboot.rom` (`qemu-system-data`) has that hash installs it and skips the container; any other host compiles. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
+| `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
 | `mkosi-build.sh` | Runs an mkosi system-image build inside the pinned builder container, so a runner needs only Docker and the toolchain cannot drift. `--image` names an output and expands to the right profile list. |
 | `mkosi-publish.sh` | Publishes an mkosi-built image to R2 from inside the same pinned container — qcow2 conversion, `.sha256` sidecar, and upload. |
 | `mkosi-common.sh` | Shared plumbing sourced by both mkosi scripts. Builds the toolchain image once, with the builder account matched to the invoking user so bind-mounted output is not written back as root. |

scripts/build-lambda-runtime-microvm-image.sh

@@ -3,28 +3,24 @@
 # + initramfs: a guest running the Lambda guest agent, with virtio-net for its
 # host control connection. It is not shipped.
 #
-# Usage: build-lambda-runtime-microvm-image.sh [agent-binary [output-dir]]
-#
 # Inputs:
 #   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
-#   agent binary, default $REPO_ROOT/bin/lambda-agent (make bin/lambda-agent)
+#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
 #
-# Outputs, in output-dir (default $REPO_ROOT/build/microvm/lambda-runtime):
-#   vmlinuz
-#   initramfs.cpio.gz
+# Outputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
+#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
 #
 # Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
 # shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-INPUT_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
-INIT_SH="$INPUT_DIR/init.sh"
-INITTAB="$INPUT_DIR/inittab"
-RUN_AGENT="$INPUT_DIR/run-agent"
-LAMBDA_AGENT_BIN="${1:-$REPO_ROOT/bin/lambda-agent}"
-# The library stages beside BUILD_DIR, so it names the output, not the inputs.
-BUILD_DIR="${2:-$INPUT_DIR}"
+BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+RUN_AGENT="$BUILD_DIR/run-agent"
+LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
 TAG="[build-lambda-runtime-microvm-image]"
 
 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
@@ -42,8 +38,7 @@ MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
 source "$REPO_ROOT/scripts/lib/microvm-image.sh"
 
 echo "$TAG repo root: $REPO_ROOT"
-echo "$TAG input dir: $INPUT_DIR"
-echo "$TAG output dir: $BUILD_DIR"
+echo "$TAG build dir: $BUILD_DIR"
 
 microvm_check_tools
 
@@ -62,9 +57,6 @@ if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
     exit 1
 fi
 
-# Absolute, because packing the initramfs runs from inside the chroot.
-mkdir -p "$BUILD_DIR"
-BUILD_DIR=$(cd "$BUILD_DIR" && pwd)
 microvm_init_dirs
 microvm_make_rootfs
 microvm_empty_dev

scripts/build-lambda-runtime-microvm-image_test.sh

@@ -1,81 +0,0 @@
-#!/bin/sh
-# build-lambda-runtime-microvm-image_test.sh — a build into a chosen output
-# directory stages and publishes there, and leaves the default input directory
-# untouched.
-#
-# The script runs in a fixture repo against the real library, with only the
-# rootfs, kernel and packing steps replaced so no image is downloaded.
-set -eu
-
-SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-echo "-- build-lambda-runtime-microvm-image.sh output directory"
-
-REPO="$TMP/repo"
-IN="$REPO/build/microvm/lambda-runtime"
-mkdir -p "$REPO/scripts/lib" "$IN" "$REPO/bin" "$TMP/bin"
-cp "$SCRIPTS_DIR/build-lambda-runtime-microvm-image.sh" "$REPO/scripts/"
-for f in init.sh inittab run-agent; do printf '%s\n' "$f" > "$IN/$f"; done
-printf 'default image\n' > "$IN/vmlinuz"
-printf '#!/bin/sh\n' > "$REPO/bin/agent-testlink"
-chmod +x "$REPO/bin/agent-testlink"
-printf '#!/bin/sh\necho "ELF 64-bit, statically linked"\n' > "$TMP/bin/file"
-chmod +x "$TMP/bin/file"
-BEFORE=$(cd "$IN" && ls -A | cksum)
-
-# The real library, with the steps that need a rootfs or kernel replaced.
-# Packing runs from inside the chroot as the real step does, and records where
-# the stage is while it still exists.
-cat > "$REPO/scripts/lib/microvm-image.sh" <<EOF
-. "$SCRIPTS_DIR/lib/microvm-image.sh"
-microvm_check_tools() { :; }
-microvm_make_rootfs() { mkdir -p "\$CHROOT_DIR/etc"; }
-microvm_empty_dev() { :; }
-microvm_locate_kernel() { :; }
-microvm_strip_modules() { :; }
-microvm_strip_rootfs() { :; }
-microvm_stage_kernel() { printf 'kernel\n' > "\$STAGE_DIR/vmlinuz"; }
-microvm_pack_initramfs() {
-    (cd "\$CHROOT_DIR" && printf 'initramfs\n' | gzip > "\$STAGE_DIR/initramfs.cpio.gz")
-    dirname "\$STAGE_DIR" > "$TMP/stage-parent"
-    ls -A "$IN" > "$TMP/input-during"
-}
-EOF
-
-# The output is given relative to the repo root, as make passes it.
-OUT="$REPO/build/microvm/lambda-testlink-runtime"
-if ! (cd "$REPO" && PATH="$TMP/bin:$PATH" bash scripts/build-lambda-runtime-microvm-image.sh bin/agent-testlink build/microvm/lambda-testlink-runtime) > "$TMP/log" 2>&1; then
-    fail "build into output dir: $(tr '\n' ' ' < "$TMP/log")"
-else
-    if [ "$(cat "$TMP/stage-parent")" = "$OUT" ]; then
-        pass "stage created in the output directory"
-    else
-        fail "stage created in $(cat "$TMP/stage-parent"), want $OUT"
-    fi
-    if grep -q '^\.stage\.' "$TMP/input-during"; then
-        fail "stage present in the default input directory during the build"
-    else
-        pass "no stage in the default input directory during the build"
-    fi
-    if [ "$(cd "$IN" && ls -A | cksum)" = "$BEFORE" ] && grep -qx 'default image' "$IN/vmlinuz"; then
-        pass "default input directory unchanged"
-    else
-        fail "default input directory changed: $(cd "$IN" && ls -A | tr '\n' ' ')"
-    fi
-    if grep -qx kernel "$OUT/vmlinuz" && gzip -t "$OUT/initramfs.cpio.gz" && [ "$(ls -A "$OUT" | wc -l)" -eq 2 ]; then
-        pass "output directory holds exactly the published pair"
-    else
-        fail "output directory: $(ls -A "$OUT" | tr '\n' ' ')"
-    fi
-fi
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi

scripts/build-microvm-image.sh

@@ -209,7 +209,6 @@ echo "[build-microvm-image] nginx: binary + stream module present"
 #   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
 #   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
 #   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
-#   af_packet.ko*  — AF_PACKET sockets (lb-agent's per-NIC gateway ARP probe)
 # All other loadable modules are removed. Drivers compiled into the kernel
 # (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
 echo "[build-microvm-image] stripping kernel modules to virtio+fw_cfg only..."
@@ -223,7 +222,6 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     find "$kernel_dir" -name "virtio*.ko.gz" -o -name "virtio*.ko" \
                        -o -name "*fw_cfg*.ko.gz" -o -name "*fw_cfg*.ko" \
                        -o -name "*failover*.ko.gz" -o -name "*failover*.ko" \
-                       -o -name "af_packet.ko.gz" -o -name "af_packet.ko" \
         2>/dev/null | while read -r mod; do
         rel="${mod#"$kernel_dir/"}"
         dest_dir="$tmp_keep/$(dirname "$rel")"
@@ -242,15 +240,6 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     done)
     rm -rf "$tmp_keep"
 
-    # lb-agent's data-path probe needs AF_PACKET; without it every LB would
-    # report a dead NIC. Accept a kernel with CONFIG_PACKET built in.
-    if ! find "$kernel_dir" -name "af_packet.ko*" | grep -q .; then
-        if ! grep -q "CONFIG_PACKET=y" "$KERNEL_CONFIG" 2>/dev/null; then
-            echo "ERROR: af_packet.ko missing from the kept modules — lb-agent's data-path probe cannot open its socket" >&2
-            exit 1
-        fi
-    fi
-
     # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
     # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
     # produces an EMPTY modules.dep from compressed modules — the guest's

scripts/build-qboot.sh

@@ -9,8 +9,7 @@
 #
 # The build runs in a digest-pinned Debian container, so its output is
 # reproducible and is checked against a recorded hash. That hash is also the
-# sha256 of Debian 13's /usr/share/qemu/qboot.rom, which is built from the same
-# commit, so a host whose copy matches installs it and skips the container.
+# sha256 of Debian 13's /usr/share/qemu/qboot.rom, which is built from the same commit.
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -21,13 +20,18 @@ QBOOT_REPO="${QBOOT_REPO:-https://gitlab.com/qemu-project/qboot.git}"
 QBOOT_COMMIT="8ca302e86d685fa05b16e2b208888243da319941"
 QBOOT_SHA256="5c9c4b12c4438e3cf11d087263afe6519c8c05dd5896727b8b169b8f5dbf9877"
 BUILD_IMAGE="debian:trixie@sha256:4c25275446ca371766b8b093c696782eabce06263fccacb8d5d813b3dc44b308"
-# Debian 13's qemu-system-data ships this exact ROM, so a host that has it
-# skips the container. QBOOT_HOST_ROM moves the path for tests.
-QBOOT_HOST_ROM="${QBOOT_HOST_ROM:-/usr/share/qemu/qboot.rom}"
 
 log() { echo "[build-qboot] $*"; }
 fail() { echo "ERROR: $*" >&2; exit 1; }
 
+CONTAINER_TOOL=""
+for tool in docker podman; do
+    if command -v "$tool" >/dev/null 2>&1 && "$tool" info >/dev/null 2>&1; then
+        CONTAINER_TOOL=$tool
+        break
+    fi
+done
+[ -n "$CONTAINER_TOOL" ] || fail "qboot builds in a container; install docker or podman"
 command -v git >/dev/null 2>&1 || fail "required tool not found: git"
 
 WORK=$(mktemp -d)
@@ -44,42 +48,19 @@ mkdir -p "$SRC_DIR"
 src_tarball="$SRC_DIR/qboot-${QBOOT_COMMIT}.tar.gz"
 git -C "$WORK/src" archive --format=tar.gz --prefix="qboot-${QBOOT_COMMIT}/" -o "$src_tarball" "$QBOOT_COMMIT"
 
-build_in_container() {
-    local CONTAINER_TOOL=""
-    for tool in docker podman; do
-        if command -v "$tool" >/dev/null 2>&1 && "$tool" info >/dev/null 2>&1; then
-            CONTAINER_TOOL=$tool
-            break
-        fi
-    done
-    [ -n "$CONTAINER_TOOL" ] || fail "qboot builds in a container; install docker or podman"
-    log "building in $BUILD_IMAGE with $CONTAINER_TOOL"
-    # The firmware comes back on stdout, so nothing the container writes is left
-    # root-owned on the host.
-    "$CONTAINER_TOOL" run --rm --platform linux/amd64 -v "$WORK/src:/src:ro" \
-        -e DEBIAN_FRONTEND=noninteractive "$BUILD_IMAGE" sh -euc '
-            {
-                apt-get update -qq
-                apt-get install -y -qq --no-install-recommends gcc libc6-dev meson ninja-build binutils
-                meson setup /tmp/build /src
-                ninja -C /tmp/build
-            } >/dev/null
-            cat /tmp/build/bios.bin
-        ' > "$WORK/qboot.rom"
-}
-
-host_sha=""
-[ -r "$QBOOT_HOST_ROM" ] && host_sha=$(sha256sum "$QBOOT_HOST_ROM" | cut -d' ' -f1)
-if [ "$host_sha" = "$QBOOT_SHA256" ]; then
-    log "using host ROM $QBOOT_HOST_ROM (sha256 matches)"
-    cp "$QBOOT_HOST_ROM" "$WORK/qboot.rom"
-elif [ -n "$host_sha" ]; then
-    log "host ROM $QBOOT_HOST_ROM has sha256 $host_sha, not $QBOOT_SHA256; building"
-    build_in_container
-else
-    log "no host ROM at $QBOOT_HOST_ROM; building"
-    build_in_container
-fi
+log "building in $BUILD_IMAGE with $CONTAINER_TOOL"
+# The firmware comes back on stdout, so nothing the container writes is left
+# root-owned on the host.
+"$CONTAINER_TOOL" run --rm --platform linux/amd64 -v "$WORK/src:/src:ro" \
+    -e DEBIAN_FRONTEND=noninteractive "$BUILD_IMAGE" sh -euc '
+        {
+            apt-get update -qq
+            apt-get install -y -qq --no-install-recommends gcc libc6-dev meson ninja-build binutils
+            meson setup /tmp/build /src
+            ninja -C /tmp/build
+        } >/dev/null
+        cat /tmp/build/bios.bin
+    ' > "$WORK/qboot.rom"
 
 got_sha=$(sha256sum "$WORK/qboot.rom" | cut -d' ' -f1)
 if [ "$got_sha" != "$QBOOT_SHA256" ]; then

scripts/build-qboot_test.sh

@@ -1,88 +0,0 @@
-#!/bin/sh
-# build-qboot_test.sh — a host ROM matching the recorded hash is installed
-# without the container; any other host ROM, or none, builds in it, and a
-# container ROM with the wrong hash still fails.
-set -eu
-
-SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-echo "-- build-qboot.sh host ROM"
-
-COMMIT=$(sed -n 's/^QBOOT_COMMIT="\(.*\)"$/\1/p' "$SCRIPTS_DIR/build-qboot.sh")
-printf 'pinned rom\n' > "$TMP/good.rom"
-printf 'other rom\n' > "$TMP/other.rom"
-GOOD_SHA=$(sha256sum "$TMP/good.rom" | cut -d' ' -f1)
-
-# git and docker stubs. git fakes a checkout of the pinned commit; docker
-# prints $STUB_ROM as the firmware and records that it ran.
-mkdir -p "$TMP/bin"
-cat > "$TMP/bin/git" <<STUB
-#!/bin/sh
-case "\$1" in
-init) mkdir -p "\$3"; printf 'GPL-2.0\n' > "\$3/LICENSE"; exit 0 ;;
-esac
-shift 2
-case "\$1" in
-rev-parse) echo "$COMMIT" ;;
-archive) while [ "\$1" != "-o" ]; do shift; done; printf 'source\n' > "\$2" ;;
-esac
-STUB
-cat > "$TMP/bin/docker" <<'STUB'
-#!/bin/sh
-[ "$1" = info ] && exit 0
-echo ran >> "$DOCKER_LOG"
-cat "$STUB_ROM"
-STUB
-chmod +x "$TMP/bin/git" "$TMP/bin/docker"
-
-# run <name> <host-rom> <container-rom>: build in a fresh fixture repo.
-run() {
-    repo="$TMP/$1"
-    mkdir -p "$repo/scripts"
-    sed "s/^QBOOT_SHA256=.*/QBOOT_SHA256=\"$GOOD_SHA\"/" "$SCRIPTS_DIR/build-qboot.sh" > "$repo/scripts/build-qboot.sh"
-    : > "$TMP/$1.docker"
-    PATH="$TMP/bin:$PATH" QBOOT_HOST_ROM="$2" STUB_ROM="$3" DOCKER_LOG="$TMP/$1.docker" \
-        bash "$repo/scripts/build-qboot.sh" > "$TMP/$1.log" 2>&1
-}
-
-# outputs <name>: the ROM is the pinned one and the licence and source exist.
-outputs() {
-    cmp -s "$TMP/$1/build/microvm/qboot.rom" "$TMP/good.rom" &&
-        [ -s "$TMP/$1/build/microvm/qboot.LICENSE" ] &&
-        [ -s "$TMP/$1/build/qboot/qboot-$COMMIT.tar.gz" ]
-}
-
-if run match "$TMP/good.rom" "$TMP/other.rom" && outputs match && [ ! -s "$TMP/match.docker" ]; then
-    pass "matching host ROM installed, container not run"
-else
-    fail "matching host ROM: $(tr '\n' ' ' < "$TMP/match.log")"
-fi
-
-if run mismatch "$TMP/other.rom" "$TMP/good.rom" && outputs mismatch && [ -s "$TMP/mismatch.docker" ] &&
-    grep -q 'has sha256' "$TMP/mismatch.log"; then
-    pass "mismatched host ROM builds in the container"
-else
-    fail "mismatched host ROM: $(tr '\n' ' ' < "$TMP/mismatch.log")"
-fi
-
-if run missing "$TMP/absent.rom" "$TMP/good.rom" && outputs missing && [ -s "$TMP/missing.docker" ]; then
-    pass "missing host ROM builds in the container"
-else
-    fail "missing host ROM: $(tr '\n' ' ' < "$TMP/missing.log")"
-fi
-
-if run badbuild "$TMP/absent.rom" "$TMP/other.rom"; then
-    fail "container ROM with the wrong hash was accepted"
-elif [ -e "$TMP/badbuild/build/microvm/qboot.rom" ]; then
-    fail "container ROM with the wrong hash was installed"
-else
-    pass "container ROM with the wrong hash fails and installs nothing"
-fi
-
-[ "$fails" -eq 0 ] || { echo "$fails failure(s)"; exit 1; }

scripts/install-node.sh

@@ -260,14 +260,6 @@ out() {
     ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@" 2>/dev/null || true
 }
 
-# remote_join_cmd prints the command that joins one node. The token is not in it: it arrives on
-# stdin and is read after sudo has logged its command, so sudo's log never holds it. The other
-# values are positional arguments of a literal script, never spliced into it.
-remote_join_cmd() {
-    local script='exec spx admin join --force --node "$1" --bind "$2" --cluster-bind "$2" --advertise "$3" --host "$4" --token "$(cat)" --region "$5" --az "$6"'
-    printf 'sudo bash -c %q _ %q %q %q %q %q %q' "$script" "$@"
-}
-
 SETUP_OVN="/usr/local/share/spinifex/setup-ovn.sh"
 
 # --- Preflight -------------------------------------------------------------
@@ -660,18 +652,14 @@ fi
 [ -n "$EXT_IFACE" ] && init_args+=(--external-iface="$EXT_IFACE")
 [ -n "$EMAIL" ] && init_args+=(--email="$EMAIL")
 
-# init prints the join token, so its log is root-only and outside the world-writable /tmp.
-INIT_LOG=/var/log/spx-init.log
-
-# redact_tokens hides join tokens in text about to be shown to the operator.
-redact_tokens() { sed -E 's/spx_join_[A-Za-z0-9_-]+/spx_join_[redacted]/g'; }
+INIT_LOG=/tmp/spx-init.log
 
 log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s) to join)"
 
 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid sh -c 'spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
     exit 0" || fail "${HOSTS[0]}: could not start spx admin init"
 
 TOKEN=""
@@ -687,7 +675,7 @@ else
         elapsed=$((elapsed + 5))
     done
     if [ -z "$TOKEN" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: spx admin init produced no join token within ${TOKEN_TIMEOUT}s"
     fi
     log "  join token issued"
@@ -703,9 +691,11 @@ fi
 # waiting on a shell that node 2 is still holding.
 JOIN_PIDS=()
 for i in $(seq 1 $((N - 1))); do
-    # The token goes to the node on stdin, so it is in no ssh argument and not in sudo's log.
-    # spx admin join takes it only as --token, so it is still in that process's arguments.
-    on "${HOSTS[$i]}" "$(remote_join_cmd "${NODE_NAMES[$i]}" "${LAN_IPS[$i]}" "${WAN_IPS[$i]}" "${LAN_IPS[0]}:$PORT" "$REGION" "$AZ")" <<<"$TOKEN" &
+    on "${HOSTS[$i]}" "sudo spx admin join --force \
+        --node ${NODE_NAMES[$i]} \
+        --bind ${LAN_IPS[$i]} --cluster-bind ${LAN_IPS[$i]} --advertise ${WAN_IPS[$i]} \
+        --host ${LAN_IPS[0]}:$PORT --token $TOKEN \
+        --region $REGION --az $AZ" &
     JOIN_PIDS+=("$!")
 done
 
@@ -734,7 +724,7 @@ if ! $DRY_RUN; then
         elapsed=$((elapsed + 5))
     done
     if [ "$elapsed" -ge "$FORMATION_TIMEOUT" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
         fail "${HOSTS[0]}: formation did not complete within ${FORMATION_TIMEOUT}s"
     fi
     log "  formation complete"

scripts/install-node_test.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
-# sudo's log (which holds the command it was given) never contains it, and the other values must
-# arrive as plain arguments even when they hold shell syntax.
-# preflight runs this with sh
-[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
-set -euo pipefail
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-mkdir "$work/bin"
-cd "$work"
-
-cat > "$work/bin/sudo" <<'SH'
-#!/bin/sh
-echo "$*" >> "$FAKE_SUDO_LOG"
-exec "$@"
-SH
-cat > "$work/bin/spx" <<'SH'
-#!/bin/sh
-for a in "$@"; do printf '<%s>' "$a"; done > "$FAKE_SPX_ARGV"
-SH
-chmod +x "$work/bin/sudo" "$work/bin/spx"
-export FAKE_SUDO_LOG="$work/sudo.log" FAKE_SPX_ARGV="$work/spx.argv" PATH="$work/bin:$PATH"
-
-eval "$(sed -n '/^remote_join_cmd()/,/^}/p' "$here/install-node.sh")"
-
-fails=0
-check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-
-TOKEN=spx_join_FAKETOKEN-abc_1
-cmd="$(remote_join_cmd node2 10.0.0.2 203.0.113.2 '10.0.0.1:4432;touch PWNED' us-east-1 'az $(touch PWNED2)')"
-bash -c "$cmd" <<<"$TOKEN"
-
-check "the token is not in the command" "$(printf '%s' "$cmd" | grep -c FAKETOKEN || true)" 0
-check "the token is not in sudo's log" "$(grep -c FAKETOKEN "$FAKE_SUDO_LOG" || true)" 0
-check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--token><$TOKEN>"
-check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
-check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
-
-[ "$fails" -eq 0 ]

scripts/lib/lambda-testlink-build_test.sh

@@ -1,111 +0,0 @@
-#!/bin/sh
-# lambda-testlink-build_test.sh — the test-link make targets refuse a malformed
-# image digest and an unsupported host or GOARCH before touching any output.
-#
-# Every case is refused on the recipe's first lines, so nothing is compiled.
-set -eu
-
-LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-REPO_ROOT=$(dirname -- "$(dirname -- "$LIB_DIR")")
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-echo "-- make build-lambda-testlink / build-lambda-testlink-image guards"
-
-PWN="$TMP/pwn"
-OUT="$TMP/out"
-HEX=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-mkdir -p "$OUT"
-printf 'old daemon\n' > "$OUT/spx"
-printf 'old initializer\n' > "$OUT/hardkill.so"
-PAIR=$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)
-
-# unchanged <name> — the existing pair is intact, no scratch was made and no
-# injected command ran.
-unchanged() {
-    if [ -e "$PWN" ]; then
-        fail "$1: injected command ran"
-        rm -f "$PWN"
-    elif [ -e "$OUT.tmp" ]; then
-        fail "$1: scratch directory created"
-    elif [ "$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)" != "$PAIR" ]; then
-        fail "$1: existing pair replaced"
-    else
-        pass "$1"
-    fi
-}
-
-# refused <name> <want> <log> <status> — make failed with the expected message.
-refused() {
-    if [ "$4" -eq 0 ]; then
-        fail "$1: make succeeded"
-    elif ! grep -q "$2" "$3"; then
-        fail "$1: wrong refusal: $(tr '\n' ' ' < "$3")"
-    else
-        unchanged "$1"
-    fi
-}
-
-testlink() {
-    make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" "$@" > "$TMP/log" 2>&1
-}
-
-DIGEST_MSG="LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"
-NL='
-'
-for v in '' "\$(shell touch $PWN)" "\$\$(touch $PWN)" "\`touch $PWN\`" "$HEX; touch $PWN" \
-    "$HEX\" -X main.x=\"y" "$HEX -X main.x=y" "$HEX${NL}touch $PWN" \
-    0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF "${HEX%?}" "${HEX}0"; do
-    # GNU make itself expands a command-line $(shell ...) while parsing, for
-    # any target, so that one literal is tested only from the environment.
-    case "$v" in
-    "\$(shell"*) ;;
-    *)
-        status=0
-        testlink LAMBDA_IMAGE_DIGEST="$v" || status=$?
-        refused "command-line digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
-        ;;
-    esac
-    status=0
-    LAMBDA_IMAGE_DIGEST="$v" testlink || status=$?
-    refused "environment digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
-done
-
-status=0
-env -u LAMBDA_IMAGE_DIGEST make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" > "$TMP/log" 2>&1 || status=$?
-refused "digest unset" "$DIGEST_MSG" "$TMP/log" "$status"
-
-ARCH_MSG="supports only an x86_64 host building linux/amd64"
-SHIM="$TMP/shim"
-mkdir -p "$SHIM"
-printf '#!/bin/sh\necho aarch64\n' > "$SHIM/uname"
-chmod +x "$SHIM/uname"
-
-status=0
-GOARCH=arm64 testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
-refused "GOARCH=arm64" "$ARCH_MSG" "$TMP/log" "$status"
-status=0
-PATH="$SHIM:$PATH" testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
-refused "aarch64 host" "$ARCH_MSG" "$TMP/log" "$status"
-
-# The image target refuses before building qboot or any agent binary.
-for c in "GOARCH=arm64" "PATH=$SHIM:$PATH"; do
-    status=0
-    env "$c" make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink-image > "$TMP/log" 2>&1 || status=$?
-    if [ "$status" -eq 0 ]; then
-        fail "image target [${c%%=*}]: make succeeded"
-    elif ! grep -q "$ARCH_MSG" "$TMP/log" || grep -q "qboot\|go build" "$TMP/log"; then
-        fail "image target [${c%%=*}]: did not refuse first: $(tr '\n' ' ' < "$TMP/log")"
-    else
-        pass "image target [${c%%=*}] refused first"
-    fi
-done
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi

scripts/terraform/latitude-spx/.gitignore

@@ -1,10 +0,0 @@
-.terraform/
-terraform.tfstate
-terraform.tfstate.*
-*.tfplan
-*.auto.tfvars
-crash.log
-terraform.tfvars
-*.pem
-*.log
-.terraform.lock.hcl

scripts/terraform/latitude-spx/README.md

@@ -1,137 +0,0 @@
-# latitude-spx
-
-Builds a Spinifex deployment on latitude.sh bare metal from your laptop with `deploy-latitude.sh`.
-The commands, flags, step markers and state layout follow `vultr-spx`.
-
-**Status: not yet run end to end.**
-The script was written from the plan doc findings (`docs/development/feature/latitude-bm.md`, L1 to L26) and from the provider schema.
-No latitude.sh API call or host run was made while building it, so treat the first `up` as the test.
-One node has been the only shape run live, and only in part (see Unverified). The three-node flow, including the floating pool, has not been run: Ben takes that over.
-
-## What it creates
-
-| Resource | Notes |
-|---|---|
-| Project | `spinifex-<name>`, unless `--project` names an existing one (then it is never deleted) |
-| SSH key | `spinifex-<name>`, account scoped |
-| User data | `spinifex-<name>`, host preparation only (no `{{ }}`, latitude.sh substitutes them) |
-| Servers | `m4-metal-medium`, site `DAL`, Ubuntu 26.04, hourly billing, hostnames `spx-<name>-nodeN` |
-| Elastic IP | one by default (`elastic_ip_count` in `variables.tf`, 1 to 5; set `TF_VAR_elastic_ip_count` to change it). With one node it is a static pool address and the count must stay 1, because a second pool is never tried when the first is full; with three nodes the addresses form the `latitude` floating pool, which Spinifex moves between servers. `nginx-webserver` needs one address and runs on one node; `nginx-alb` needs two public addresses and is a three-node workbook |
-| Virtual network | three nodes only: `spinifex-<name>`, one VLAN assignment per server |
-
-## Requirements
-
-OpenTofu (or Terraform) 1.5 or later, `aws` CLI, `python3`, `curl`, `ssh`, an SSH key pair.
-
-| Variable | When | Meaning |
-|---|---|---|
-| `LATITUDESH_AUTH_TOKEN` | always | the account API token; used by the provider and this script only, never sent to a node |
-| `LATITUDESH_NODE_TOKEN` | `--nodes 3` | a second, different, writable API key, copied to each node (`/etc/spinifex/latitude/api-token`, owned by `spinifex-daemon` with mode 0400, in a root-owned 0711 directory; the other services share the `spinifex` group, so no group may read it; ownership and mode are set on every `up`; a symlink or any other owner there is refused) so the daemon can move Elastic IPs |
-
-The script refuses a node key it can see is read-only (it lists `/auth/api_keys` and matches the key's last five characters). If the key is not among the account's keys, write authority is first proven by the first Elastic IP move.
-
-Exposure: a latitude.sh API key is team-wide. It is not scoped to Elastic IPs, so anyone who reads it from a node can do whatever the key's owner can in the team (the API exposes `read_only` and `allowed_ips` on a key, and no narrower scope). Create the node key with `allowed_ips` set to the nodes' public addresses, and revoke it after `destroy`.
-
-Nodes are reached as `ubuntu` with the deployment's SSH key, and remote commands run through passwordless `sudo`: latitude.sh puts the key on the image's login user and root login is not enabled.
-
-## Usage
-
-```
-export LATITUDESH_AUTH_TOKEN=...        # from the latitude.sh dashboard
-./deploy-latitude.sh up --name demo     # one node
-
-export LATITUDESH_NODE_TOKEN=...        # a second, writable key
-./deploy-latitude.sh up --name demo3 --nodes 3
-./deploy-latitude.sh status --name demo
-./deploy-latitude.sh ssh --name demo node1 'systemctl status spinifex.target'
-./deploy-latitude.sh credentials --name demo
-./deploy-latitude.sh stop node1 --name demo
-./deploy-latitude.sh start node1 --name demo
-./deploy-latitude.sh destroy --name demo
-```
-
-Options: `--name`, `--site` (alias `--region`), `--plan`, `--os`, `--project ID|SLUG`, `--distro TARBALL`, `--channel`, `--spx-region`, `--spx-az`, `--ssh-key`, `--image`, `--data-disk-id`, `--vpc-subnet` (the VLAN's /24, three nodes), `--nodes 1|3`, `--yes`.
-`./deploy-latitude.sh --help` lists them with defaults.
-
-Three nodes need `--distro` with a tarball built from this branch (`make distro-amd64`) until a release carries the latitude provider.
-A released spx rejects `source="latitude"` and its daemon restarts in a loop, so `up --nodes 3` without `--distro` stops in preflight.
-Before the pool is written, each node's `/usr/local/bin/spx` is checked for the `latitude_elastic_ip_ids` key name; spx has no validate subcommand, so this is a string search of the binary, not a run of its config check.
-
-## What `up` does
-
-`terraform` then, per node: `host` (wait for SSH as `ubuntu` and cloud-init, `/dev/kvm`), `disk`, `vlan` (three nodes), `install`, `form`, `start`; then `pool` (one node) or `cluster` (three nodes: `install-node.sh` forms the cluster); three nodes then install the node token on every `up` and add the `latitude` pool (`pool`); then `restart` (three nodes wait for the current daemon to log "floating pool ready" for the pool), `public`, `image` and the AWS profile.
-Each step writes `step.<name>[.nodeN]` in the state directory, and a rerun resumes at the first step without one.
-
-- **Data disk.** `host-data-disk.sh` picks the blank NVMe by content and by-id EUI, never by `nvmeXn1` (L1, L11).
-A candidate is at least 100 GB with no signature, partitions, holders or mounts, and not in the root filesystem's ancestry; with none it stops and lists the disks.
-With several (LON2 has two 1.7 TB and one spare 447 GB blank NVMe next to the root disk) it takes the largest, the lowest by-id path among equals, prints what it chose and the blank disks it left, and never touches the others (`--data-disk-id` chooses another).
-It mounts by UUID with `nofail` and the script records the UUID in `data_uuid.<node>`.
-- **Mount check.** Before anything writes under `/var/lib/spinifex`, and on every run, the mounted filesystem UUID must equal the recorded one.
-The units that write there also get a `RequiresMountsFor=/var/lib/spinifex` drop-in.
-The mount is by UUID in fstab, so renamed disks do not change it, and there is no check at service start.
-- **Firewall.** `setup.sh --firewall on` runs before Spinifex first starts.
-- **One node.** The Elastic IP is a static pool (`gateway_ip = "192.0.2.1"`, the node's next hop as `gateway`, no `prefix_len`) with the IMDS addresses moved off the ones latitude.sh routes.
-- **Three nodes.** The `vlan` step makes `eno2.<vid>` with `ip link` and persists it as systemd-networkd files for that link plus a drop-in on the `eno2` unit.
-It never runs `netplan apply`, refuses a subnet that overlaps the SSH peer or a route, and removes the VLAN again if the public interface's address, default route or the route to the SSH peer differ afterwards.
-`form_cluster` then puts the cluster and Geneve planes on that VLAN with `install-node.sh` (`--user ubuntu`); its output is not kept, as it can hold the join token.
-- **Floating pool (three nodes).** The `latitude` pool names the Elastic IP IDs and the project ID from the Terraform outputs, and each node's `provider_host_id` is its server ID. `spinifex.toml` is edited idempotently and read back. Failover moves an address with one `PATCH /elastic_ips/{id}` and waits for it to be active; latitude.sh has no unassign, so a node that loses an address is not told to release it.
-- **State** is in `~/.spinifex/latitude/<name>/`: Terraform state, `ids` (every `latitudesh_*` object in the state, merged on each apply and before each destroy), `step.*` markers, logs and the data disk UUIDs.
-- **AWS profile** `spinifex-latitude-<name>` is replaced on each `up`, with the CA at `~/.spinifex/latitude-<name>-ca.pem`.
-The secret key is written to `~/.aws/credentials` (0600, other profiles kept, the old file saved beside it) and never appears in a process argument.
-
-## Token handling
-
-`LATITUDESH_AUTH_TOKEN` is read from the environment by the provider and by the script's API calls.
-It is not a Terraform variable, is on no command line (curl reads it from stdin), is not sent to a node, and is written to no file.
-`LATITUDESH_NODE_TOKEN` is checked with curl reading it from stdin and reaches each node on stdin of the install script, so it is in no argument on either side.
-`TF_LOG` is unset by the script.
-The join token that `install-node.sh` creates is shared with Vultr and OCI: its init log is root-only, and `spx admin join` still takes it as `--token` on the joining node (tracked separately).
-
-## Recovery
-
-If SSH is lost once the firewall is armed, use the serial console.
-`up` enables the server's out-of-band connection with the deployment's SSH key and prints the `ssh` command it returns before it arms the firewall, falling back to the Out-of-Band tab in the dashboard.
-Logging in there needs an OS user password, which you set yourself while SSH still works.
-A fresh deployment that cannot be recovered is cheaper to `destroy` and `up` again.
-The VLAN step prints the console command first.
-It refuses a subnet that holds the SSH peer or overlaps an existing route, and removes the VLAN again if the route to the SSH peer or the public interface changes.
-
-## destroy
-
-`destroy` asks you to type the deployment name (`--yes` skips only that), then records every ID in the Terraform state and stops without destroying anything if it cannot.
-It runs `terraform destroy`, fetches every recorded ID by itself (2xx present, 404 gone, anything else fails and keeps the records), and deletes what remains of those.
-Servers, keys, user data, virtual networks and projects whose names match the deployment but that are not recorded are listed by ID and deleted only after you type `delete-untracked` for exactly those IDs, which `--yes` never supplies; an ID that shows up later needs its own confirmation, and without a terminal they are kept and `destroy` exits 1.
-Elastic IPs are touched only by recorded ID.
-Elastic IP release is asynchronous and only works once the address is active or in error, so the sweep retries.
-Because the Elastic IP list ignores pagination, `destroy` ends by asking you to check the dashboard for leftovers and type `checked`.
-An `up` that failed while creating can leave an object nothing recorded, such as an Elastic IP; that dashboard check is the only thing that finds it.
-
-## Unverified
-
-Nothing in the three-node flow has run against real servers. For one node, only the parts the person ran have been exercised, and the script has changed since.
-
-- The OS slug `ubuntu_26_04_x64_lts` (the `latitudesh_operating_system` data source fails the plan if it is wrong).
-- That the deploy key is on `ubuntu` and `sudo -n` works for it, and that `sudo` resets nothing the script needs (the SSH peer address is read before `sudo` for that reason).
-- The power action request (`POST /servers/{id}/actions` with `power_off` and `power_on`).
-- Resolving `--project` as a slug through `GET /projects/{slug}`, and the `in_stock` shape of `GET /plans` used by the stock check, which only warns when it cannot read it.
-- The `GET` and `DELETE` paths for `/user_data/{id}` and `/virtual_networks/{id}`.
-- That the Spinifex API certificate names the node's public address, which the profile check needs.
-- That the private NIC is `eno2`, its tagging and MTU (the script uses 1500), and that the networkd files persist the VLAN across a reboot.
-- `GET /auth/api_keys` and its `token_last_slice` and `read_only` fields, which the node key check relies on; and that a key with `allowed_ips` set to the nodes' addresses still works from them.
-- The Elastic IP move itself: `PATCH /elastic_ips/{id}` with `server_id`, its 202 then `active`, the error codes, and how long a move takes; the client waits up to its move timeout and makes one write per call.
-- That the daemon reads the node token and logs "floating pool ready", then fails over when a node stops (`stop node1`) and the address lands on another node.
-- `install-node.sh` as `--user ubuntu` on latitude.sh servers.
-
-## Not exercised
-
-Any real API call from the three-node flow, `destroy`, `host-data-disk.sh` on a host and the VLAN step on a server.
-Guest launch, public reachability of the Elastic IP, the Terraform workbooks and failover are not part of `up`; prove them by hand.
-`e2e-latitude.sh` and removing the old `install-node.sh` in this directory are not done.
-
-## Tests
-
-`sh node-scripts_test.sh` checks the node token install (install, unchanged, rotation, modes, an older install converging, symlinks and owners refused; run it as root too, with `sudo sh node-scripts_test.sh`, to check that only the daemon's user can read the file) and the readiness check (old success then current failure) with stand-in tools.
-`bash deploy-argv_test.sh` checks that neither API token, nor the five characters used to find the node key, is an argument of any external command in the node token paths, and that the SSH peer address comes from the same connection after sudo's environment reset.
-`scripts/install-node_test.sh` (part of `make preflight`) checks that the join token reaches `spx admin join` only on stdin, so sudo's log never holds it.
-The first two are not part of `make preflight`.
-The Go side: `cd spinifex && GOWORK=off go test ./providers/cloud/latitude/ ./domains/network/external/floating/ ./bootstrap/config/ ./runtime/daemon/`.

scripts/terraform/latitude-spx/cloud-init/user-data.yaml

@@ -1,23 +0,0 @@
-#cloud-config
-# Host preparation only. Spinifex is installed and formed over SSH by deploy-latitude.sh.
-# latitude.sh substitutes double-brace variables in user data, so this file holds none.
-write_files:
-  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
-    content: |
-      network: {config: disabled}
-  # The image's netplan lists eno2 with no address and without optional, so boot waits two minutes
-  # for it. --any returns once one link, eno1, is online; it changes the wait, not the network.
-  # UNVERIFIED: the binary path is the Ubuntu one and the drop-in first applies on the next boot.
-  - path: /etc/systemd/system/systemd-networkd-wait-online.service.d/10-spinifex-any.conf
-    content: |
-      [Service]
-      ExecStart=
-      ExecStart=/usr/lib/systemd/systemd-networkd-wait-online --any
-# runcmd carries on past a failing command, so each line stands alone.
-runcmd:
-  - [sh, -c, "command -v ufw >/dev/null && ufw --force disable || true"]
-  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
-  # The image ships empty apt lists and no git, which the workbooks need.
-  - [sh, -c, "apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git || true"]
-  - [sh, -c, "systemctl daemon-reload || true"]
-  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]

scripts/terraform/latitude-spx/compute.tf

@@ -1,70 +0,0 @@
-locals {
-  hostname_prefix = "spx-${var.deployment_name}-node"
-  project         = var.project != "" ? var.project : latitudesh_project.deploy[0].slug
-  project_id      = var.project != "" ? var.project_id : latitudesh_project.deploy[0].id
-}
-
-# Both lookups fail the plan when a slug does not exist, which is the only check of the OS slug.
-data "latitudesh_operating_system" "os" {
-  slug = var.operating_system
-}
-
-data "latitudesh_plan" "plan" {
-  slug = var.plan
-}
-
-resource "latitudesh_project" "deploy" {
-  count             = var.project == "" ? 1 : 0
-  name              = "spinifex-${var.deployment_name}"
-  description       = "Spinifex deployment ${var.deployment_name}"
-  environment       = "Development"
-  provisioning_type = "on_demand"
-}
-
-# The key is team scoped. The server login user is ubuntu, not root.
-resource "latitudesh_ssh_key" "deploy" {
-  name       = "spinifex-${var.deployment_name}"
-  public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
-}
-
-# Host preparation only. Latitude substitutes double-brace variables inside user data, so the file
-# must hold none. Spinifex is installed over SSH by deploy-latitude.sh.
-resource "latitudesh_user_data" "host" {
-  description = "spinifex-${var.deployment_name}"
-  content     = filebase64("${path.module}/cloud-init/user-data.yaml")
-}
-
-# billing must be hourly: the default is monthly, which bills upfront with no refund.
-resource "latitudesh_server" "node" {
-  count            = var.node_count
-  project          = local.project
-  hostname         = "${local.hostname_prefix}${count.index + 1}"
-  site             = var.site
-  plan             = data.latitudesh_plan.plan.slug
-  operating_system = data.latitudesh_operating_system.os.slug
-  billing          = "hourly"
-  ssh_keys         = [latitudesh_ssh_key.deploy.id]
-  user_data        = latitudesh_user_data.host.id
-
-  timeouts = {
-    create = "60m"
-  }
-}
-
-# Elastic IPs are bought here and never created or deleted by Spinifex. Each starts on a server, spread
-# over the nodes, and server_id is ignored afterwards so a move by Spinifex is not undone by a later apply.
-resource "latitudesh_elastic_ip" "pool" {
-  count = var.elastic_ip_count
-  # The provider documents ID or slug but posts the value as project_id unresolved: a slug gave 404.
-  project   = local.project_id
-  server_id = latitudesh_server.node[count.index % var.node_count].id
-
-  lifecycle {
-    ignore_changes = [server_id]
-  }
-}
-
-moved {
-  from = latitudesh_elastic_ip.node1
-  to   = latitudesh_elastic_ip.pool[0]
-}

scripts/terraform/latitude-spx/deploy-argv_test.sh

@@ -1,228 +0,0 @@
-#!/usr/bin/env bash
-# Checks that neither API token is ever an argument of an external command in the node token
-# paths of deploy-latitude.sh. External commands are stand-ins that record their arguments and
-# their standard input; the test fails if a token is in the first and proves it is in the second.
-# preflight runs this with sh
-[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
-set -euo pipefail
-exec </dev/null
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-mkdir "$work/bin" "$work/home"
-real_python="$(command -v python3)"
-
-AUTH=FAKEAUTHTOKEN9
-NODE=FAKENODETOKEN7
-
-# Every command the script runs outside bash records "name args" and, for the first line of
-# standard input, "name stdin: ...".
-for c in curl ssh scp aws tofu ssh-keygen; do
-    cat > "$work/bin/$c" <<'SH'
-#!/bin/sh
-echo "$(basename "$0") $*" >> "$FAKE_ARGV"
-if [ ! -t 0 ]; then
-    in="$(cat)"
-    [ -z "$in" ] || echo "$(basename "$0") stdin: $in" >> "$FAKE_STDIN"
-fi
-case "$(basename "$0")" in
-    curl) printf '{"data":[{"attributes":{"token_last_slice":"%s","read_only":false}}]}\n200' "$(printf '%s' "$FAKE_NODE" | tail -c 5)" ;;
-    ssh)
-        if [ -n "${FAKE_SSH_EVAL:-}" ]; then
-            # Behave like a login on a node: skip the options and host, run the rest with the
-            # connection variable of this connection, and let sudo reset the environment.
-            while [ "$#" -gt 0 ]; do case "$1" in ubuntu@*) shift; break ;; *) shift ;; esac; done
-            SSH_CONNECTION="$FAKE_SSH_CONNECTION" bash -c "$*"
-        else
-            echo installed
-        fi ;;
-esac
-SH
-    chmod +x "$work/bin/$c"
-done
-cat > "$work/bin/sudo" <<'SH'
-#!/bin/sh
-while [ "$#" -gt 0 ]; do case "$1" in -*) shift ;; *) break ;; esac; done
-exec env -i PATH="$PATH" "$@"
-SH
-chmod +x "$work/bin/sudo"
-cat > "$work/bin/python3" <<SH
-#!/bin/sh
-echo "python3 \$*" >> "\$FAKE_ARGV"
-exec "$real_python" "\$@"
-SH
-chmod +x "$work/bin/python3"
-
-export FAKE_ARGV="$work/argv" FAKE_STDIN="$work/stdin" FAKE_NODE="$NODE" PATH="$work/bin:$PATH" HOME="$work/home"
-: > "$FAKE_ARGV"
-: > "$FAKE_STDIN"
-export LATITUDESH_AUTH_TOKEN="$AUTH" LATITUDESH_NODE_TOKEN="$NODE"
-
-# shellcheck source=deploy-latitude.sh
-source "$here/deploy-latitude.sh"
-
-# shellcheck disable=SC2034 # read by the sourced functions
-NODES=3 IP=192.0.2.10 NODE_NAME=node1
-check_node_token >/dev/null
-install_key >/dev/null 2>&1
-
-fails=0
-for tok in "$AUTH" "$NODE" "${NODE: -5}"; do
-    if grep -q "$tok" "$FAKE_ARGV"; then
-        echo "FAIL token $tok is in a command's arguments:"
-        grep "$tok" "$FAKE_ARGV"
-        fails=$((fails + 1))
-    else
-        echo "ok   $tok is in no argument"
-    fi
-done
-for what in "curl stdin:.*Bearer $NODE" "ssh stdin: $NODE"; do
-    if grep -q "$what" "$FAKE_STDIN"; then
-        echo "ok   the check can see the token where it is sent ($what)"
-    else
-        echo "FAIL the token never reached stdin ($what): the test would pass without checking anything"
-        fails=$((fails + 1))
-    fi
-done
-
-# The peer address comes from the same connection that runs the command, whichever source
-# address that connection has, and survives sudo's environment reset.
-export FAKE_SSH_EVAL=1
-for src in 198.51.100.7 203.0.113.99; do
-    export FAKE_SSH_CONNECTION="$src 51000 192.0.2.10 22"
-    got="$(on 'printf %s "$SSH_PEER"')"
-    if [ "$got" = "$src" ]; then echo "ok   peer $src reaches the command"; else echo "FAIL peer for $src was '$got'"; fails=$((fails + 1)); fi
-done
-
-# A three-node run needs a build that knows the latitude source: a release would be refused by its
-# own config check. The guard needs --distro, and the node is checked before it gets the pool.
-check_result() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-check_result "three nodes without --distro stop" "$( (NODES=3 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-check_result "three nodes with --distro go on" "$( (NODES=3 DISTRO=/tmp/x.tar.gz require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node without --distro goes on" "$( (NODES=1 DISTRO="" require_branch_build) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with one address goes on" "$( (NODES=1 TF_VAR_elastic_ip_count=1 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with no setting goes on" "$( (NODES=1 TF_VAR_elastic_ip_count="" require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-check_result "one node with two addresses stops" "$( (NODES=1 TF_VAR_elastic_ip_count=2 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-check_result "three nodes with three addresses go on" "$( (NODES=3 TF_VAR_elastic_ip_count=3 require_one_address) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-printf 'binary with latitude_elastic_ip_ids inside\n' > "$work/spx-new"
-printf 'binary of a release\n' > "$work/spx-old"
-SPX_BIN="$work/spx-new"
-check_result "a build that knows latitude is accepted" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" ran
-# shellcheck disable=SC2034 # read by the sourced functions
-SPX_BIN="$work/spx-old"
-check_result "a release is refused" "$( (node_knows_latitude) >/dev/null 2>&1 && echo ran || echo stopped)" stopped
-# The profile spx writes under sudo is in the ubuntu user's home, so node-side reads name that
-# directory instead of relying on root's HOME.
-mkdir "$work/awsdir"
-printf '[other]\naws_access_key_id = X\n[spinifex]\naws_access_key_id = AK1\naws_secret_access_key = SK1\n[z]\nq = 1\n' > "$work/awsdir/credentials"
-node_ip() { printf 192.0.2.10; }
-# shellcheck disable=SC2034 # read by the sourced functions
-NODE_AWS_DIR="$work/awsdir"
-check_result "credentials come from the node's AWS dir" "$(cmd_credentials | tr '\n' ' ')" "[spinifex] aws_access_key_id = AK1 aws_secret_access_key = SK1 "
-
-# A failed API call logs its status and a documented code, never the response text, which can
-# echo the request's headers.
-mkdir "$work/bin2"
-cat > "$work/bin2/curl" <<'SH'
-#!/bin/sh
-cat > /dev/null
-printf '{"errors":[{"code":"%s","detail":"Authorization: Bearer %s"}]}\n401' "$FAKE_CODE" "$FAKE_LEAK"
-SH
-chmod +x "$work/bin2/curl"
-logged="$(FAKE_CODE=UNAUTHORIZED FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
-check_result "a failed call names its status and code" "$(grep -c 'returned 401 (UNAUTHORIZED)' <<<"$logged" || true)" 1
-check_result "a failed call never logs the token" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
-logged="$(FAKE_CODE="$AUTH" FAKE_LEAK="$AUTH" PATH="$work/bin2:$PATH" latitude GET /servers 2>&1 || true)"
-check_result "an undocumented code is not logged" "$(grep -c "$AUTH" <<<"$logged" || true)" 0
-
-# A recorded ID that now names another deployment's object is not ours to delete.
-# shellcheck disable=SC2034 # read by the sourced functions
-NAME=demo
-latitude() { printf '%s' "$FAKE_LAT"; }
-orig_expected_project_id="$(declare -f expected_project_id)"
-expected_project_id() { echo proj_1; }
-srv() {
-    local frag=',"project":{"id":"proj_1"}'
-    [ "$#" -lt 2 ] || frag="$2"
-    printf '{"data":{"id":"sv_1","attributes":{"hostname":"%s"%s}}}' "$1" "$frag"
-}
-FAKE_LAT="$(srv spx-demo-node2)"
-check_result "a recorded server with our hostname is ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" ours
-FAKE_LAT="$(srv spx-other-node1)"
-check_result "a recorded server with another hostname is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo2-node1)"
-check_result "a hostname that only starts like ours is not" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
-check_result "our hostname in another project is not ours" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=1
-FAKE_LAT="$(srv spx-demo-node1 '')"
-check_result "a server with no project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{}')"
-check_result "a project with no id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-FAKE_LAT="$(srv spx-demo-node1 ',"project":null')"
-check_result "a null project is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-expected_project_id() { return 1; }
-FAKE_LAT="$(srv spx-demo-node1)"
-check_result "no known project id is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-expected_project_id() { echo proj_1; }
-FAKE_LAT='not json'
-check_result "an unreadable object is unknown" "$(recorded_identity server sv_1 && echo ours || echo "rc=$?")" rc=2
-KNOWN_IDS="$work/ids"
-printf 'server sv_1\n' > "$KNOWN_IDS"
-lat_exists() { return 0; }
-FAKE_LAT="$(srv spx-other-node1)"
-check_result "destroy stops before Terraform when a recorded ID was reused" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1 ',"project":{"id":"proj_other"}')"
-check_result "destroy stops when the same hostname is in another project" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1 '')"
-check_result "destroy stops when the project cannot be read" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" stopped
-FAKE_LAT="$(srv spx-demo-node1)"
-check_result "destroy goes on when the recorded server is ours" "$( (check_recorded_identity) >/dev/null 2>&1 && echo went || echo stopped)" went
-leftovers_out="$( (lat_match() { return 0; }; FAKE_LAT="$(srv spx-other-node1)" leftovers) 2>/dev/null || true)"
-check_result "a reused ID is not listed as a leftover to delete" "$(grep -c sv_1 <<<"$leftovers_out" || true)" 0
-
-# After terraform destroy the state's project output is gone; the remembered ID still lets the
-# sweep delete a matching leftover and keep a foreign one.
-eval "$orig_expected_project_id"
-TF_BIN="$work/tofu"
-printf '#!/bin/sh\nif [ -f "%s/gone" ]; then exit 1; fi\nprintf proj_1\n' "$work" > "$TF_BIN"
-chmod +x "$TF_BIN"
-EXPECTED_PROJECT=""
-remember_project_id
-check_result "the project ID is remembered while the state holds it" "$EXPECTED_PROJECT" proj_1
-: > "$work/gone"
-check_result "the state no longer answers after destroy" "$("$TF_BIN" output >/dev/null 2>&1 && echo up || echo gone)" gone
-printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
-lat_match() { return 1; }
-latitude() { if [ "$2" = /servers/sv_1 ]; then srv spx-demo-node1; else srv spx-other-node1 ',"project":{"id":"proj_other"}'; fi; }
-swept="$(leftovers 2>/dev/null || true)"
-check_result "after destroy a matching leftover is still listed" "$(grep -c 'server sv_1 ' <<<"$swept" || true)" 1
-check_result "after destroy a foreign one is kept out" "$(grep -c 'server sv_2 ' <<<"$swept" || true)" 0
-EXPECTED_PROJECT=""
-swept="$(leftovers 2>/dev/null || true)"
-check_result "without the remembered ID a leftover is unknown, not matched" "$(grep -c 'server sv_1 sv_1 (unknown)' <<<"$swept" || true)" 1
-expected_project_id() { echo proj_1; }
-
-# Cleanup never deletes an object whose ownership could not be read, recorded or not.
-deleted="$work/deleted"
-: > "$deleted"
-latitude() { printf '%s %s\n' "$1" "$2" >> "$deleted"; }
-printf 'server sv_1\nserver sv_2\n' > "$KNOWN_IDS"
-delete_left $'server sv_1 sv_1 (unknown)\nserver sv_2 spx-demo-node2\n' "" >/dev/null 2>&1
-check_result "an unreadable recorded object is not deleted" "$(grep -c sv_1 "$deleted" || true)" 0
-check_result "a verified recorded object is deleted" "$(grep -c 'DELETE .*sv_2' "$deleted" || true)" 1
-
-# Both deployment sizes write the metadata remap.
-cfg="$work/spinifex.toml"
-extract() { sed -n "/^$1()/,/^}/p" "$here/deploy-latitude.sh" | sed -n "/<<'PY'/,/^PY\$/p" | sed '1d;$d' | sed "s|/etc/spinifex/spinifex.toml|$cfg|"; }
-printf '[network]\nexternal_mode = "nat"\n\n[bootstrap]\ny = 1\n' > "$cfg"
-extract configure_pool | python3 -I - 203.0.113.1 203.0.113.254
-check_result "one node gets the metadata remap" "$(grep -c 'imds_host_meta_ip = "169.254.42.254"' "$cfg")" 1
-printf '[network]\nexternal_mode = "nat"\n\n[nodes.node1]\nx = 1\n\n[nodes.node2]\nx = 1\n\n[nodes.node3]\nx = 1\n\n[bootstrap]\ny = 1\n' > "$cfg"
-extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
-check_result "three nodes get the metadata remap" "$(grep -c 'imds_host_dns_ip  = "169.254.42.253"' "$cfg")" 1
-check_result "three nodes get the latitude pool" "$(grep -c 'source                  = "latitude"' "$cfg")" 1
-extract configure_latitude_pool | python3 -I - proj_1 eip_1,eip_2 node1=sv_1,node2=sv_2,node3=sv_3
-check_result "writing the config again changes nothing" "$(grep -c 'imds_host_meta_ip' "$cfg")" 1
-unset FAKE_SSH_EVAL
-
-[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/host-data-disk.sh

@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-# host-data-disk.sh — format and mount the blank data disk at /var/lib/spinifex. Run as root ON the host.
-#
-#   host-data-disk.sh [/dev/disk/by-id/nvme-eui.<id>]
-#
-# The disk is chosen by content, never by its nvmeXn1 name, which swaps between boots. A candidate is a
-# whole-disk by-id EUI link whose device is not in the root filesystem's ancestry, has no partitions,
-# holders, swap or mounts, no filesystem signature and no partition table. With none the script stops
-# and lists the disks. With several it takes the largest, and the lowest by-id path among equals, and
-# prints what it chose and the blank disks it left alone; pass the by-id path to choose another.
-# It never reformats: a rerun on a host already done prints the UUID and exits. The last line of stdout
-# is DATA_UUID=<uuid>, which the later install phases need.
-
-set -euo pipefail
-
-MP=/var/lib/spinifex
-LABEL=spinifex-data
-FSTAB_BAK=/etc/fstab.pre-spinifex
-MIN_BYTES="${MIN_DATA_BYTES:-100000000000}"
-WANT="${1:-}"
-
-die() { echo "STOP: $*" >&2; exit 1; }
-[ "$(id -u)" = 0 ] || die "run as root"
-
-# Resume: the mount is already there and is ours.
-if mp_uuid="$(findmnt --mountpoint "$MP" -no UUID 2>/dev/null)" && [ -n "$mp_uuid" ]; then
-    src="$(findmnt --mountpoint "$MP" -no SOURCE)"
-    [ "$(blkid -s LABEL -o value "$src")" = "$LABEL" ] || die "$MP is mounted from $src, which is not labelled $LABEL; not touching it"
-    grep -q "UUID=${mp_uuid}[[:space:]]" /etc/fstab || die "$MP is mounted but /etc/fstab has no UUID=$mp_uuid entry"
-    echo "already done: $MP is mounted from $src (UUID $mp_uuid)"
-    echo "DATA_UUID=$mp_uuid"
-    exit 0
-fi
-if grep -q "$MP" /etc/fstab; then die "/etc/fstab mentions $MP but nothing is mounted there; fix that by hand first"; fi
-[ ! -e "$MP" ] || [ -z "$(ls -A "$MP")" ] || die "$MP exists and is not empty"
-
-root_src="$(findmnt -no SOURCE /)" || die "cannot find the root source"
-root_anc="$(lsblk -s -nro NAME "$root_src")" || die "cannot read the root ancestry"
-[ -n "$root_anc" ] || die "empty root ancestry"
-
-# blank_reason ID prints why the disk is not a candidate and returns 1, or returns 0.
-blank_reason() {
-    local id="$1" dev name size kids hold sw mnt rc w
-    dev="$(readlink -e "$id")" || { echo "link does not resolve"; return 1; }
-    [ -b "$dev" ] || { echo "not a block device"; return 1; }
-    name="$(basename "$dev")"
-    if grep -qx "$name" <<<"$root_anc"; then echo "in the root ancestry"; return 1; fi
-    size="$(blockdev --getsize64 "$id")" || { echo "size unreadable"; return 1; }
-    [ "$size" -ge "$MIN_BYTES" ] || { echo "only $size bytes"; return 1; }
-    kids="$(lsblk -nro NAME "$id")" || { echo "children unreadable"; return 1; }
-    [ "$kids" = "$name" ] || { echo "has children"; return 1; }
-    hold="$(ls -A "/sys/block/$name/holders/")" || { echo "holders unreadable"; return 1; }
-    [ -z "$hold" ] || { echo "has holders"; return 1; }
-    sw="$(swapon --show=NAME --noheadings)" || { echo "swapon failed"; return 1; }
-    if grep -q "$name" <<<"$sw"; then echo "is swap"; return 1; fi
-    mnt="$(lsblk -nro MOUNTPOINTS "$id")" || { echo "mounts unreadable"; return 1; }
-    [ -z "${mnt//[[:space:]]/}" ] || { echo "is mounted"; return 1; }
-    set +e; blkid -p "$id" >/dev/null 2>&1; rc=$?; set -e
-    [ "$rc" -eq 2 ] || { echo "blkid -p says it has a signature (rc $rc)"; return 1; }
-    w="$(wipefs -n "$id")" || { echo "wipefs failed"; return 1; }
-    [ -z "$w" ] || { echo "wipefs finds signatures"; return 1; }
-    return 0
-}
-
-# choose_data_disk ID... prints the largest disk's ID, the lowest path among equal sizes, and says on
-# stderr which it left. The choice depends only on size and path, so every node of one plan agrees.
-choose_data_disk() {
-    local chosen rest
-    chosen="$(for id in "$@"; do printf '%s %s\n' "$(blockdev --getsize64 "$id")" "$id"; done |
-        sort -k1,1nr -k2,2 | head -n1)"
-    chosen="${chosen#* }"
-    if [ "$#" -gt 1 ]; then
-        rest="$(printf '%s\n' "$@" | grep -vxF "$chosen" | sort | tr '\n' ' ')"
-        echo "chose $chosen ($(blockdev --getsize64 "$chosen") bytes), the largest of $# blank disks; left untouched: ${rest% }" >&2
-    fi
-    printf '%s\n' "$chosen"
-}
-
-candidates=()
-shopt -s nullglob
-for id in /dev/disk/by-id/nvme-eui.*; do
-    case "$id" in *-part*) continue ;; esac
-    if [ -n "$WANT" ] && [ "$id" != "$WANT" ]; then continue; fi
-    if why="$(blank_reason "$id")"; then
-        candidates+=("$id")
-    else
-        echo "skip $id: $why"
-    fi
-done
-shopt -u nullglob
-
-if [ "${#candidates[@]}" -eq 0 ]; then
-    lsblk -o NAME,SIZE,TYPE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS >&2 || true
-    die "no blank data disk among the by-id EUI disks"
-fi
-ID="$(choose_data_disk "${candidates[@]}")"
-DEV="$(readlink -e "$ID")"
-echo "CHECKS PASSED: $ID -> $DEV"
-
-mkdir -p "$MP"
-mkfs.ext4 -q -L "$LABEL" "$ID"
-U="$(blkid -p -o value -s UUID "$ID")" || die "blkid UUID failed"
-[ -n "$U" ] || die "no UUID after mkfs"
-[ "$(blkid -p -o value -s TYPE "$ID")" = ext4 ] || die "not ext4 after mkfs"
-
-NEW="$(mktemp /etc/fstab.spinifex.XXXXXX)"
-cp /etc/fstab "$NEW"
-echo "UUID=$U $MP ext4 defaults,nofail,x-systemd.device-timeout=30s 0 2" >> "$NEW"
-findmnt --verify --tab-file "$NEW" || { rm -f "$NEW"; die "staged fstab failed verify, /etc/fstab untouched"; }
-BAK="$FSTAB_BAK.$(date +%s)"
-cp -a /etc/fstab "$BAK"
-trap 'cp -a "$BAK" /etc/fstab; systemctl daemon-reload; echo "ROLLED BACK /etc/fstab from $BAK" >&2' EXIT
-chmod 644 "$NEW"
-mv "$NEW" /etc/fstab
-systemctl daemon-reload
-mount "$MP"
-[ "$(findmnt --mountpoint "$MP" -no UUID)" = "$U" ] || die "wrong UUID at $MP"
-[ "$(findmnt --mountpoint "$MP" -no FSTYPE)" = ext4 ] || die "wrong fstype at $MP"
-[ "$(findmnt --mountpoint "$MP" -no MAJ:MIN | tr -d ' ')" = "$(lsblk -dno MAJ:MIN "$ID" | tr -d ' ')" ] || die "wrong device at $MP"
-trap - EXIT
-findmnt --mountpoint "$MP" -o TARGET,SOURCE,MAJ:MIN,UUID,FSTYPE,OPTIONS
-echo "STORAGE OK"
-echo "DATA_UUID=$U"

scripts/terraform/latitude-spx/network.tf

@@ -1,14 +0,0 @@
-# A private network needs two or more servers, so a single node has none. The VLAN carries the cluster
-# and Geneve traffic; its address is set over SSH, since the VID is only known after apply.
-resource "latitudesh_virtual_network" "lan" {
-  count       = var.node_count > 1 ? 1 : 0
-  project     = local.project_id
-  site        = var.site
-  description = "spinifex-${var.deployment_name}"
-}
-
-resource "latitudesh_vlan_assignment" "node" {
-  count              = var.node_count > 1 ? var.node_count : 0
-  server_id          = latitudesh_server.node[count.index].id
-  virtual_network_id = latitudesh_virtual_network.lan[0].id
-}

scripts/terraform/latitude-spx/node-install-api-token.sh

@@ -1,48 +0,0 @@
-#!/bin/sh
-# Runs as root on a node. Reads the API token on stdin and installs it for the daemon.
-# The directory belongs to root and is not writable by anyone else, so no service can create or
-# swap an entry in it. The file belongs to the daemon's user alone (mode 0400): the other
-# services share the spinifex group, so group access would hand them the team-wide token. The
-# temporary file has an unpredictable name and is created exclusively. A symlink, or an owner
-# other than root or the daemon's user, where the directory or the file should be is refused.
-# Ownership and mode are set on every run, so an older install converges. The file's content is
-# replaced only when the token differs. Prints installed or unchanged.
-set -eu
-dir="${API_TOKEN_DIR:-/etc/spinifex/latitude}"
-root="${API_TOKEN_ROOT-root}"
-rootgroup="${API_TOKEN_ROOT_GROUP-root}"
-user="${API_TOKEN_USER-spinifex-daemon}"
-umask 077
-
-fail() { echo "$*" >&2; exit 1; }
-owner_of() { stat -c %U "$1"; }
-
-[ ! -L "$dir" ] || fail "$dir is a symlink"
-if [ ! -e "$dir" ]; then mkdir -p "$dir"; fi
-[ -d "$dir" ] || fail "$dir is not a directory"
-[ "$(owner_of "$dir")" = "$root" ] || fail "$dir is not owned by $root"
-chown "$root:$rootgroup" "$dir"
-chmod 0711 "$dir"
-
-target="$dir/api-token"
-[ ! -L "$target" ] || fail "$target is a symlink"
-if [ -e "$target" ]; then
-    [ -f "$target" ] || fail "$target is not a regular file"
-    holder="$(owner_of "$target")"
-    [ "$holder" = "$root" ] || [ "$holder" = "$user" ] || fail "$target is owned by $holder"
-fi
-
-tmp="$(mktemp "$dir/.api-token.XXXXXX")"
-trap 'rm -f "$tmp"' EXIT
-cat > "$tmp"
-[ -s "$tmp" ] || fail "empty token on stdin"
-if [ -f "$target" ] && cmp -s "$tmp" "$target"; then
-    chown "$user" "$target"
-    chmod 0400 "$target"
-    echo unchanged
-    exit 0
-fi
-chown "$user" "$tmp"
-chmod 0400 "$tmp"
-mv -f "$tmp" "$target"
-echo installed

scripts/terraform/latitude-spx/node-pool-state.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-# Runs as root on a node with the pool name as $1. Reads only the journal of the daemon's
-# current invocation, so a success logged before a restart cannot hide a failure after it.
-# Prints ready, refused or waiting.
-set -eu
-pool="${1:?pool name}"
-id="$(systemctl show -p InvocationID --value spinifex-daemon)"
-[ -n "$id" ] || { echo waiting; exit 0; }
-log="$(journalctl "_SYSTEMD_INVOCATION_ID=$id" -u spinifex-daemon --no-pager -o cat 2>/dev/null || true)"
-named() { grep -F -e "pool=$pool" -e "\"pool\":\"$pool\""; }
-if printf '%s\n' "$log" | grep -F "refused the pool's first call" | named >/dev/null; then
-    echo refused
-elif printf '%s\n' "$log" | grep -F "floating pool ready" | named >/dev/null; then
-    echo ready
-else
-    echo waiting
-fi

scripts/terraform/latitude-spx/node-scripts_test.sh

@@ -1,119 +0,0 @@
-#!/bin/sh
-# Tests node-install-api-token.sh and node-pool-state.sh with stand-in tools.
-set -eu
-here="$(cd "$(dirname "$0")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-fails=0
-check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
-
-# token install: A, A again, then B across a rotation; the file ends as B, owned by the daemon
-# user alone (0400) in a 0711 directory that only root may change
-me="$(id -un)"
-export API_TOKEN_DIR="$work/tok" API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
-install_token() { sh "$here/node-install-api-token.sh"; }
-check "first install" "$(printf 'TOKENA' | install_token)" installed
-check "same token" "$(printf 'TOKENA' | install_token)" unchanged
-check "rotation" "$(printf 'TOKENB' | install_token)" installed
-check "file holds B" "$(cat "$work/tok/api-token")" TOKENB
-check "file mode and owner" "$(stat -c '%U %a' "$work/tok/api-token")" "$me 400"
-check "dir mode and owner" "$(stat -c '%U %a' "$work/tok")" "$me 711"
-check "no temp left after install" "$(ls -A "$work/tok")" api-token
-check "empty refused" "$(printf '' | install_token 2>/dev/null || echo failed)" failed
-check "file kept after refusal" "$(cat "$work/tok/api-token")" TOKENB
-
-# an older install converges on the unchanged path too
-chmod 0640 "$work/tok/api-token"
-chmod 0755 "$work/tok"
-check "unchanged path repairs" "$(printf 'TOKENB' | install_token)" unchanged
-check "mode repaired on unchanged" "$(stat -c %a "$work/tok/api-token")" 400
-check "dir mode repaired on unchanged" "$(stat -c %a "$work/tok")" 711
-
-# who can read it: only the daemon's identity. This needs two real identities, so it runs as root only.
-if [ "$(id -u)" = 0 ]; then
-    chmod 0711 "$work"
-    export API_TOKEN_ROOT=root API_TOKEN_ROOT_GROUP=root API_TOKEN_USER=daemon
-    for mode in installed unchanged; do
-        [ "$mode" = installed ] && rm -f "$work/tok/api-token"
-        check "$mode path" "$(printf 'TOKENB' | install_token)" "$mode"
-        check "daemon reads ($mode)" "$(setpriv --reuid=daemon --regid=daemon --clear-groups cat "$work/tok/api-token")" TOKENB
-        check "another identity in the shared group cannot read ($mode)" \
-            "$(setpriv --reuid=nobody --regid=daemon --clear-groups cat "$work/tok/api-token" 2>/dev/null || echo denied)" denied
-    done
-    export API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
-else
-    echo "skip reading as the daemon and as another service: needs root"
-fi
-
-# a link planted where the file or the directory belongs is refused, and nothing it points at is touched
-echo SENTINEL > "$work/sentinel"
-rm -f "$work/tok/api-token"
-ln -s "$work/sentinel" "$work/tok/api-token"
-check "symlinked file refused" "$(printf 'TOKENC' | sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
-check "symlink target untouched" "$(cat "$work/sentinel")" SENTINEL
-ln -s "$work/tok" "$work/tok-link"
-check "symlinked dir refused" "$(printf 'TOKENC' | API_TOKEN_DIR="$work/tok-link" sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
-check "dir owned by someone else refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=nobody-such install_token 2>/dev/null || echo refused)" refused
-rm -f "$work/tok/api-token"
-printf 'TOKENB' | install_token >/dev/null
-check "file owned by a stranger refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=$me API_TOKEN_USER=nobody-such install_token 2>/dev/null || echo refused)" refused
-check "no temp left" "$(ls -A "$work/tok")" api-token
-
-# data disk choice: the largest blank disk, the lowest path among equals, the rest left alone
-choose_dir="$work/choose"
-mkdir -p "$choose_dir/bin"
-cat > "$choose_dir/bin/blockdev" <<'SH'
-#!/bin/sh
-# blockdev --getsize64 <id>: the fixture's size for that disk
-cat "$CHOOSE_DIR/size.$(basename "$2")"
-SH
-chmod +x "$choose_dir/bin/blockdev"
-eval "$(sed -n '/^choose_data_disk()/,/^}/p' "$here/host-data-disk.sh")"
-size_of() { printf '%s' "$2" > "$choose_dir/size.$1"; }
-pick() { CHOOSE_DIR="$choose_dir" PATH="$choose_dir/bin:$PATH" choose_data_disk "$@" 2>"$choose_dir/err"; }
-size_of nvme-eui.0b 1700000000000
-size_of nvme-eui.0a 1700000000000
-size_of nvme-eui.09 447000000000
-size_of nvme-eui.0c 1900000000000
-check "largest of three wins" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0c)" /dev/disk/by-id/nvme-eui.0c
-check "equal sizes: the lowest path" "$(pick /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a)" /dev/disk/by-id/nvme-eui.0a
-check "order of the list does not matter" "$(pick /dev/disk/by-id/nvme-eui.0b /dev/disk/by-id/nvme-eui.0a /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.0a
-check "what was left is said" "$(grep -c 'left untouched: /dev/disk/by-id/nvme-eui.09 /dev/disk/by-id/nvme-eui.0b' "$choose_dir/err")" 1
-check "one disk is just chosen" "$(pick /dev/disk/by-id/nvme-eui.09)" /dev/disk/by-id/nvme-eui.09
-check "one disk says nothing" "$(wc -c < "$choose_dir/err" | tr -d ' ')" 0
-
-# pool state: the stand-in journal serves only the lines of the invocation asked for
-mkdir "$work/bin"
-cat > "$work/bin/systemctl" <<'SH'
-#!/bin/sh
-cat "$FAKE_DIR/invocation"
-SH
-cat > "$work/bin/journalctl" <<'SH'
-#!/bin/sh
-for a in "$@"; do
-    case "$a" in _SYSTEMD_INVOCATION_ID=*) id="${a#_SYSTEMD_INVOCATION_ID=}" ;; esac
-done
-cat "$FAKE_DIR/journal.$id" 2>/dev/null
-SH
-chmod +x "$work/bin/systemctl" "$work/bin/journalctl"
-export FAKE_DIR="$work" PATH="$work/bin:$PATH"
-state() { sh "$here/node-pool-state.sh" latitude; }
-
-echo old > "$work/invocation"
-echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.old"
-check "current success" "$(state)" ready
-
-echo new > "$work/invocation"
-echo 'ERROR floating: the provider refused the pool'"'"'s first call; public addresses will fail until fixed pool=latitude' > "$work/journal.new"
-check "old success then current failure" "$(state)" refused
-
-echo 'INFO starting' > "$work/journal.new"
-check "current invocation not yet ready" "$(state)" waiting
-
-echo 'INFO floating pool ready pool=other source=latitude' > "$work/journal.new"
-check "another pool does not count" "$(state)" waiting
-
-echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.new"
-check "current success" "$(state)" ready
-
-[ "$fails" -eq 0 ]

scripts/terraform/latitude-spx/outputs.tf

@@ -1,56 +0,0 @@
-output "deployment_name" {
-  value = var.deployment_name
-}
-
-output "site" {
-  value = var.site
-}
-
-output "project" {
-  value = local.project
-}
-
-# provider_host_id is the server ID. public_ip is the server's own /31 address, which SSH and the
-# Spinifex API use; the Elastic IP is separate.
-output "nodes" {
-  value = [for n in latitudesh_server.node : {
-    name             = trimprefix(n.hostname, "spx-${var.deployment_name}-")
-    provider_host_id = n.id
-    public_ip        = n.primary_ipv4
-  }]
-}
-
-# The first address, for a single node's static pool.
-output "elastic_ip" {
-  value = latitudesh_elastic_ip.pool[0].address
-}
-
-# IDs as well as addresses: destroy deletes only these IDs, and the floating pool lists them.
-output "elastic_ips" {
-  value = [for e in latitudesh_elastic_ip.pool : { id = e.id, address = e.address }]
-}
-
-# The VLAN ID of the cluster network; null for one node, which has no private network.
-output "vid" {
-  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].vid
-}
-
-output "ssh_key_id" {
-  value = latitudesh_ssh_key.deploy.id
-}
-
-output "user_data_id" {
-  value = latitudesh_user_data.host.id
-}
-
-output "project_id" {
-  value = local.project_id
-}
-
-output "project_created" {
-  value = length(latitudesh_project.deploy) > 0
-}
-
-output "virtual_network_id" {
-  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].id
-}

scripts/terraform/latitude-spx/provider.tf

@@ -1,4 +0,0 @@
-# The token comes from LATITUDESH_AUTH_TOKEN in the environment, which the provider reads
-# itself (its auth_token argument documents that variable). It is never a variable here, so it
-# is not written to the state or to a tfvars file. The default project is set per resource.
-provider "latitudesh" {}

scripts/terraform/latitude-spx/terraform.tfvars.example

@@ -1,5 +0,0 @@
-deployment_name     = "l1"
-site                = "DAL"
-plan                = "m4-metal-medium"
-node_count          = 1
-ssh_public_key_path = "~/.ssh/id_ed25519.pub"

scripts/terraform/latitude-spx/variables.tf

@@ -1,63 +0,0 @@
-variable "deployment_name" {
-  type        = string
-  description = "Names every resource and the server hostnames, so two deployments in one account never collide."
-  validation {
-    # A hostname is at most 32 characters: "spx-" + name + "-node1".
-    condition     = can(regex("^[a-z0-9][a-z0-9-]{0,15}$", var.deployment_name))
-    error_message = "deployment_name must be 1-16 lowercase letters, digits or hyphens."
-  }
-}
-
-variable "project" {
-  type        = string
-  default     = ""
-  description = "ID or slug of an existing latitude.sh project to deploy into. Empty creates a project spinifex-<deployment_name> and removes it on destroy."
-}
-
-variable "project_id" {
-  type        = string
-  default     = ""
-  description = "ID of the project named by project. The Elastic IP create sends it as project_id without resolving a slug (provider 4.8.0), so it must be an ID. Unused when project is empty."
-}
-
-variable "site" {
-  type        = string
-  default     = "DAL"
-  description = "latitude.sh site slug. Pick one with stock for the plan and Elastic IP support; not SAO."
-}
-
-variable "plan" {
-  type        = string
-  default     = "m4-metal-medium"
-  description = "Server plan slug (the pricing page calls it m4.metal.medium)."
-}
-
-variable "operating_system" {
-  type        = string
-  default     = "ubuntu_26_04_x64_lts"
-  description = "OS slug. UNVERIFIED: the 26.04 slug follows the 24.04 pattern (ubuntu_24_04_x64_lts) and is checked by a data source at plan time."
-}
-
-variable "node_count" {
-  type    = number
-  default = 1
-  validation {
-    condition     = contains([1, 3], var.node_count)
-    error_message = "node_count must be 1 or 3."
-  }
-}
-
-variable "elastic_ip_count" {
-  type        = number
-  default     = 1
-  description = "Elastic IPs to buy, each 4 USD a month. Every one starts on a server and keeps it unless Spinifex moves it. latitude.sh allows 5 per team."
-  validation {
-    condition     = var.elastic_ip_count >= 1 && var.elastic_ip_count <= 5 && floor(var.elastic_ip_count) == var.elastic_ip_count
-    error_message = "elastic_ip_count must be a whole number from 1 to 5."
-  }
-}
-
-variable "ssh_public_key_path" {
-  type    = string
-  default = "~/.ssh/id_ed25519.pub"
-}

scripts/terraform/latitude-spx/versions.tf

@@ -1,7 +0,0 @@
-terraform {
-  required_version = ">= 1.5.0"
-  required_providers {
-    # Pinned exactly: 4.8.0 is the version whose schema these files were written against.
-    latitudesh = { source = "latitudesh/latitudesh", version = "= 4.8.0" }
-  }
-}

spec/EBSProvider/Capabilities.tla

@@ -1,27 +0,0 @@
-------------------------------------------------- MODULE Capabilities -------------------------------------------------
-(*********************************************************************************************************************)
-(* The optional behaviour that an implementation advertises.                                                         *)
-(*********************************************************************************************************************)
-EXTENDS EBSProvider
-
------------------------------------------------------------------------------------------------------------------------
-\* Interface methods
-
-GetCapabilitiesRequests == {<<>>}
-
-GetCapabilities(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* The implementation reports which optional behaviour it supports.                                                *)
-  (*******************************************************************************************************************)
-     \* The provider failed to answer.
-  \/ /\ val = None
-     /\ err \in ProviderErrors
-     /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \* Every answer is the same, so an earlier one may be cached.
-  \/ /\ val = Capabilities
-     /\ err = None
-     /\ UNCHANGED vars
-
-=======================================================================================================================

spec/EBSProvider/EBSProvider.tla

@@ -1,163 +0,0 @@
-------------------------------------------------- MODULE EBSProvider --------------------------------------------------
-(*********************************************************************************************************************)
-(* The state that an implementation exposes, and the types and constants that every other module shares.             *)
-(*********************************************************************************************************************)
-EXTENDS
-  Naturals,
-  Sequences,
-  FiniteSets,
-  Functions,
-  TLC
-
------------------------------------------------------------------------------------------------------------------------
-\* Constants
-
-CONSTANTS
-  MaxBytes,
-  VolumeIDs, VolumeHandles,
-  NodeIDs, NBDURIs, DevicePaths,
-  SnapshotIDs, SnapshotHandles,
-  AvailabilityZones, Timestamps,
-  Parameters, ProviderData,
-  ListTokens, MaxListResults,
-  \* The optional behaviour that the implementation advertises, which never changes.
-  Capabilities,
-  \* Whether the provider may fail to answer at any point in a behaviour.
-  ProviderMayFail,
-  \* Stands in for an optional field that a request or response omitted.
-  None
-
-ASSUME MaxBytes \in Nat \ {0}
-ASSUME VolumeIDs /= {} /\ VolumeHandles /= {}
-ASSUME NodeIDs /= {} => NBDURIs /= {} \/ DevicePaths /= {}
-ASSUME SnapshotIDs /= {} => SnapshotHandles /= {} /\ Timestamps /= {}
-ASSUME ListTokens /= {}
-ASSUME MaxListResults \in Nat \ {0}
-ASSUME None \notin UNION {VolumeIDs, VolumeHandles, NodeIDs, SnapshotIDs, SnapshotHandles,
-                          AvailabilityZones, Timestamps, Parameters, ProviderData, ListTokens}
-ASSUME Capabilities \in [online_expansion          : BOOLEAN,
-                         sparse_extent_reporting   : BOOLEAN,
-                         crash_consistent_snapshot : BOOLEAN,
-                         volume_seeding            : BOOLEAN,
-                         read_only_publish         : BOOLEAN,
-                         volume_enumeration        : BOOLEAN,
-                         snapshot_enumeration      : BOOLEAN,
-                         owner_routing             : BOOLEAN,
-                         exclusion                 : [scope             : {"none", "node", "cluster"},
-                                                      claim_ttl_seconds : Nat,
-                                                      fences_lost_claim : BOOLEAN]]
-ASSUME ProviderMayFail \in BOOLEAN
-
------------------------------------------------------------------------------------------------------------------------
-\* Variables
-
-VARIABLES
-  \* Every volume that the implementation exposes, keyed by volume ID.
-  volumes,
-  \* Every publication that the implementation exposes, keyed by the volume and the node it is published to.
-  publications,
-  \* Every snapshot that the implementation exposes, keyed by snapshot ID.
-  snapshots
-
-vars == <<volumes, publications, snapshots>>
-
------------------------------------------------------------------------------------------------------------------------
-\* Miscellaneous
-
-IsPartialFn(f, D, R) == DOMAIN f \subseteq D /\ \A x \in DOMAIN f : f[x] \in R
-
-Optional(S) == S \union {None}
-
-InsertKey(f, x, y) == f @@ (x :> y)
-
-RemoveKey(f, x) == RestrictDomain(f, LAMBDA x_2 : x_2 /= x)
-
-KeyExists(f, x) == x \in DOMAIN f
-
------------------------------------------------------------------------------------------------------------------------
-\* Types
-
-Bytes == 0..MaxBytes
-
-\* The errors that an implementation may return when its provider fails to answer.
-ProviderErrors == IF ProviderMayFail
-                  THEN {"internal", "unavailable"}
-                  \* Otherwise an implementation could conform by refusing to answer.
-                  ELSE {}
-
-CapacityRanges == [required_bytes : Bytes, limit_bytes : Bytes]
-
-VolumeStates == {"available", "in-use"}
-
-Volumes == [capacity_bytes     : 1..MaxBytes,
-            state              : VolumeStates,
-            handle             : VolumeHandles,
-            availability_zone  : Optional(AvailabilityZones),
-
-            \* Never returned, but kept to tell a retry from a conflicting request.
-            source_snapshot_id : Optional(SnapshotIDs),
-            parameters         : Optional(Parameters),
-            provider_data      : Optional(ProviderData)]
-
-\* How a published volume is reached on its node, either over NBD or as a host block device.
-Exports == [kind    : {"nbd"},
-            nbd_uri : NBDURIs]
-           \union
-           [kind        : {"block"},
-            device_path : DevicePaths,
-            size_bytes  : 1..MaxBytes]
-
-\* A volume published to a node, and whether it was published read-only.
-Publications == [export    : Exports,
-                 read_only : BOOLEAN]
-
-SnapshotStates == {"pending", "completed", "error"}
-
-Snapshots == [size_bytes       : 1..MaxBytes,
-              created_at       : Timestamps,
-              state            : SnapshotStates,
-              handle           : SnapshotHandles,
-              source_volume_id : VolumeIDs]
-
------------------------------------------------------------------------------------------------------------------------
-\* State helpers
-
-\* The part of a volume that the implementation returns.
-VolumeView(volID) ==
-  ("id" :> volID) @@ Restrict(volumes[volID], {"capacity_bytes",
-                                               "state",
-                                               "handle",
-                                               "availability_zone",
-                                               "provider_data"})
-
-\* The nodes that a volume is published to.
-PublishedNodes(volID) == {n \in NodeIDs : KeyExists(publications, <<volID, n>>)}
-
-\* The part of a publication that the implementation returns.
-PublicationView(volID, nodeID) == [volume_id |-> volID,
-                                   node_id   |-> nodeID,
-                                   export    |-> publications[volID, nodeID].export]
-
-\* The part of a snapshot that the implementation returns.
-SnapshotView(snapID) == ("id" :> snapID) @@ snapshots[snapID]
-
-\* The number of refs that a page may hold, with an omitted or oversized request clamped to the maximum.
-PageSize(m) == IF m <= 0 \/ m > MaxListResults THEN MaxListResults ELSE m
-
------------------------------------------------------------------------------------------------------------------------
-\* Invariants
-
-TypeOK ==
-  /\ IsPartialFn(volumes, VolumeIDs, Volumes)
-  /\ IsPartialFn(publications, VolumeIDs \X NodeIDs, Publications)
-  /\ IsPartialFn(snapshots, SnapshotIDs, Snapshots)
-
------------------------------------------------------------------------------------------------------------------------
-\* Initial state
-
-Init ==
-  /\ volumes      = <<>>
-  /\ publications = <<>>
-  /\ snapshots    = <<>>
-
-=======================================================================================================================

spec/EBSProvider/README.md

@@ -1,140 +0,0 @@
-# EBSProvider specification
-
-The contract that every implementation of the `EBSProvider` interface must meet.
-
-## Terminology
-
-- **Provider:** the service that backs an implementation's operations, such as Viperblock.
-  The memory implementation has none.
-- **Implementation:** a Go implementation of the `EBSProvider` interface, which adapts between the shapes of the contract and the shapes that its provider expects.
-- **Contract:** the behaviour that this specification requires of every implementation, as observed through the interface.
-
-## Open questions
-
-### EBSProvider
-
-- What may a failed call change?
-  A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
-- Is the schema version part of the contract?
-  Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
-- Are `internal` and `unavailable` the only errors that a failed provider may cause?
-- Must a present but malformed ID be refused with `invalid_argument`?
-  IDs are opaque here, so only an omitted one is refused.
-
-### Capabilities
-
-#### GetCapabilities
-
-- Is it a lasting promise that capabilities never change, even when an implementation or its provider is upgraded or reconfigured after an earlier answer was cached?
-
-### Volumes
-
-- Who decides whether a volume is `in-use`?
-  An implementation may track attachments itself, or derive the state from whether a publication claim is held.
-
-#### CreateVolume
-
-- May the capacity be anywhere in the requested range, as CSI allows?
-  An implementation that rounds up would then fail the comparison that tells a retry from a conflicting request.
-- May seed data be combined with a source snapshot?
-  The Viperblock implementation refuses, the memory implementation accepts.
-- Must the source snapshot be completed?
-  AWS refuses one that is not.
-- Must an implementation store the availability zone, parameters and source snapshot, so that it can detect a conflicting retry?
-  The Viperblock implementation does not.
-- Should a retry be compared against the original request rather than the current volume?
-  A retry after ExpandVolume conflicts on capacity.
-- Should a retry with a different source snapshot volume ID conflict?
-  It is not compared.
-- Is byte equality the meaning of equal parameters, or may an implementation compare them semantically?
-- Should `MaxSeedBytes` be modelled?
-  Seed data larger than it is refused with `invalid_argument`.
-
-#### GetVolume
-
-- Must an implementation compare the handle?
-  The Viperblock implementation ignores it.
-- Is `not_found` the right error for a volume that was deleted and recreated under the same ID?
-
-#### ListVolumes
-
-- Must following next tokens list every volume that exists throughout the walk exactly once?
-  The conformance suite tests it, but the specification describes each call alone, so it does not say how one page relates to the next.
-- What should a starting token that the implementation never issued do?
-  Every implementation treats any token as a position to resume after, while CSI refuses an invalid one with `ABORTED`.
-- May a page hold fewer refs than requested yet carry a next token?
-  AWS and CSI allow it, but here a next token comes only with a full page, so that a walk cannot stall on empty pages.
-
-#### ExpandVolume
-
-- Should a request below the current capacity succeed without change, as CSI recommends, rather than be refused with `invalid_argument`?
-  It is refused here, as AWS refuses `ModifyVolume` to a smaller size and the conformance suite tests, while CSI says the plugin should reply OK when the volume is already at least the requested size.
-- Must a block export's size follow an online expansion?
-  A published volume keeps its export's `size_bytes`, so after it grows the export reports the old size.
-
-#### PublishVolume
-
-- Should `volume_in_use` depend on the exclusion scope?
-  An implementation whose scope is `none` lets two writers in, and one whose scope is `node` cannot see another node, yet both are refused here.
-- Must a republish to the same node with a different `read_only` be refused?
-  The Viperblock implementation refuses it with `volume_in_use`, while the memory implementation ignores the flag.
-- May two volumes on one node share an NBD URI or a device path?
-- Must a block export's size equal the volume's capacity?
-
-#### UnpublishVolume
-
-- Must an implementation compare the handle?
-  Both the Viperblock and memory implementations ignore it.
-- Is a provider that fails to seal the volume a failed provider?
-  The Viperblock implementation answers `internal` and leaves the volume published, which the specification allows only when the provider may fail.
-- Should a claim that lapses through its TTL be modelled, as a step that happens without a call?
-- Should a volume that does not exist be refused with `not_found`?
-  It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DetachVolume` on a missing volume with `InvalidVolume.NotFound`.
-
-#### DeleteVolume
-
-- Must an implementation compare the handle before refusing a published volume?
-  The memory implementation does, and succeeds without deleting it, while the Viperblock implementation ignores the handle and refuses with `volume_in_use`.
-- Should a volume that does not exist be refused with `not_found`?
-  It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteVolume` on a missing volume with `InvalidVolume.NotFound`.
-
-### Snapshots
-
-- Should a pending snapshot's move to `completed` or `error` be modelled, as a step that happens without a call?
-  Without it a pending snapshot never completes, and no snapshot is ever in `error`.
-
-#### CreateSnapshot
-
-- May success return a `pending` snapshot?
-  AWS starts a snapshot as `pending`, while the memory implementation returns `completed` and the NATS implementation waits for completion and refuses anything else.
-- Must a retry compare the volume handle?
-  It is ignored, as in the memory implementation, so a volume deleted and recreated under the same ID returns the old snapshot.
-- Should a copy's ID count as a retry?
-  A copy keeps its source's volume ID, so creating a snapshot of that volume under the copy's ID returns the copy.
-- Should `crash_consistent_snapshot` be modelled?
-  The specification does not model volume data, so it cannot say what a snapshot of a published volume holds.
-
-#### CopySnapshot
-
-- Should a retried copy return the existing copy, as a retried CreateVolume does, rather than `already_exists`?
-- May a source that is not completed be copied?
-  AWS refuses until it is.
-- What state does a copy start in?
-  The memory implementation always returns `completed`, while AWS starts a copy as `pending`.
-
-#### ListSnapshots
-
-- The questions for ListVolumes apply here too.
-- Must a ref carry the source volume ID?
-  The Viperblock implementation leaves it empty, since reading it costs a fetch per snapshot, yet reconciliation needs it to act on a snapshot whose metadata is lost.
-
-#### DeleteSnapshot
-
-- Is the handle check part of the contract?
-  The Viperblock implementation ignores the handle and deletes anyway.
-- May a pending snapshot be deleted?
-  AWS allows it.
-- What does success promise?
-  The Go docs say the snapshot is gone, but a snapshot whose handle did not match is left in place.
-- Should a snapshot that does not exist be refused with `not_found`?
-  It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteSnapshot` on a missing snapshot with `InvalidSnapshot.NotFound`.

spec/EBSProvider/Snapshots.tla

@@ -1,159 +0,0 @@
--------------------------------------------------- MODULE Snapshots ---------------------------------------------------
-(*********************************************************************************************************************)
-(* The lifecycle of a snapshot, from its creation to its deletion.                                                   *)
-(*********************************************************************************************************************)
-EXTENDS EBSProvider
-
------------------------------------------------------------------------------------------------------------------------
-\* Interface methods
-
-CreateSnapshotRequests == [snapshot_id   : Optional(SnapshotIDs),
-                           volume_id     : Optional(VolumeIDs),
-                           volume_handle : Optional(VolumeHandles)]
-
-CreateSnapshot(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. A retry of a call that         *)
-  (* already succeeded returns the same snapshot.                                                                    *)
-  (*******************************************************************************************************************)
-  LET snapID == req.snapshot_id
-      volID  == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> None \in {snapID, volID},
-                       already_exists   |-> /\ KeyExists(snapshots, snapID)
-                                            /\ snapshots[snapID].source_volume_id /= volID,
-                       \* Only checked for a new snapshot, so a retry outlives the volume.
-                       not_found        |-> /\ ~KeyExists(snapshots, snapID)
-                                            /\ \/ ~KeyExists(volumes, volID)
-                                               \/ req.volume_handle \notin {None, volumes[volID].handle}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-           \* A retry returns the existing snapshot.
-        /\ \/ /\ KeyExists(snapshots, snapID)
-              /\ UNCHANGED snapshots
-
-           \/ /\ ~KeyExists(snapshots, snapID)
-              \* The handle, creation time and state are the implementation's choice.
-              /\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
-                    t \in Timestamps,
-                    s \in {"pending", "completed"} :
-                   snapshots' = InsertKey(snapshots, snapID, [size_bytes       |-> volumes[volID].capacity_bytes,
-                                                              created_at       |-> t,
-                                                              state            |-> s,
-                                                              handle           |-> h,
-                                                              source_volume_id |-> volID])
-        /\ val = SnapshotView(snapID)'
-        /\ err = None
-        /\ UNCHANGED <<volumes, publications>>
-
-
-CopySnapshotRequests == [source_snapshot_id      : Optional(SnapshotIDs),
-                         destination_snapshot_id : Optional(SnapshotIDs),
-                         volume_id               : Optional(VolumeIDs)]
-
-CopySnapshot(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A snapshot is copied under a new ID, sharing its frozen data, and the copy can be deleted independently         *)
-  (* of it.                                                                                                          *)
-  (*******************************************************************************************************************)
-  LET srcSnapID == req.source_snapshot_id
-      dstSnapID == req.destination_snapshot_id
-      volID     == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> \/ None \in {srcSnapID, dstSnapID, volID}
-                                            \/ srcSnapID = dstSnapID
-                                            \* Only compared, not looked up, since a snapshot outlives its volume.
-                                            \/ /\ KeyExists(snapshots, srcSnapID)
-                                               /\ snapshots[srcSnapID].source_volume_id /= volID,
-                       already_exists   |-> KeyExists(snapshots, dstSnapID),
-                       not_found        |-> srcSnapID \notin Optional(DOMAIN snapshots)]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        \* The handle and creation time are the implementation's choice.
-        /\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
-              t \in Timestamps :
-             snapshots' = InsertKey(snapshots, dstSnapID, [created_at |-> t,
-                                                           handle     |-> h]
-                                                           @@ snapshots[srcSnapID])
-        /\ val = SnapshotView(dstSnapID)'
-        /\ err = None
-        /\ UNCHANGED <<volumes, publications>>
-
-
-ListSnapshotsRequests == [max_results    : -1..(MaxListResults + 1),
-                          starting_token : Optional(ListTokens)]
-
-ListSnapshots(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* Some of the snapshots are listed, a page at a time, and nothing changes. A call with no starting token that     *)
-  (* returns no next token lists every snapshot.                                                                     *)
-  (*******************************************************************************************************************)
-  LET Errors ==
-    LET checks == [unsupported_capability |-> ~Capabilities.snapshot_enumeration]
-    IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ LET refs == {[id               |-> id,
-                         source_volume_id |-> snapshots[id].source_volume_id,
-                         handle           |-> snapshots[id].handle] : id \in DOMAIN snapshots}
-           \* The page and the next token are the implementation's choice.
-           IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
-                /\ Cardinality(page) <= PageSize(req.max_results)
-                /\ next /= None => Cardinality(page) = PageSize(req.max_results)
-                /\ req.starting_token = None /\ next = None => page = refs
-                /\ val = [snapshots |-> page, next_token |-> next]
-        /\ err = None
-        /\ UNCHANGED vars
-
-
-DeleteSnapshotRequests == [snapshot_id : Optional(SnapshotIDs),
-                           handle      : Optional(SnapshotHandles)]
-
-DeleteSnapshot(req, err) ==
-  (*******************************************************************************************************************)
-  (* A snapshot is deleted, so that no new volume can be restored from it. Volumes already restored from it,         *)
-  (* and copies of it, are unaffected.                                                                               *)
-  (*******************************************************************************************************************)
-  LET snapID == req.snapshot_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> snapID = None]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ snapshots' = IF /\ KeyExists(snapshots, snapID)
-                           /\ req.handle \notin {None, snapshots[snapID].handle}
-                        THEN snapshots
-                        \* Record removal is idempotent so that retries succeed.
-                        ELSE RemoveKey(snapshots, snapID)
-        /\ err = None
-        /\ UNCHANGED <<volumes, publications>>
-
-=======================================================================================================================

spec/EBSProvider/Volumes.tla

@@ -1,286 +0,0 @@
---------------------------------------------------- MODULE Volumes ----------------------------------------------------
-(*********************************************************************************************************************)
-(* The lifecycle of a volume, from its creation to its deletion.                                                     *)
-(*********************************************************************************************************************)
-EXTENDS EBSProvider
-
------------------------------------------------------------------------------------------------------------------------
-\* Interface methods
-
-CreateVolumeRequests == [volume_id                 : Optional(VolumeIDs),
-                         capacity_range            : CapacityRanges,
-                         availability_zone         : Optional(AvailabilityZones),
-                         source_snapshot_id        : Optional(SnapshotIDs),
-                         source_snapshot_volume_id : Optional(VolumeIDs),
-                         parameters                : Optional(Parameters),
-                         seed_data                 : Bytes]
-
-CreateVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume of the requested capacity is created, either blank or holding a snapshot's data, optionally            *)
-  (* with seed data written at its start. A retry of a call that already succeeded returns the same volume.          *)
-  (*******************************************************************************************************************)
-  LET volID     == req.volume_id
-      snapID    == req.source_snapshot_id
-      snapVolID == req.source_snapshot_volume_id
-      capacity  == req.capacity_range
-
-      \* Any warranted error may be returned, since the order of checks is not part of the contract.
-      Errors ==
-        LET checks == [invalid_argument       |-> \/ volID = None
-                                                  \/ capacity.required_bytes <= 0
-                                                  \/ /\ capacity.limit_bytes > 0
-                                                     /\ capacity.required_bytes > capacity.limit_bytes
-                                                  \/ req.seed_data > capacity.required_bytes
-                                                  \/ snapID /= None /\ snapVolID = None
-                                                  \* Only checked for a new volume, so a retry outlives the snapshot.
-                                                  \/ /\ ~KeyExists(volumes, volID)
-                                                     /\ KeyExists(snapshots, snapID)
-                                                     /\ \/ capacity.required_bytes < snapshots[snapID].size_bytes
-                                                        \/ snapshots[snapID].source_volume_id /= snapVolID,
-                       \* Seed data is not compared, since it only applies on creation.
-                       already_exists         |-> /\ KeyExists(volumes, volID)
-                                                  /\ \/ volumes[volID].capacity_bytes     /= capacity.required_bytes
-                                                     \/ volumes[volID].availability_zone  /= req.availability_zone
-                                                     \/ volumes[volID].source_snapshot_id /= snapID
-                                                     \/ volumes[volID].parameters         /= req.parameters,
-                       not_found              |-> /\ ~KeyExists(volumes, volID)
-                                                  /\ snapID \notin Optional(DOMAIN snapshots),
-                       unsupported_capability |-> /\ req.seed_data > 0
-                                                  /\ ~Capabilities.volume_seeding]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-           \* A retry returns the existing volume.
-        /\ \/ /\ KeyExists(volumes, volID)
-              /\ UNCHANGED volumes
-
-           \/ /\ ~KeyExists(volumes, volID)
-              \* The handle and provider data are the implementation's choice.
-              /\ \E h \in VolumeHandles \ {volumes[x].handle : x \in DOMAIN volumes},
-                    pd \in Optional(ProviderData) :
-                   volumes' = InsertKey(volumes, volID, [capacity_bytes     |-> capacity.required_bytes,
-                                                         state              |-> "available",
-                                                         handle             |-> h,
-                                                         availability_zone  |-> req.availability_zone,
-                                                         source_snapshot_id |-> snapID,
-                                                         parameters         |-> req.parameters,
-                                                         provider_data      |-> pd])
-        /\ val = VolumeView(volID)'
-        /\ err = None
-        /\ UNCHANGED <<publications, snapshots>>
-
-
-GetVolumeRequests == [volume_id : Optional(VolumeIDs),
-                      handle    : Optional(VolumeHandles)]
-
-GetVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's current capacity, state and handle are read, and nothing changes.                                    *)
-  (*******************************************************************************************************************)
-  LET volID == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> volID = None,
-                       not_found        |-> \/ ~KeyExists(volumes, volID)
-                                            \/ req.handle \notin {None, volumes[volID].handle}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ val = VolumeView(volID)
-        /\ err = None
-        /\ UNCHANGED vars
-
-
-ListVolumesRequests == [max_results    : -1..(MaxListResults + 1),
-                        starting_token : Optional(ListTokens)]
-
-ListVolumes(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* Some of the volumes are listed, a page at a time, and nothing changes. A call with no starting token that       *)
-  (* returns no next token lists every volume.                                                                       *)
-  (*******************************************************************************************************************)
-  LET Errors ==
-    LET checks == [unsupported_capability |-> ~Capabilities.volume_enumeration]
-    IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ LET refs == {[id |-> id, handle |-> volumes[id].handle] : id \in DOMAIN volumes}
-           \* The page and the next token are the implementation's choice.
-           IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
-                /\ Cardinality(page) <= PageSize(req.max_results)
-                /\ next /= None => Cardinality(page) = PageSize(req.max_results)
-                /\ req.starting_token = None /\ next = None => page = refs
-                /\ val = [volumes |-> page, next_token |-> next]
-        /\ err = None
-        /\ UNCHANGED vars
-
-
-ExpandVolumeRequests == [volume_id      : Optional(VolumeIDs),
-                         handle         : Optional(VolumeHandles),
-                         capacity_range : CapacityRanges]
-
-ExpandVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume grows to the requested capacity, keeping its data. A published volume grows only if the                *)
-  (* implementation supports online expansion, and a request for the current capacity succeeds without change.       *)
-  (*******************************************************************************************************************)
-  LET volID    == req.volume_id
-      capacity == req.capacity_range
-
-      Errors ==
-        LET checks == [invalid_argument |-> \/ volID = None
-                                            \/ capacity.required_bytes = 0
-                                            \/ /\ capacity.limit_bytes > 0
-                                               /\ capacity.required_bytes > capacity.limit_bytes
-                                            \/ /\ KeyExists(volumes, volID)
-                                               /\ capacity.required_bytes < volumes[volID].capacity_bytes,
-                       not_found        |-> \/ ~KeyExists(volumes, volID)
-                                            \/ req.handle \notin {None, volumes[volID].handle},
-                       volume_in_use    |-> /\ PublishedNodes(volID) /= {}
-                                            /\ ~Capabilities.online_expansion
-                                            /\ capacity.required_bytes > volumes[volID].capacity_bytes]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ volumes' = [volumes EXCEPT ![volID].capacity_bytes = capacity.required_bytes]
-        /\ val = VolumeView(volID)'
-        /\ err = None
-        /\ UNCHANGED <<publications, snapshots>>
-
-
-PublishVolumeRequests == [volume_id : Optional(VolumeIDs),
-                          handle    : Optional(VolumeHandles),
-                          node_id   : Optional(NodeIDs),
-                          read_only : BOOLEAN]
-
-PublishVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's data becomes reachable on a node, through an export that a VM there can attach, either an NBD        *)
-  (* URI or a host block device. Reads and writes through the export reach the volume, and a read-only export        *)
-  (* refuses writes. Only one node at a time may reach a volume.                                                     *)
-  (*******************************************************************************************************************)
-  LET volID  == req.volume_id
-      nodeID == req.node_id
-
-      Errors ==
-        LET checks == [invalid_argument       |-> None \in {volID, nodeID},
-                       not_found              |-> \/ ~KeyExists(volumes, volID)
-                                                  \/ req.handle \notin {None, volumes[volID].handle},
-                       unsupported_capability |-> /\ req.read_only
-                                                  /\ ~Capabilities.read_only_publish,
-                       volume_in_use          |-> PublishedNodes(volID) \ {nodeID} /= {}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-           \* A retry returns the existing publication.
-        /\ \/ /\ KeyExists(publications, <<volID, nodeID>>)
-              /\ UNCHANGED publications
-
-           \/ /\ ~KeyExists(publications, <<volID, nodeID>>)
-              \* The type of export is the implementation's choice.
-              /\ \E x \in Exports :
-                   publications' = InsertKey(publications, <<volID, nodeID>>, [export    |-> x,
-                                                                               read_only |-> req.read_only])
-        /\ volumes' = [volumes EXCEPT ![volID].state = "in-use"]
-        /\ val = PublicationView(volID, nodeID)'
-        /\ err = None
-        /\ UNCHANGED snapshots
-
-
-UnpublishVolumeRequests == [volume_id : Optional(VolumeIDs),
-                            handle    : Optional(VolumeHandles),
-                            node_id   : Optional(NodeIDs)]
-
-UnpublishVolume(req, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's export on a node is torn down, so that its data is no longer reachable there. Once no node           *)
-  (* can reach the volume, it is available again.                                                                    *)
-  (*******************************************************************************************************************)
-  LET volID  == req.volume_id
-      nodeID == req.node_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> None \in {volID, nodeID}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        \* Record removal is idempotent so that retries succeed.
-        /\ publications' = RemoveKey(publications, <<volID, nodeID>>)
-        \* The volume is available again once no node holds it.
-        /\ volumes' = IF \/ ~KeyExists(volumes, volID)
-                         \/ PublishedNodes(volID)' /= {}
-                      THEN volumes
-                      ELSE [volumes EXCEPT ![volID].state = "available"]
-        /\ err = None
-        /\ UNCHANGED snapshots
-
-
-DeleteVolumeRequests == [volume_id : Optional(VolumeIDs),
-                         handle    : Optional(VolumeHandles)]
-
-DeleteVolume(req, err) ==
-  (*******************************************************************************************************************)
-  (* A volume is deleted, so that its data is gone. Snapshots taken from it are unaffected.                          *)
-  (*******************************************************************************************************************)
-  LET volID == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> volID = None,
-                       volume_in_use    |-> /\ PublishedNodes(volID) /= {}
-                                            /\ req.handle \in {None, volumes[volID].handle}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        \* Record removal is idempotent so that retries succeed.
-        /\ volumes' = IF /\ KeyExists(volumes, volID)
-                         /\ req.handle \notin {None, volumes[volID].handle}
-                      THEN volumes
-                      ELSE RemoveKey(volumes, volID)
-        /\ err = None
-        /\ UNCHANGED <<publications, snapshots>>
-
------------------------------------------------------------------------------------------------------------------------
-\* Specification
-
-=======================================================================================================================

spinifex/agents/elbv2/agent.go

@@ -21,7 +21,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -95,18 +94,6 @@ type Agent struct {
 	statsFn func(socketPath string) ([]ServerStatus, error)
 	// For testing: override the active health prober (nginx/NLB).
 	probeFn func(targets []HealthTarget) []ServerStatus
-
-	// Data-path probe: netcfg and sysfs locations, the ARP exchange, the
-	// instance ID lookup and per-NIC miss counters (overridable in tests).
-	netcfgPath   string
-	sysNetDir    string
-	probeWait    time.Duration
-	arpProbeFn   arpProbeFn
-	instanceIDFn func() (string, error)
-	instanceID   string
-	idSkip       int
-	probeMu      sync.Mutex
-	probeStates  map[string]*nicProbeState
 }
 
 // New creates a new LB agent for the given load balancer.
@@ -163,12 +150,6 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error)
 		reloadNginxFn: reloadNginx,
 		statsFn:       queryHAProxyStats,
 		probeFn:       probeHealthTargets,
-		netcfgPath:    netcfgPath,
-		sysNetDir:     sysClassNet,
-		probeWait:     probeWait,
-		arpProbeFn:    arpProbe,
-		instanceIDFn:  fetchInstanceID,
-		probeStates:   map[string]*nicProbeState{},
 	}, nil
 }
 
@@ -226,7 +207,7 @@ func (a *Agent) tick() {
 		}
 	}
 
-	resp, err := a.sendHeartbeat(servers, a.dataPathEvidence())
+	resp, err := a.sendHeartbeat(servers)
 	if err != nil {
 		// Include the dial target so the serial console disambiguates a
 		// transport failure (never reached AWSGW — "send request: dial ...")
@@ -307,7 +288,7 @@ func (a *Agent) enginePaths(engine string) (configPath, pidPath, certDir string,
 }
 
 // sendHeartbeat sends a heartbeat with health report to the gateway.
-func (a *Agent) sendHeartbeat(servers []ServerStatus, ev dataPathEvidence) (*heartbeatResponse, error) {
+func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error) {
 	params := url.Values{}
 	params.Set("Action", "LBAgentHeartbeat")
 	params.Set("Version", "2015-12-01")
@@ -319,7 +300,6 @@ func (a *Agent) sendHeartbeat(servers []ServerStatus, ev dataPathEvidence) (*hea
 		params.Set("Servers.member."+idx+".Server", s.Server)
 		params.Set("Servers.member."+idx+".Status", s.Status)
 	}
-	ev.addTo(params)
 
 	body, err := a.signedPost(params)
 	if err != nil {

spinifex/agents/elbv2/arp_linux.go

@@ -1,62 +0,0 @@
-//go:build linux
-
-package elbv2
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"net/netip"
-	"syscall"
-	"time"
-)
-
-// arpProbe sends one ARP request for gateway out of iface on an AF_PACKET
-// socket bound to it and waits for the gateway's reply on the same socket.
-func arpProbe(iface string, mac net.HardwareAddr, src, gateway netip.Addr, wait time.Duration) error {
-	ifi, err := net.InterfaceByName(iface)
-	if err != nil {
-		return fmt.Errorf("interface %s: %w", iface, err)
-	}
-
-	proto := htons(syscall.ETH_P_ARP)
-	fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, int(proto))
-	if err != nil {
-		return fmt.Errorf("packet socket: %w", err)
-	}
-	defer syscall.Close(fd)
-
-	addr := &syscall.SockaddrLinklayer{Protocol: proto, Ifindex: ifi.Index}
-	if err := syscall.Bind(fd, addr); err != nil {
-		return fmt.Errorf("bind %s: %w", iface, err)
-	}
-
-	deadline := time.Now().Add(wait)
-	if err := syscall.Sendto(fd, buildARPRequest(mac, src, gateway), 0, addr); err != nil {
-		return fmt.Errorf("send ARP request: %w", err)
-	}
-
-	buf := make([]byte, 128)
-	for {
-		left := time.Until(deadline)
-		if left <= 0 {
-			return errARPNoReply
-		}
-		tv := syscall.NsecToTimeval(int64(left))
-		if err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv); err != nil {
-			return fmt.Errorf("set receive deadline: %w", err)
-		}
-		n, _, err := syscall.Recvfrom(fd, buf, 0)
-		if err != nil {
-			if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
-				continue
-			}
-			return fmt.Errorf("receive ARP reply: %w", err)
-		}
-		if isARPReplyFrom(buf[:n], mac, gateway) {
-			return nil
-		}
-	}
-}
-
-func htons(v uint16) uint16 { return v<<8 | v>>8 }

spinifex/agents/elbv2/arp_other.go

@@ -1,15 +0,0 @@
-//go:build !linux
-
-package elbv2
-
-import (
-	"errors"
-	"net"
-	"net/netip"
-	"time"
-)
-
-// arpProbe is only meaningful inside the Linux guest.
-func arpProbe(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-	return errors.New("ARP probe is only supported on linux")
-}

spinifex/agents/elbv2/datapath.go

@@ -1,303 +0,0 @@
-package elbv2
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"net"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-)
-
-const (
-	// netcfgPath is the fw_cfg blob the microVM init script reads to configure NICs.
-	netcfgPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw"
-
-	sysClassNet = "/sys/class/net"
-
-	// probeFailureThreshold is the number of consecutive missed probes after
-	// which a NIC is reported as failing (about 15 s at the poll interval).
-	probeFailureThreshold = 3
-
-	// arpExchangeWait bounds one ARP exchange. probeWait bounds how long a tick
-	// waits for all probes, so a stuck probe never delays the heartbeat.
-	arpExchangeWait = time.Second
-	probeWait       = 1500 * time.Millisecond
-)
-
-// nicConfig is one NIC of the netcfg blob that has a gateway to probe.
-type nicConfig struct {
-	MAC     string
-	Addr    netip.Addr
-	Gateway netip.Addr
-}
-
-// NICReport is the per-NIC data-path evidence carried in the heartbeat.
-type NICReport struct {
-	MAC                 string
-	ProbeOK             bool
-	ConsecutiveFailures int64
-	RxPackets           int64
-	TxPackets           int64
-	RxDropped           int64
-	TxDropped           int64
-}
-
-// arpProbeFn sends one ARP request for gateway out of iface and returns nil
-// when the gateway's reply arrives within timeout.
-type arpProbeFn func(iface string, mac net.HardwareAddr, src, gateway netip.Addr, timeout time.Duration) error
-
-// parseNetcfg reads the KEY=value netcfg blob and returns the NICs that have a
-// gateway, in NIC order. The mgmt NIC carries no gateway and is skipped, as
-// is any NIC without a usable MAC, address or gateway.
-func parseNetcfg(blob []byte) []nicConfig {
-	type raw struct{ mac, cidr, gw string }
-	byIndex := map[int]*raw{}
-
-	sc := bufio.NewScanner(bytes.NewReader(blob))
-	for sc.Scan() {
-		line := strings.TrimSpace(sc.Text())
-		key, val, ok := strings.Cut(line, "=")
-		if !ok || !strings.HasPrefix(key, "NIC") {
-			continue
-		}
-		idxStr, field, ok := strings.Cut(strings.TrimPrefix(key, "NIC"), "_")
-		if !ok {
-			continue
-		}
-		idx, err := strconv.Atoi(idxStr)
-		if err != nil {
-			continue
-		}
-		r := byIndex[idx]
-		if r == nil {
-			r = &raw{}
-			byIndex[idx] = r
-		}
-		switch field {
-		case "MAC":
-			r.mac = strings.ToLower(val)
-		case "CIDR":
-			r.cidr = val
-		case "GW":
-			r.gw = val
-		}
-	}
-
-	idxs := make([]int, 0, len(byIndex))
-	for i := range byIndex {
-		idxs = append(idxs, i)
-	}
-	sort.Ints(idxs)
-
-	var out []nicConfig
-	for _, i := range idxs {
-		r := byIndex[i]
-		if r.mac == "" || r.cidr == "" || r.gw == "" {
-			continue
-		}
-		prefix, err := netip.ParsePrefix(r.cidr)
-		if err != nil || !prefix.Addr().Is4() {
-			continue
-		}
-		gw, err := netip.ParseAddr(r.gw)
-		if err != nil || !gw.Is4() {
-			continue
-		}
-		out = append(out, nicConfig{MAC: r.mac, Addr: prefix.Addr(), Gateway: gw})
-	}
-	return out
-}
-
-// ifaceByMAC returns the interface under sysNetDir whose address is mac.
-func ifaceByMAC(sysNetDir, mac string) (string, error) {
-	entries, err := os.ReadDir(sysNetDir)
-	if err != nil {
-		return "", err
-	}
-	for _, e := range entries {
-		b, err := os.ReadFile(filepath.Join(sysNetDir, e.Name(), "address"))
-		if err != nil {
-			continue
-		}
-		if strings.EqualFold(strings.TrimSpace(string(b)), mac) {
-			return e.Name(), nil
-		}
-	}
-	return "", fmt.Errorf("no interface with MAC %s", mac)
-}
-
-// readCounters fills the sysfs statistics of iface into r.
-func readCounters(sysNetDir, iface string, r *NICReport) {
-	read := func(name string) int64 {
-		b, err := os.ReadFile(filepath.Join(sysNetDir, iface, "statistics", name))
-		if err != nil {
-			return 0
-		}
-		v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
-		return v
-	}
-	r.RxPackets = read("rx_packets")
-	r.TxPackets = read("tx_packets")
-	r.RxDropped = read("rx_dropped")
-	r.TxDropped = read("tx_dropped")
-}
-
-var errARPNoReply = errors.New("no ARP reply from the gateway")
-
-// buildARPRequest returns the Ethernet frame asking who has target.
-func buildARPRequest(mac net.HardwareAddr, src, target netip.Addr) []byte {
-	f := make([]byte, 0, 42)
-	f = append(f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff)
-	f = append(f, mac...)
-	f = binary.BigEndian.AppendUint16(f, 0x0806)
-	f = binary.BigEndian.AppendUint16(f, 1)      // Ethernet
-	f = binary.BigEndian.AppendUint16(f, 0x0800) // IPv4
-	f = append(f, 6, 4)
-	f = binary.BigEndian.AppendUint16(f, 1) // request
-	f = append(f, mac...)
-	s := src.As4()
-	f = append(f, s[:]...)
-	f = append(f, 0, 0, 0, 0, 0, 0)
-	t := target.As4()
-	f = append(f, t[:]...)
-	return f
-}
-
-// isARPReplyFrom reports whether frame is an ARP reply sent by sender to mac.
-func isARPReplyFrom(frame []byte, mac net.HardwareAddr, sender netip.Addr) bool {
-	if len(frame) < 42 || binary.BigEndian.Uint16(frame[12:14]) != 0x0806 {
-		return false
-	}
-	if binary.BigEndian.Uint16(frame[20:22]) != 2 || !bytes.Equal(frame[32:38], mac) {
-		return false
-	}
-	s := sender.As4()
-	return bytes.Equal(frame[28:32], s[:])
-}
-
-// nicProbeState carries the miss counter of one NIC across ticks. A probe that
-// has not returned is still in flight, and counts as a miss without starting
-// another goroutine behind it.
-type nicProbeState struct {
-	mu       sync.Mutex
-	failures int64
-	inflight bool
-}
-
-type probeResult struct {
-	ok  bool
-	set bool
-}
-
-// probeNICs probes every gateway-bearing data NIC once, in parallel, and
-// returns within probeWait however long a probe blocks.
-func (a *Agent) probeNICs() []NICReport {
-	blob, err := os.ReadFile(a.netcfgPath)
-	if err != nil {
-		return nil
-	}
-	nics := parseNetcfg(blob)
-	if len(nics) == 0 {
-		return nil
-	}
-
-	reports := make([]NICReport, len(nics))
-	results := make([]chan probeResult, len(nics))
-	for i, n := range nics {
-		results[i] = make(chan probeResult, 1)
-		st := a.probeState(n.MAC)
-
-		st.mu.Lock()
-		busy := st.inflight
-		if !busy {
-			st.inflight = true
-		}
-		st.mu.Unlock()
-		if busy {
-			results[i] <- probeResult{set: true}
-			continue
-		}
-
-		go func(n nicConfig, st *nicProbeState, out chan<- probeResult) {
-			ok := a.probeOne(n)
-			st.mu.Lock()
-			st.inflight = false
-			st.mu.Unlock()
-			out <- probeResult{ok: ok, set: true}
-		}(n, st, results[i])
-	}
-
-	deadline := time.NewTimer(a.probeWait)
-	defer deadline.Stop()
-	expired := false
-	for i, n := range nics {
-		var res probeResult
-		if expired {
-			select {
-			case res = <-results[i]:
-			default:
-			}
-		} else {
-			select {
-			case res = <-results[i]:
-			case <-deadline.C:
-				// Out of time: an unfinished probe is a miss for this tick.
-				expired = true
-			}
-		}
-
-		st := a.probeState(n.MAC)
-		st.mu.Lock()
-		if res.ok {
-			st.failures = 0
-		} else {
-			st.failures++
-		}
-		failures := st.failures
-		st.mu.Unlock()
-
-		r := NICReport{
-			MAC:                 n.MAC,
-			ProbeOK:             failures < probeFailureThreshold,
-			ConsecutiveFailures: failures,
-		}
-		if iface, err := ifaceByMAC(a.sysNetDir, n.MAC); err == nil {
-			readCounters(a.sysNetDir, iface, &r)
-		}
-		reports[i] = r
-	}
-	return reports
-}
-
-// probeOne runs one ARP exchange for n, false on any failure.
-func (a *Agent) probeOne(n nicConfig) bool {
-	iface, err := ifaceByMAC(a.sysNetDir, n.MAC)
-	if err != nil {
-		return false
-	}
-	hw, err := net.ParseMAC(n.MAC)
-	if err != nil {
-		return false
-	}
-	return a.arpProbeFn(iface, hw, n.Addr, n.Gateway, arpExchangeWait) == nil
-}
-
-func (a *Agent) probeState(mac string) *nicProbeState {
-	a.probeMu.Lock()
-	defer a.probeMu.Unlock()
-	st := a.probeStates[mac]
-	if st == nil {
-		st = &nicProbeState{}
-		a.probeStates[mac] = st
-	}
-	return st
-}

spinifex/agents/elbv2/datapath_test.go

@@ -1,484 +0,0 @@
-package elbv2
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/netip"
-	"net/url"
-	"os"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-const (
-	testDataMAC = "02:00:00:aa:00:01"
-	testMgmtMAC = "02:00:00:aa:00:02"
-	testNetcfg  = "NIC0_MAC=" + testDataMAC + "\nNIC0_CIDR=10.0.1.5/24\nNIC0_GW=10.0.1.1\nNIC0_DEFAULT=1\n" +
-		"NIC1_MAC=" + testMgmtMAC + "\nNIC1_CIDR=10.255.0.9/24\nNIC1_DEFAULT=0\n"
-)
-
-// datapathAgent returns a test agent whose netcfg and sysfs live in a temp dir.
-func datapathAgent(t *testing.T, gwURL, netcfg string) *Agent {
-	t.Helper()
-	a := newTestAgent(t, gwURL)
-	dir := t.TempDir()
-
-	a.netcfgPath = filepath.Join(dir, "netcfg")
-	if err := os.WriteFile(a.netcfgPath, []byte(netcfg), 0o600); err != nil {
-		t.Fatal(err)
-	}
-
-	a.sysNetDir = filepath.Join(dir, "net")
-	for name, mac := range map[string]string{"ens3": testDataMAC, "ens4": testMgmtMAC} {
-		stats := filepath.Join(a.sysNetDir, name, "statistics")
-		if err := os.MkdirAll(stats, 0o755); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.WriteFile(filepath.Join(a.sysNetDir, name, "address"), []byte(mac+"\n"), 0o600); err != nil {
-			t.Fatal(err)
-		}
-		for file, v := range map[string]string{"rx_packets": "100", "tx_packets": "200", "rx_dropped": "3", "tx_dropped": "4"} {
-			if err := os.WriteFile(filepath.Join(stats, file), []byte(v+"\n"), 0o600); err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	a.instanceIDFn = func() (string, error) { return "i-0123456789abcdef0", nil }
-	return a
-}
-
-func TestParseNetcfg_SkipsMgmtNICAndMissingGateway(t *testing.T) {
-	blob := testNetcfg +
-		"NIC2_MAC=02:00:00:aa:00:03\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\nNIC2_DEFAULT=0\n" +
-		"NIC3_MAC=02:00:00:aa:00:04\nNIC3_DHCP=1\nNIC3_DEFAULT=0\n" +
-		"NIC4_CIDR=10.0.4.5/24\nNIC4_GW=10.0.4.1\n" +
-		"NIC5_MAC=02:00:00:aa:00:06\nNIC5_CIDR=not-a-cidr\nNIC5_GW=10.0.5.1\n"
-
-	got := parseNetcfg([]byte(blob))
-	if len(got) != 2 {
-		t.Fatalf("got %d NICs, want the 2 with a gateway: %+v", len(got), got)
-	}
-	if got[0].MAC != testDataMAC || got[0].Gateway != netip.MustParseAddr("10.0.1.1") || got[0].Addr != netip.MustParseAddr("10.0.1.5") {
-		t.Errorf("first NIC = %+v", got[0])
-	}
-	if got[1].MAC != "02:00:00:aa:00:03" || got[1].Gateway != netip.MustParseAddr("10.0.2.1") {
-		t.Errorf("second NIC = %+v", got[1])
-	}
-}
-
-func TestParseNetcfg_OrdersByNICIndexAndLowercasesMAC(t *testing.T) {
-	blob := "NIC10_MAC=02:00:00:AA:00:0A\nNIC10_CIDR=10.0.9.5/24\nNIC10_GW=10.0.9.1\n" +
-		"NIC2_MAC=02:00:00:AA:00:02\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\n"
-	got := parseNetcfg([]byte(blob))
-	if len(got) != 2 || got[0].MAC != "02:00:00:aa:00:02" || got[1].MAC != "02:00:00:aa:00:0a" {
-		t.Fatalf("order or case wrong: %+v", got)
-	}
-}
-
-func TestIfaceByMACAndCounters(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-
-	iface, err := ifaceByMAC(a.sysNetDir, "02:00:00:AA:00:01")
-	if err != nil || iface != "ens3" {
-		t.Fatalf("iface = %q, err = %v", iface, err)
-	}
-	if _, err := ifaceByMAC(a.sysNetDir, "02:00:00:ff:ff:ff"); err == nil {
-		t.Error("an unknown MAC must not resolve to an interface")
-	}
-
-	var r NICReport
-	readCounters(a.sysNetDir, iface, &r)
-	if r.RxPackets != 100 || r.TxPackets != 200 || r.RxDropped != 3 || r.TxDropped != 4 {
-		t.Errorf("counters = %+v", r)
-	}
-}
-
-func TestARPFrames_RequestShapeAndReplyMatching(t *testing.T) {
-	mac, _ := net.ParseMAC(testDataMAC)
-	src, gw := netip.MustParseAddr("10.0.1.5"), netip.MustParseAddr("10.0.1.1")
-
-	req := buildARPRequest(mac, src, gw)
-	if len(req) != 42 || req[12] != 0x08 || req[13] != 0x06 || req[21] != 1 {
-		t.Fatalf("request frame malformed: % x", req)
-	}
-	if string(req[38:42]) != string(gw.AsSlice()) {
-		t.Errorf("target IP not the gateway: % x", req[38:42])
-	}
-
-	reply := make([]byte, 42)
-	copy(reply[0:6], mac)
-	reply[12], reply[13] = 0x08, 0x06
-	reply[21] = 2
-	copy(reply[28:32], gw.AsSlice())
-	copy(reply[32:38], mac)
-	if !isARPReplyFrom(reply, mac, gw) {
-		t.Error("a reply from the gateway to our MAC must match")
-	}
-	if isARPReplyFrom(reply, mac, netip.MustParseAddr("10.0.1.99")) {
-		t.Error("a reply from another host must not match")
-	}
-	reply[21] = 1
-	if isARPReplyFrom(reply, mac, gw) {
-		t.Error("a request must not count as a reply")
-	}
-	if isARPReplyFrom(reply[:20], mac, gw) {
-		t.Error("a short frame must not match")
-	}
-}
-
-func TestProbeNICs_ProbesOnlyTheGatewayNIC(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	var mu sync.Mutex
-	var probed []string
-	a.arpProbeFn = func(iface string, _ net.HardwareAddr, src, gw netip.Addr, _ time.Duration) error {
-		mu.Lock()
-		defer mu.Unlock()
-		probed = append(probed, fmt.Sprintf("%s %s>%s", iface, src, gw))
-		return nil
-	}
-
-	reports := a.probeNICs()
-	if len(reports) != 1 || reports[0].MAC != testDataMAC || !reports[0].ProbeOK || reports[0].ConsecutiveFailures != 0 {
-		t.Fatalf("reports = %+v", reports)
-	}
-	if len(probed) != 1 || probed[0] != "ens3 10.0.1.5>10.0.1.1" {
-		t.Errorf("probed = %v, want only the data NIC", probed)
-	}
-	if reports[0].RxPackets != 100 || reports[0].TxDropped != 4 {
-		t.Errorf("counters missing from report: %+v", reports[0])
-	}
-}
-
-func TestProbeNICs_TwoMissesThenSuccessNeverReportsFailing(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	results := []error{errARPNoReply, errARPNoReply, nil, errARPNoReply, errARPNoReply, nil}
-	var n atomic.Int32
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		return results[n.Add(1)-1]
-	}
-
-	for i, want := range []int64{1, 2, 0, 1, 2, 0} {
-		r := a.probeNICs()
-		if len(r) != 1 || !r[0].ProbeOK || r[0].ConsecutiveFailures != want {
-			t.Fatalf("tick %d: report = %+v, want ok with %d failures", i, r, want)
-		}
-	}
-}
-
-func TestProbeNICs_ThreeMissesReportFailingAndRecoveryClears(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	var fail atomic.Bool
-	fail.Store(true)
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		if fail.Load() {
-			return errARPNoReply
-		}
-		return nil
-	}
-
-	for i := 1; i <= 4; i++ {
-		r := a.probeNICs()
-		wantOK := i < 3 // failing from the third consecutive miss
-		if r[0].ProbeOK != wantOK || r[0].ConsecutiveFailures != int64(i) {
-			t.Fatalf("miss %d: report = %+v, want ProbeOK=%v", i, r[0], wantOK)
-		}
-	}
-
-	fail.Store(false)
-	if r := a.probeNICs(); !r[0].ProbeOK || r[0].ConsecutiveFailures != 0 {
-		t.Errorf("a successful probe must clear the failure: %+v", r[0])
-	}
-}
-
-func TestProbeNICs_BlockedProbeIsBoundedAndCountsAsMiss(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	a.probeWait = 50 * time.Millisecond
-	release := make(chan struct{})
-	defer close(release)
-	var calls atomic.Int32
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		calls.Add(1)
-		<-release
-		return nil
-	}
-
-	for i := 1; i <= 3; i++ {
-		start := time.Now()
-		r := a.probeNICs()
-		if d := time.Since(start); d > time.Second {
-			t.Fatalf("tick %d waited %s on a stuck probe", i, d)
-		}
-		if len(r) != 1 || r[0].ConsecutiveFailures != int64(i) {
-			t.Fatalf("tick %d: report = %+v, want %d misses", i, r, i)
-		}
-	}
-	if got := calls.Load(); got != 1 {
-		t.Errorf("a stuck probe was started %d times, want 1 (no goroutine pile-up)", got)
-	}
-}
-
-func TestTick_BlockedProbeDoesNotDelayHeartbeatBeyondOneTick(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		if r.FormValue("Action") == "LBAgentHeartbeat" {
-			v := r.Form
-			got.Store(&v)
-		}
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.probeWait = 50 * time.Millisecond
-	release := make(chan struct{})
-	defer close(release)
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		<-release
-		return nil
-	}
-
-	start := time.Now()
-	a.tick()
-	if d := time.Since(start); d >= pollInterval {
-		t.Fatalf("tick took %s with a stuck probe, want under one %s tick", d, pollInterval)
-	}
-	if got.Load() == nil {
-		t.Fatal("the heartbeat was never sent")
-	}
-}
-
-func TestHeartbeat_EncodesEvidenceWhenPresent(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		v := r.Form
-		got.Store(&v)
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return errARPNoReply }
-
-	a.tick()
-	v := got.Load()
-	if v == nil {
-		t.Fatal("no heartbeat received")
-	}
-	want := map[string]string{
-		"InstanceID":                        "i-0123456789abcdef0",
-		"NICs.member.1.MAC":                 testDataMAC,
-		"NICs.member.1.ProbeOK":             "true",
-		"NICs.member.1.ConsecutiveFailures": "1",
-		"NICs.member.1.RxPackets":           "100",
-		"NICs.member.1.TxPackets":           "200",
-		"NICs.member.1.RxDropped":           "3",
-		"NICs.member.1.TxDropped":           "4",
-	}
-	for k, w := range want {
-		if g := v.Get(k); g != w {
-			t.Errorf("%s = %q, want %q", k, g, w)
-		}
-	}
-	if v.Get("NICs.member.2.MAC") != "" {
-		t.Error("the mgmt NIC must not be reported")
-	}
-}
-
-func TestHeartbeat_OmitsEvidenceWithoutNetcfg(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		v := r.Form
-		got.Store(&v)
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.netcfgPath = filepath.Join(t.TempDir(), "absent")
-	a.instanceIDFn = func() (string, error) { return "", errors.New("must not be asked without NICs") }
-
-	a.tick()
-	v := got.Load()
-	if v == nil {
-		t.Fatal("no heartbeat received")
-	}
-	for k := range *v {
-		if k == "InstanceID" || len(k) >= 4 && k[:4] == "NICs" {
-			t.Errorf("unexpected evidence parameter %s", k)
-		}
-	}
-}
-
-func TestDataPathEvidence_InstanceIDRetriedUntilKnown(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
-	var calls atomic.Int32
-	a.instanceIDFn = func() (string, error) {
-		if calls.Add(1) < 3 {
-			return "", errors.New("metadata not ready")
-		}
-		return "i-late", nil
-	}
-
-	// A failure backs the lookup off for imdsRetryTicks-1 ticks.
-	ticks := func(n int) string {
-		var id string
-		for range n {
-			id = a.dataPathEvidence().InstanceID
-		}
-		return id
-	}
-	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 1 {
-		t.Fatalf("after %d ticks: id %q, calls %d; want no id and one lookup", imdsRetryTicks, got, calls.Load())
-	}
-	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 2 {
-		t.Fatalf("second window: id %q, calls %d; want no id and two lookups", got, calls.Load())
-	}
-	if got := ticks(imdsRetryTicks); got != "i-late" || calls.Load() != 3 {
-		t.Fatalf("third window: id %q, calls %d; want i-late and three lookups", got, calls.Load())
-	}
-	ticks(imdsRetryTicks)
-	if calls.Load() != 3 {
-		t.Errorf("metadata asked %d times, want 3 (stop once known)", calls.Load())
-	}
-}
-
-func TestTick_BlockedInstanceIDLookupDoesNotDelayHeartbeat(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		if r.FormValue("Action") == "LBAgentHeartbeat" {
-			v := r.Form
-			got.Store(&v)
-		}
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
-	release := make(chan struct{})
-	defer close(release)
-	var calls atomic.Int32
-	a.instanceIDFn = func() (string, error) {
-		calls.Add(1)
-		<-release
-		return "", errors.New("unreachable")
-	}
-
-	for i := range 3 {
-		start := time.Now()
-		a.tick()
-		if d := time.Since(start); d >= pollInterval {
-			t.Fatalf("tick %d took %s with IMDS blocked, want under one %s tick", i, d, pollInterval)
-		}
-	}
-	v := got.Load()
-	if v == nil {
-		t.Fatal("the heartbeat was never sent")
-	}
-	if v.Get("InstanceID") != "" {
-		t.Error("InstanceID must be omitted while the lookup fails")
-	}
-	if v.Get("NICs.member.1.MAC") == "" {
-		t.Error("NIC evidence must still be sent")
-	}
-	if calls.Load() != 1 {
-		t.Errorf("blocked lookup started %d times in 3 ticks, want 1 (backoff)", calls.Load())
-	}
-}
-
-func TestFetchInstanceID_SharesOneDeadline(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(3 * imdsWait)
-	}))
-	defer srv.Close()
-	old := imdsBase
-	imdsBase = srv.URL + "/latest"
-	defer func() { imdsBase = old }()
-
-	start := time.Now()
-	if _, err := fetchInstanceID(); err == nil {
-		t.Fatal("a hung metadata service must fail the lookup")
-	}
-	if d := time.Since(start); d > imdsWait+imdsWait/2 {
-		t.Fatalf("lookup took %s, want about one %s deadline", d, imdsWait)
-	}
-}
-
-func TestARPProbe_NoResponderFailsClosed(t *testing.T) {
-	mac, _ := net.ParseMAC(testDataMAC)
-	// Nobody answers an ARP request sent on loopback, so a probe that returned
-	// nil here would mask a dead data path. It fails either on the socket (no
-	// CAP_NET_RAW) or on the missing reply.
-	err := arpProbe("lo", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond)
-	if err == nil {
-		t.Fatal("a probe with no responder must fail")
-	}
-	if err := arpProbe("no-such-iface0", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond); err == nil {
-		t.Fatal("a probe on a missing interface must fail")
-	}
-}
-
-func TestFetchInstanceID(t *testing.T) {
-	serve := func(tokenStatus, idStatus int, wantToken string) *httptest.Server {
-		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			switch {
-			case r.Method == http.MethodPut && r.URL.Path == "/latest/api/token":
-				w.WriteHeader(tokenStatus)
-				fmt.Fprint(w, "tok123")
-			case r.URL.Path == "/latest/meta-data/instance-id":
-				if got := r.Header.Get("X-Aws-Ec2-Metadata-Token"); got != wantToken {
-					t.Errorf("token header = %q, want %q", got, wantToken)
-				}
-				w.WriteHeader(idStatus)
-				fmt.Fprint(w, " i-abc123 \n")
-			default:
-				http.NotFound(w, r)
-			}
-		}))
-	}
-	saved := imdsBase
-	defer func() { imdsBase = saved }()
-
-	for _, tc := range []struct {
-		name                  string
-		tokenStatus, idStatus int
-		wantToken             string
-		wantID                string
-		wantErr               bool
-	}{
-		{"token then id", http.StatusOK, http.StatusOK, "tok123", "i-abc123", false},
-		{"token refused falls back to v1", http.StatusNotFound, http.StatusOK, "", "i-abc123", false},
-		{"id endpoint error", http.StatusOK, http.StatusServiceUnavailable, "tok123", "", true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			srv := serve(tc.tokenStatus, tc.idStatus, tc.wantToken)
-			defer srv.Close()
-			imdsBase = srv.URL + "/latest"
-			id, err := fetchInstanceID()
-			if (err != nil) != tc.wantErr || id != tc.wantID {
-				t.Fatalf("id = %q, err = %v", id, err)
-			}
-		})
-	}
-
-	imdsBase = "http://127.0.0.1:1/latest"
-	if _, err := fetchInstanceID(); err == nil {
-		t.Error("an unreachable metadata service must fail")
-	}
-}

spinifex/agents/elbv2/evidence.go

@@ -1,133 +0,0 @@
-package elbv2
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-)
-
-const (
-	imdsWait       = time.Second
-	imdsRetryTicks = 6
-)
-
-// imdsBase is the in-VM metadata endpoint (a variable so tests can point it at a fake).
-var imdsBase = "http://169.254.169.254/latest"
-
-// dataPathEvidence is the optional part of a heartbeat: which VM the agent runs
-// in and what each data NIC's gateway probe saw. An empty value adds nothing,
-// which is what an agent without a netcfg blob sends.
-type dataPathEvidence struct {
-	InstanceID string
-	NICs       []NICReport
-}
-
-// addTo encodes the evidence as additional heartbeat query parameters.
-func (e dataPathEvidence) addTo(params url.Values) {
-	if e.InstanceID != "" {
-		params.Set("InstanceID", e.InstanceID)
-	}
-	for i, n := range e.NICs {
-		p := "NICs.member." + strconv.Itoa(i+1) + "."
-		params.Set(p+"MAC", n.MAC)
-		params.Set(p+"ProbeOK", strconv.FormatBool(n.ProbeOK))
-		params.Set(p+"ConsecutiveFailures", strconv.FormatInt(n.ConsecutiveFailures, 10))
-		params.Set(p+"RxPackets", strconv.FormatInt(n.RxPackets, 10))
-		params.Set(p+"TxPackets", strconv.FormatInt(n.TxPackets, 10))
-		params.Set(p+"RxDropped", strconv.FormatInt(n.RxDropped, 10))
-		params.Set(p+"TxDropped", strconv.FormatInt(n.TxDropped, 10))
-	}
-}
-
-// dataPathEvidence gathers the probe results and instance ID for one heartbeat.
-// Neither step can fail the heartbeat: missing evidence is simply omitted.
-func (a *Agent) dataPathEvidence() dataPathEvidence {
-	nics := a.probeNICs()
-	if len(nics) == 0 {
-		return dataPathEvidence{}
-	}
-	if a.instanceID == "" {
-		a.lookupInstanceID()
-	}
-	return dataPathEvidence{InstanceID: a.instanceID, NICs: nics}
-}
-
-// lookupInstanceID asks for the instance ID without ever holding up a tick: the
-// call is bounded by imdsWait and, after a failure, retried only every
-// imdsRetryTicks ticks.
-func (a *Agent) lookupInstanceID() {
-	if a.idSkip > 0 {
-		a.idSkip--
-		return
-	}
-	type result struct {
-		id  string
-		err error
-	}
-	ch := make(chan result, 1)
-	go func() {
-		id, err := a.instanceIDFn()
-		ch <- result{id, err}
-	}()
-	timer := time.NewTimer(imdsWait)
-	defer timer.Stop()
-	select {
-	case r := <-ch:
-		if r.err == nil && r.id != "" {
-			a.instanceID = r.id
-			return
-		}
-	case <-timer.C:
-	}
-	a.idSkip = imdsRetryTicks - 1
-}
-
-// fetchInstanceID reads the instance ID from the in-VM metadata service. The
-// token request and the read share one deadline.
-func fetchInstanceID() (string, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), imdsWait)
-	defer cancel()
-	c := &http.Client{}
-
-	var token string
-	if req, err := http.NewRequestWithContext(ctx, http.MethodPut, imdsBase+"/api/token", nil); err == nil {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "60")
-		if resp, err := c.Do(req); err == nil {
-			b, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
-			resp.Body.Close()
-			if resp.StatusCode == http.StatusOK {
-				token = strings.TrimSpace(string(b))
-			}
-		}
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsBase+"/meta-data/instance-id", nil)
-	if err != nil {
-		return "", err
-	}
-	if token != "" {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
-	}
-	resp, err := c.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("instance-id: status %d", resp.StatusCode)
-	}
-	b, err := io.ReadAll(io.LimitReader(resp.Body, 256))
-	if err != nil {
-		return "", err
-	}
-	id := strings.TrimSpace(string(b))
-	if id == "" {
-		return "", fmt.Errorf("instance-id: empty")
-	}
-	return id, nil
-}

spinifex/agents/lambda/agent/agent.go

@@ -12,10 +12,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
 )
 
-// resultWriteGrace is how long past the invocation deadline a result may take
-// to reach the host.
-const resultWriteGrace = 2 * time.Second
-
 const (
 	errTypeTimedOut = "Sandbox.Timedout"
 	errTypeUnknown  = "Runtime.Unknown"
@@ -57,11 +53,6 @@ func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error
 			Payload:     req.Payload,
 		})
 		timedOut := errors.Is(err, runtimeapi.ErrDeadlineExceeded)
-		if errors.Is(err, runtimeapi.ErrResponseBodyFailed) {
-			// The admitted body was lost: report nothing, so the host's
-			// outcome stays unknown.
-			return err
-		}
 		if err != nil {
 			if ctx.Err() != nil {
 				return cause(ctx, err)
@@ -73,15 +64,9 @@ func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error
 			AttemptID:     req.AttemptID,
 			RequestID:     req.RequestID,
 			Payload:       res.Payload,
-			Segments:      res.Segments,
 			FunctionError: res.FunctionError,
 		}
-		// A timed-out invocation writes its result after the deadline, so the
-		// write is bounded from whichever is later, plus the grace.
-		wctx, wcancel := context.WithDeadline(ctx, laterOf(time.UnixMilli(req.DeadlineMs), time.Now()).Add(resultWriteGrace))
-		err = link.Complete(wctx, resp)
-		wcancel()
-		if err != nil {
+		if err := link.Complete(ctx, resp); err != nil {
 			return cause(ctx, err)
 		}
 		if timedOut {
@@ -90,13 +75,6 @@ func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error
 	}
 }
 
-func laterOf(a, b time.Time) time.Time {
-	if a.After(b) {
-		return a
-	}
-	return b
-}
-
 func cause(ctx context.Context, err error) error {
 	if ctx.Err() != nil {
 		return context.Cause(ctx)

spinifex/agents/lambda/agent/agent_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -24,7 +23,6 @@ const api = "/2018-06-01/runtime/"
 type fakeLink struct {
 	reqs []hostlink.Request
 	got  chan hostlink.Response
-	ctxs chan context.Context // when set, the context of each Complete
 }
 
 var _ hostlink.Link = (*fakeLink)(nil)
@@ -39,37 +37,16 @@ func (l *fakeLink) Next(ctx context.Context) (hostlink.Request, error) {
 }
 
 func (l *fakeLink) Complete(ctx context.Context, r hostlink.Response) error {
-	if l.ctxs != nil {
-		l.ctxs <- ctx
-	}
 	l.got <- r
 	return nil
 }
 
-// runtime serves srv and polls next, answering using reply. Cleanup cancels the
-// poll before closing the server: an in-flight next long-poll otherwise holds
-// its connection active and blocks Close forever.
-func runtime(t *testing.T, srv *runtimeapi.Server, reply func(id string, body []byte) (path string, out string)) {
+// runtime polls next and answers using reply.
+func runtime(t *testing.T, base string, reply func(id string, body []byte) (path string, out string)) {
 	t.Helper()
-	ts := httptest.NewServer(srv.Handler())
-	ctx, cancel := context.WithCancel(context.Background())
-	done := make(chan struct{})
-	t.Cleanup(func() {
-		cancel()
-		<-done
-		ts.Close()
-	})
-	do := func(method, url string, body io.Reader) (*http.Response, error) {
-		req, err := http.NewRequestWithContext(ctx, method, url, body)
-		if err != nil {
-			return nil, err
-		}
-		return ts.Client().Do(req)
-	}
 	go func() {
-		defer close(done)
 		for {
-			resp, err := do(http.MethodGet, ts.URL+api+"invocation/next", nil)
+			resp, err := http.Get(base + api + "invocation/next")
 			if err != nil {
 				return
 			}
@@ -77,7 +54,7 @@ func runtime(t *testing.T, srv *runtimeapi.Server, reply func(id string, body []
 			resp.Body.Close()
 			id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id")
 			path, out := reply(id, body)
-			r, err := do(http.MethodPost, ts.URL+api+"invocation/"+id+path, strings.NewReader(out))
+			r, err := http.Post(base+api+"invocation/"+id+path, "application/json", strings.NewReader(out))
 			if err != nil {
 				return
 			}
@@ -90,7 +67,9 @@ func deadline(d time.Duration) int64 { return time.Now().Add(d).UnixMilli() }
 
 func TestRunServesRequestsAndEchoesIDs(t *testing.T) {
 	srv := runtimeapi.New()
-	runtime(t, srv, func(id string, body []byte) (string, string) {
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	runtime(t, ts.URL, func(id string, body []byte) (string, string) {
 		if string(body) == "bad" {
 			return "/error", `{"errorMessage":"no"}`
 		}
@@ -161,56 +140,3 @@ func TestRunReturnsLinkError(t *testing.T) {
 	err := Run(context.Background(), &errLink{}, runtimeapi.New())
 	require.EqualError(t, err, "link down")
 }
-
-func TestRunBoundsTheResultWriteByTheDeadlinePlusGrace(t *testing.T) {
-	t.Run("on time", func(t *testing.T) {
-		srv := runtimeapi.New()
-		runtime(t, srv, func(string, []byte) (string, string) { return "/response", "ok" })
-		dl := deadline(time.Minute)
-		link := &fakeLink{got: make(chan hostlink.Response, 1), ctxs: make(chan context.Context, 1),
-			reqs: []hostlink.Request{{RequestID: "r", DeadlineMs: dl}}}
-		require.NoError(t, Run(context.Background(), link, srv))
-		d, ok := (<-link.ctxs).Deadline()
-		require.True(t, ok)
-		assert.Equal(t, time.UnixMilli(dl).Add(resultWriteGrace), d)
-	})
-	t.Run("a timed-out result is written after the deadline", func(t *testing.T) {
-		srv := runtimeapi.New() // nothing answers: the invocation expires
-		link := &fakeLink{got: make(chan hostlink.Response, 1), ctxs: make(chan context.Context, 1),
-			reqs: []hostlink.Request{{RequestID: "r", DeadlineMs: deadline(30 * time.Millisecond)}}}
-		require.ErrorIs(t, Run(context.Background(), link, srv), ErrInvocationTimedOut)
-		wctx := <-link.ctxs
-		d, ok := wctx.Deadline()
-		require.True(t, ok)
-		assert.Greater(t, time.Until(d), -resultWriteGrace, "the write deadline is not already past")
-		assert.Equal(t, "Sandbox.Timedout", (<-link.got).FunctionError)
-	})
-}
-
-// A body lost after admission sends the host nothing, so it reports the
-// outcome unknown, whether or not the bootstrap is still alive.
-func TestRunSendsNoResultWhenTheBodyIsLost(t *testing.T) {
-	srv := runtimeapi.New()
-	ts := httptest.NewServer(srv.Handler())
-	defer ts.Close()
-	link := &fakeLink{got: make(chan hostlink.Response, 1), reqs: []hostlink.Request{{RequestID: "r", DeadlineMs: deadline(time.Minute)}}}
-	done := make(chan error, 1)
-	go func() { done <- Run(context.Background(), link, srv) }()
-
-	resp, err := http.Get(ts.URL + api + "invocation/next")
-	require.NoError(t, err)
-	resp.Body.Close()
-	conn, err := net.Dial("tcp", ts.Listener.Addr().String())
-	require.NoError(t, err)
-	_, err = conn.Write([]byte("POST " + api + "invocation/r/response HTTP/1.1\r\nHost: x\r\nContent-Length: 100\r\n\r\npartial"))
-	require.NoError(t, err)
-	conn.Close()
-
-	select {
-	case err := <-done:
-		require.ErrorIs(t, err, runtimeapi.ErrResponseBodyFailed)
-	case <-time.After(10 * time.Second):
-		t.Fatal("Run waited for the invocation deadline")
-	}
-	assert.Empty(t, link.got, "no result may be reported")
-}

spinifex/agents/lambda/agent/agent_testlink_test.go

@@ -1,61 +0,0 @@
-//go:build lambdatestlink
-
-package agent_test
-
-import (
-	"context"
-	"net"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
-	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
-)
-
-// An expired invocation's timeout result is written over a real connection
-// after the deadline, within the grace, and the host decodes it.
-func TestRunWritesTheTimeoutFrameAfterTheDeadline(t *testing.T) {
-	guestConn, hostConn := net.Pipe()
-	t.Cleanup(func() {
-		_ = guestConn.Close()
-		_ = hostConn.Close()
-	})
-	guest := hostlink.NewTestStream(guestConn)
-	host := hostlink.NewTestStream(hostConn)
-
-	dl := time.Now().Add(50 * time.Millisecond)
-	req := hostlink.Request{GrantID: "g", AttemptID: "a", RequestID: "r", DeadlineMs: dl.UnixMilli()}
-	sent := make(chan error, 1)
-	go func() { sent <- host.HostSend(req) }()
-
-	done := make(chan error, 1)
-	go func() { done <- agent.Run(t.Context(), guest, runtimeapi.New()) }()
-	select {
-	case err := <-sent:
-		require.NoError(t, err)
-	case <-time.After(5 * time.Second):
-		t.Fatal("the request was not delivered")
-	}
-
-	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
-	defer cancel()
-	resp, err := host.HostResult(ctx)
-	got := time.Now()
-	require.NoError(t, err)
-	assert.Equal(t, "Sandbox.Timedout", resp.FunctionError)
-	assert.Equal(t, "r", resp.RequestID)
-	assert.Equal(t, "g", resp.GrantID)
-	assert.Equal(t, "a", resp.AttemptID)
-	assert.False(t, got.Before(time.UnixMilli(req.DeadlineMs)), "the result was read before the deadline")
-
-	select {
-	case err := <-done:
-		require.ErrorIs(t, err, agent.ErrInvocationTimedOut)
-	case <-time.After(5 * time.Second):
-		t.Fatal("Run did not return after writing the timeout result")
-	}
-}

spinifex/agents/lambda/hostlink/configdisk.go

@@ -1,71 +0,0 @@
-package hostlink
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
-	"golang.org/x/sys/unix"
-)
-
-// On a VMM without fw_cfg the executor attaches the same entries as a tar
-// archive on a second disk, which the runtime image's init extracts here in
-// the fw_cfg sysfs layout, so the fw_cfg readers work unchanged.
-const (
-	ConfigDiskRoot   = "/spinifex-cfg"
-	ConfigDiskDir    = ConfigDiskRoot + "/by_name/" + lambdav1.FwCfgPrefix
-	ConfigDiskDevice = "/dev/vdb"
-)
-
-// zeroChunk bounds each write while erasing the configuration disk.
-const zeroChunk = 64 << 10
-
-// unmount is a var so tests without privilege can stand in for the tmpfs.
-var unmount = unix.Unmount
-
-// SealConfigDisk overwrites the whole configuration disk with zeros, then
-// removes the extracted tree. The tree stays if the disk cannot be erased, so
-// SealFwCfg's check on root fails closed.
-func SealConfigDisk(device, root string) error {
-	if err := zeroDevice(device); err != nil {
-		return fmt.Errorf("hostlink: erase configuration disk: %w", err)
-	}
-	if err := unmount(root, 0); err != nil && !errors.Is(err, unix.EINVAL) {
-		return fmt.Errorf("hostlink: unmount %s: %w", root, err)
-	}
-	if err := os.RemoveAll(root); err != nil {
-		return fmt.Errorf("hostlink: remove %s: %w", root, err)
-	}
-	return nil
-}
-
-func zeroDevice(device string) error {
-	f, err := os.OpenFile(device, os.O_WRONLY, 0)
-	if err != nil {
-		return err
-	}
-	size, err := f.Seek(0, io.SeekEnd)
-	if err == nil {
-		_, err = f.Seek(0, io.SeekStart)
-	}
-	if err != nil {
-		_ = f.Close()
-		return err
-	}
-	zeros := make([]byte, zeroChunk)
-	for left := size; left > 0; {
-		n := min(left, int64(len(zeros)))
-		if _, err := f.Write(zeros[:n]); err != nil {
-			_ = f.Close()
-			return err
-		}
-		left -= n
-	}
-	if err := f.Sync(); err != nil {
-		_ = f.Close()
-		return err
-	}
-	return f.Close()
-}

spinifex/agents/lambda/hostlink/configdisk_test.go

@@ -1,63 +0,0 @@
-//test:in-package reaches the unexported configDisk and fwCfgTree readers to seal and read a real archive
-
-package hostlink
-
-import (
-	"bytes"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/sys/unix"
-)
-
-// configDisk writes a fake disk holding secret and an extracted tree; the
-// stand-in unmount reports the tree is not a mount, as on a plain directory.
-func configDisk(t *testing.T, secret []byte) (device, root string) {
-	t.Helper()
-	dir := t.TempDir()
-	device = filepath.Join(dir, "vdb")
-	require.NoError(t, os.WriteFile(device, append(bytes.Repeat([]byte{0xa5}, 3*zeroChunk+7), secret...), 0o600))
-	root = fwCfgTree(t)
-	require.NoError(t, os.WriteFile(filepath.Join(root, "by_name", "key"), secret, 0o600))
-	old := unmount
-	unmount = func(string, int) error { return unix.EINVAL }
-	t.Cleanup(func() { unmount = old })
-	return device, root
-}
-
-func TestSealConfigDisk(t *testing.T) {
-	secret := []byte("guest-key-material")
-
-	t.Run("erases the disk and removes the tree", func(t *testing.T) {
-		device, root := configDisk(t, secret)
-		before, err := os.Stat(device)
-		require.NoError(t, err)
-		require.NoError(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
-
-		disk, err := os.ReadFile(device)
-		require.NoError(t, err)
-		assert.Len(t, disk, int(before.Size()))
-		assert.Equal(t, make([]byte, len(disk)), disk, "configuration disk not fully erased")
-		_, err = os.Stat(root)
-		assert.ErrorIs(t, err, os.ErrNotExist)
-	})
-	t.Run("an unwritable disk keeps the tree and fails closed", func(t *testing.T) {
-		device, root := configDisk(t, secret)
-		require.NoError(t, os.Chmod(device, 0o400))
-		err := SealFwCfg(root, func() error { return SealConfigDisk(device, root) })
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "still readable")
-		_, err = os.Stat(root)
-		assert.NoError(t, err)
-	})
-	t.Run("a failed unmount keeps the tree", func(t *testing.T) {
-		device, root := configDisk(t, secret)
-		unmount = func(string, int) error { return unix.EBUSY }
-		require.Error(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
-		_, err := os.Stat(root)
-		assert.NoError(t, err)
-	})
-}

spinifex/agents/lambda/hostlink/link.go

@@ -5,9 +5,7 @@ package hostlink
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
-	"log/slog"
 )
 
 // ErrShutdown is returned by Link.Next when the host asks the agent to stop.
@@ -23,52 +21,15 @@ type Request struct {
 	TraceID     string `json:"trace_id,omitempty"`
 	DeadlineMs  int64  `json:"deadline_ms"`
 	Payload     []byte `json:"payload"`
-	// Credential is the environment's execution-role session. Only test-link
-	// hosts send it; release hosts leave it nil.
-	Credential *Credential `json:"credential,omitempty"`
 }
 
-// Credential holds one execution-role session and its function environment.
-type Credential struct {
-	AccessKeyID     string            `json:"access_key_id"`
-	SecretAccessKey string            `json:"secret_access_key"`
-	SessionToken    string            `json:"session_token"`
-	Environment     map[string]string `json:"environment,omitempty"`
-}
-
-// LogValue keeps every value out of logs.
-func (Credential) LogValue() slog.Value { return slog.StringValue("[redacted]") }
-
-// Complete reports whether all three values are present.
-func (c Credential) Complete() bool {
-	return c.AccessKeyID != "" && c.SecretAccessKey != "" && c.SessionToken != ""
-}
-
-// Response is the outcome of a Request. FunctionError is empty on success. A
-// non-nil Segments is the payload and takes precedence over Payload; only a
-// streaming carrier can send it, and encoding/json refuses it.
+// Response is the outcome of a Request. FunctionError is empty on success.
 type Response struct {
-	GrantID       string   `json:"grant_id"`
-	AttemptID     string   `json:"attempt_id"`
-	RequestID     string   `json:"request_id"`
-	Payload       []byte   `json:"payload"`
-	Segments      [][]byte `json:"-"`
-	FunctionError string   `json:"function_error,omitempty"`
-}
-
-// ErrSegmentedNotMarshalable is returned when a segmented payload reaches
-// encoding/json, which would otherwise drop it.
-var ErrSegmentedNotMarshalable = errors.New("hostlink: a segmented payload can only be streamed by a carrier")
-
-type responseAlias Response
-
-// MarshalJSON encodes r as the plain struct, and fails for a segmented
-// payload rather than emit a frame without it.
-func (r Response) MarshalJSON() ([]byte, error) {
-	if r.Segments != nil {
-		return nil, ErrSegmentedNotMarshalable
-	}
-	return json.Marshal(responseAlias(r))
+	GrantID       string `json:"grant_id"`
+	AttemptID     string `json:"attempt_id"`
+	RequestID     string `json:"request_id"`
+	Payload       []byte `json:"payload"`
+	FunctionError string `json:"function_error,omitempty"`
 }
 
 // Link carries invocations from the host to the guest and results back.

spinifex/agents/lambda/hostlink/protocol_release.go

@@ -1,8 +0,0 @@
-//go:build !lambdatestlink
-
-package hostlink
-
-import lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
-
-// Protocol is the ALPN protocol release hosts and guests offer.
-const Protocol = lambdav1.ALPNv1

spinifex/agents/lambda/hostlink/protocol_testlink.go

@@ -1,8 +0,0 @@
-//go:build lambdatestlink
-
-package hostlink
-
-// Protocol is the only ALPN protocol a test-link host and guest offer. It
-// differs from the release protocol because the first frame carries the
-// execution credential and function environment, which an older guest would ignore.
-const Protocol = "spinifex-lambda-testlink/3"

spinifex/agents/lambda/hostlink/test_stream.go

@@ -3,17 +3,13 @@
 package hostlink
 
 import (
-	"bufio"
 	"context"
-	"encoding/base64"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"net"
-	"sync/atomic"
-	"time"
+	"sync"
 )
 
 // maxFrame bounds one frame: a 6 MiB payload as base64 plus headroom.
@@ -37,14 +33,9 @@ type frame struct {
 // TestStream is a test-only carrier over an io.ReadWriter, framing JSON with
 // a 4-byte big-endian length. It is not the guest protocol.
 type TestStream struct {
-	rw    io.ReadWriter
-	conn  net.Conn // rw when it is a net.Conn; results are written only then
-	in    chan frame
-	wslot chan struct{}
-
-	// closed is set, under the write slot, once the connection was closed
-	// with a result possibly half sent.
-	closed atomic.Bool
+	rw  io.ReadWriter
+	in  chan frame
+	wmu sync.Mutex
 
 	err error // set before in is closed
 }
@@ -53,8 +44,7 @@ var _ Link = (*TestStream)(nil)
 
 // NewTestStream starts reading frames from rw.
 func NewTestStream(rw io.ReadWriter) *TestStream {
-	s := &TestStream{rw: rw, in: make(chan frame), wslot: make(chan struct{}, 1)}
-	s.conn, _ = rw.(net.Conn)
+	s := &TestStream{rw: rw, in: make(chan frame)}
 	go s.readLoop()
 	return s
 }
@@ -102,218 +92,12 @@ func (s *TestStream) Next(ctx context.Context) (Request, error) {
 	}
 }
 
-// ErrNoDeadlineTransport is returned by Complete when the stream's transport
-// is not a net.Conn, since a blocked write could then never be interrupted.
-var ErrNoDeadlineTransport = errors.New("hostlink: result needs a net.Conn transport")
-
-// ErrStreamClosed is returned by Complete after a failed write closed the
-// connection.
-var ErrStreamClosed = errors.New("hostlink: stream closed after a failed result write")
-
-// pastDeadline is a write deadline that is always already over.
-var pastDeadline = time.Unix(1, 0)
-
-// acquire takes the write slot or gives up when ctx ends.
-func (s *TestStream) acquire(ctx context.Context) error {
-	select {
-	case s.wslot <- struct{}{}:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}
-
-func (s *TestStream) release() { <-s.wslot }
-
-// Complete implements Link. The result is streamed as the same JSON frame
-// json.Marshal makes for it, without holding a copy of the payload.
-func (s *TestStream) Complete(ctx context.Context, resp Response) (err error) {
-	if s.conn == nil {
-		return ErrNoDeadlineTransport
-	}
-	if s.closed.Load() {
-		return ErrStreamClosed
-	}
+// Complete implements Link.
+func (s *TestStream) Complete(ctx context.Context, resp Response) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
-	rf, err := newResultFrame(resp)
-	if err != nil {
-		return err
-	}
-	if rf.size > maxFrame {
-		return ErrFrameTooLarge
-	}
-	if err := s.acquire(ctx); err != nil {
-		return err
-	}
-	// The slot is released last, after the deadline has been cleared.
-	defer s.release()
-	if s.closed.Load() {
-		return ErrStreamClosed
-	}
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-	// Once any frame byte may be out, the connection is closed before the
-	// slot is released so no later writer can append to a partial frame.
-	closeConn := func() {
-		s.closed.Store(true)
-		_ = s.conn.Close()
-	}
-
-	dl, _ := ctx.Deadline()
-	if err := s.conn.SetWriteDeadline(dl); err != nil {
-		closeConn()
-		return fmt.Errorf("hostlink: set write deadline: %w", err)
-	}
-	stop, watched, watchErr := make(chan struct{}), make(chan struct{}), make(chan error, 1)
-	go func() {
-		defer close(watched)
-		select {
-		case <-ctx.Done():
-			if err := s.conn.SetWriteDeadline(pastDeadline); err != nil {
-				// Closing is what unblocks the pending write.
-				s.closed.Store(true)
-				_ = s.conn.Close()
-				watchErr <- err
-			}
-		case <-stop:
-		}
-	}()
-	cw := &countingWriter{w: s.conn}
-	werr := rf.writeTo(cw)
-	close(stop)
-	<-watched
-	select {
-	case e := <-watchErr:
-		return fmt.Errorf("hostlink: interrupt blocked write: %w", errors.Join(e, werr))
-	default:
-	}
-	if werr == nil && cw.n != int64(4+rf.size) {
-		werr = fmt.Errorf("hostlink: frame wrote %d bytes, expected %d", cw.n, 4+rf.size)
-	}
-	if werr != nil {
-		closeConn()
-		return werr
-	}
-	if cerr := s.conn.SetWriteDeadline(time.Time{}); cerr != nil {
-		closeConn()
-		return fmt.Errorf("hostlink: clear write deadline: %w", errors.Join(cerr, werr))
-	}
-	return nil
-}
-
-// countingWriter writes everything to w, in full, and counts the bytes.
-type countingWriter struct {
-	w io.Writer
-	n int64
-}
-
-func (c *countingWriter) Write(p []byte) (int, error) {
-	done := 0
-	for done < len(p) {
-		n, err := c.w.Write(p[done:])
-		done += n
-		c.n += int64(n)
-		if err != nil {
-			return done, err
-		}
-		if n == 0 {
-			return done, io.ErrShortWrite
-		}
-	}
-	return done, nil
-}
-
-// resultFrame is the result frame in the field order json.Marshal gives
-// frame{Type, Response}, split around the payload value.
-type resultFrame struct {
-	head, tail []byte
-	payload    [][]byte
-	null       bool
-	size       int // JSON length, excluding the 4-byte prefix
-}
-
-func newResultFrame(r Response) (*resultFrame, error) {
-	q := func(v string) ([]byte, error) { return json.Marshal(v) }
-	var parts [3][]byte
-	for i, v := range []string{r.GrantID, r.AttemptID, r.RequestID} {
-		b, err := q(v)
-		if err != nil {
-			return nil, err
-		}
-		parts[i] = b
-	}
-	rf := &resultFrame{}
-	rf.head = append(rf.head, `{"type":"result","response":{"grant_id":`...)
-	rf.head = append(rf.head, parts[0]...)
-	rf.head = append(rf.head, `,"attempt_id":`...)
-	rf.head = append(rf.head, parts[1]...)
-	rf.head = append(rf.head, `,"request_id":`...)
-	rf.head = append(rf.head, parts[2]...)
-	rf.head = append(rf.head, `,"payload":`...)
-	if r.FunctionError != "" {
-		b, err := q(r.FunctionError)
-		if err != nil {
-			return nil, err
-		}
-		rf.tail = append(rf.tail, `,"function_error":`...)
-		rf.tail = append(rf.tail, b...)
-	}
-	rf.tail = append(rf.tail, `}}`...)
-
-	total := 0
-	switch {
-	case r.Segments != nil:
-		rf.payload = r.Segments
-	case r.Payload != nil:
-		rf.payload = [][]byte{r.Payload}
-	default:
-		rf.null = true
-	}
-	for _, seg := range rf.payload {
-		total += len(seg)
-	}
-	rf.size = len(rf.head) + len(rf.tail)
-	if rf.null {
-		rf.size += len("null")
-	} else {
-		// A payload too large for maxFrame is refused before its length matters.
-		if total > maxFrame {
-			rf.size = maxFrame + 1
-			return rf, nil
-		}
-		rf.size += 2 + base64.StdEncoding.EncodedLen(total)
-	}
-	return rf, nil
-}
-
-// writeTo writes the length prefix and the frame, in order, through a small
-// buffer so the payload is never held a second time.
-func (rf *resultFrame) writeTo(w io.Writer) error {
-	bw := bufio.NewWriterSize(w, 32<<10)
-	var hdr [4]byte
-	binary.BigEndian.PutUint32(hdr[:], uint32(rf.size))
-	_, _ = bw.Write(hdr[:])
-	_, _ = bw.Write(rf.head)
-	if rf.null {
-		_, _ = bw.WriteString("null")
-	} else {
-		_ = bw.WriteByte('"')
-		enc := base64.NewEncoder(base64.StdEncoding, bw)
-		for _, seg := range rf.payload {
-			if _, err := enc.Write(seg); err != nil {
-				return err
-			}
-		}
-		if err := enc.Close(); err != nil {
-			return err
-		}
-		_ = bw.WriteByte('"')
-	}
-	_, _ = bw.Write(rf.tail)
-	return bw.Flush()
+	return s.write(frame{Type: typeResult, Response: &resp})
 }
 
 // HostSend writes an invoke frame, as the host side of the carrier.
@@ -321,19 +105,6 @@ func (s *TestStream) HostSend(req Request) error {
 	return s.write(frame{Type: typeInvoke, Request: &req})
 }
 
-// CheckInvokeFits encodes req as HostSend would and returns ErrFrameTooLarge
-// when the frame exceeds the bound, so a caller can refuse before launch.
-func CheckInvokeFits(req Request) error {
-	b, err := json.Marshal(frame{Type: typeInvoke, Request: &req})
-	if err != nil {
-		return err
-	}
-	if len(b) > maxFrame {
-		return ErrFrameTooLarge
-	}
-	return nil
-}
-
 // HostShutdown writes a shutdown frame, as the host side of the carrier.
 func (s *TestStream) HostShutdown() error {
 	return s.write(frame{Type: typeShutdown})
@@ -362,8 +133,8 @@ func (s *TestStream) write(f frame) error {
 	buf := make([]byte, 4+len(b))
 	binary.BigEndian.PutUint32(buf, uint32(len(b)))
 	copy(buf[4:], b)
-	_ = s.acquire(context.Background())
-	defer s.release()
+	s.wmu.Lock()
+	defer s.wmu.Unlock()
 	_, err = s.rw.Write(buf)
 	return err
 }

spinifex/agents/lambda/hostlink/test_stream_complete_test.go

@@ -1,612 +0,0 @@
-//go:build lambdatestlink
-
-package hostlink
-
-//test:in-package reaches the unexported write slot, frame types and readFrame to force orderings
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"encoding/json"
-	"errors"
-	"io"
-	"net"
-	"os"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
-)
-
-// fakeConn is a net.Conn that records what is written and the deadlines set,
-// and can fail or block on demand.
-type fakeConn struct {
-	mu        sync.Mutex
-	buf       bytes.Buffer
-	deadlines []time.Time
-	maxWrite  int
-
-	failDeadline func(n int, t time.Time) error
-	blockWrites  bool
-	started      chan struct{}
-	unblock      chan struct{}
-	closedCh     chan struct{}
-	startOnce    sync.Once
-	unblockOnce  sync.Once
-	closeOnce    sync.Once
-	closes       atomic.Int32
-}
-
-var _ net.Conn = (*fakeConn)(nil)
-
-func newFakeConn() *fakeConn {
-	return &fakeConn{started: make(chan struct{}), unblock: make(chan struct{}), closedCh: make(chan struct{})}
-}
-
-func (c *fakeConn) Read([]byte) (int, error) { <-c.closedCh; return 0, io.EOF }
-
-func (c *fakeConn) Write(p []byte) (int, error) {
-	if c.blockWrites {
-		c.startOnce.Do(func() { close(c.started) })
-		select {
-		case <-c.unblock:
-			return 0, os.ErrDeadlineExceeded
-		case <-c.closedCh:
-			return 0, net.ErrClosed
-		}
-	}
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.maxWrite = max(c.maxWrite, len(p))
-	return c.buf.Write(p)
-}
-
-func (c *fakeConn) Close() error {
-	c.closes.Add(1)
-	c.closeOnce.Do(func() { close(c.closedCh) })
-	return nil
-}
-
-func (c *fakeConn) SetWriteDeadline(t time.Time) error {
-	c.mu.Lock()
-	c.deadlines = append(c.deadlines, t)
-	n := len(c.deadlines)
-	c.mu.Unlock()
-	if c.failDeadline != nil {
-		if err := c.failDeadline(n, t); err != nil {
-			return err
-		}
-	}
-	if !t.IsZero() && !t.After(time.Now()) {
-		c.unblockOnce.Do(func() { close(c.unblock) })
-	}
-	return nil
-}
-
-func (c *fakeConn) LocalAddr() net.Addr             { return nil }
-func (c *fakeConn) RemoteAddr() net.Addr            { return nil }
-func (c *fakeConn) SetDeadline(time.Time) error     { return nil }
-func (c *fakeConn) SetReadDeadline(time.Time) error { return nil }
-
-func (c *fakeConn) written() []byte {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return bytes.Clone(c.buf.Bytes())
-}
-
-func (c *fakeConn) deadlineLog() []time.Time {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return append([]time.Time(nil), c.deadlines...)
-}
-
-// await bounds a receive so a regression fails promptly instead of hanging.
-func await[T any](t *testing.T, ch <-chan T) T {
-	t.Helper()
-	select {
-	case v := <-ch:
-		return v
-	case <-time.After(15 * time.Second):
-		t.Fatal("timed out waiting on a channel")
-		panic("unreachable")
-	}
-}
-
-// pattern returns n deterministic bytes.
-func pattern(n int) []byte {
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = byte(i*7 + i/251)
-	}
-	return b
-}
-
-func split(b []byte, seg int) [][]byte {
-	out := [][]byte{}
-	for len(b) > 0 {
-		n := min(seg, len(b))
-		out = append(out, b[:n])
-		b = b[n:]
-	}
-	return out
-}
-
-func segmented(b []byte, n int) [][]byte { return split(b, n) }
-
-func joined(r Response) []byte {
-	if r.Segments != nil {
-		return append([]byte{}, bytes.Join(r.Segments, nil)...)
-	}
-	return r.Payload
-}
-
-// assertTodaysFrame checks the written bytes are exactly the length prefix
-// and the json.Marshal of the frame, and that readFrame decodes it.
-func assertTodaysFrame(t *testing.T, got []byte, r Response) {
-	t.Helper()
-	plain := r
-	plain.Segments = nil
-	plain.Payload = joined(r)
-	want, err := json.Marshal(frame{Type: typeResult, Response: &plain})
-	require.NoError(t, err)
-	require.GreaterOrEqual(t, len(got), 4)
-	assert.Equal(t, uint32(len(got)-4), binary.BigEndian.Uint32(got[:4]), "length prefix must match the bytes after it")
-	if !assert.True(t, bytes.Equal(want, got[4:]), "frame differs from json.Marshal (%d vs %d bytes)", len(want), len(got)-4) {
-		return
-	}
-	f, err := readFrame(bytes.NewReader(got))
-	require.NoError(t, err)
-	var wf frame
-	require.NoError(t, json.Unmarshal(want, &wf))
-	assert.Equal(t, *wf.Response, *f.Response)
-}
-
-func complete(t *testing.T, r Response) []byte {
-	t.Helper()
-	c := newFakeConn()
-	defer c.Close()
-	require.NoError(t, NewTestStream(c).Complete(context.Background(), r))
-	return c.written()
-}
-
-func TestCompleteMatchesJSONMarshal(t *testing.T) {
-	meta := Response{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1"}
-	t.Run("6 MiB from one buffer and from many segments", func(t *testing.T) {
-		p := pattern(6 << 20)
-		one := meta
-		one.Payload = p
-		assertTodaysFrame(t, complete(t, one), one)
-		many := meta
-		many.Segments = segmented(p, 256<<10)
-		assertTodaysFrame(t, complete(t, many), many)
-		odd := meta
-		odd.Segments = segmented(p, 256<<10+1)
-		odd.FunctionError = "Unhandled"
-		assertTodaysFrame(t, complete(t, odd), odd)
-	})
-	t.Run("segment lengths against totals of each length mod 3", func(t *testing.T) {
-		for _, seg := range []int{1, 2, 3, 4, 5, 256<<10 + 1} {
-			totals := []int{0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66}
-			if seg > 5 {
-				totals = append(totals, seg-1, seg, seg+1, 2*seg+1, 2*seg+2)
-			}
-			for _, total := range totals {
-				r := meta
-				r.Segments = segmented(pattern(total), seg)
-				assertTodaysFrame(t, complete(t, r), r)
-			}
-		}
-	})
-	t.Run("nil and empty payloads", func(t *testing.T) {
-		nilp := meta
-		got := complete(t, nilp)
-		assert.Contains(t, string(got[4:]), `"payload":null`)
-		assertTodaysFrame(t, got, nilp)
-
-		empty := meta
-		empty.Payload = []byte{}
-		got = complete(t, empty)
-		assert.Contains(t, string(got[4:]), `"payload":""`)
-		assertTodaysFrame(t, got, empty)
-
-		noSegs := meta
-		noSegs.Segments = [][]byte{}
-		assertTodaysFrame(t, complete(t, noSegs), noSegs)
-	})
-	t.Run("metadata is escaped as json.Marshal escapes it", func(t *testing.T) {
-		for _, s := range []string{
-			`quo"te`, `back\slash`, "ctl\x00\x01\x1f\n\t\r", "naïve 日本語 😀", "<script>&amp;</script>",
-			`","payload":"x"`, `{"type":"invoke"}`, `"payload":`, "bad\xffutf8", ``,
-		} {
-			r := Response{GrantID: s, AttemptID: s + "a", RequestID: "r" + s, Payload: []byte("p"), FunctionError: s}
-			assertTodaysFrame(t, complete(t, r), r)
-			r.FunctionError = ""
-			r.Segments = [][]byte{[]byte("a"), []byte("bc")}
-			assertTodaysFrame(t, complete(t, r), r)
-		}
-	})
-	t.Run("a payload landing exactly on maxFrame", func(t *testing.T) {
-		for pad := range 4 {
-			r := Response{GrantID: "g", AttemptID: "a", RequestID: strings.Repeat("r", 1+pad)}
-			rf, err := newResultFrame(r)
-			require.NoError(t, err)
-			room := maxFrame - rf.size + 2
-			if room%4 != 0 {
-				continue
-			}
-			r.Payload = pattern(room / 4 * 3)
-			rf, err = newResultFrame(r)
-			require.NoError(t, err)
-			require.Equal(t, maxFrame, rf.size)
-			assertTodaysFrame(t, complete(t, r), r)
-			return
-		}
-		t.Fatal("no request id length lands on maxFrame")
-	})
-}
-
-func TestCompleteRefusesOverMaxFrameBeforeWriting(t *testing.T) {
-	c := newFakeConn()
-	defer c.Close()
-	s := NewTestStream(c)
-	err := s.Complete(context.Background(), Response{Segments: segmented(make([]byte, maxFrame), 1<<20)})
-	require.ErrorIs(t, err, ErrFrameTooLarge)
-	assert.Empty(t, c.written())
-	assert.Empty(t, c.deadlineLog(), "nothing is touched before the size check")
-}
-
-// plainRW has no deadline support at all.
-type plainRW struct {
-	io.Reader
-	wrote atomic.Int32
-}
-
-func (p *plainRW) Write(b []byte) (int, error) { p.wrote.Add(1); return len(b), nil }
-
-// deadlineRW has SetWriteDeadline but is not a net.Conn.
-type deadlineRW struct{ plainRW }
-
-func (*deadlineRW) SetWriteDeadline(time.Time) error { return nil }
-
-func TestCompleteRefusesNonNetConn(t *testing.T) {
-	p := &plainRW{Reader: strings.NewReader("")}
-	d := &deadlineRW{plainRW{Reader: strings.NewReader("")}}
-	for name, tc := range map[string]struct {
-		rw    io.ReadWriter
-		wrote *atomic.Int32
-	}{
-		"plain":                 {p, &p.wrote},
-		"with SetWriteDeadline": {d, &d.wrote},
-	} {
-		t.Run(name, func(t *testing.T) {
-			s := NewTestStream(tc.rw)
-			err := s.Complete(context.Background(), Response{Payload: []byte{1}})
-			require.ErrorIs(t, err, ErrNoDeadlineTransport)
-			assert.Zero(t, tc.wrote.Load(), "no byte may be written")
-		})
-	}
-}
-
-// sigCtx reports the first time something selects on its Done channel.
-type sigCtx struct {
-	context.Context
-	waiting chan struct{}
-	once    sync.Once
-}
-
-func (c *sigCtx) Done() <-chan struct{} {
-	c.once.Do(func() { close(c.waiting) })
-	return c.Context.Done()
-}
-
-func TestCompleteCancelledWhileAnotherWriteHoldsTheSlot(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	s := NewTestStream(c)
-	require.NoError(t, s.acquire(context.Background())) // another writer holds the slot
-
-	base, cancel := context.WithCancel(context.Background())
-	ctx := &sigCtx{Context: base, waiting: make(chan struct{})}
-	done := make(chan error, 1)
-	go func() { done <- s.Complete(ctx, Response{Payload: []byte("x")}) }()
-	await(t, ctx.waiting)
-	cancel()
-	require.ErrorIs(t, await(t, done), context.Canceled)
-	assert.Empty(t, c.written(), "the peer receives no bytes")
-	assert.Empty(t, c.deadlineLog())
-
-	s.release()
-	require.NoError(t, s.Complete(context.Background(), Response{Payload: []byte("x")}), "the slot is still usable")
-	c.Close()
-}
-
-func TestCompletePreCancelledReleasesSlot(t *testing.T) {
-	c := newFakeConn()
-	defer c.Close()
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	require.ErrorIs(t, s.Complete(ctx, Response{Payload: []byte("x")}), context.Canceled)
-	assert.Empty(t, c.written())
-	assert.Zero(t, len(s.wslot), "slot released")
-}
-
-func TestCompleteInitialDeadlineFailureClosesAndWritesNothing(t *testing.T) {
-	c := newFakeConn()
-	c.failDeadline = func(int, time.Time) error { return errors.New("no deadlines") }
-	s := NewTestStream(c)
-	err := s.Complete(context.Background(), Response{Payload: []byte("x")})
-	require.ErrorContains(t, err, "set write deadline")
-	assert.Equal(t, int32(1), c.closes.Load())
-	assert.Empty(t, c.written())
-	assert.Zero(t, len(s.wslot))
-}
-
-func TestCompleteInterruptFailureClosesAndReportsLinkError(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	c.blockWrites = true
-	c.failDeadline = func(_ int, t time.Time) error {
-		if !t.IsZero() && t.Before(time.Now()) {
-			return errors.New("cannot interrupt")
-		}
-		return nil
-	}
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	done := make(chan error, 1)
-	go func() { done <- s.Complete(ctx, Response{Payload: []byte("x")}) }()
-	await(t, c.started)
-	cancel()
-	var err error
-	select {
-	case err = <-done:
-	case <-time.After(5 * time.Second):
-		c.Close()
-		t.Fatal("Complete stayed blocked: the watcher did not close the connection")
-	}
-	require.ErrorContains(t, err, "interrupt blocked write")
-	assert.Equal(t, int32(1), c.closes.Load())
-	assert.Zero(t, len(s.wslot))
-}
-
-func TestCompleteClearFailureAfterWriteClosesButFrameWasSent(t *testing.T) {
-	c := newFakeConn()
-	c.failDeadline = func(n int, _ time.Time) error {
-		if n == 2 {
-			return errors.New("cannot clear")
-		}
-		return nil
-	}
-	s := NewTestStream(c)
-	r := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Payload: []byte("done")}
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-	defer cancel()
-	err := s.Complete(ctx, r)
-	require.ErrorContains(t, err, "clear write deadline")
-	assert.Equal(t, int32(1), c.closes.Load())
-	// The host already holds a complete valid frame and may accept it.
-	assertTodaysFrame(t, c.written(), r)
-}
-
-func TestCompleteReleasesSlotOnlyAfterDeadlineCleared(t *testing.T) {
-	c := newFakeConn()
-	clearing, proceed := make(chan struct{}), make(chan struct{})
-	c.failDeadline = func(n int, t time.Time) error {
-		if n == 2 { // writer A clearing its deadline
-			close(clearing)
-			<-proceed
-		}
-		return nil
-	}
-	s := NewTestStream(c)
-	ctxA, cancelA := context.WithTimeout(context.Background(), time.Minute)
-	defer cancelA()
-	aDone := make(chan error, 1)
-	go func() { aDone <- s.Complete(ctxA, Response{Payload: []byte("a")}) }()
-	await(t, clearing)
-
-	base, cancelB := context.WithTimeout(context.Background(), 2*time.Minute)
-	defer cancelB()
-	ctxB := &sigCtx{Context: base, waiting: make(chan struct{})}
-	bDone := make(chan error, 1)
-	go func() { bDone <- s.Complete(ctxB, Response{Payload: []byte("b")}) }()
-	await(t, ctxB.waiting)
-	assert.Len(t, c.deadlineLog(), 2, "B has set no deadline while A is still cleaning up")
-
-	close(proceed)
-	require.NoError(t, await(t, aDone))
-	require.NoError(t, await(t, bDone))
-	log := c.deadlineLog()
-	require.Len(t, log, 4)
-	assert.False(t, log[0].IsZero())
-	assert.True(t, log[1].IsZero(), "A clears first")
-	assert.False(t, log[2].IsZero())
-	assert.True(t, log[3].IsZero())
-	c.Close()
-}
-
-func TestCompleteStopsAtDeadlineWhenPeerNeverReads(t *testing.T) {
-	g, h := net.Pipe()
-	defer h.Close()
-	s := NewTestStream(g)
-	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
-	defer cancel()
-	err := s.Complete(ctx, Response{GrantID: "g", Payload: []byte("x")})
-	require.Error(t, err)
-	assert.ErrorIs(t, err, os.ErrDeadlineExceeded)
-	assert.Zero(t, len(s.wslot))
-
-	// The host never received a frame, so it has nothing to accept.
-	host := NewTestStream(h)
-	g.Close()
-	_, err = host.HostResult(context.Background())
-	require.Error(t, err)
-}
-
-func TestCompleteCancelInterruptsBlockedWrite(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	c.blockWrites = true
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	done := make(chan error, 1)
-	go func() { done <- s.Complete(ctx, Response{Payload: []byte("x")}) }()
-	await(t, c.started)
-	cancel()
-	require.ErrorIs(t, await(t, done), os.ErrDeadlineExceeded)
-	assert.Equal(t, int32(1), c.closes.Load(), "a frame may be half sent, so the connection is closed")
-	assert.ErrorIs(t, s.Complete(context.Background(), Response{Payload: []byte("x")}), ErrStreamClosed)
-}
-
-func TestCompleteCancelAfterWriteDoesNotTouchConn(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	c := newFakeConn()
-	s := NewTestStream(c)
-	ctx, cancel := context.WithCancel(context.Background())
-	require.NoError(t, s.Complete(ctx, Response{Payload: []byte("x")}))
-	before := len(c.deadlineLog())
-	cancel()
-	assert.Len(t, c.deadlineLog(), before, "no watcher is left to move the deadline")
-	assert.Zero(t, c.closes.Load())
-	c.Close()
-}
-
-func TestCompleteWritesNeverInterleave(t *testing.T) {
-	c := newFakeConn()
-	defer c.Close()
-	s := NewTestStream(c)
-	const n = 16
-	var wg sync.WaitGroup
-	for i := range n {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if i%2 == 0 {
-				assert.NoError(t, s.Complete(context.Background(), Response{GrantID: "g", Segments: segmented(pattern(100000+i), 7001)}))
-			} else {
-				assert.NoError(t, s.HostSend(Request{GrantID: "g", Payload: pattern(100000 + i)}))
-			}
-		}()
-	}
-	wg.Wait()
-	r := bytes.NewReader(c.written())
-	var results, invokes int
-	for r.Len() > 0 {
-		f, err := readFrame(r)
-		require.NoError(t, err)
-		switch f.Type {
-		case typeResult:
-			results++
-		case typeInvoke:
-			invokes++
-		}
-	}
-	assert.Equal(t, n/2, results)
-	assert.Equal(t, n/2, invokes)
-}
-
-// discardConn drops writes so a benchmark does not hold the frame.
-type discardConn struct{ *fakeConn }
-
-func (discardConn) Write(p []byte) (int, error) { return len(p), nil }
-
-func TestCompleteMakesNoPayloadSizedAllocation(t *testing.T) {
-	r := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Segments: segmented(pattern(6<<20), 256<<10)}
-	c := newFakeConn()
-	defer c.Close()
-	require.NoError(t, NewTestStream(c).Complete(context.Background(), r))
-	assert.Less(t, c.maxWrite, 64<<10, "no single write is payload sized")
-
-	d := discardConn{newFakeConn()}
-	defer d.Close()
-	bs := NewTestStream(d)
-	br := testing.Benchmark(func(b *testing.B) {
-		b.ReportAllocs()
-		for range b.N {
-			if err := bs.Complete(context.Background(), r); err != nil {
-				b.Fatal(err)
-			}
-		}
-	})
-	assert.Less(t, br.AllocedBytesPerOp(), int64(1<<20), "writing a 6 MiB result")
-}
-
-func TestResponseJSONCompatibility(t *testing.T) {
-	plain := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Payload: []byte("p<&>"), FunctionError: "Unhandled"}
-
-	t.Run("nil segments encode as the plain struct", func(t *testing.T) {
-		got, err := json.Marshal(plain)
-		require.NoError(t, err)
-		want := `{"grant_id":"g","attempt_id":"a","request_id":"r","payload":"cDwmPg==","function_error":"Unhandled"}`
-		assert.Equal(t, want, string(got))
-		inFrame, err := json.Marshal(frame{Type: typeResult, Response: &plain})
-		require.NoError(t, err)
-		assert.Equal(t, `{"type":"result","response":`+want+`}`, string(inFrame))
-		nilPayload, err := json.Marshal(Response{GrantID: "g"})
-		require.NoError(t, err)
-		assert.Equal(t, `{"grant_id":"g","attempt_id":"","request_id":"","payload":null}`, string(nilPayload))
-	})
-	t.Run("a legacy frame decodes as before", func(t *testing.T) {
-		legacy, err := json.Marshal(frame{Type: typeResult, Response: &plain})
-		require.NoError(t, err)
-		var f frame
-		require.NoError(t, json.Unmarshal(legacy, &f))
-		assert.Equal(t, plain, *f.Response)
-	})
-	t.Run("unknown fields including segments are ignored", func(t *testing.T) {
-		var f frame
-		in := `{"type":"result","extra":1,"response":{"grant_id":"g","segments":{"x":1},"unknown":[1],"payload":"cA=="}}`
-		require.NoError(t, json.Unmarshal([]byte(in), &f))
-		assert.Equal(t, Response{GrantID: "g", Payload: []byte("p")}, *f.Response)
-	})
-	t.Run("a segmented payload cannot be marshalled", func(t *testing.T) {
-		r := Response{GrantID: "g", Segments: segmented([]byte("data"), 2)}
-		_, err := json.Marshal(r)
-		require.ErrorIs(t, err, ErrSegmentedNotMarshalable)
-		_, err = json.Marshal(frame{Type: typeResult, Response: &r})
-		require.ErrorIs(t, err, ErrSegmentedNotMarshalable)
-		_, err = json.Marshal(Response{GrantID: "g", Segments: [][]byte{}})
-		require.ErrorIs(t, err, ErrSegmentedNotMarshalable)
-	})
-}
-
-// partialConn accepts n bytes of a write and then fails.
-type partialConn struct {
-	*fakeConn
-	budget int
-}
-
-func (p *partialConn) Write(b []byte) (int, error) {
-	if p.budget < len(b) {
-		n := p.budget
-		p.budget = 0
-		p.fakeConn.Write(b[:n])
-		return n, errors.New("link broke")
-	}
-	p.budget -= len(b)
-	return p.fakeConn.Write(b)
-}
-
-func TestCompleteAfterPartialWriteSendsNothingMore(t *testing.T) {
-	pc := &partialConn{fakeConn: newFakeConn(), budget: 20}
-	s := NewTestStream(pc)
-	err := s.Complete(context.Background(), Response{GrantID: "g", Payload: pattern(100)})
-	require.ErrorContains(t, err, "link broke")
-	assert.Equal(t, int32(1), pc.closes.Load(), "closed while the slot was held")
-	assert.Zero(t, len(s.wslot))
-	partial := len(pc.written())
-	assert.Less(t, partial, 4+100, "the host holds an incomplete frame")
-
-	pc.budget = 1 << 20
-	require.ErrorIs(t, s.Complete(context.Background(), Response{GrantID: "g2", Payload: []byte("again")}), ErrStreamClosed)
-	assert.Len(t, pc.written(), partial, "the peer receives no further bytes")
-	_, err = readFrame(bytes.NewReader(pc.written()))
-	assert.Error(t, err, "the host has no valid frame to accept")
-}

spinifex/agents/lambda/hostlink/tls.go

@@ -86,11 +86,11 @@ func (id Identity) ClientConfig() *tls.Config {
 		MinVersion:         tls.VersionTLS13,
 		MaxVersion:         tls.VersionTLS13,
 		Certificates:       []tls.Certificate{id.Cert},
-		NextProtos:         []string{Protocol},
+		NextProtos:         []string{lambdav1.ALPNv1},
 		InsecureSkipVerify: true, //nolint:gosec // VerifyConnection performs full validation against the pinned host certificate.
 		VerifyConnection: func(cs tls.ConnectionState) error {
-			if cs.NegotiatedProtocol != Protocol {
-				return fmt.Errorf("hostlink: application protocol %q, want %q", cs.NegotiatedProtocol, Protocol)
+			if cs.NegotiatedProtocol != lambdav1.ALPNv1 {
+				return fmt.Errorf("hostlink: application protocol %q, want %q", cs.NegotiatedProtocol, lambdav1.ALPNv1)
 			}
 			return pki.VerifyPinned(cs.PeerCertificates, id.HostCert, x509.ExtKeyUsageServerAuth, id.Scope.URI(lambdav1.RoleHost))
 		},

spinifex/agents/lambda/hostlink/tls_test.go

@@ -69,7 +69,7 @@ func newTestIdent(t *testing.T, scope lambdav1.Scope, role string) testIdent {
 func hostConfig(host testIdent, pinGuest *x509.Certificate, scope lambdav1.Scope) *tls.Config {
 	return &tls.Config{
 		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{host.cert},
-		ClientAuth: tls.RequireAnyClientCert, NextProtos: []string{Protocol},
+		ClientAuth: tls.RequireAnyClientCert, NextProtos: []string{lambdav1.ALPNv1},
 		VerifyConnection: func(cs tls.ConnectionState) error {
 			return pki.VerifyPinned(cs.PeerCertificates, pinGuest, x509.ExtKeyUsageClientAuth, scope.URI(lambdav1.RoleGuest))
 		},

spinifex/agents/lambda/runtimeapi/server.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"strconv"
@@ -43,9 +42,6 @@ var (
 	ErrDeadlineExceeded = errors.New("runtimeapi: invocation deadline exceeded")
 	// ErrInvalidInvocation is returned by Invoke for an invocation with no request ID.
 	ErrInvalidInvocation = errors.New("runtimeapi: invocation has no request id")
-	// ErrResponseBodyFailed is returned by Invoke when a response or error
-	// body that was admitted could not be read in full. No result exists.
-	ErrResponseBodyFailed = errors.New("runtimeapi: admitted response body was not received")
 )
 
 // Invocation is one event handed to the runtime.
@@ -58,11 +54,9 @@ type Invocation struct {
 }
 
 // Result is what the runtime answered. FunctionError is empty on success and
-// otherwise the error type. A non-nil Segments is the payload, in order, and
-// takes precedence over Payload.
+// otherwise the error type.
 type Result struct {
 	Payload       []byte
-	Segments      [][]byte
 	FunctionError string
 }
 
@@ -72,25 +66,10 @@ type InitError struct {
 	Body []byte
 }
 
-type flightState int
-
-const (
-	stWaiting flightState = iota
-	stRunning
-	stReceiving
-	stCompleted
-	stExited
-)
-
-type outcome struct {
-	res Result
-	err error
-}
-
 type flight struct {
 	inv   Invocation
-	state flightState
-	done  chan outcome
+	taken bool
+	done  chan Result
 }
 
 // Server implements the Runtime API for a single in-flight invocation.
@@ -99,11 +78,6 @@ type Server struct {
 	cur  *flight
 	wake chan struct{}
 
-	// exited records that the bootstrap process has ended; settled that the
-	// last flight has its result, so a later exit no longer matters.
-	exited  bool
-	settled bool
-
 	initOnce sync.Once
 	initDone chan struct{}
 	initErr  InitError
@@ -133,16 +107,12 @@ func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
 	if inv.RequestID == "" {
 		return Result{}, ErrInvalidInvocation
 	}
-	f := &flight{inv: inv, done: make(chan outcome, 1)}
+	f := &flight{inv: inv, done: make(chan Result, 1)}
 	s.mu.Lock()
 	if s.cur != nil {
 		s.mu.Unlock()
 		return Result{}, ErrBusy
 	}
-	if s.exited {
-		f.state = stExited
-	}
-	s.settled = false
 	s.cur = f
 	s.mu.Unlock()
 	select {
@@ -154,8 +124,8 @@ func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
 	defer timer.Stop()
 	var cause error
 	select {
-	case out := <-f.done:
-		return out.res, out.err
+	case res := <-f.done:
+		return res, nil
 	case <-timer.C:
 		cause = ErrDeadlineExceeded
 	case <-ctx.Done():
@@ -170,27 +140,7 @@ func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
 	}
 	s.mu.Unlock()
 	// Completed while we were giving up; the result is already queued.
-	out := <-f.done
-	return out.res, out.err
-}
-
-// BootstrapExited records that the runtime process has ended and reports
-// whether the caller should cancel the invocation. Before the response is
-// admitted it should; afterwards the result stands.
-func (s *Server) BootstrapExited() bool {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.exited = true
-	f := s.cur
-	switch {
-	case f == nil:
-		return !s.settled
-	case f.state < stReceiving:
-		f.state = stExited
-		return true
-	default:
-		return false
-	}
+	return <-f.done, nil
 }
 
 // Handler returns the Runtime API routes.
@@ -207,8 +157,8 @@ func (s *Server) next(w http.ResponseWriter, r *http.Request) {
 	for {
 		s.mu.Lock()
 		f := s.cur
-		if f != nil && f.state == stWaiting {
-			f.state = stRunning
+		if f != nil && !f.taken {
+			f.taken = true
 			s.mu.Unlock()
 			h := w.Header()
 			h.Set(headerRequestID, f.inv.RequestID)
@@ -229,149 +179,58 @@ func (s *Server) next(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// admit claims the single response slot of the invocation named id, so only
-// one POST per invocation is ever read. It returns nil if id is not the
-// delivered, unclaimed invocation of a runtime that is still running.
-func (s *Server) admit(id string) *flight {
+// claimable reports whether id names the delivered, uncompleted invocation.
+func (s *Server) claimable(id string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.cur != nil && s.cur.taken && s.cur.inv.RequestID == id
+}
+
+// complete finishes the invocation named id, reporting whether it was live.
+func (s *Server) complete(id string, res Result) bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	f := s.cur
-	if f == nil || f.state != stRunning || f.inv.RequestID != id {
-		return nil
-	}
-	f.state = stReceiving
-	return f
-}
-
-// complete records res for an admitted flight, reporting whether the
-// invocation was still waiting for it.
-func (s *Server) complete(f *flight, res Result) bool {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.cur != f {
+	if f == nil || !f.taken || f.inv.RequestID != id {
 		return false
 	}
 	s.cur = nil
-	s.settled = true
-	f.state = stCompleted
-	f.done <- outcome{res: res}
+	f.done <- res
 	return true
 }
 
-// readFailed ends an admitted flight whose body was lost. The outcome is
-// published before the flight is detached, so Invoke never waits on it.
-func (s *Server) readFailed(f *flight, err error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.cur != f {
-		return
-	}
-	s.cur = nil
-	s.settled = true
-	f.state = stCompleted
-	f.done <- outcome{err: fmt.Errorf("%w: %w", ErrResponseBodyFailed, err)}
-}
-
 func (s *Server) response(w http.ResponseWriter, r *http.Request) {
 	id := r.PathValue("id")
-	f := s.admit(id)
-	if f == nil {
+	if !s.claimable(id) {
 		invalidID(w, id)
 		return
 	}
-	s.finishResponse(w, r, f)
-}
-
-// segmentBytes is the largest piece an undeclared-length body is read into.
-const segmentBytes = 256 << 10
-
-// finishResponse reads the admitted response body and completes f.
-func (s *Server) finishResponse(w http.ResponseWriter, r *http.Request, f *flight) {
-	var res Result
-	var oversize bool
-	var err error
-	switch cl := r.ContentLength; {
-	case cl > MaxResponseBytes:
-		oversize, err = discardToCap(r.Body)
-	case cl >= 0:
-		res.Payload = make([]byte, cl)
-		_, err = io.ReadFull(r.Body, res.Payload)
-	default:
-		res.Segments, oversize, err = readSegments(r.Body)
-	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes+1))
 	if err != nil {
-		s.readFailed(f, err)
 		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
 		return
 	}
-	if oversize {
+	if len(body) > MaxResponseBytes {
 		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
-		s.complete(f, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeUnhandled})
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
 		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
 		return
 	}
-	if !s.complete(f, res) {
-		invalidID(w, f.inv.RequestID)
+	if !s.complete(id, Result{Payload: body}) {
+		invalidID(w, id)
 		return
 	}
 	writeStatus(w)
 }
 
-// discardToCap counts body through a small buffer. It reports oversize once
-// byte MaxResponseBytes+1 arrives and an error if the body ends first.
-func discardToCap(body io.Reader) (bool, error) {
-	n, err := io.CopyN(io.Discard, body, MaxResponseBytes+1)
-	if n == MaxResponseBytes+1 {
-		return true, nil
-	}
-	if err == nil || errors.Is(err, io.EOF) {
-		err = io.ErrUnexpectedEOF
-	}
-	return false, err
-}
-
-// readSegments reads body into segments of at most segmentBytes, never
-// concatenating them. It reports oversize and drops them at byte
-// MaxResponseBytes+1.
-func readSegments(body io.Reader) ([][]byte, bool, error) {
-	segs := [][]byte{}
-	total := 0
-	for {
-		n := min(segmentBytes, MaxResponseBytes+1-total)
-		seg := make([]byte, n)
-		got := 0
-		var err error
-		for got < n && err == nil {
-			var c int
-			c, err = body.Read(seg[got:])
-			got += c
-		}
-		total += got
-		if total > MaxResponseBytes {
-			return nil, true, nil
-		}
-		if got > 0 {
-			segs = append(segs, seg[:got])
-		}
-		if errors.Is(err, io.EOF) {
-			return segs, false, nil
-		}
-		if err != nil {
-			return nil, false, err
-		}
-	}
-}
-
 func (s *Server) invocationError(w http.ResponseWriter, r *http.Request) {
 	id := r.PathValue("id")
-	f := s.admit(id)
-	if f == nil {
+	if !s.claimable(id) {
 		invalidID(w, id)
 		return
 	}
 	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes))
 	if err != nil {
-		s.readFailed(f, err)
 		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
 		return
 	}
@@ -379,7 +238,7 @@ func (s *Server) invocationError(w http.ResponseWriter, r *http.Request) {
 	if errType == "" {
 		errType = ErrTypeUnhandled
 	}
-	if !s.complete(f, Result{Payload: body, FunctionError: errType}) {
+	if !s.complete(id, Result{Payload: body, FunctionError: errType}) {
 		invalidID(w, id)
 		return
 	}

spinifex/agents/lambda/runtimeapi/server_admission_test.go

@@ -1,497 +0,0 @@
-package runtimeapi
-
-//test:in-package drives the unexported admit and exit transitions to force both orders
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"runtime"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
-)
-
-// await bounds a receive so a regression fails promptly instead of hanging.
-func await[T any](t *testing.T, ch <-chan T) T {
-	t.Helper()
-	select {
-	case v := <-ch:
-		return v
-	case <-time.After(15 * time.Second):
-		t.Fatal("timed out waiting on a channel")
-		panic("unreachable")
-	}
-}
-
-// allocBytesPerOp is the mean heap allocated per call of f, measured the same
-// way testing.Benchmark does but over a fixed number of runs.
-func allocBytesPerOp(runs int, f func()) int64 {
-	f()
-	runtime.GC()
-	var before, after runtime.MemStats
-	runtime.ReadMemStats(&before)
-	for range runs {
-		f()
-	}
-	runtime.ReadMemStats(&after)
-	return int64(after.TotalAlloc-before.TotalAlloc) / int64(runs)
-}
-
-// arm puts a delivered, unclaimed flight in place without any HTTP traffic.
-func arm(s *Server, id string) *flight {
-	f := &flight{inv: inv(id, time.Minute), state: stRunning, done: make(chan outcome, 1)}
-	s.mu.Lock()
-	s.cur = f
-	s.mu.Unlock()
-	return f
-}
-
-func postTo(s *Server, kind, id string, body io.Reader, length int64) *httptest.ResponseRecorder {
-	r := httptest.NewRequest(http.MethodPost, base+"invocation/"+id+"/"+kind, body)
-	r.ContentLength = length
-	r.SetPathValue("id", id)
-	rec := httptest.NewRecorder()
-	if kind == "response" {
-		s.response(rec, r)
-	} else {
-		s.invocationError(rec, r)
-	}
-	return rec
-}
-
-// patterned yields n bytes of a repeating pattern without holding them.
-type patterned struct{ n, off int }
-
-func (p *patterned) Read(b []byte) (int, error) {
-	if p.off >= p.n {
-		return 0, io.EOF
-	}
-	c := min(len(b), p.n-p.off)
-	for i := range c {
-		b[i] = byte((p.off + i) % 251)
-	}
-	p.off += c
-	return c, nil
-}
-
-func pattern(n int) []byte {
-	b := make([]byte, n)
-	_, _ = io.ReadFull(&patterned{n: n}, b)
-	return b
-}
-
-func concat(r Result) []byte {
-	if r.Segments == nil {
-		return r.Payload
-	}
-	return bytes.Join(r.Segments, nil)
-}
-
-// counting records whether the handler read the body at all.
-type counting struct{ reads atomic.Int32 }
-
-func (c *counting) Read([]byte) (int, error) { c.reads.Add(1); return 0, io.EOF }
-
-// gated blocks its first Read until released and reports that it started.
-type gated struct {
-	started chan struct{}
-	release chan struct{}
-	rest    io.Reader
-	once    atomic.Bool
-}
-
-func (g *gated) Read(b []byte) (int, error) {
-	if g.once.CompareAndSwap(false, true) {
-		close(g.started)
-		<-g.release
-	}
-	return g.rest.Read(b)
-}
-
-func TestOnlyOnePostPerInvocationIsRead(t *testing.T) {
-	for _, first := range []string{"response", "error"} {
-		for _, second := range []string{"response", "error"} {
-			t.Run(first+"_then_"+second, func(t *testing.T) {
-				s := New()
-				f := arm(s, "req-1")
-				g := &gated{started: make(chan struct{}), release: make(chan struct{}), rest: bytes.NewReader([]byte("body"))}
-				firstDone := make(chan *httptest.ResponseRecorder, 1)
-				go func() { firstDone <- postTo(s, first, "req-1", g, 4) }()
-				await(t, g.started)
-
-				c := &counting{}
-				rec := postTo(s, second, "req-1", c, 4)
-				assert.Equal(t, http.StatusBadRequest, rec.Code)
-				assert.Equal(t, "InvalidRequestID", errType(t, rec.Body.Bytes()))
-				assert.Zero(t, c.reads.Load(), "the refused POST's body was read")
-				assert.Empty(t, f.done, "nothing completes while the admitted body is still arriving")
-
-				close(g.release)
-				assert.Equal(t, http.StatusAccepted, (await(t, firstDone)).Code)
-				assert.Equal(t, "body", string(concat((await(t, f.done)).res)))
-			})
-		}
-	}
-}
-
-func TestDeclaredOversizeCompletesWithoutPayloadBuffer(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	n := MaxResponseBytes + 10
-	rec := postTo(s, "response", "req-1", &patterned{n: n}, int64(n))
-	assert.Equal(t, http.StatusRequestEntityTooLarge, rec.Code)
-	res := (await(t, f.done)).res
-	assert.Equal(t, ErrTypeUnhandled, res.FunctionError)
-	assert.Equal(t, ErrTypeResponseTooLarge, errType(t, res.Payload))
-
-	perOp := allocBytesPerOp(8, func() {
-		s := New()
-		f := arm(s, "req-1")
-		postTo(s, "response", "req-1", &patterned{n: n}, int64(n))
-		await(t, f.done)
-	})
-	assert.Less(t, perOp, int64(256<<10), "an oversize body must not be buffered")
-}
-
-func TestDeclaredOversizeCutShortIsUnknown(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes}, MaxResponseBytes+10)
-	assert.Equal(t, http.StatusBadRequest, rec.Code)
-	assert.ErrorIs(t, (await(t, f.done)).err, ErrResponseBodyFailed, "no result is made from a partial body")
-
-	rec = postTo(s, "response", "req-1", &patterned{n: 1}, 1)
-	assert.Equal(t, "InvalidRequestID", errType(t, rec.Body.Bytes()), "the slot stays consumed")
-}
-
-func TestKnownLengthWithinCapAllocatesOnce(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes}, MaxResponseBytes)
-	assert.Equal(t, http.StatusAccepted, rec.Code)
-	res := (await(t, f.done)).res
-	assert.Nil(t, res.Segments)
-	assert.Equal(t, pattern(MaxResponseBytes), res.Payload)
-
-	s = New()
-	f = arm(s, "req-1")
-	rec = postTo(s, "response", "req-1", &patterned{n: 100}, 200)
-	assert.Equal(t, http.StatusBadRequest, rec.Code, "a body ending short of its length is truncated")
-	out := await(t, f.done)
-	assert.ErrorIs(t, out.err, ErrResponseBodyFailed)
-	assert.Empty(t, out.res.Payload)
-}
-
-func TestChunkedBodyCap(t *testing.T) {
-	t.Run("exactly the cap is accepted in order", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes}, -1)
-		assert.Equal(t, http.StatusAccepted, rec.Code)
-		res := (await(t, f.done)).res
-		assert.Empty(t, res.FunctionError)
-		require.NotNil(t, res.Segments)
-		for _, seg := range res.Segments {
-			assert.LessOrEqual(t, len(seg), segmentBytes)
-		}
-		assert.Equal(t, pattern(MaxResponseBytes), concat(res))
-	})
-	t.Run("one byte over completes as too large", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		rec := postTo(s, "response", "req-1", &patterned{n: MaxResponseBytes + 1}, -1)
-		assert.Equal(t, http.StatusRequestEntityTooLarge, rec.Code)
-		res := (await(t, f.done)).res
-		assert.Equal(t, ErrTypeResponseTooLarge, errType(t, res.Payload))
-	})
-	t.Run("empty body is an empty non-nil payload", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		postTo(s, "response", "req-1", &patterned{}, -1)
-		res := (await(t, f.done)).res
-		require.NotNil(t, res.Segments)
-		assert.Empty(t, concat(res))
-	})
-	t.Run("peak allocation stays within the cap plus one segment", func(t *testing.T) {
-		for _, n := range []int{MaxResponseBytes, MaxResponseBytes + 1} {
-			perOp := allocBytesPerOp(8, func() {
-				s := New()
-				f := arm(s, "req-1")
-				postTo(s, "response", "req-1", &patterned{n: n}, -1)
-				await(t, f.done)
-			})
-			assert.Less(t, perOp, int64(MaxResponseBytes+segmentBytes+64<<10), "body of %d bytes", n)
-		}
-	})
-}
-
-func TestErrorPostOverCapKeepsTruncatedPayload(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	n := MaxResponseBytes + 100
-	rec := postTo(s, "error", "req-1", &patterned{n: n}, int64(n))
-	assert.Equal(t, http.StatusAccepted, rec.Code)
-	res := (await(t, f.done)).res
-	assert.Equal(t, ErrTypeUnhandled, res.FunctionError)
-	assert.Equal(t, pattern(MaxResponseBytes), res.Payload)
-}
-
-func TestBootstrapExitBeforeAdmissionCancels(t *testing.T) {
-	t.Run("before the flight exists", func(t *testing.T) {
-		s := New()
-		assert.True(t, s.BootstrapExited())
-		ctx := t.Context()
-		go func() { _, _ = s.Invoke(ctx, inv("req-1", time.Minute)) }()
-		require.Eventually(t, func() bool {
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			return s.cur != nil && s.cur.state == stExited
-		}, time.Second, time.Millisecond, "the flight is created already exited")
-	})
-	t.Run("before next", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		f.state = stWaiting
-		assert.True(t, s.BootstrapExited())
-		assert.Equal(t, stExited, f.state)
-	})
-	t.Run("after next", func(t *testing.T) {
-		s := New()
-		f := arm(s, "req-1")
-		assert.True(t, s.BootstrapExited())
-		assert.Equal(t, stExited, f.state)
-	})
-}
-
-func TestExitWinsTheRace(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	require.True(t, s.BootstrapExited())
-
-	c := &counting{}
-	rec := postTo(s, "response", "req-1", c, 4)
-	assert.Equal(t, http.StatusBadRequest, rec.Code)
-	assert.Equal(t, "InvalidRequestID", errType(t, rec.Body.Bytes()))
-	assert.Zero(t, c.reads.Load())
-	assert.Empty(t, f.done)
-}
-
-func TestAdmissionWinsTheRace(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	adm := s.admit("req-1")
-	require.NotNil(t, adm)
-	assert.False(t, s.BootstrapExited(), "an admitted response is not cancelled by the exit")
-
-	r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader([]byte("late body")))
-	rec := httptest.NewRecorder()
-	s.finishResponse(rec, r, adm)
-	assert.Equal(t, http.StatusAccepted, rec.Code)
-	assert.Equal(t, "late body", string((await(t, f.done)).res.Payload))
-	assert.False(t, s.BootstrapExited(), "a recorded result stands")
-}
-
-func TestExitedAfterResultIsIgnored(t *testing.T) {
-	s := New()
-	f := arm(s, "req-1")
-	postTo(s, "response", "req-1", bytes.NewReader([]byte("ok")), 2)
-	await(t, f.done)
-	assert.False(t, s.BootstrapExited())
-}
-
-// failing yields some bytes and then a read error, as a reset connection does.
-type failing struct{ left int }
-
-func (f *failing) Read(b []byte) (int, error) {
-	if f.left == 0 {
-		return 0, io.ErrClosedPipe
-	}
-	n := min(len(b), f.left)
-	f.left -= n
-	return n, nil
-}
-
-// invokeReady runs Invoke on a real flight that the runtime has taken.
-func invokeReady(t *testing.T, s *Server, ctx context.Context) <-chan invokeOut {
-	t.Helper()
-	out := make(chan invokeOut, 1)
-	go func() {
-		res, err := s.Invoke(ctx, inv("req-1", time.Minute))
-		out <- invokeOut{res, err}
-	}()
-	require.Eventually(t, func() bool {
-		s.mu.Lock()
-		defer s.mu.Unlock()
-		if s.cur != nil && s.cur.state == stWaiting {
-			s.cur.state = stRunning
-		}
-		return s.cur != nil && s.cur.state == stRunning
-	}, time.Second, time.Millisecond)
-	return out
-}
-
-func TestLiveBootstrapTruncatedBodyEndsInvokeAtOnce(t *testing.T) {
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-	s := New()
-	out := invokeReady(t, s, context.Background()) // a minute-long deadline
-	adm := s.admit("req-1")
-	require.NotNil(t, adm)
-	s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 3}), adm)
-	select {
-	case got := <-out:
-		require.ErrorIs(t, got.err, ErrResponseBodyFailed)
-		assert.Empty(t, got.res.Payload)
-		assert.Empty(t, got.res.FunctionError)
-	case <-time.After(5 * time.Second):
-		t.Fatal("Invoke waited for its deadline")
-	}
-}
-
-func TestIncompleteBodyNeverStrandsInvoke(t *testing.T) {
-	// A body that fails after admission loses the result. The flight must end
-	// with an explicit outcome whichever of the read failure, the bootstrap
-	// exit and the invocation ending comes first.
-	steps := map[string]func(s *Server, adm *flight, cancel context.CancelFunc){
-		"read fails": func(s *Server, adm *flight, _ context.CancelFunc) {
-			s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 3}), adm)
-		},
-		"bootstrap exits":      func(s *Server, _ *flight, _ context.CancelFunc) { s.BootstrapExited() },
-		"invocation cancelled": func(_ *Server, _ *flight, cancel context.CancelFunc) { cancel() },
-	}
-	orders := [][]string{
-		{"read fails", "bootstrap exits", "invocation cancelled"},
-		{"bootstrap exits", "read fails", "invocation cancelled"},
-		{"invocation cancelled", "read fails", "bootstrap exits"},
-		{"invocation cancelled", "bootstrap exits", "read fails"},
-		{"read fails", "invocation cancelled", "bootstrap exits"},
-		{"bootstrap exits", "invocation cancelled", "read fails"},
-	}
-	for _, order := range orders {
-		t.Run(order[0]+"_"+order[1]+"_"+order[2], func(t *testing.T) {
-			defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-			s := New()
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-			out := invokeReady(t, s, ctx)
-			adm := s.admit("req-1")
-			require.NotNil(t, adm)
-			// Whichever of the read failure and the cancellation comes first
-			// decides the outcome; the exit changes nothing once admitted.
-			want := ErrResponseBodyFailed
-			for _, step := range order {
-				if step == "invocation cancelled" {
-					want = context.Canceled
-				}
-				if step != "bootstrap exits" {
-					break
-				}
-			}
-			var got invokeOut
-			returned := false
-			for _, step := range order {
-				steps[step](s, adm, cancel)
-				if step == "invocation cancelled" {
-					// Settle the cancellation before the next transition.
-					select {
-					case got = <-out:
-						returned = true
-					case <-time.After(5 * time.Second):
-						t.Fatal("Invoke did not return after cancel")
-					}
-				}
-			}
-			if !returned {
-				select {
-				case got = <-out:
-				case <-time.After(5 * time.Second):
-					t.Fatal("Invoke did not return")
-				}
-			}
-			require.ErrorIs(t, got.err, want)
-			assert.Empty(t, got.res.Payload)
-			assert.Empty(t, got.res.FunctionError)
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			assert.Nil(t, s.cur, "the flight must be detached")
-		})
-	}
-}
-
-func TestIncompleteBodyAndDeadlineInBothOrders(t *testing.T) {
-	t.Run("deadline first", func(t *testing.T) {
-		defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-		s := New()
-		out := make(chan invokeOut, 1)
-		go func() {
-			res, err := s.Invoke(context.Background(), inv("req-1", 20*time.Millisecond))
-			out <- invokeOut{res, err}
-		}()
-		require.Eventually(t, func() bool {
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			if s.cur != nil && s.cur.state == stWaiting {
-				s.cur.state = stRunning
-			}
-			return s.cur != nil
-		}, time.Second, time.Millisecond)
-		adm := s.admit("req-1")
-		require.NotNil(t, adm)
-		select {
-		case got := <-out:
-			require.ErrorIs(t, got.err, ErrDeadlineExceeded)
-		case <-time.After(5 * time.Second):
-			t.Fatal("Invoke did not time out")
-		}
-		s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 1}), adm)
-		assert.Empty(t, adm.done, "a late failure publishes nothing")
-	})
-	t.Run("body failure first", func(t *testing.T) {
-		defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-		s := New()
-		out := invokeReady(t, s, context.Background())
-		adm := s.admit("req-1")
-		require.NotNil(t, adm)
-		s.finishResponse(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/", &failing{left: 1}), adm)
-		require.ErrorIs(t, (await(t, out)).err, ErrResponseBodyFailed)
-	})
-}
-
-// The reply a bootstrap sees to an oversize body is recorded here, not
-// claimed to match AWS.
-func TestOversizeBootstrapVisibleReply(t *testing.T) {
-	cases := map[string]struct {
-		length int64
-		size   int
-	}{
-		"declared length": {MaxResponseBytes + 2, MaxResponseBytes + 2},
-		"chunked":         {-1, MaxResponseBytes + 2},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			s, ts := start(t)
-			out := invokeAsync(s, inv("req-1", 10*time.Second))
-			get(t, ts.URL+base+"invocation/next")
-			req, err := http.NewRequest(http.MethodPost, ts.URL+base+"invocation/req-1/response", io.NopCloser(&patterned{n: tc.size}))
-			require.NoError(t, err)
-			req.ContentLength = tc.length
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				t.Logf("bootstrap-visible: transport error: %v", err)
-			} else {
-				defer resp.Body.Close()
-				t.Logf("bootstrap-visible: status %d", resp.StatusCode)
-				assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
-			}
-			got := await(t, out)
-			require.NoError(t, got.err)
-			assert.Equal(t, ErrTypeResponseTooLarge, errType(t, got.res.Payload))
-		})
-	}
-}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -170,7 +170,7 @@ func TestOversizeResponse(t *testing.T) {
 
 	got := <-out
 	require.NoError(t, got.err)
-	assert.Equal(t, "Unhandled", got.res.FunctionError)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
 	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
 }
 

spinifex/agents/rds/agent/agent_test.go

@@ -459,7 +459,6 @@ func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
 // instance's volume carries the source instance's receipt. Acknowledging on one
 // would destroy the ciphertext this instance still needs.
 func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
-	t.Parallel()
 	tests := map[string]struct{ payloadID, dbID string }{
 		"another payload":  {payloadID: "bp-somebody-else", dbID: "db-resolved"},
 		"another instance": {payloadID: testPayloadID, dbID: "db-the-snapshot-source"},
@@ -467,7 +466,6 @@ func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
 
 	for name, tc := range tests {
 		t.Run(name, func(t *testing.T) {
-			t.Parallel()
 			cp := newFakeControlPlane()
 			cfg := testConfig(t)
 			cfg.DataMount = t.TempDir()
@@ -570,7 +568,6 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 }
 
 func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cp.bootstrapOut = bootstrapOutput(nil)
 	cfg := testConfig(t)
@@ -857,7 +854,6 @@ func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
 // window. Reporting that would put a failure on every healthy create's beat, and
 // then into the reason a create that timed out for some other cause records.
 func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()

spinifex/agents/rds/agent/command.go

@@ -126,8 +126,6 @@ type commander struct {
 	id       identity
 	registry commandRegistry
 	wait     time.Duration
-	// The wait after a failed poll, pollErrorBackoff unless a test shortens it.
-	pollBackoff time.Duration
 	// The control plane's budget per type, capped by the remaining budget the
 	// issuer stamps on each copy and measured from receipt. IssuedAt is not
 	// compared: the guest clock is not trusted against the host's.
@@ -154,13 +152,12 @@ func newCommander(cp controlPlane, registry commandRegistry, wait time.Duration)
 		wait = defaultPollWait
 	}
 	return &commander{
-		cp:          cp,
-		registry:    registry,
-		wait:        wait,
-		pollBackoff: pollErrorBackoff,
-		budget:      controlPlaneBudget,
-		serial:      make(chan queuedCommand, serialQueueDepth),
-		inFlight:    make(map[string]struct{}),
+		cp:       cp,
+		registry: registry,
+		wait:     wait,
+		budget:   controlPlaneBudget,
+		serial:   make(chan queuedCommand, serialQueueDepth),
+		inFlight: make(map[string]struct{}),
 	}
 }
 
@@ -190,7 +187,7 @@ func (c *commander) Run(ctx context.Context) {
 			select {
 			case <-ctx.Done():
 				return
-			case <-time.After(c.pollBackoff):
+			case <-time.After(pollErrorBackoff):
 			}
 			continue
 		}