chore: bump the go-dependencies group with 13 updates (#1214)

Bumps the go-dependencies group with 13 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.47.1` | `1.47.3` |
| [github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream](https://github.com/aws/aws-sdk-go-v2) | `1.7.20` | `1.7.22` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.33.7` | `1.33.9` |
| [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.20.7` | `1.20.9` |
| [github.com/aws/aws-sdk-go-v2/feature/ec2/imds](https://github.com/aws/aws-sdk-go-v2) | `1.20.1` | `1.20.3` |
| [github.com/containerd/containerd/v2](https://github.com/containerd/containerd) | `2.4.1` | `2.4.2` |
| [github.com/go-chi/chi/v5](https://github.com/go-chi/chi) | `5.3.2` | `5.5.0` |
| [github.com/klauspost/compress](https://github.com/klauspost/compress) | `1.20.1` | `1.20.3` |
| [github.com/klauspost/cpuid/v2](https://github.com/klauspost/cpuid) | `2.4.0` | `2.4.1` |
| [github.com/nats-io/nats-server/v2](https://github.com/nats-io/nats-server) | `2.15.0` | `2.15.1` |
| [github.com/oracle/oci-go-sdk/v65](https://github.com/oracle/oci-go-sdk) | `65.126.2` | `65.127.0` |
| [github.com/pterm/pterm](https://github.com/pterm/pterm) | `0.12.83` | `0.12.84` |
| [golang.org/x/tools](https://github.com/golang/tools) | `0.51.0` | `0.52.0` |


Updates `github.com/aws/aws-sdk-go-v2` from 1.47.1 to 1.47.3
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.47.1...v1.47.3)

Updates `github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream` from 1.7.20 to 1.7.22
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/notifications/v1.7.20...service/notifications/v1.7.22)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.33.7 to 1.33.9
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.33.7...config/v1.33.9)

Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.20.7 to 1.20.9
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/mq/v1.20.7...service/kms/v1.20.9)

Updates `github.com/aws/aws-sdk-go-v2/feature/ec2/imds` from 1.20.1 to 1.20.3
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.20.1...v1.20.3)

Updates `github.com/containerd/containerd/v2` from 2.4.1 to 2.4.2
- [Release notes](https://github.com/containerd/containerd/releases)
- [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md)
- [Commits](https://github.com/containerd/containerd/compare/v2.4.1...v2.4.2)

Updates `github.com/go-chi/chi/v5` from 5.3.2 to 5.5.0
- [Release notes](https://github.com/go-chi/chi/releases)
- [Changelog](https://github.com/go-chi/chi/blob/master/CHANGELOG.md)
- [Commits](https://github.com/go-chi/chi/compare/v5.3.2...v5.5.0)

Updates `github.com/klauspost/compress` from 1.20.1 to 1.20.3
- [Release notes](https://github.com/klauspost/compress/releases)
- [Commits](https://github.com/klauspost/compress/compare/v1.20.1...v1.20.3)

Updates `github.com/klauspost/cpuid/v2` from 2.4.0 to 2.4.1
- [Release notes](https://github.com/klauspost/cpuid/releases)
- [Commits](https://github.com/klauspost/cpuid/compare/v2.4.0...v2.4.1)

Updates `github.com/nats-io/nats-server/v2` from 2.15.0 to 2.15.1
- [Release notes](https://github.com/nats-io/nats-server/releases)
- [Changelog](https://github.com/nats-io/nats-server/blob/main/RELEASES.md)
- [Commits](https://github.com/nats-io/nats-server/compare/v2.15.0...v2.15.1)

Updates `github.com/oracle/oci-go-sdk/v65` from 65.126.2 to 65.127.0
- [Release notes](https://github.com/oracle/oci-go-sdk/releases)
- [Changelog](https://github.com/oracle/oci-go-sdk/blob/master/CHANGELOG.md)
- [Commits](https://github.com/oracle/oci-go-sdk/compare/v65.126.2...v65.127.0)

Updates `github.com/pterm/pterm` from 0.12.83 to 0.12.84
- [Release notes](https://github.com/pterm/pterm/releases)
- [Commits](https://github.com/pterm/pterm/compare/v0.12.83...v0.12.84)

Updates `golang.org/x/tools` from 0.51.0 to 0.52.0
- [Release notes](https://github.com/golang/tools/releases)
- [Commits](https://github.com/golang/tools/compare/v0.51.0...v0.52.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2
  dependency-version: 1.47.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream
  dependency-version: 1.7.22
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.33.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/credentials
  dependency-version: 1.20.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/ec2/imds
  dependency-version: 1.20.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/containerd/containerd/v2
  dependency-version: 2.4.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/go-chi/chi/v5
  dependency-version: 5.5.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: go-dependencies
- dependency-name: github.com/klauspost/compress
  dependency-version: 1.20.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/klauspost/cpuid/v2
  dependency-version: 2.4.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/nats-io/nats-server/v2
  dependency-version: 2.15.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: github.com/oracle/oci-go-sdk/v65
  dependency-version: 65.127.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: go-dependencies
- dependency-name: github.com/pterm/pterm
  dependency-version: 0.12.84
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: go-dependencies
- dependency-name: golang.org/x/tools
  dependency-version: 0.52.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: go-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/actions/junit-report/README.md

@@ -8,7 +8,7 @@ Actions job summary and emits one `::error` annotation per failed test case.
 ```yaml
 - name: Publish test summary
   if: always()
-  uses: ./spinifex/.github/actions/junit-report   # or your own checkout path
+  uses: $/.github/actions/junit-report
   with:
     junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
     title: "Go E2E results"

.github/dependabot.yml

@@ -23,11 +23,6 @@ updates:
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
-    ignore:
-      - dependency-name: "*"
-        update-types:
-          - "version-update:semver-minor"
-          - "version-update:semver-patch"
     groups:
       github-actions:
         patterns:
@@ -50,3 +45,28 @@ updates:
       npm-dependencies:
         patterns:
           - "*"
+
+  # Base images are pinned by digest, so this is what moves them. The golang
+  # version is deliberate and changes with go.mod; only its digest moves here.
+  - package-ecosystem: docker
+    target-branch: dev
+    directories:
+      - /build
+      - /scripts
+      - /docs/terraform-workbooks/demo-app
+    schedule:
+      interval: weekly
+      day: sunday
+      timezone: Australia/Sydney
+    cooldown:
+      default-days: 3
+    ignore:
+      - dependency-name: golang
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+          - "version-update:semver-patch"
+    groups:
+      docker-images:
+        patterns:
+          - "*"

.github/pull_request_template.md

@@ -5,3 +5,17 @@
 ## Changes
 
 <!-- The notable changes, as bullets. -->
+
+## Account-scoped lifecycle impact
+
+<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
+
+- Lifecycle owner and resource family:
+- Owner-local ordering, recovery, retained-state, or force impact:
+- Cross-domain dependency or ownership impact:
+- Evidence added or rechecked:
+- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
+
+## Community Contribution License
+
+- [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.

.github/resolve-release-sibling-refs.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Resolve the tag of each sibling repo (viperblock, predastore, northstar,
+# bluebottle) a release build checks out. A release ships what go.mod requires,
+# so each sibling is built at the version go.mod names, never a branch tip. A
+# pseudo-version has no tag to check out, so it fails the release rather than
+# silently building something else.
+#
+# Takes the path to spinifex's go.mod and writes viperblock=<tag>,
+# predastore=<tag>, northstar=<tag> and bluebottle=<tag> to $GITHUB_OUTPUT for
+# the checkout steps to consume.
+set -euo pipefail
+
+gomod="${1:?usage: $0 <path to spinifex go.mod>}"
+: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
+
+for repo in viperblock predastore northstar bluebottle; do
+  module="github.com/mulgadc/${repo}"
+  version=$(awk -v m="$module" '$1 == m { print $2; exit }' "$gomod")
+  if [[ -z "$version" ]]; then
+    echo "::error::${gomod} does not require ${module}"
+    exit 1
+  fi
+  if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ || "$version" =~ [0-9]{14}-[0-9a-f]{12}$ ]]; then
+    echo "::error::${module} is required at ${version}, which is not a release tag; tag ${repo} and bump go.mod before releasing"
+    exit 1
+  fi
+  echo "${repo}: building against ${version}"
+  echo "${repo}=${version}" >>"$GITHUB_OUTPUT"
+done

.github/workflows/actions.lock

@@ -0,0 +1,137 @@
+# This file is machine-generated by `gh actions-lock`.
+# Do not edit by hand; run `gh actions-lock` to update.
+# Docs: https://gh.io/actions-lockfile
+version: 'v0.0.3'
+workflows:
+    '.github/workflows/apt-packages.yml':
+        - 'actions/checkout@v7.0.1'
+    '.github/workflows/dev-release.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+        - 'docker/setup-buildx-action@v4.4.1'
+    '.github/workflows/e2e-baremetal.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e-cloudvendor-nightly.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/upload-artifact@v7.0.2'
+        - 'hashicorp/setup-terraform@v4.0.1'
+    '.github/workflows/e2e-ddil.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e-nightly.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/download-artifact@v8.0.2'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e-suites.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/download-artifact@v8.0.2'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/gpu-e2e.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/image-scripts.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+    '.github/workflows/perf-3node.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/publish-system-ami.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+    '.github/workflows/release.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/download-artifact@v8.0.2'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+        - 'docker/setup-buildx-action@v4.4.1'
+        - 'softprops/action-gh-release@v3.0.3'
+    '.github/workflows/segscan-oracle.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/setup-go@v7.0.0'
+    '.github/workflows/spinifex.yml':
+        - 'actions/cache@v6.1.0'
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+        - 'golangci/golangci-lint-action@v9.3.0'
+    '.github/workflows/ui.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-node@v7.1.0'
+        - 'pnpm/action-setup@v6.0.10'
+dependencies:
+    'actions/cache@v6.1.0':
+        ref: 'v6.1.0'
+        commit: 'sha1-55cc8345863c7cc4c66a329aec7e433d2d1c52a9'
+        owner_id: 44036562
+        repo_id: 215566462
+    'actions/checkout@v7.0.1':
+        ref: 'v7.0.1'
+        commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
+        owner_id: 44036562
+        repo_id: 197814629
+    'actions/create-github-app-token@v3.3.0':
+        ref: 'v3.3.0'
+        commit: 'sha1-64ab729a6294000f5aa4a4bc0b73d1f32bc45510'
+        owner_id: 44036562
+        repo_id: 642580244
+    'actions/download-artifact@v8.0.2':
+        ref: 'v8.0.2'
+        commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333'
+        owner_id: 44036562
+        repo_id: 192626254
+    'actions/setup-go@v7.0.0':
+        ref: 'v7.0.0'
+        commit: 'sha1-b7ad1dad31e06c5925ef5d2fc7ad053ef454303e'
+        owner_id: 44036562
+        repo_id: 192624594
+    'actions/setup-node@v7.1.0':
+        ref: 'v7.1.0'
+        commit: 'sha1-949feb2413d6458794dcd2491c4babbbce0c15c1'
+        owner_id: 44036562
+        repo_id: 189476904
+    'actions/upload-artifact@v7.0.2':
+        ref: 'v7.0.2'
+        commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9'
+        owner_id: 44036562
+        repo_id: 192625955
+    'docker/setup-buildx-action@v4.4.1':
+        ref: 'v4.4.1'
+        commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069'
+        owner_id: 5429470
+        repo_id: 288485773
+    'golangci/golangci-lint-action@v9.3.0':
+        ref: 'v9.3.0'
+        commit: 'sha1-ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a'
+        owner_id: 35628013
+        repo_id: 232539879
+    'hashicorp/setup-terraform@v4.0.1':
+        ref: 'v4.0.1'
+        commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
+        owner_id: 761456
+        repo_id: 258271226
+    'pnpm/action-setup@v6.0.10':
+        ref: 'v6.0.10'
+        commit: 'sha1-0977fd99725f1db4007ccb2928dbb4e90d06cc86'
+        owner_id: 21320719
+        repo_id: 262289747
+    'softprops/action-gh-release@v3.0.3':
+        ref: 'v3.0.3'
+        commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
+        owner_id: 2242
+        repo_id: 204253808

.github/workflows/apt-packages.yml

@@ -40,7 +40,7 @@ jobs:
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
 
       - name: Resolve runtime package list
         run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}

.github/workflows/dev-release.yml

@@ -34,7 +34,7 @@ jobs:
       # Full history and tags, because the tag name is git describe and a shallow
       # clone has no tag to describe against.
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           fetch-depth: 0
@@ -42,40 +42,41 @@ jobs:
       # dev, not each repository's default branch, which is what release.yml takes.
       # A dev build of spinifex against main of the others is a pairing nobody runs.
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: dev
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
           ref: dev
           path: viperblock
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
           ref: dev
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
           ref: dev
           path: bluebottle
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
+        uses: docker/setup-buildx-action@v4.4.1
 
       - name: Set up Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
+          cache: false
 
       # The tag carries the describe string so the commit is recoverable from the
       # version alone, and the -dev suffix is what release.yml's own convention
@@ -167,11 +168,7 @@ jobs:
 
       # Kept rather than unbounded: the Worker reads the first page of releases to
       # resolve the channel, so enough prereleases would eventually push the real
-      # ones off it, and each build carries two assets.
-      #
-      # Twenty, not five. A pinned dev tag is how an external evaluator installs a
-      # specific build, and five is about a week of them -- short enough that the tag
-      # someone was given stops resolving while they are still using it.
+      # ones off it. Five leaves a few recent builds to roll back to.
       - name: Prune old dev prereleases
         if: steps.version.outputs.exists != 'true'
         env:
@@ -179,7 +176,7 @@ jobs:
         run: |
           set -euo pipefail
           gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
-              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[20:] | .[].tag_name' \
+              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \
               > stale.txt
           while read -r tag; do
               [ -n "$tag" ] || continue

.github/workflows/e2e-baremetal.yml

@@ -77,18 +77,18 @@ jobs:
           if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           ref: ${{ inputs.ref || github.ref }}
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -151,7 +151,8 @@ jobs:
           fi
 
       - name: Install go-junit-report on runner
-        run: go install github.com/jstemmer/go-junit-report/v2@latest
+        # Version from go.mod's tool directive, checked against go.sum.
+        run: GOWORK=off go install github.com/jstemmer/go-junit-report/v2
 
       # The eks suite's CreateCluster boots a control-plane VM from the eks-node
       # AMI. Built here because libguestfs needs source and network, which the
@@ -221,20 +222,20 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always()
         continue-on-error: true
-        uses: ./.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           cell: ${{ env.BM_CELL }}
 
       - name: Analyze e2e failures
         if: always()
-        uses: ./.github/actions/e2e-analyze
+        uses: $/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
           title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}

.github/workflows/e2e-cloudvendor-nightly.yml

@@ -76,11 +76,11 @@ jobs:
       - name: Put the Terraform state outside the checkout
         run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
@@ -92,14 +92,14 @@ jobs:
 
       # build-install-artifacts.sh lives in mulga. The workbooks and their driver
       # are in spinifex, so only the one directory is needed here.
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           path: mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
 
-      - uses: hashicorp/setup-terraform@v4
+      - uses: hashicorp/setup-terraform@v4.0.1
         with:
           terraform_wrapper: false
 
@@ -273,7 +273,7 @@ jobs:
       # collect step above chose, so terraform.tfstate cannot reach an artifact.
       - name: Upload the run
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: e2e-oci-${{ github.run_id }}
           path: |

.github/workflows/e2e-ddil.yml

@@ -38,19 +38,19 @@ jobs:
       - name: Clean Workspace
         run: sudo rm -rf "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         name: Checkout Spinifex
         with:
           path: spinifex
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         name: Checkout Mulga (tofu-cluster)
         with:
           repository: mulgadc/mulga
@@ -81,7 +81,7 @@ jobs:
       # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
       # the VMs, so a telemetry doc points back at this job's log directly.
       - name: Resolve CI job id
-        uses: ./spinifex/.github/actions/otel-job-id
+        uses: $/.github/actions/otel-job-id
         with:
           job-name: DDIL Real Multi-Node
 
@@ -175,7 +175,7 @@ jobs:
 
       - name: Publish test summary
         if: always()
-        uses: ./spinifex/.github/actions/junit-report
+        uses: $/.github/actions/junit-report
         with:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
@@ -185,7 +185,7 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always()
         continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           cell: ddil
@@ -214,7 +214,7 @@ jobs:
 
       - name: Upload DDIL Results
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: ddil-results-${{ github.run_id }}
           path: ${{ runner.temp }}/ddil-results
@@ -234,7 +234,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-ddil-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}

.github/workflows/e2e-nightly.yml

@@ -43,7 +43,7 @@ jobs:
       cells: ${{ steps.pick.outputs.cells }}
       any: ${{ steps.pick.outputs.any }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: |
             .github/e2e-cells.json
@@ -128,21 +128,21 @@ jobs:
           if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
@@ -151,7 +151,7 @@ jobs:
       # links each cell's telemetry to the job log that produced it. Runs
       # before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
       - name: Resolve CI job id
-        uses: ./spinifex/.github/actions/otel-job-id
+        uses: $/.github/actions/otel-job-id
         with:
           job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
 
@@ -795,7 +795,8 @@ jobs:
       - name: Install go-junit-report
         if: always()
         run: |
-          go install github.com/jstemmer/go-junit-report/v2@latest
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Convert test logs to JUnit XML
@@ -812,20 +813,20 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always()
         continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           cell: ${{ matrix.cell }}
 
       - name: Analyze e2e failures
         if: always()
-        uses: ./spinifex/.github/actions/e2e-analyze
+        uses: $/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
           title: "E2E failure analysis (cell ${{ matrix.cell }})"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -837,7 +838,7 @@ jobs:
       # pulling nineteen journals it has no use for.
       - name: Upload junit only
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
@@ -868,7 +869,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
@@ -973,20 +974,20 @@ jobs:
             { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
           echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           ref: ${{ env.GIT_BRANCH }}
           path: mulga/spinifex
@@ -998,25 +999,25 @@ jobs:
         run: bash .github/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
           ref: ${{ steps.siblings.outputs.viperblock }}
           path: mulga/viperblock
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
           path: mulga/predastore
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
           ref: ${{ steps.siblings.outputs.northstar }}
           path: mulga/northstar
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
           ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -1151,7 +1152,7 @@ jobs:
                  /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
           [ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-artifacts-cell-21-${{ github.run_id }}
@@ -1220,20 +1221,20 @@ jobs:
           # disks, which is what the tmpfs assertion below is about.
           echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           ref: ${{ env.GIT_BRANCH }}
           path: mulga/spinifex
@@ -1245,25 +1246,25 @@ jobs:
         run: bash .github/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
           ref: ${{ steps.siblings.outputs.viperblock }}
           path: mulga/viperblock
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
           path: mulga/predastore
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
           ref: ${{ steps.siblings.outputs.northstar }}
           path: mulga/northstar
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
           ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -1317,7 +1318,7 @@ jobs:
           cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
             echo "no per-node serial logs — the run passed and cleaned up after itself"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-artifacts-cell-22-${{ github.run_id }}
@@ -1369,7 +1370,7 @@ jobs:
           echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
 
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
@@ -1384,7 +1385,7 @@ jobs:
         working-directory: spinifex
 
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
@@ -1437,7 +1438,7 @@ jobs:
       # Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
       # until the run-to-run spread is known from real nights, a threshold would
       # be a guess, and a gate that cries wolf gets muted.
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: predastore-perf-${{ github.run_id }}
@@ -1496,7 +1497,7 @@ jobs:
           echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
 
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
@@ -1508,7 +1509,7 @@ jobs:
         working-directory: spinifex
 
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
@@ -1563,7 +1564,7 @@ jobs:
           mkdir -p "$STRESS_RESULTS_ROOT"
           make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: predastore-stress-${{ github.run_id }}
@@ -1600,13 +1601,13 @@ jobs:
     if: always()
     runs-on: ubuntu-26.04
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: .github/scripts/e2e-run-summary.py
           sparse-checkout-cone-mode: false
 
       - name: Collect junit
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         with:
           pattern: junit-cell-*-${{ github.run_id }}
           path: junit

.github/workflows/e2e-suites.yml

@@ -103,25 +103,25 @@ jobs:
             | sort -r | xargs -r -I{} sudo umount -l -R {} || true
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -157,7 +157,7 @@ jobs:
       # the env: the suite legs share this env and its baked telemetry.env, so
       # the provisioner is the only job the daemon telemetry can belong to.
       - name: Resolve CI job id
-        uses: ./spinifex/.github/actions/otel-job-id
+        uses: $/.github/actions/otel-job-id
         with:
           job-name: Single env · provision
 
@@ -300,7 +300,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-single-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
@@ -323,18 +323,19 @@ jobs:
       SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
       ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
 
       - name: Install go-junit-report
         run: |
-          go install github.com/jstemmer/go-junit-report/v2@latest
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Run ${{ matrix.suite }}
@@ -390,7 +391,7 @@ jobs:
 
       - name: Publish test summary
         if: always()
-        uses: ./spinifex/.github/actions/junit-report
+        uses: $/.github/actions/junit-report
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
@@ -400,20 +401,20 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always()
         continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           cell: single
 
       - name: Analyze e2e failures
         if: always()
-        uses: ./spinifex/.github/actions/e2e-analyze
+        uses: $/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
           title: "E2E ${{ matrix.suite }} failure analysis"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
@@ -444,7 +445,7 @@ jobs:
       # Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
       # Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
       # again would wipe the local tofu state and orphan the VM.
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
@@ -467,7 +468,7 @@ jobs:
             | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
           true
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-single-diag-${{ github.run_id }}
@@ -523,7 +524,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-single-td-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
@@ -571,25 +572,25 @@ jobs:
             | sort -r | xargs -r -I{} sudo umount -l -R {} || true
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -606,7 +607,7 @@ jobs:
 
       # See the matching note on single_provision.
       - name: Resolve CI job id
-        uses: ./spinifex/.github/actions/otel-job-id
+        uses: $/.github/actions/otel-job-id
         with:
           job-name: Multi env · provision
 
@@ -714,7 +715,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-multi-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
@@ -737,18 +738,19 @@ jobs:
       NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
 
       - name: Install go-junit-report
         run: |
-          go install github.com/jstemmer/go-junit-report/v2@latest
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Run ${{ matrix.suite }}
@@ -804,7 +806,7 @@ jobs:
 
       - name: Publish test summary
         if: always()
-        uses: ./spinifex/.github/actions/junit-report
+        uses: $/.github/actions/junit-report
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
@@ -814,20 +816,20 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always()
         continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           cell: multi
 
       - name: Analyze e2e failures
         if: always()
-        uses: ./spinifex/.github/actions/e2e-analyze
+        uses: $/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
           title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
@@ -854,7 +856,7 @@ jobs:
       SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
       NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
@@ -908,7 +910,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-multi-td-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}

.github/workflows/e2e.yml

@@ -54,7 +54,7 @@ jobs:
       single_timeout: ${{ steps.budgets.outputs.single_timeout }}
       multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           sparse-checkout: .github/scripts
@@ -142,25 +142,25 @@ jobs:
           # stale suite-status.txt / analysis dirs can't leak into this attempt.
           rm -rf "${ARTIFACT_DIR}"
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -178,7 +178,7 @@ jobs:
       # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
       # the VM, so a telemetry doc points back at this job's log directly.
       - name: Resolve CI job id
-        uses: ./spinifex/.github/actions/otel-job-id
+        uses: $/.github/actions/otel-job-id
         with:
           job-name: Single-Node E2E
 
@@ -372,7 +372,8 @@ jobs:
 
       - name: Install go-junit-report on runner
         run: |
-          go install github.com/jstemmer/go-junit-report/v2@latest
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Ship test binaries to VM
@@ -512,7 +513,7 @@ jobs:
 
       - name: Publish test summary
         if: always()
-        uses: ./spinifex/.github/actions/junit-report
+        uses: $/.github/actions/junit-report
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
@@ -522,20 +523,20 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always()
         continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           cell: single
 
       - name: Analyze e2e failures
         if: always()
-        uses: ./spinifex/.github/actions/e2e-analyze
+        uses: $/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
           title: "E2E failure analysis"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-artifacts-${{ github.run_id }}
@@ -579,7 +580,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-single-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
@@ -642,25 +643,25 @@ jobs:
           # stale suite-status.txt / analysis dirs can't leak into this attempt.
           rm -rf "${ARTIFACT_DIR}"
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -677,7 +678,7 @@ jobs:
 
       # See the matching note on e2e_single.
       - name: Resolve CI job id
-        uses: ./spinifex/.github/actions/otel-job-id
+        uses: $/.github/actions/otel-job-id
         with:
           job-name: Multi-Node E2E
 
@@ -758,7 +759,8 @@ jobs:
 
       - name: Install go-junit-report on runner
         run: |
-          go install github.com/jstemmer/go-junit-report/v2@latest
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Ship test binaries to node1
@@ -965,7 +967,7 @@ jobs:
 
       - name: Publish test summary
         if: always()
-        uses: ./spinifex/.github/actions/junit-report
+        uses: $/.github/actions/junit-report
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
@@ -975,20 +977,20 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always()
         continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           cell: multi
 
       - name: Analyze e2e failures
         if: always()
-        uses: ./spinifex/.github/actions/e2e-analyze
+        uses: $/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
           title: "Multi-Node E2E failure analysis"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: e2e-multi-artifacts-${{ github.run_id }}
@@ -1035,7 +1037,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-multi-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
@@ -1105,14 +1107,14 @@ jobs:
       # the aggregation script below treats an absent suite-status.txt as
       # "no failures from that leg".
       - name: Download single-node artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         continue-on-error: true
         with:
           name: e2e-artifacts-${{ github.run_id }}
           path: ${{ github.workspace }}/es-summary-single
 
       - name: Download multi-node artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         continue-on-error: true
         with:
           name: e2e-multi-artifacts-${{ github.run_id }}

.github/workflows/gpu-e2e.yml

@@ -95,10 +95,10 @@ jobs:
         if: steps.gate.outputs.paused != 'true'
         run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         if: steps.gate.outputs.paused != 'true'
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         if: steps.gate.outputs.paused != 'true'
         with:
           go-version-file: go.mod
@@ -187,7 +187,8 @@ jobs:
       - name: Install go-junit-report on runner
         if: steps.gate.outputs.paused != 'true'
         run: |
-          go install github.com/jstemmer/go-junit-report/v2@latest
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go install github.com/jstemmer/go-junit-report/v2
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Ship GPU e2e test binary to wattle
@@ -243,7 +244,7 @@ jobs:
 
       - name: Publish test summary
         if: always() && steps.gate.outputs.paused != 'true'
-        uses: ./.github/actions/junit-report
+        uses: $/.github/actions/junit-report
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
@@ -251,20 +252,20 @@ jobs:
       - name: Ship test results to Elasticsearch
         if: always() && steps.gate.outputs.paused != 'true'
         continue-on-error: true
-        uses: ./.github/actions/junit-to-es
+        uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           cell: gpu
 
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
-        uses: ./.github/actions/e2e-analyze
+        uses: $/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
           title: "GPU E2E failure analysis"
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always() && steps.gate.outputs.paused != 'true'
         with:
           name: gpu-e2e-artifacts-${{ github.run_id }}

.github/workflows/image-scripts.yml

@@ -29,13 +29,16 @@ jobs:
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
 
       - name: Run image script tests + shellcheck
         run: make test-images
 
   eks_mkosi_image:
     name: Build mkosi EKS image
+    # A fork PR's code must never run on a persistent self-hosted runner: it
+    # shares disk, caches and credentials with the release jobs.
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: [self-hosted, ci-single]
     concurrency:
       group: eks-mkosi-image-${{ github.ref }}
@@ -43,10 +46,10 @@ jobs:
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
 
       - name: Set up Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: "go.mod"
           # Self-hosted: the module + build cache already persists on local

.github/workflows/perf-3node.yml

@@ -80,25 +80,25 @@ jobs:
           echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
           echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -116,7 +116,7 @@ jobs:
         run: bash .github/resolve-sibling-refs.sh
         working-directory: spinifex
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         id: predastore
         with:
           repository: mulgadc/predastore
@@ -366,7 +366,7 @@ jobs:
       # Report-only, matching cell-23: the baseline comparison and any
       # regression gate are future work once run-to-run spread is known from
       # real bare-metal runs.
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
           name: predastore-perf-3node-${{ github.run_id }}
@@ -407,7 +407,7 @@ jobs:
 
       - name: Upload tofu log
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: tofu-log-perf3node-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}

.github/workflows/publish-system-ami.yml

@@ -17,28 +17,12 @@ name: Publish System AMIs
 #   spx admin images import --name spinifex-rds-postgres
 #   spx admin images import --name spinifex-rds-mariadb
 #
-# Publishing overwrites the mirror artifact, so it is a deliberate action, not
-# something every push should trigger: it runs manually, or from release.yml
-# once a tag has passed its gates. The `variant` input scopes which images
-# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
-# DKMS build), so publishing only the CPU AMIs stays cheap.
+# Publishing overwrites the mirror artifact, so it runs only from release.yml
+# once a tag has passed its gates, with siblings at the tags go.mod requires.
+# The `variant` input scopes which images build — the Ubuntu GPU legs are 16G
+# and much slower (NVIDIA driver download + DKMS build).
 
 on:
-  workflow_dispatch:
-    inputs:
-      variant:
-        description: "Which images to build"
-        type: choice
-        default: all
-        options:
-          - all
-          - cpu
-          - gpu
-          - rds
-      dry_run:
-        description: "Build + convert only; skip the R2 upload"
-        type: boolean
-        default: false
   workflow_call:
     inputs:
       variant:
@@ -94,40 +78,44 @@ jobs:
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
-          ref: dev
+          ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
-          ref: dev
+          ref: ${{ steps.siblings.outputs.bluebottle }}
           path: bluebottle
 
       - name: Setup Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: "spinifex/go.mod"
           # Self-hosted runner: the Go module + build cache already persists on

.github/workflows/release.yml

@@ -5,48 +5,57 @@ on:
     tags: [ "v*" ]
 
 permissions:
-  contents: write
+  contents: read
 
 jobs:
   distro-amd64:
     runs-on: ubuntu-26.04
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
           path: bluebottle
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
+        uses: docker/setup-buildx-action@v4.4.1
 
       - name: Set up Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
+          cache: false
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -57,7 +66,7 @@ jobs:
         run: cd spinifex && make distro-amd64 VERSION="$VERSION"
 
       - name: Upload amd64 Artifacts
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: distro-amd64
           path: |
@@ -69,36 +78,44 @@ jobs:
     runs-on: ubuntu-26.04-arm
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
           path: bluebottle
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
+        uses: docker/setup-buildx-action@v4.4.1
 
       - name: Build arm64 Tarball
         env:
@@ -106,7 +123,7 @@ jobs:
         run: cd spinifex && make distro-arm64 VERSION="$VERSION"
 
       - name: Upload arm64 Artifacts
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: distro-arm64
           path: |
@@ -116,21 +133,23 @@ jobs:
   release:
     needs: [ distro-amd64, distro-arm64 ]
     runs-on: ubuntu-26.04
+    permissions:
+      contents: write
     steps:
       - name: Download amd64 Artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         with:
           name: distro-amd64
           path: artifacts
 
       - name: Download arm64 Artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         with:
           name: distro-arm64
           path: artifacts
 
       - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v3
+        uses: softprops/action-gh-release@v3.0.3
         with:
           draft: true
           files: |
@@ -140,7 +159,7 @@ jobs:
           generate_release_notes: true
 
       - name: Upload Tarball for Verify Job
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: release-assets
           path: artifacts/
@@ -167,14 +186,14 @@ jobs:
             | sort -r | xargs -r -I{} sudo umount -l -R {} || true
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -183,13 +202,13 @@ jobs:
 
       # Only the shared shell helpers: this job builds nothing from source, but
       # it drives the same libvirt provider the e2e cells retry around.
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: .github/scripts
           path: spinifex
 
       - name: Download Release Assets
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         with:
           name: release-assets
           path: release-assets
@@ -231,6 +250,8 @@ jobs:
   promote:
     needs: [ verify, baremetal ]
     runs-on: ubuntu-26.04
+    permissions:
+      contents: write
     env:
       TAG: ${{ github.ref_name }}
       REPO: ${{ github.repository }}
@@ -263,6 +284,8 @@ jobs:
     needs: [ verify, baremetal ]
     if: failure()
     runs-on: ubuntu-26.04
+    permissions:
+      contents: write
     env:
       TAG: ${{ github.ref_name }}
       REPO: ${{ github.repository }}
@@ -280,6 +303,8 @@ jobs:
   iso:
     needs: release
     runs-on: [ self-hosted, ci-single ]
+    permissions:
+      contents: write
     # ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
     # upload gate then waits for baremetal to finish, so the job must outlast it.
     timeout-minutes: 180
@@ -294,7 +319,7 @@ jobs:
           sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
@@ -302,37 +327,45 @@ jobs:
           repositories: mulga
 
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           ref: ${{ github.ref }}
 
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
           path: bluebottle
 
       - name: Checkout Mulga iso-builder
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -343,7 +376,7 @@ jobs:
         run: sudo make -C mulga/scripts/iso-builder clean
 
       - name: Set up Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false

.github/workflows/segscan-oracle.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           fetch-depth: 2
@@ -38,34 +38,34 @@ jobs:
         working-directory: spinifex
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
           ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
           ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
           ref: ${{ steps.siblings.outputs.bluebottle }}
           path: bluebottle
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
@@ -73,7 +73,7 @@ jobs:
           repositories: mulga
 
       - name: Checkout Mulga (scripts/segscan only)
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -81,7 +81,7 @@ jobs:
           path: mulga
 
       - name: Setup Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"

.github/workflows/spinifex.yml

@@ -27,7 +27,7 @@ jobs:
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           fetch-depth: 2
@@ -40,28 +40,28 @@ jobs:
         working-directory: spinifex
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
           ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
           ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
           ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -71,7 +71,7 @@ jobs:
       # the last go.sum change and never refreshed it. Both caches are managed
       # below instead.
       - name: Setup Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: "spinifex/go.mod"
           cache: false
@@ -80,7 +80,7 @@ jobs:
       # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
       # dependencies out of the key and re-downloading on every run.
       - name: Cache Go modules
-        uses: actions/cache@v6
+        uses: actions/cache@v6.1.0
         with:
           path: ~/go/pkg/mod
           key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
@@ -93,7 +93,7 @@ jobs:
       # a PR reads the base branch's entry without spending a minute uploading
       # one only that PR would ever read.
       - name: Restore Go build cache
-        uses: actions/cache/restore@v6
+        uses: actions/cache/restore@v6.1.0
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -150,7 +150,7 @@ jobs:
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
+        uses: actions/cache/save@v6.1.0
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -161,7 +161,7 @@ jobs:
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
@@ -173,28 +173,28 @@ jobs:
         working-directory: spinifex
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
           ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
           ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
           ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -204,7 +204,7 @@ jobs:
       # the last go.sum change and never refreshed it. Both caches are managed
       # below instead.
       - name: Setup Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: "spinifex/go.mod"
           cache: false
@@ -213,7 +213,7 @@ jobs:
       # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
       # dependencies out of the key and re-downloading on every run.
       - name: Cache Go modules
-        uses: actions/cache@v6
+        uses: actions/cache@v6.1.0
         with:
           path: ~/go/pkg/mod
           key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
@@ -226,7 +226,7 @@ jobs:
       # a PR reads the base branch's entry without spending a minute uploading
       # one only that PR would ever read.
       - name: Restore Go build cache
-        uses: actions/cache/restore@v6
+        uses: actions/cache/restore@v6.1.0
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -251,7 +251,7 @@ jobs:
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
+        uses: actions/cache/save@v6.1.0
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -262,7 +262,7 @@ jobs:
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           fetch-depth: 2
@@ -275,28 +275,28 @@ jobs:
         working-directory: spinifex
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
           ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Northstar
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/northstar
           ref: ${{ steps.siblings.outputs.northstar }}
           path: northstar
 
       - name: Checkout Bluebottle
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/bluebottle
           ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -306,7 +306,7 @@ jobs:
       # the last go.sum change and never refreshed it. Both caches are managed
       # below instead.
       - name: Setup Go
-        uses: actions/setup-go@v7
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: "spinifex/go.mod"
           cache: false
@@ -315,7 +315,7 @@ jobs:
       # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
       # dependencies out of the key and re-downloading on every run.
       - name: Cache Go modules
-        uses: actions/cache@v6
+        uses: actions/cache@v6.1.0
         with:
           path: ~/go/pkg/mod
           key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
@@ -328,7 +328,7 @@ jobs:
       # a PR reads the base branch's entry without spending a minute uploading
       # one only that PR would ever read.
       - name: Restore Go build cache
-        uses: actions/cache/restore@v6
+        uses: actions/cache/restore@v6.1.0
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -338,9 +338,9 @@ jobs:
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Lint
-        uses: golangci/golangci-lint-action@v9
+        uses: golangci/golangci-lint-action@v9.3.0
         with:
-          version: v2.13.0
+          version: v2.14.0
           working-directory: spinifex
 
       - name: Govulncheck
@@ -363,7 +363,7 @@ jobs:
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
+        uses: actions/cache/save@v6.1.0
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}

.github/workflows/ui.yml

@@ -30,17 +30,17 @@ jobs:
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v7
+        uses: actions/checkout@v7.0.1
         with:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6
+        uses: pnpm/action-setup@v6.0.10
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 
       - name: Setup Node.js
-        uses: actions/setup-node@v7
+        uses: actions/setup-node@v7.1.0
         with:
           cache: "pnpm"
           cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
@@ -58,5 +58,14 @@ jobs:
       - name: Build Frontend
         run: pnpm build
 
-      - name: Check for generated route tree drift
-        run: git diff --exit-code -- src/routeTree.gen.ts
+      # spx embeds the committed dist/, so a bundle that differs from a clean
+      # build of this source would ship JavaScript CI never built. Porcelain
+      # rather than git diff, so added and deleted chunks count too.
+      - name: Check for generated file drift
+        run: |
+          drift=$(git status --porcelain -- src/routeTree.gen.ts src/lib/cn-tables.gen.ts dist)
+          if [ -n "$drift" ]; then
+            echo "::error::Generated files differ from a clean build. Run pnpm build and commit the result."
+            echo "$drift"
+            exit 1
+          fi

CONTRIBUTING.md

@@ -54,11 +54,11 @@ AWS_PROFILE=spinifex aws ec2 describe-instance-types
 
 A list of instance types means the cluster is up.
 
-`make preflight` also needs `shellcheck` and `golangci-lint` (CI pins v2.13.0):
+`make preflight` also needs `shellcheck` and `golangci-lint`
 
 ```bash
 sudo apt-get install -y shellcheck
-curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.13.0
+curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.14.0
 ```
 
 The web console additionally needs Node.js 24 and `pnpm`.

Makefile

@@ -262,17 +262,6 @@ test-cover:
 	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
-# Refresh the README coverage badge. Measured as awesome-go documents
-# (-coverpkg=./... over ./...), with cmd/ entrypoints left out of the total.
-coverage-badge:
-	@echo -e "\n....Measuring whole-module coverage for the README badge...."
-	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 600s -covermode=atomic -coverpkg=./... -coverprofile=coverage-badge.out ./...
-	@grep -v '/cmd/' coverage-badge.out > coverage-badge-nocmd.out
-	@pct=$$(go tool cover -func=coverage-badge-nocmd.out | tail -1 | awk '{print $$NF}' | tr -d '%'); \
-	color=$$(awk -v p="$$pct" 'BEGIN { print (p >= 80) ? "brightgreen" : (p >= 70) ? "yellow" : "red" }'); \
-	sed -i -E "s|badge/coverage-[0-9.]+%25-[a-z]+|badge/coverage-$${pct}%25-$${color}|" README.md; \
-	echo "Coverage $${pct}% ($${color}); README badge updated"
-
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
@@ -390,9 +379,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.2 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.2.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -475,7 +464,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover coverage-badge test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -4,225 +4,136 @@
 
 <p align="center">
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
-  <a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
-  <a href="https://github.com/mulgadc/spinifex/blob/main/Makefile"><img src="https://img.shields.io/badge/coverage-82.9%25-brightgreen?style=flat-square" alt="Coverage"></a>
+  <a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-Reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
-  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
+  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/Site-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
 </p>
 
+
 <p align="center">
-  <a href="#what-is-spinifex">What is Spinifex?</a> ·
-  <a href="#the-platform">Platform</a> ·
-  <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS Compatibility</a> ·
-  <a href="#live-demo">Live Demo</a> ·
-  <a href="#core-components">Components</a> ·
-  <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#self-hosted-installation">Installation</a> ·
-  <a href="https://docs.mulgadc.com">Docs</a>
+  <a href="#try-it-live">Live Demo</a> ·
+  <a href="#services">Services</a> ·
+  <a href="#deploy-anywhere">Deploy</a> ·
+  <a href="#install">Install</a> ·
+  <a href="#architecture">Architecture</a>
 </p>
 
 ---
 
 # Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line.
 
-## What is Spinifex?
-
-Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
-
-Built for teams that need:
-
-- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
-- Full control of the stack: your hardware, your network, your data, your keys
-- A cloud that keeps running offline, through disconnection, with no external control plane
-- An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
+- **Your AWS tooling, unchanged.** The AWS CLI, SDKs, and Terraform work as they are.
+- **Full control of the stack.** Your hardware, data, and keys, with no external control plane, running through disconnection.
+- **Open core under AGPL-3.0.** Auditable, yours to keep, with no lock-in.
 
 You change *where* your software runs, not *what* it is.
 
-## The Platform
+## Try It Live
+
+Get a real Spinifex endpoint in under a minute, with no card and no install. The sandbox is yours for 72 hours, with access keys, the web console, and EC2, EBS, S3, VPC, IAM, EKS, and RDS ready to call. Point your existing AWS CLI or Terraform straight at it.
+
+**[Create your sandbox →](https://mulgadc.com/signup)**
+
+## Same Commands, Your Cloud
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-images
+aws ec2 run-instances --image-id ami-xxxxxxxx --instance-type t3.micro
+aws s3 mb s3://my-bucket
+```
+
+For Terraform, start from the [example workbooks](https://docs.mulgadc.com/docs/nginx-webserver).
+
+## Why Not LocalStack or Floci?
+
+[LocalStack](https://github.com/localstack/localstack) and [Floci](https://github.com/floci-io/floci) are emulators for local development and CI testing. An emulator returns a plausible response; Spinifex boots the machine. A call launches a real KVM guest, attaches a replicated Viperblock volume, or writes an erasure-coded object into Predastore, so Spinifex can run your production workloads on hardware you own. Test against an emulator, then run in production on Spinifex.
+
+## Services
+
+The platform serves **392 AWS API operations** across the services below. See exactly which ones in the [coverage report](https://docs.mulgadc.com/coverage).
+
+| Service | What it is | Built on | Status |
+| --- | --- | --- | --- |
+| **EC2** | Compute | QEMU/KVM, with spot instances and GPU passthrough | Available |
+| **EBS** | Block storage | [Viperblock](https://github.com/mulgadc/viperblock): replicated, with a WAL on local NVMe | Available |
+| **S3** | Object storage | [Predastore](https://github.com/mulgadc/predastore): Reed-Solomon erasure coding | Available |
+| **VPC** | Networking | OVN, with security groups | Available |
+| **IAM** | Identity & auth | SigV4 across every service | Available |
+| **ALB / NLB** | Load balancers | HAProxy (L7), nginx `stream` (L4), with ACM certificates | Available |
+| **EKS** | Kubernetes | k3s, with HA control planes and managed node groups | Available |
+| **ECR** | Container registry | Predastore, deduplicated per account | Available |
+| **ECS** | Container service | Spinifex instances | Available |
+| **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
+| **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
+| **Lambda** | Serverless functions | | Q1 2027 |
+
+Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+
+## Deploy Anywhere
 
 <p align="center">
   <img src=".github/assets/platform.svg" alt="The Spinifex stack: your apps and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU and GPU hardware, deployable to Neocloud, on-premise, or the edge." width="900">
 </p>
 
-From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
+- **Neocloud:** lift and shift onto partner GPU capacity (H100 / H200 / B200). Change an endpoint, keep the software.
+- **On-premise:** a multi-node HA cluster in your own data centre, so the stack, the data, and the jurisdiction stay yours.
+- **Edge:** air-gapped sites, vehicles, vessels, factories, and clinics, with compute next to the data.
 
-## Three Ways To Deploy
+## Install
 
-The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
+Spinifex runs on Ubuntu 26.04 or Debian 13. The full guides are at [docs.mulgadc.com](https://docs.mulgadc.com).
 
-### Neocloud
+>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
 
-Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
-
-### On-premise
-
-Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
-
-### Edge
-
-Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
-
-## AWS Compatibility
-
-Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
-
-| Service | What it is | Status |
-| --- | --- | --- |
-| **EC2** | Compute | Available |
-| **EBS** | Block Storage | Available |
-| **S3** | Object Storage | Available |
-| **VPC** | Networking | Available |
-| **IAM** | Identity & Auth | Available |
-| **ALB / NLB** | Load Balancers | Available |
-| **EKS** | Kubernetes | Available |
-| **ECR** | Container Registry | Available |
-| **ECS** | Container Service | Available |
-| **RDS** | Databases | Available |
-| **Bedrock** | AI Deployment | Q4 2026 |
-
-Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
-
-## Live Demo
-
-Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
-
-* Live in under a minute. No card, no install.
-* Yours for 72 hours to test and experiment.
-* Real AWS-compatible endpoint, access keys and console.
-* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
-* Point your existing AWS CLI or Terraform straight at it.
-
-Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
-
-## Core Components
-
-### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
-
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
-
-The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
-
-- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
-- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
-- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
-- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
-- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
-- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
-- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
-
-### Viperblock (Block Storage – EBS Alternative)
-
-[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It's built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
-
-- Fast, durable virtual disks
-- Replication for resilience
-- Exposed over NBD or embedded in VMs
-- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
-- In-memory read/write block cache
-
-### Predastore (Object Storage – S3-Compatible)
-
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
-
-- S3-compatible API and auth
-- Multipart uploads, streaming reads/writes
-- Data redundancy with Reed-Solomon encoding
-
-### Northstar (Authoritative DNS)
-
-[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
-
-- Authoritative DNS over UDP, TCP, and DNS-over-TLS
-- Service discovery for cluster components via SRV records
-- Zone files stored locally or in S3-compatible storage
-
-## Architecture at a Glance
-
-<p align="center">
-  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
-</p>
-
-Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
-
-## Self-hosted Installation
-
-Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
-
-### Bare Metal ISO
-
-The recommended installation is a [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) for bare-metal hardware.
+**Bare metal (recommended):** download the [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) and follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb).
 
 ```bash
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
-
-### Single Node Install
-
->*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
+**Single node:**
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
-
 sudo /usr/local/share/spinifex/setup-ovn.sh --management
-
 sudo spx admin init --node node1 --nodes 1
-
 sudo systemctl start spinifex.target
-
 export AWS_PROFILE=spinifex
-
 aws ec2 describe-instance-types
 ```
 
-### Development Setup
+**From source:** see the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
-
-## Spinifex UI
-
-Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+## Web Console
 
 <p align="center">
   <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
 </p>
 
-Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
+Spinifex ships with a built-in web console for your instances, volumes, buckets, networks, and IAM. See [Launching the Web UI](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui).
 
-- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
-- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
+## Architecture
 
-For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+<p align="center">
+  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
+</p>
 
-## Development Philosophy
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless, so you scale horizontally by starting more. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-### Built by Engineers, For Engineers
+## How We Build Spinifex
 
-Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
+Spinifex is built by infrastructure engineers with deep AWS experience, including former AWS team members, drawing on decades of combined work across AWS, enterprise infrastructure, and edge computing. Engineers architect and implement it, with **Claude Code** accelerating parts of the work.
 
-### AI-Assisted Development
-
-Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
-
-**How We Use Claude Code:**
-
-- **Code Generation**: Boilerplate AWS API structures and handlers
-- **Documentation**: Comprehensive development guides and API documentation
-- **Testing**: Test case generation and validation scenarios
-- **Refactoring**: Large-scale code restructuring and optimization
-
-**What Remains Human-Driven:**
-
-- **Architecture Decisions**: Core system design and scalability choices
-- **Security Implementation**: Authentication, encryption, and threat modeling
-- **Performance Optimization**: Real-world performance tuning and benchmarking
-- **Production Operations**: Deployment strategies and operational procedures
+| Claude Code assists with | Engineers own |
+| --- | --- |
+| Boilerplate AWS API structures and handlers | Architecture and scalability decisions |
+| Development guides and API documentation | Authentication, encryption, and threat modelling |
+| Test case generation and validation scenarios | Real-world performance tuning and benchmarking |
+| Large-scale restructuring and refactoring | Deployment strategy and production operations |
 
 ## Trademarks
 
@@ -230,4 +141,4 @@ Spinifex is architected and implemented by experienced engineers, with **Claude
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available.

architecture/layering/allowlist_test.go

@@ -122,9 +122,6 @@ var allowlist = []allowance{
 
 	{From: "gateway/sts", To: "handlers/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "GetCallerIdentity takes IAMService; needs an IAM capability"},
 
-	{From: "gateway/tagging", To: "domains/ec2/tags", Debt: "close-out: gateway/tagging", Reason: "cross-service GetResources; clears when domains/tagging aggregates through per-domain capabilities"},
-	{From: "gateway/tagging", To: "handlers/elbv2", Debt: "close-out: gateway/tagging", Reason: "cross-service GetResources; clears when domains/tagging aggregates through per-domain capabilities"},
-
 	{From: "handlers/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
 	{From: "handlers/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
 	{From: "handlers/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},

architecture/layering/layers_test.go

@@ -67,7 +67,7 @@ var legacyRoots = map[string]bool{
 // Bedrock is the AWS surface of the Ochre domain.
 var gatewayDomains = map[string]string{
 	"bedrock": "ochre", "ecs": "ecs", "eks": "eks", "elbv2": "elbv2", "iam": "iam",
-	"rds": "rds", "sts": "sts", "tagging": "tagging",
+	"rds": "rds", "sts": "sts",
 }
 
 // runtimeLegacy and operatorLegacy are the legacy roots whose ADR-0001 target

architecture/layering/legacy_test.go

@@ -21,7 +21,6 @@ var frozenLegacyPackages = []string{
 	"gateway/rds",
 	"gateway/spx",
 	"gateway/sts",
-	"gateway/tagging",
 	"handlers/ecs",
 	"handlers/eks",
 	"handlers/elbv2",

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.27-bookworm AS builder
+FROM golang:1.27.2-bookworm@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS builder
 
 ARG VERSION
 

contracts/ec2/v1/README.md

@@ -36,3 +36,27 @@ contract or an approved compatibility plan; it is not a package refactor.
 `EnsureDefaultVpcSubject` (`ec2.EnsureDefaultVpc`) is the request/reply route on which the gateway's account creation asks a daemon in the `spinifex-workers` queue group to build the account's default VPC.
 Only the subject is owned here today.
 The request (`account_id`) and reply (`vpc_id`, `error`) bodies are still declared as anonymous structs on each side; moving them into this package is a contract change that needs its own characterization first.
+
+## Platform inventory
+
+`PlatformInventorySubject` (`ec2.v1.PlatformInventory`) is a request/reply route on which a daemon in the `spinifex-workers` queue group reports one account's key pairs, placement groups, launch templates and their `ec2-tags` entries from the EC2 owner stores.
+EC2 owns it: `domains/ec2/accountinventory` produces the reply and the daemon serves it; the only consumer today is `accountteardown`, which binds an adapter but does not call it.
+
+The request names the account in its body (`account_id`, a 12-digit ID) and may carry an opaque `correlation_ref` that is echoed for logs.
+The daemon ignores `X-Account-ID` for this subject.
+Neither field grants anything: the subject is read-only, writes to no store, and its reply is never authority to delete.
+
+The reply has one `kinds` entry per kind (`key-pair`, `placement-group`, `launch-template`, `tag-index`) with a status and scan start and end times.
+The times are zero when no owner scan ran: a consumer-synthesised `scan-incomplete` report for no responder, a timeout or a transport failure carries zero timestamps and must not be read as a scan.
+`observed-clean-at-scan-time` means every store for that kind was read completely and held nothing at scan time; writers are not fenced, so it is a snapshot and never evidence of absence.
+`items` carry only a resource ID, kind and disposition (`live`, `dependency-held`, or tag-index `residue`), which are the identities the per-kind Describe subjects already return.
+A `dependency-held` placement group does not name its member instances, which may be stale and are not exposed by the existing subjects.
+Records no existing subject exposes (public-key objects without metadata, launch-template version bodies and name-index entries without a header, and every unreadable record) appear only as aggregate `counts` per kind, disposition and class.
+The reply never carries an object key, raw KV key, key or template name, tag key or value, public-key bytes, launch-template data, user data, record body or store error text; `incomplete_reasons` are fixed categories.
+
+### Compatibility boundary
+
+The subject name carries the contract major; `platform_inventory_test.go` preserves the version-1 JSON shape and wire vocabulary.
+Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
+A daemon that predates the subject has no subscription, so the request gets no responder; the consumer treats no responder, a timeout, an error reply, an undecodable reply, a reply for another account, and a missing, repeated or unknown-status kind as `scan-incomplete`, never clean.
+An incompatible change to the subject, fields or vocabulary needs a new major.

contracts/ec2/v1/platform_inventory.go

@@ -0,0 +1,123 @@
+package ec2v1
+
+import "time"
+
+// PlatformInventorySubject is the request/reply subject on which a daemon in
+// the spinifex-workers queue group reports one account's key pairs, placement
+// groups, launch templates and their tag-index entries. It never writes.
+const PlatformInventorySubject = "ec2.v1.PlatformInventory"
+
+// Kinds reported by a platform inventory. The tag index is its own kind
+// because it is a separate store with its own completeness.
+const (
+	PlatformKindKeyPair        = "key-pair"
+	PlatformKindPlacementGroup = "placement-group"
+	PlatformKindLaunchTemplate = "launch-template"
+	PlatformKindTagIndex       = "tag-index"
+)
+
+// PlatformKinds lists every kind a complete reply reports, in reply order.
+func PlatformKinds() []string {
+	return []string{PlatformKindKeyPair, PlatformKindPlacementGroup, PlatformKindLaunchTemplate, PlatformKindTagIndex}
+}
+
+// Per-kind statuses. Only PlatformStatusObservedClean says a kind's stores
+// were read completely and held nothing, and only at scan time: writers are
+// not fenced, so it is a snapshot and never evidence of absence.
+const (
+	PlatformStatusObservedClean  = "observed-clean-at-scan-time"
+	PlatformStatusRecordsFound   = "records-found"
+	PlatformStatusScanIncomplete = "scan-incomplete"
+)
+
+// Item and count dispositions. A store failure is reported once per kind as
+// PlatformStatusScanIncomplete rather than per record.
+const (
+	PlatformDispositionLive           = "live"
+	PlatformDispositionResidue        = "residue"
+	PlatformDispositionUnreadable     = "unreadable"
+	PlatformDispositionDependencyHeld = "dependency-held"
+)
+
+// Record classes that are reported only as aggregate counts.
+const (
+	PlatformClassPublicKeyWithoutMetadata  = "public-key-object-without-metadata"
+	PlatformClassPublicKeyUnresolved       = "public-key-object-unresolved"
+	PlatformClassUnreadableRecord          = "unreadable-record"
+	PlatformClassVersionBodyWithoutHeader  = "version-body-without-header"
+	PlatformClassNameIndexWithoutHeader    = "name-index-without-header"
+	PlatformClassUnrecognisedTemplateEntry = "unrecognised-template-entry"
+)
+
+// Generic reasons a kind is PlatformStatusScanIncomplete. They are categories,
+// never store error text, which can quote stored content.
+const (
+	PlatformReasonStoreListFailed        = "store-list-failed"
+	PlatformReasonRecordReadFailed       = "record-read-failed"
+	PlatformReasonOwnerRecordsUnresolved = "owner-records-unresolved"
+	PlatformReasonNoResponder            = "no-responder"
+	PlatformReasonTimeout                = "timeout"
+	PlatformReasonTransportFailed        = "transport-failed"
+	PlatformReasonErrorReply             = "error-reply"
+	PlatformReasonInvalidReply           = "invalid-reply"
+	PlatformReasonKindNotReported        = "kind-not-reported"
+)
+
+// PlatformInventoryRequest asks for one account's platform inventory.
+// CorrelationRef is an opaque caller reference echoed for logs; it grants
+// nothing and is never a fence or authority to write.
+type PlatformInventoryRequest struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+}
+
+// PlatformInventoryReply carries per-kind status, the identities a direct
+// caller of the per-kind Describe subjects can already read, and counts for
+// every record class those subjects do not expose. It carries no record
+// keys, names, tags, key material, template data or record bodies.
+type PlatformInventoryReply struct {
+	AccountID      string                `json:"account_id"`
+	CorrelationRef string                `json:"correlation_ref,omitempty"`
+	Kinds          []PlatformKindReport  `json:"kinds"`
+	Items          []PlatformItem        `json:"items,omitempty"`
+	Counts         []PlatformRecordCount `json:"counts,omitempty"`
+}
+
+// PlatformKindReport is one kind's scan status and when its scan ran. The
+// timestamps are zero when no owner scan ran, as in a consumer-synthesised
+// scan-incomplete report for no responder, a timeout or a transport failure.
+type PlatformKindReport struct {
+	Kind              string    `json:"kind"`
+	Status            string    `json:"status"`
+	IncompleteReasons []string  `json:"incomplete_reasons,omitempty"`
+	ScanStartedAt     time.Time `json:"scan_started_at"`
+	ScanEndedAt       time.Time `json:"scan_ended_at"`
+}
+
+// PlatformItem is one resource by its AWS resource ID. A dependency-held
+// placement group does not name its member instances.
+type PlatformItem struct {
+	ResourceID  string `json:"resource_id"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+}
+
+// PlatformRecordCount is the number of records of one class in one kind.
+type PlatformRecordCount struct {
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Class       string `json:"class"`
+	Count       int    `json:"count"`
+}
+
+// IncompletePlatformInventory is the reply a consumer substitutes when no
+// usable answer arrived: every kind is scan-incomplete for reason.
+func IncompletePlatformInventory(accountID, correlationRef, reason string) PlatformInventoryReply {
+	reply := PlatformInventoryReply{AccountID: accountID, CorrelationRef: correlationRef}
+	for _, kind := range PlatformKinds() {
+		reply.Kinds = append(reply.Kinds, PlatformKindReport{
+			Kind: kind, Status: PlatformStatusScanIncomplete, IncompleteReasons: []string{reason},
+		})
+	}
+	return reply
+}

contracts/ec2/v1/platform_inventory_test.go

@@ -0,0 +1,102 @@
+package ec2v1
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPlatformInventorySubject(t *testing.T) {
+	require.Equal(t, "ec2.v1.PlatformInventory", PlatformInventorySubject)
+}
+
+func TestPlatformInventoryRequestJSONContract(t *testing.T) {
+	data, err := json.Marshal(PlatformInventoryRequest{AccountID: "111122223333", CorrelationRef: "req-1"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333","correlation_ref":"req-1"}`, string(data))
+
+	data, err = json.Marshal(PlatformInventoryRequest{AccountID: "111122223333"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333"}`, string(data))
+}
+
+func TestPlatformInventoryReplyJSONContract(t *testing.T) {
+	start := time.Date(2026, 10, 16, 1, 2, 3, 0, time.UTC)
+	reply := PlatformInventoryReply{
+		AccountID:      "111122223333",
+		CorrelationRef: "req-1",
+		Kinds: []PlatformKindReport{
+			{Kind: PlatformKindKeyPair, Status: PlatformStatusObservedClean, ScanStartedAt: start, ScanEndedAt: start.Add(time.Second)},
+			{Kind: PlatformKindPlacementGroup, Status: PlatformStatusRecordsFound, ScanStartedAt: start, ScanEndedAt: start},
+			{Kind: PlatformKindLaunchTemplate, Status: PlatformStatusScanIncomplete, IncompleteReasons: []string{PlatformReasonStoreListFailed}, ScanStartedAt: start, ScanEndedAt: start},
+			{Kind: PlatformKindTagIndex, Status: PlatformStatusRecordsFound, ScanStartedAt: start, ScanEndedAt: start},
+		},
+		Items: []PlatformItem{
+			{ResourceID: "pg-0held", Kind: PlatformKindPlacementGroup, Disposition: PlatformDispositionDependencyHeld},
+			{ResourceID: "lt-0gone", Kind: PlatformKindTagIndex, Disposition: PlatformDispositionResidue},
+		},
+		Counts: []PlatformRecordCount{
+			{Kind: PlatformKindKeyPair, Disposition: PlatformDispositionResidue, Class: PlatformClassPublicKeyWithoutMetadata, Count: 2},
+		},
+	}
+
+	data, err := json.Marshal(reply)
+	require.NoError(t, err)
+	require.JSONEq(t, `{
+		"account_id": "111122223333",
+		"correlation_ref": "req-1",
+		"kinds": [
+			{"kind": "key-pair", "status": "observed-clean-at-scan-time", "scan_started_at": "2026-10-16T01:02:03Z", "scan_ended_at": "2026-10-16T01:02:04Z"},
+			{"kind": "placement-group", "status": "records-found", "scan_started_at": "2026-10-16T01:02:03Z", "scan_ended_at": "2026-10-16T01:02:03Z"},
+			{"kind": "launch-template", "status": "scan-incomplete", "incomplete_reasons": ["store-list-failed"], "scan_started_at": "2026-10-16T01:02:03Z", "scan_ended_at": "2026-10-16T01:02:03Z"},
+			{"kind": "tag-index", "status": "records-found", "scan_started_at": "2026-10-16T01:02:03Z", "scan_ended_at": "2026-10-16T01:02:03Z"}
+		],
+		"items": [
+			{"resource_id": "pg-0held", "kind": "placement-group", "disposition": "dependency-held"},
+			{"resource_id": "lt-0gone", "kind": "tag-index", "disposition": "residue"}
+		],
+		"counts": [
+			{"kind": "key-pair", "disposition": "residue", "class": "public-key-object-without-metadata", "count": 2}
+		]
+	}`, string(data))
+}
+
+// The wire values of every status, disposition, class and reason.
+func TestPlatformInventoryVocabulary(t *testing.T) {
+	require.Equal(t, []string{"key-pair", "placement-group", "launch-template", "tag-index"}, PlatformKinds())
+	require.Equal(t, []string{"live", "residue", "unreadable", "dependency-held"},
+		[]string{PlatformDispositionLive, PlatformDispositionResidue, PlatformDispositionUnreadable, PlatformDispositionDependencyHeld})
+	require.Equal(t, []string{
+		"public-key-object-without-metadata", "public-key-object-unresolved", "unreadable-record",
+		"version-body-without-header", "name-index-without-header", "unrecognised-template-entry",
+	}, []string{
+		PlatformClassPublicKeyWithoutMetadata, PlatformClassPublicKeyUnresolved, PlatformClassUnreadableRecord,
+		PlatformClassVersionBodyWithoutHeader, PlatformClassNameIndexWithoutHeader, PlatformClassUnrecognisedTemplateEntry,
+	})
+	require.Equal(t, []string{
+		"store-list-failed", "record-read-failed", "owner-records-unresolved", "no-responder",
+		"timeout", "transport-failed", "error-reply", "invalid-reply", "kind-not-reported",
+	}, []string{
+		PlatformReasonStoreListFailed, PlatformReasonRecordReadFailed, PlatformReasonOwnerRecordsUnresolved, PlatformReasonNoResponder,
+		PlatformReasonTimeout, PlatformReasonTransportFailed, PlatformReasonErrorReply, PlatformReasonInvalidReply, PlatformReasonKindNotReported,
+	})
+}
+
+// A substituted reply never reports a kind clean.
+func TestIncompletePlatformInventoryMarksEveryKindIncomplete(t *testing.T) {
+	reply := IncompletePlatformInventory("111122223333", "req-1", PlatformReasonNoResponder)
+	require.Equal(t, "111122223333", reply.AccountID)
+	require.Equal(t, "req-1", reply.CorrelationRef)
+	require.Len(t, reply.Kinds, len(PlatformKinds()))
+	for i, kind := range PlatformKinds() {
+		require.Equal(t, kind, reply.Kinds[i].Kind)
+		require.Equal(t, PlatformStatusScanIncomplete, reply.Kinds[i].Status)
+		require.Equal(t, []string{PlatformReasonNoResponder}, reply.Kinds[i].IncompleteReasons)
+		require.True(t, reply.Kinds[i].ScanStartedAt.IsZero(), "no owner scan ran")
+		require.True(t, reply.Kinds[i].ScanEndedAt.IsZero(), "no owner scan ran")
+	}
+	require.Empty(t, reply.Items)
+	require.Empty(t, reply.Counts)
+}

contracts/operator/v1/storage.go

@@ -33,6 +33,9 @@ type StorageMetaNode struct {
 	ID   int    `json:"id"`
 	Host string `json:"host"`
 	Port int    `json:"port"`
+	// AdminAddr is set only on a single-host cluster, which opens no QUIC socket:
+	// its host's admin listener, whose /readyz answers for the node's health.
+	AdminAddr string `json:"admin_addr,omitempty"`
 }
 
 // StorageBlobNode describes a Predastore blob node that holds erasure-coded

docs/DESIGN.md

@@ -305,22 +305,32 @@ Configs are generated by `spx admin init` (leader) or `spx admin join` (follower
 
 ## File Structure
 
+The layout follows Mulga ADR-0001 (Spinifex package boundary and layout).
+`architecture/layering` enforces its dependency rules in `make preflight`.
+
 ```
 spinifex/
-├── cmd/
-│   ├── spinifex/            # spx CLI: service entry points and `spx admin`
-│   ├── installer/           # Platform installer
-│   └── ...                  # In-guest agents (lb, ecs, rds, eks helpers) and dev tools
+├── cmd/                     # Thin executable mains: spx, installer, in-guest agents, dev tools
+├── contracts/               # Versioned cross-process wire contracts (contracts/<domain>/vN)
 ├── spinifex/
-│   ├── services/            # Long-running services (awsgw, spinifex, nats, predastore, ...)
-│   ├── gateway/             # AWS API surface: routing, SigV4 auth, per-service dispatch
-│   ├── handlers/            # NATS-side service implementations (ec2, iam, eks, rds, ...)
-│   ├── daemon/              # Per-node daemon and its NATS subscription table
-│   ├── vm/, qmp/            # VM lifecycle and the QEMU Machine Protocol client
-│   ├── vpcd/, network/      # VPC daemon and the layered OVN/OVS network stack
-│   ├── admin/               # Cluster bootstrap (init/join, master.key, CA, tokens)
-│   └── config/              # Configuration structures
-├── internal/                # In-guest SigV4 clients and the AWS Smithy models
+│   ├── foundation/          # Shared primitives: AWS errors/ARNs/filters/tags, KV state, messaging, telemetry, PKI
+│   ├── providers/           # Backend adapters: EBS (Viperblock), object store, cloud (OCI, Exoscale)
+│   ├── domains/             # Service domains (ec2, acm, ecr, dns, network, ochre, rds, tagging, ...)
+│   │   └── <domain>/awsapi/ # A domain's AWS action adapters
+│   ├── ingress/             # AWS protocol ingress: query/REST parsing, dispatch registry, rate limiting
+│   ├── runtime/
+│   │   ├── roles/           # Long-running service roles (awsgw, spinifex, vpcd, nats, predastore, ...)
+│   │   └── compute/         # VM lifecycle, QMP, NBD, GPU
+│   ├── bootstrap/           # Cluster and node configuration structures, preflight checks
+│   ├── operator/            # spx CLI commands, admin use cases, installer, upgrade
+│   ├── agents/              # In-guest agent libraries (ecs, eks, rds)
+│   ├── gateway/             # Legacy root: HTTP route assembly, SigV4 auth, legacy per-service dispatch
+│   ├── handlers/            # Legacy root: ecs, eks, elbv2, iam, rds, sts service implementations
+│   ├── daemon/              # Legacy root: per-node daemon and its NATS subscription table
+│   ├── accountteardown/     # Legacy root: account deletion cleanup
+│   └── lbagent/             # Legacy root: load-balancer agent
+├── internal/                # In-guest SigV4 clients, AWS Smithy models and test kits
+├── architecture/            # Executable architecture rules (package layering)
 ├── build/                   # systemd units, node helper scripts, microVM build
 ├── scripts/                 # Install, dev-environment, image-build scripts
 └── tests/
@@ -328,6 +338,9 @@ spinifex/
     └── e2e/                 # Live-cluster suites (see docs/testing/e2e-coverage.md)
 ```
 
+The five legacy roots predate ADR-0001 and are frozen: they may shrink but not gain packages.
+[`PACKAGE_BOUNDARY_MIGRATION.md`](PACKAGE_BOUNDARY_MIGRATION.md) records what remains in each and what blocks its move.
+
 ## Development
 
 ### Running the Stack

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -337,6 +337,9 @@ while the branch is in flight; they do not describe the umbrella repository's
 | `3fdaa54d9` | `spinifex/admin` (`DefaultAccountID`, `DefaultAccountName`) | `spinifex/foundation/aws/identifiers` | The fixed, platform-reserved bootstrap account identifiers join `GlobalAccountID`: the ID and name of the first human-facing account created during bootstrap, used by `daemon`, `gateway`, `runtime/roles/awsgw`, `accountteardown` and `operator/cli`. Function names and values are unchanged, and they stay fixed rather than configurable. Callers use the existing `awsidentifiers` alias; `daemon`, `gateway`, `runtime/roles/awsgw`, `accountteardown` and the other `operator/cli` files drop their `admin` import, and `operator/cli/admin.go` keeps `legacyadmin` only for the credential generators. The `TestDefaultAccountID` assertion moves with the function, plus a name pin and a check that the ID is valid and distinct from `GlobalAccountID`. `admin` now holds only the IAM credential generators. |
 | `254d898a3` | `spinifex/handlers/sts` (`oidc_jwks.go`: the `handlers_eks.AccountBucketName`/`OIDCJWKSKey` import) | `spinifex/handlers/eks` (`OIDCJWKSReader`, a `ClusterJWKSResolver` implementation), wired by `spinifex/runtime/roles/awsgw` | The Q-22 STS-to-EKS dependency is cut by a consumer-owned resolver: `handlers/sts` declares `ClusterJWKSResolver` (raw-bytes lookup, the same found/error split `FetchClusterJWKS` already had), and `handlers/eks` implements it beside `AccountBucketName`/`OIDCJWKSKey` with the same bucket-open-then-get, timeouts and not-found classification `oidc_jwks.go` used. STS still decodes the JWKS, applies the empty-keys rule and maps every outcome to `InvalidIdentityToken`, unchanged. `runtime/roles/awsgw` and `tests/integration/harness.go` construct the resolver and pass it to `NewSTSServiceImpl`, now a required, nil-checked argument; the compile-time interface check lives in `awsgw` because `handlers/eks` must not import `handlers/sts`. The literal bucket/key, KV-dependency-failure and cluster-status-not-consulted characterisation tests move to `handlers/eks`, next to the layout they pin; every remaining `handlers/sts` test, including the web-identity fixtures and `newTestSetup`, is rewritten against a test-owned `fakeJWKSResolver` and imports no EKS package. The one test proving an EKS-minted issuer URL parses through STS's `ParseEKSIssuerURL` moves to `tests/integration` as a composition test. The stale `handlers/sts` → `handlers/eks` allowlist entry is deleted. |
 | `0368e861a` | `spinifex/admin` (`accounts.go`: `GenerateAWSAccessKey`, `GenerateAWSSecretKey`, and their format and uniqueness tests) | `spinifex/foundation/aws/credentials` (`awscredentials`) | A stateless AWS credential-format primitive: it generates an `AKIA` + 16-alphanumeric access key ID and a 40-character base64 secret key, and issues, stores and authorises nothing. It is not IAM issuance and does not implement ADR-0005 S5. Output, randomness source, errors and names are unchanged; only the package line and doc comment changed. It is deliberately not unified with `handlers/iam`'s internal `generateAccessKeyID`, whose IDs are `AKIA` + 20 uppercase hex characters. `handlers/iam` and `operator/cli/admin.go` import `awscredentials`; `operator/cli` no longer imports any `legacyadmin`. The `spinifex/admin` directory is deleted: `admin` leaves the legacy-root, frozen-package and operator-legacy lists, joins the retired roots so it cannot return, and its two allowlist entries (`handlers/iam` → `admin`, `operator/cli` → `admin`) are removed. Operator bootstrap still generates the admin and Northstar credentials directly, recorded as debt below. |
+| `0d830e901` | — | `spinifex/domains/tagging` (`TaggedLister`, `TaggedResource`) | The read-only list capability for the Resource Groups Tagging aggregation, declared by its consumer per ADR-0001. `ListTagged` takes the caller's account and the lower-cased `ResourceTypeFilters` set, where an entry is a full `service:type` or a bare service, and returns ARNs with read-only tag maps. The package imports no domain, handler or contract package and the layering ratchet classifies it with no allowlist row. It carries no mutation, limit or ever-tagged surface. |
+| `f3545e3ea` | `spinifex/gateway/tagging` (`listEC2`, `buildEC2Mappings`, `ec2ARN`, `elbv2Resources`, `matchesType`), copied | `spinifex/runtime/roles/awsgw` (`ec2IndexLister`, `elbv2Lister`), `spinifex/domains/tagging` (`MatchesType`) | AWSGW-side NATS client adapters implement `tagging.TaggedLister`, each with its compile-time check beside it in the composition root; no owner package carries one. The EC2 index adapter wraps `ec2.DescribeTags` and builds each ARN in the gateway's region from the stored resource type, including the unresolved-ID shapes today's code produces. The ELBv2 adapter lists load balancers and target groups admitted by the type filter, tagged or not, and reads tags in `DescribeTags` batches of 20. Both reuse existing subjects and clients, so no wire contract is added, and each call builds fresh tag maps. Type matching is exported once from `domains/tagging` so no copy survives the move. Embedded-JetStream tests cover type filters including a bare service, untagged ELBv2 resources, EC2 ARN mapping, account scoping, batch sizes and owner error propagation. |
+| `aedd0b2c4` | `spinifex/gateway/tagging` (`GetResources` parsing, tag filtering, sorting, offset token, authorization, response writing) | `spinifex/domains/tagging` (`Aggregator`, `NewAggregator`, `HasScope`, `ScopedActions`), wired by `spinifex/runtime/roles/awsgw` | The aggregation moves unchanged and consumes `[]TaggedLister`: ELBv2 is called before EC2, the non-stable ARN sort is kept, lister errors propagate with the same codes, and tag maps return as key-sorted tag lists with `[]` for untagged resources. AWSGW builds the aggregator with both adapters and sets the new `GatewayConfig.TaggingResources` field; the dispatch in `gateway/tagging.go` stays and still answers `ServerInternal` when NATS or the aggregator is not wired. The `TestCurrentBehaviour_Tagging_GetResources*` tests move to `runtime/roles/awsgw` with unchanged assertions, seeding EC2 tags through the existing EC2 tags client over NATS; pure aggregator tests move to `domains/tagging` against fake listers; the gateway keeps dispatch and error-path tests. The gateway's private copies of the listing helpers and the legacy NATS lister are deleted with their tests, `gateway/tagging` is removed, and its two allowlist rows (`gateway/tagging` → `domains/ec2/tags`, `gateway/tagging` → `handlers/elbv2`) and its frozen-legacy entry go with it. |
 The EC2 contract row is intentionally different from the directory moves: it creates
 a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
 subscription and permission shape, while callers use
@@ -417,6 +420,7 @@ These residuals were recorded by individual slices and must be resolved once the
 - `domains/dns`: imports `runtime/compute/vm` (domain-to-runtime), `domains/network/reconcile` for the reconcile lock and `bootstrap/config` for node configuration.
 - ACM: the IAM AES-GCM helper import (ACM relocation residual).
 - `runtime/roles/awsgw`: the quota reconcile loop belongs to `domains/admission/quota`, and the EKS token-verify responder belongs with STS or EKS identity.
+- `runtime/roles/awsgw` → `handlers/elbv2` (inherited from `gateway/tagging`): this is a new physical import in AWSGW that inherits the former `gateway/tagging` dependency, because the `elbv2Lister` adapter builds its client with `handlers_elbv2.NewNATSELBv2Service` and calls the `ELBv2Service` interface. It is permitted temporarily because AWSGW is the composition root, which may wire legacy packages, so the layering test needs no allowlist row. Removal condition: it must clear when ELBv2 exposes a domain-owned boundary (its NATS client or contract under its domain owner) and the adapter consumes that instead.
 - `runtime/roles/vpcd` (placement): the role still mixes process/role supervision with network-controller responsibilities. `lease_reaper.go` (orphaned external-pool DHCP lease sweep), `ovn_controller_watchdog.go` (OVN controller liveness and restart), `host_eip_loop.go` (host-EIP reconcile loop) and `lease_datapath_reconcile.go` (lease datapath reconcile) are periodic network-control loops, not role wiring; `vpcd.go` itself also carries bridge-mode detection and verification, chassis-list parsing, external-CIDR resolution, DHCP manager startup and lease/IP-change handling, and gateway/host-EIP allocator and neighbour flush/prime construction alongside `Service.Start`. The future placement question is whether these belong in `domains/network`. Removal condition: an accepted decision on network-controller placement, and extraction of those loops and helpers behind the role.
 - ECR control-plane registration: `awsapi.Deps.NATS` is a raw `*nats.Conn`, not a narrow capability. Every NATS-backed `Handler` passes it to `domains/ecr.NewNATSMetaStore`, which calls `natsmsg.NATSRequest`, so narrowing it means changing the exported `Handler` type, every handler and the meta store; it must not be broadened further meanwhile.
 - ACM control-plane registration: `domains/acm/awsapi.Deps.NATS` (`registration.go`) is likewise a raw `*nats.Conn` carried by every `Handler`, for the same reason; it must not be broadened further meanwhile.
@@ -442,7 +446,7 @@ These residuals were recorded by individual slices and must be resolved once the
 - `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
 - `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
 - `gateway/requestaudit.go`: the outermost request-audit middleware records AWS fields and is updated from four places in the gateway's authentication flow through the gateway's own context key. Moving it would export about seven names used only by that flow, so it stays in `gateway` until the generic ingress registration seam defines how ingress middleware receives authentication results.
-- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves.
+- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; no engine path calls it yet, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters.
 - `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `gateway/elbv2` and `handlers/rds` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
 - `domains/ec2/awsapi/instance` (dependency): imports `handlers/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
 - `domains/ec2/awsapi/spotinstance` (dependency): imports `handlers/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.
@@ -481,6 +485,13 @@ These are security corrections found during discovery, not lifecycle work.
 - Static-credential control-plane VMs (IAM wiring failed) are now denied every internal route and cannot bootstrap; that fallback is unsupported.
 
 Preflight exception (branch only): from `ee13be950`, full `make preflight` on this branch is red solely because the vulnerability gate reports GO-2026-6599 to GO-2026-6617 against the pinned Go 1.27.0 toolchain. Every other gate passes. This branch is not production. No advisory is accepted or suppressed, and the accepted-advisory list, `go.mod` and the toolchain pins are unchanged; the toolchain is updated through Dependabot. Normal full-preflight enforcement resumes when the toolchain baseline is updated.
+Resolved by `b42373efb`: merging `dev` brought the Go 1.27.2 toolchain, the vulnerability gate passes with no advisory accepted or suppressed, full `make preflight` is green, and normal enforcement has resumed.
+
+Merge from `dev` (`b42373efb`): `dev`'s lint fix replacing `strings.LastIndex` with `strings.CutLast` in `actionFromTarget` touched `spinifex/gateway/acm.go` and `spinifex/gateway/ecrapi.go`, which this branch had deleted when it moved those registrations. The change was ported to the moved code in `spinifex/domains/acm/awsapi/registration.go` and `spinifex/domains/ecr/awsapi/registration.go`, and the deletions were kept. The same merge carried `dev`'s ACM tag private-key fix into `spinifex/domains/acm` with import paths adjusted to this branch's layout.
+
+Merge from `dev` (`316a1b7c0`): `dev`'s IMDS PUT hop-limit enforcement was ported into `spinifex/domains/ec2/guestmetadata` with the package renamed. `dev`'s fail-closed instance inventory for account teardown (`DescribeAccountInstancesComplete`) landed in `spinifex/domains/ec2/awsapi/instance`, and its callers and tests in `spinifex/accountteardown`, `spinifex/gateway` and `spinifex/operator/cli` were adjusted to this branch's import paths (`natsmsg.NodeIDHeader`, `awserrors.GenerateErrorPayload`, `ec2instanceapi`). The spinifexui frontend source and `dist` were taken from `dev` at `spinifex/runtime/roles/spinifexui/frontend`, keeping only this branch's path-comment update in `src/types/rds.ts`.
+
+Merge from `dev` (`62870d350`): `dev`'s identity-checked volume holder proof for account teardown landed in `spinifex/accountteardown` and its `DeleteVolume` guard in `spinifex/domains/ec2/awsapi/volume`, with call sites renamed to this branch's packages (`natsmsg`, `ec2volumeapi`, `ec2instanceapi`). The new gateway guard test and the teardown test fixtures were adjusted to this branch's import paths, and public `DeleteVolume` passes `gw.NodeIDs` as on `dev`.
 
 Gates outside preflight and CI (follow-up, not merge-blocking): `make test-package-check`, which the `Makefile` keeps out of preflight, reports 87 test files added on this branch in their own package rather than `<pkg>_test`; they include the contract, agent, new-domain, ingress, layering and split-out ECR tests. Converting each to an external test package, or marking it `//test:in-package` with a reason, is follow-up work before that gate returns to preflight. `make nilaway` was killed by the host on both local attempts (15 GB, including `GOGC=25 GOMAXPROCS=2`) before it reported anything, so this branch has no nilaway evidence.
 
@@ -650,13 +661,13 @@ series above; "needs decision" is open.
 | `gateway/bedrock` | The Ochre domain core, its stores and the inference engine, together with the Bedrock action layer, in one package with internal cycles. | Deferred: needs the split plan recorded above — domain core and stores to `domains/ochre/<resource>`, inference to an Ochre-owned package, and only the action layer to `domains/ochre/awsapi`. |
 | `gateway/{ecs,eks,rds,iam,sts,elbv2}` | Each service's AWS action adapters and dispatch, not yet registered through ADR-0002. | Deferred: the ADR-0002 registration contract has only reached ECR and ACM so far; each service needs its own registration slice. |
 | `gateway/spx` | `promote_image.go` and `spx.go` dispatch (`storage.go` already consumes `contracts/operator/v1`). | Needs decision: which package owns operator-plane action dispatch registration — an `ingress/aws`-style seam for the operator plane, or `gateway` stays its home. |
-| `gateway/tagging` | The cross-service tagging API (`GetResources`) and its authorization. | Decided: a new `domains/tagging` boundary owns the cross-resource tagging API and aggregation; individual resource domains keep ownership of their own tag storage and expose narrow tagging capabilities. Stays debt until those per-domain capabilities exist; clears when `domains/tagging` aggregates only through them. |
+| `gateway/tagging` | — | Done. `domains/tagging` owns the read-only cross-resource `GetResources` aggregation and its API semantics (`0d830e901`, `aedd0b2c4`); AWSGW composes the owner listers, the EC2 tag index and ELBv2 (`f3545e3ea`); the root `gateway` keeps only the legacy dispatch in `gateway/tagging.go`. The package is removed, and its two allowlist rows and frozen-legacy entry are gone. Owner tag mutation capabilities remain future direction, so the RDS and ECS tag-writing edges above stay open. |
 | `services/viperblockd` | — | Done. The adapter is `providers/ebs/viperblock` (with the read-only `dirty` reader), the shared vocabulary is `contracts/viperblockd/legacy/v1`, and the role is `runtime/roles/viperblockd`. `services/` is empty. |
 | `utils` | — | Done and deleted. `ec2_subject.go` went to `contracts/ec2/v1`, the vpcd protocol to `contracts/network/v1` with the typed client in `domains/network/projection`, and the master-key loader to `domains/ec2/ebs/encryption`. |
 | `vpcd` | — | Done. The IMDS local-file reader and record-space opener moved to `domains/ec2/instance/state` and are injected through `guestmetadata.LocalStateReader`/`RecordLoader` (recorded above), so `vpcd` no longer imports `daemon` directly; its inter-process boundary now exists (`contracts/network/v1`, with `domains/network/subscribers` as the consumer and `domains/network/projection` as the producer client). The package moved unchanged to `runtime/roles/vpcd` (recorded below), and the legacy directory is removed. |
 | `daemon` | Reserved as `runtime/daemon`'s home only; not split or moved. | Deferred: Q-28's write-side consolidation is a prerequisite ADR-0001 names explicitly. |
 | `admin` | — | Done and deleted. The operator use cases moved to `operator/admin`, image promotion to `domains/ec2/image`, the bootstrap account identifiers to `foundation/aws/identifiers`, and the credential-format generators to `foundation/aws/credentials` (recorded above). Operator bootstrap's direct credential generation stays recorded as debt. |
-| `accountteardown` | Direct calls into `domains/ec2/awsapi/*` action adapters (recorded above). | Deferred: the ADR's `accountteardown` row replaces this with domain-owned cleanup capabilities. |
+| `accountteardown` | The whole package, a frozen legacy root. Its direct cleanup calls into `domains/ec2/awsapi/*` action adapters, other domain and handler services, the object-store provider, raw NATS subjects and a raw KV bucket stay as recorded debt. | Assessed for relocation and not moved. ADR-0001's `accountteardown` row requires conversion to domain-owned cleanup capabilities before moving, and ADR-0008 does not authorise a mechanical rename. It remains a legacy root until its direct cleanup calls are converted to domain-owned capabilities; its destination after that conversion is `operator/accountlifecycle`. |
 | `lbagent` | The whole package; `handlers/elbv2` imports it as a domain-to-agent implementation edge (recorded above). | Deferred: Q-30 reserves `agents/elbv2` as its destination but forbids moving it as an isolated rename ahead of the common-agent design. |
 | `migrate` | Nothing at the old path. | Split, not an exact ADR row: ADR-0001 maps `migrate` to `foundation/state/migrate`, but the package held two responsibilities. The KV migration engine and its tests are in `foundation/state/migrate` (`8cd9cff75`); the config-file and object-store upgrade workflow moved to `operator/upgrade` (`919d5bfa6`) by owner decision. Needs an ADR-0001 map clarification before this row can be claimed as target conformance. |
 
@@ -670,7 +681,7 @@ ADR-0001 completion is structural, so none of these is cleared by remediating se
 |---|---|---|
 | Explicitly deferred by ADR-0001 | `daemon` (Q-28), `lbagent` (Q-30) | Closed for this refactor; the ADR names the prerequisite. |
 | No missing boundary (candidate moves) | `handlers/ecs/bus` → `contracts/ecs/v1` (`a130f6b8e`); `admin/images_promote.go` → `domains/ec2/image` (`263c58167`); `admin` bootstrap account identifiers → `foundation/aws/identifiers` (`3fdaa54d9`); `admin` credential-format generators → `foundation/aws/credentials` `0368e861a`, retiring `admin` | Done. |
-| One narrow missing boundary | Done: `handlers/sts` EKS JWKS lookup → `handlers/eks` `ClusterJWKSResolver` (`254d898a3`). Open: `gateway/tagging` (per-domain tagging capabilities, owner decided); `gateway/spx` (owner of operator-plane dispatch registration) | Each needs one decision or one capability, then a mechanical move. |
+| One narrow missing boundary | Done: `handlers/sts` EKS JWKS lookup → `handlers/eks` `ClusterJWKSResolver` (`254d898a3`); `gateway/tagging` → `domains/tagging` with AWSGW-composed listers (`0d830e901`, `f3545e3ea`, `aedd0b2c4`). Open: `gateway/spx` (owner of operator-plane dispatch registration) | Each needs one decision or one capability, then a mechanical move. |
 | Broader boundary or follows another package | `handlers/iam` and `handlers/iam/mock` (per-consumer IAM capabilities); `handlers/{ecs,eks,rds}` (ADR-0004 blocked-dependency lists); `handlers/elbv2` (data-plane boundary evidence); `accountteardown` (domain cleanup capabilities); `gateway` root (ADR-0002 auth and request-context phase); `gateway/{iam,sts,ecs,eks,rds,elbv2}` (each follows its `handlers/<svc>`, because a new-path `awsapi` cannot import a legacy handler); `gateway/bedrock` (the recorded Ochre split plan) | Recorded debt with removal conditions above. |
 
 Per-consumer IAM capabilities are the widest unblocker: they gate `handlers/iam` itself, STS's IAM reads, Ochre's IAM reads, the ELBv2 role path and the ECS, EKS and RDS ensurers.

docs/compute/vpc-networking/README.md

@@ -9,6 +9,7 @@ tags:
   - ovn
   - public-subnet
   - security-groups
+  - ipsec
 ---
 
 # VPC Networking
@@ -42,6 +43,107 @@ Spinifex maps AWS VPC concepts directly to OVN constructs:
 
 Cross-host traffic uses **Geneve tunnels** (UDP 6081) over the management/overlay NIC. Each host runs `ovn-controller` which programs OpenFlow rules on `br-int` (the integration bridge where all VM TAP devices connect).
 
+## Overlay Encryption (IPsec)
+
+**IPsec is on by default.** Spinifex encrypts the Geneve overlay with OVN native IPsec — ESP transport mode, `rfc4106(gcm(aes))` 128, authenticated with each node's certificate under the cluster CA. Every packet between two instances on different hosts is encrypted on the wire.
+
+This is the right default and matches the AWS trust model: east-west guest traffic crosses a physical fabric that the VPC abstraction otherwise asks you to trust implicitly. It is also, on a fabric you already control, the single largest performance cost in the stack.
+
+### When disabling is appropriate
+
+Only where the underlay between nodes is **physically trusted end to end**:
+
+| Topology | Reasonable to disable? |
+| --- | --- |
+| Single rack, dedicated top-of-rack switch, no other tenants | Yes |
+| One datacenter, private VLAN, switch ports under your control | Yes |
+| Nodes sharing a switch with equipment you do not administer | No |
+| Any leg of the overlay crossing a shared or provider network | No |
+| Nodes in different datacenters, or over a WAN or VPN of any kind | No |
+
+The trade is explicit and it is not partial: with IPsec off, **all east-west guest traffic is in plaintext on the wire**, including traffic between instances belonging to different tenants. Anything with access to a span port, a mirrored uplink or the switch itself reads it. Tenant isolation still holds in the datapath — security groups and per-VPC routing are unchanged — but confidentiality against a fabric-level observer is gone.
+
+Note that this setting governs the overlay only. It has no effect on TLS terminated inside a guest, on the API endpoint, or on north-south traffic through the Internet Gateway.
+
+### What it costs
+
+Measured on the reference cluster: 4 nodes, AMD EPYC 7513 (128 threads), Mellanox **ConnectX-5** (MT27800, firmware 16.35.8002) on a 100 GbE fabric, `t3.xlarge` guests (4 vCPU), Ubuntu 26.04, kernel 6.12. Three client guests running `iperf` at 4 streams for 60s into one server guest, over their private addresses.
+
+| | IPsec on (default) | IPsec off | |
+| --- | --- | --- | --- |
+| aggregate into one guest | 3.17 Gbit/s | **21.82 Gbit/s** | **6.9x** |
+| per client, mean | 1.06 Gbit/s | **7.27 Gbit/s** | |
+| guest MTU | 1408 | 1442 | applied automatically |
+| NIC queue pairs | 1 | 4 | applied automatically |
+| busiest receive core | 97.4% busy, 71.8% softirq | under 15%, spread over 16 cores | |
+
+Disk throughput is unaffected in either direction — Predastore and Viperblock traffic does not ride the VPC overlay.
+
+The cost is not raw crypto throughput, it is **CPU scaling**. ESP wraps each Geneve packet in an outer IP header with protocol 50, which has no L4 ports, so a NIC's receive-side scaling can only hash on source and destination IP. All traffic between a given pair of nodes therefore lands in one RSS bucket and is processed by one core, regardless of how many flows or streams it carries. Without ESP the outer header is UDP 6081 and OVN varies the source port per inner flow, which spreads the work across every core.
+
+**ConnectX-5 cannot offload this.** ASAP² handles OVS flow offload, but IPsec inline crypto requires ConnectX-6 Dx or BlueField. Adding cores or streams does not help either; throughput with IPsec on is flat in stream count because everything funnels through the same core.
+
+The MTU and multiqueue rows above are not separate knobs. Both follow `ipsec_enabled` on their own, because the ESP header consumes 34 bytes of the path budget and because the sign of the multiqueue result flips with encryption. Turning IPsec off is the whole change.
+
+Full analysis, including what does *not* fix it and the measurements behind each row: [Performance Tuning → IPsec on the overlay](/docs/performance-tuning#2-ipsec-on-the-overlay).
+
+### Disabling it
+
+Four steps, and all four are required. The daemon owns step 2 but not step 3.
+
+**1. Set the config on every node.** In `/etc/spinifex/spinifex.toml`:
+
+```toml
+[network]
+ipsec_enabled = false
+```
+
+A cluster where the nodes disagree is a cluster where some tunnels are encrypted and some are not, so change it everywhere before restarting anything.
+
+**2. Restart the daemon on every node.**
+
+```bash
+systemctl restart spinifex-daemon
+```
+
+This stops `openvswitch-ipsec` and tears down the SAs. Confirm on every node — both must be `0`:
+
+```bash
+ip xfrm state  | grep -c '^src'
+ip xfrm policy | grep -c '^src'
+```
+
+**3. Clear the OVN and OVS flags by hand.** The reconciler returns early when IPsec is disabled, so it stops requesting encryption but never *clears* what it previously set. `NB_Global.ipsec` and `other_config:ipsec_encapsulation` are left reading `true` — inert, because nothing is left to act on them, but they make the control plane disagree with the config and mislead the next person to inspect it.
+
+```bash
+# Use the NB raft endpoints. The local unix socket fails with
+# "database connection failed" on a clustered deployment.
+NBDB='tcp:10.2.0.2:6641,tcp:10.2.0.3:6641,tcp:10.2.0.4:6641'
+ovn-nbctl --db="$NBDB" set NB_Global . ipsec=false
+
+# On every node:
+ovs-vsctl remove Open_vSwitch . other_config ipsec_encapsulation
+```
+
+Leave `ca_cert`, `certificate` and `private_key` in `other_config` alone. They are inert without an IPsec daemon, and keeping them means re-enabling is a config change rather than a re-enrolment.
+
+This step is manual today. It should move into the reconciler so that disabling converges the same way enabling does.
+
+**4. Verify.**
+
+```bash
+ovn-nbctl --db="$NBDB" get NB_Global . ipsec     # false
+ovs-vsctl get Open_vSwitch . other_config        # no ipsec_encapsulation key
+systemctl is-active openvswitch-ipsec            # inactive
+ip xfrm policy | grep -c '^src'                  # 0
+```
+
+### Restart guests after changing this in either direction
+
+The guest MTU moves with the setting, but a running guest holds its DHCP lease for up to an hour and its device-level `host_mtu` until it is stopped and started. Going from off to on narrows the path from 1442 to 1408 while the guest still believes 1442, which blackholes large segments until it restarts.
+
+**Expect launches to fail for a minute or two after re-enabling.** `RunInstances` returns `ServerInternal` with `vpc.add-nat` timing out, because the OVN flows-ready barrier cannot confirm while `ovn-controller` reprograms every tunnel for the new SAs. It clears once `ip xfrm state` settles. The error is reported as a capacity problem, which it is not.
+
 ## Private vs Public Subnets
 
 A subnet's behavior depends on three things: whether the VPC has an Internet Gateway, whether the subnet's route table has a default route to that IGW, and whether the subnet has `MapPublicIpOnLaunch` enabled.

docs/containers/ecr/README.md

@@ -66,6 +66,9 @@ sudo update-ca-certificates
 sudo systemctl restart docker
 ```
 
+  This reaches dockerd and anything running directly on the host.
+  A tool that runs in its own container, or a buildx builder using the `docker-container` driver, has a separate trust store; see [Troubleshooting](#troubleshooting).
+
 - **The registry endpoint** — the Spinifex gateway `host:9999`. `aws ecr get-login-password` and `aws ecr describe-repositories` return the exact `host:port`; copy it from there rather than constructing it by hand.
 - **For EKS pulls:** worker nodes need the `AmazonEC2ContainerRegistryReadOnly` policy on their node IAM role (the EKS prerequisites already include this) and network egress to the registry endpoint.
 
@@ -179,6 +182,44 @@ Then authenticate Docker and push using the `repository_url` output, following s
 
 **`docker push` cannot reach the registry.** Use the exact host from `aws ecr describe-repositories` — the Spinifex gateway `host:9999`. Docker dials it directly, so the `:9999` port must be present (without it docker tries `:443` and fails). If you are using the AWS-parity `<account-id>.dkr.ecr…` hostname instead, that path needs DNS resolving it to the gateway — prefer the gateway host the API returns.
 
+**A containerised tool fails with `x509: certificate signed by unknown authority`, while `docker login` and `docker push` work from the host.**
+The registry token exchange (`POST /v2/token`) runs in the process that issued the build or push.
+When that process is itself a container, such as `ghcr.io/basecamp/kamal` or a containerised CI runner, it checks the registry certificate against the container's trust store, which does not hold the Spinifex CA.
+Bind-mount the host bundle, which already includes the CA after `update-ca-certificates`, over the container's bundle:
+
+```bash
+docker run --rm -it \
+  -v /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v "$PWD:/workdir" \
+  ghcr.io/basecamp/kamal:latest deploy
+```
+
+Mounting the whole bundle keeps the public roots, so pulls of public base images still verify.
+Pointing `SSL_CERT_FILE` at `ca.pem` alone replaces them and breaks those pulls.
+
+**`docker buildx build --push` fails with an x509 error, while `docker push` of the same image works.**
+The buildx driver decides whose trust store BuildKit uses.
+The `docker` driver runs the BuildKit built into dockerd, which uses the host trust store, so the host CA install above covers it.
+The `docker-container` driver, the default for a builder made with `docker buildx create`, runs BuildKit in its own container, and a CA installed on the host or mounted into the CLI container does not reach it.
+
+Use the `docker` driver when you can (`docker buildx use default`, or `builder: driver: docker` in Kamal's `config/deploy.yml`).
+The `docker-container` route is unavoidable for multi-platform builds and remote builders, which the `docker` driver cannot do.
+For those, give BuildKit the CA through its own config:
+
+```toml
+# buildkitd.toml
+[registry."<spinifex-gateway-host>:9999"]
+  ca = ["/etc/spinifex/ca.pem"]
+```
+
+```bash
+docker buildx create --name spinifex --driver docker-container \
+  --buildkitd-config buildkitd.toml --use
+```
+
+`docker buildx create` copies the referenced CA file into the builder container, so recreate the builder after a CA rotation.
+
 **EKS workers cannot pull an image.** Confirm the node IAM role has `AmazonEC2ContainerRegistryReadOnly` and that the workers have egress to the registry endpoint (an Internet Gateway or NAT Gateway route). See the [EKS prerequisites](/docs/eks).
 
 **Image-scanning commands return an error.** Image scanning is not supported on Spinifex; the scan APIs intentionally reject these calls. Remove scan-on-push configuration from your tooling.

docs/install/performance-tuning/README.md

@@ -7,7 +7,10 @@ tags:
   - tuning
   - networking
   - sysctl
+  - ipsec
 resources:
+  - title: "VPC Networking"
+    url: "/docs/vpc-networking"
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
 ---
@@ -152,8 +155,7 @@ east-west guest traffic goes on the wire unencrypted.
 
 ### Disabling it for trusted links
 
-Only for topologies where the underlay is physically trusted — a single rack, a
-private VLAN, no untrusted tenant on the fabric.
+Only for topologies where the underlay is physically trusted — a single rack, a private VLAN, no untrusted tenant on the fabric. The trade-off, the topologies it does and does not apply to, and the same procedure in more detail are in the VPC networking guide: [Overlay Encryption (IPsec)](/docs/vpc-networking#overlay-encryption-ipsec).
 
 In `/etc/spinifex/spinifex.toml` on **every** node:
 
@@ -162,12 +164,21 @@ In `/etc/spinifex/spinifex.toml` on **every** node:
 ipsec_enabled = false
 ```
 
-Then restart the daemon: `systemctl restart spinifex-daemon`. Confirm with
-`ip xfrm state | grep -c '^src'`, which must reach 0 on every node.
+Then restart the daemon: `systemctl restart spinifex-daemon`. Confirm with `ip xfrm state | grep -c '^src'`, which must reach 0 on every node.
 
-Note that `ovn-nbctl set NB_Global . ipsec=false` is **not** sufficient on its own —
-the daemon reconciles that value, so a manual override is transient. Change the
-config file.
+**Then clear the OVN and OVS flags by hand.** The reconciler returns early when IPsec is disabled: it stops requesting encryption, but never clears what it previously set, so `NB_Global.ipsec` and `other_config:ipsec_encapsulation` are left reading `true`. They are inert — nothing remains to act on them once charon and the SAs are gone — but the control plane then disagrees with the config, and the next person to inspect it will read the cluster as encrypted when it is not.
+
+```bash
+# Use the NB raft endpoints. The local unix socket fails with
+# "database connection failed" on a clustered deployment.
+NBDB='tcp:10.2.0.2:6641,tcp:10.2.0.3:6641,tcp:10.2.0.4:6641'
+ovn-nbctl --db="$NBDB" set NB_Global . ipsec=false
+
+# On every node:
+ovs-vsctl remove Open_vSwitch . other_config ipsec_encapsulation
+```
+
+This is manual today and should move into the reconciler. Note that these commands are a *complement* to the config change, not a substitute for it — setting `NB_Global.ipsec=false` without `ipsec_enabled = false` leaves the daemon reconciling encryption back on, and leaves the guest MTU and queue-pair count on their encrypted values.
 
 **Restart guests after changing this in either direction.** The guest MTU moves
 with the setting, but a running guest holds its DHCP lease for up to an hour and

docs/package-boundary/README.md

@@ -0,0 +1,4 @@
+# Package-boundary contracts
+
+This directory holds code-adjacent evidence that versions with the code: present lifecycle contracts and boundary notes describing a package as it stands.
+Plans, discovery reports, inventories and draft designs belong in the Mulga repository; the earlier ones are in `docs/development/reference/spinifex-package-boundary/` there.

docs/package-boundary/adr-0004-inventory-ecs.md

@@ -1,388 +0,0 @@
-# ADR-0004 source inventory — ECS
-
-Code-read inventory of `refactor/spinifex-package-boundaries` at `6c50349bd`.
-It is not verified against a live system, and the branch is 59 commits behind `dev` at the time of writing, so recheck any file it cites before acting on it.
-Items marked "verify" or "uncertain" are unconfirmed.
-Gaps record observed behaviour, not target behaviour.
-
-
-Repo: `spinifex` submodule, branch `refactor/spinifex-package-boundaries`; paths below are relative to `spinifex/spinifex/` unless stated otherwise.
-Binding inputs: ADR-0001..0004 and Q-78 at `origin/main`.
-This inventory reports what the code does; it accepts or closes no gap.
-"Verify" marks claims about AWS behaviour, or runtime behaviour, that a test or live system has not confirmed here.
-
-## 1. File table
-
-Totals: `handlers/ecs` is 7,760 production lines over 28 files, `gateway/ecs` + `gateway/ecs.go` is 1,236 lines, and `daemon/ecs_deps.go` + `accountteardown/reapers_ecs.go` is 132 lines.
-
-| File | Lines | Class | Reason / split |
-|---|---|---|---|
-| `handlers/ecs/bus/messages.go` | 173 | guest/controller wire contract (part temporary residue) | `Assign`, `AssignContainer`, `StopDirective`, `TaskState`, `ContainerStatus`, `InstanceCapacity`, `PortMapping` and `SystemControl` are live payloads of the agent and gateway path. `RegisterInstance` (:19) and `Heartbeat` (:33) have no production publisher, so they are residue. Split: live types go to a versioned `contracts/ecs/v1`; the residue types are deleted once the scheduler bus subscriptions are removed. |
-| `handlers/ecs/bus/subjects.go` | 45 | temporary residue (prerequisite: drop scheduler bus subs; removal: delete unused builders) | `RegisterSubject`, `HeartbeatSubject`, `AssignSubject` and `TaskStateSubject` (:19-38) have no production publisher. `ServiceReconcileSubject` (:43) is live and is an internal cross-node wake owned by the service reconcile. |
-| `handlers/ecs/capacity.go` | 223 | cross-resource orchestration (container-instance capacity provisioning) | `ProvisionCapacity` (:70) is a non-AWS action. It composes an IAM system role, EC2 AMI lookup and EC2 `RunInstances`, and records nothing durable in ECS. |
-| `handlers/ecs/capacity_providers.go` | 233 | resource owner (`capacityprovider`), mixed | Create, Describe and Delete belong to `capacityprovider`. `PutClusterCapacityProviders` (:16-40) mutates the cluster record and belongs to `cluster`. |
-| `handlers/ecs/cluster_teardown.go` | 171 | mixed: cross-resource orchestration + resource owner (`containerinstance`) | `DeleteCluster` (:24-53) is a cascade over every child, which is orchestration. `DeregisterContainerInstance` (:60), `UpdateContainerInstancesState` (:103), `instanceActiveTasks` (:143) and `drainInstanceServiceTasks` (:160) are container-instance commands that also stop tasks. Split three ways: cluster delete orchestration, container-instance owner, and task-stop calls through a task capability. |
-| `handlers/ecs/conv.go` | 291 | mixed | `ClusterShortName`, `ContainerInstanceShortID`, `TaskShortID` and `CapacityProviderShortName` (:27-60) are per-resource identity parsers that the gateway authz also uses. The `containerDefsFromAWS`, `runtimePlatformFromAWS` and `ContainerDef.toAWS` group (:87-265) is the `taskdefinition` projection. `toAssignContainer` (:268) is wire-contract projection. |
-| `handlers/ecs/deployments.go` | 267 | resource owner (`service`) | This is pure deployment, rollout and circuit-breaker logic on `ServiceRecord`, with no KV I/O. |
-| `handlers/ecs/eni.go` | 155 | cross-resource orchestration (task realization, network capability) | The consumer-owned `eniController` interface (:37) belongs with `task`. `natsENIController` (:46-128) binds it to EC2 subjects and belongs at the composition root. |
-| `handlers/ecs/gpu_report.go` | 63 | resource owner (`task`), mixed with wire contract | `ReportTaskGPU` (:31) writes the task record. `ReportTaskGPUInput`, `ContainerGPUReport` and `ReportTaskGPUOutput` (:10-25) are agent wire shapes. |
-| `handlers/ecs/instancerole.go` | 48 | cross-resource orchestration (identity for container instances) | It ensures and converges the `ecsInstanceRole` through `handlers_iam` (:37-48). |
-| `handlers/ecs/instances.go` | 532 | mixed: resource owner (`containerinstance`) + `task` + orchestration | Register, Describe, List, `upsertInstance`, `recordRegister` and `recordHeartbeat` (:19-357) belong to `containerinstance`. `recordTaskState` (:361) and `SubmitTaskStateChange` (:445) belong to `task`. `releaseReservation` (:505) is placement against the instance record. `recordDeploymentFailure` (:481) belongs to `service`. |
-| `handlers/ecs/netcfg.go` | 78 | resource owner (`task`) | It resolves the network mode and parses awsvpc input for task launch. |
-| `handlers/ecs/placement.go` | 144 | cross-resource orchestration (placement) | It contains pure fit and strategy selection over container-instance capacity, with no I/O. |
-| `handlers/ecs/poll.go` | 107 | guest/controller wire contract | It holds the agent inbox drain and ack (`PollAssignments` :37), and the task-owned inbox reclaim (:93-107). |
-| `handlers/ecs/records.go` | 525 | mixed (all resources' records) | It holds the status constants, ARN builders and every record type for every resource. Split each record, its ARN builder and its statuses into the owning resource. |
-| `handlers/ecs/scheduler.go` | 449 | cross-resource orchestration + temporary residue | Lease, service reconcile loop, instance reaper and role converge are orchestration. `onRegister`, `onHeartbeat` and `onTaskState` (:236-320) are residue because nothing publishes those subjects. |
-| `handlers/ecs/service.go` | 706 | mixed: facade + `cluster` + `taskdefinition` + shared KV helpers | `ECSService` (:27-81) is an AWS-shaped facade that also serves as an internal API. `Deps` (:98) is composition. Cluster CRUD (:231-411) belongs to `cluster`. Task-definition CRUD and `ParseTaskDefRef` (:416-706) belong to `taskdefinition`. `getJSON`, `putJSON` and `keysWithPrefix` (:164-225) are shared persistence helpers. |
-| `handlers/ecs/service_nats.go` | 177 | AWS protocol adapter (gateway→daemon NATS hop) | `NATSECSService` forwards 34 methods onto `ecs.<Action>`. It is also used as an internal API by `accountteardown` and by gateway PassRole resolution. |
-| `handlers/ecs/services.go` | 786 | mixed: resource owner (`service`) + orchestration bindings | Service CRUD and reconcile (:128-665) belong to `service`. `targetRegistrar` / `natsTargetRegistrar` (:28-59, ELBv2) and `eipManager` / `natsEIPManager` (:64-121, EC2 EIP) are consumer-owned capabilities, but their NATS bindings belong at the composition root. `listTaskRecords` (:605) reads task records directly. |
-| `handlers/ecs/store.go` | 227 | temporary residue (shared account bucket; prerequisite: decide per-owner key namespaces; removal: key helpers move to owners) | One bucket holds the keys of all six resources (:36-150). It also holds the leader bucket factory (:221) and the account enumeration used by every sweep (:186). |
-| `handlers/ecs/tags.go` | 277 | cross-resource orchestration (tag dispatch) | It parses the ARN and then reads and writes each resource's record directly (:76-203). It should become one tag command per owner behind ARN dispatch. |
-| `handlers/ecs/task_stop.go` | 301 | mixed: resource owner (`task`) + orchestration | `StopTask`, `StartTask`, `reserveOnInstance`, `requestStopTask` and `forceStopTask` (:28-209) belong to `task`, with placement mixed into `reserveOnInstance`. `applyServiceTargets` (:235) and `assignTaskPublicIP` / `releaseTaskPublicIP` (:264-301) are ELBv2 and EIP orchestration. |
-| `handlers/ecs/tasks.go` | 438 | mixed: resource owner (`task`) + placement orchestration + wire | `RunTask`, Describe, List and the record/projection code belong to `task`. `reservePlacement` (:180) does a CAS on the container-instance record. `publishAssign` (:249) writes the agent inbox. |
-| `handlers/ecs/tasks_gc.go` | 130 | resource owner (`task`) reconciliation | It retains STOPPED tasks, retries owed ENI releases and prunes old tasks. It is hosted on `Scheduler` (:21). |
-| `handlers/ecs/userdata.go` | 58 | guest/controller wire contract | It writes the cloud-config that seeds the agent env (`ECS_GATEWAY_URL`, `ECS_GATEWAY_CA`, `ECS_REGION`, `ECS_CLUSTER`) (:31-58). |
-| `gateway/ecs.go` | 80 | AWS protocol adapter | `ECS_Request` (:26) does target parsing, the scope gate (:54-60), the PassRole closure (:65-67) and response encoding. |
-| `gateway/ecs/handler.go` | 165 | AWS protocol adapter | It holds the authoritative action table (:50-112), `StubbedActionNames` (:115), the raw-JSON action list (:131) and the JSON 1.1 writers. |
-| `gateway/ecs/actions.go` | 452 | AWS protocol adapter (mixed: PassRole resolution) | Each action decodes and forwards over NATS. `checkTaskLaunchRoles` (:44) queries `DescribeTaskDefinition` over the ECS NATS facade to find role ARNs. |
-| `gateway/ecs/authz.go` | 350 | AWS protocol adapter | The per-action resource-ARN scope table (:57-124) and resolver (:139-350) are built on the `handlers_ecs` parsers and ARN builders. |
-| `gateway/ecs/azrebalancing.go` | 103 | AWS protocol adapter | It rejects `availabilityZoneRebalancing=ENABLED` (:28) and adds the SDK-gap field to service responses (:54). |
-| `gateway/ecs/clienttoken.go` | 86 | AWS protocol adapter holding durable retry state (flag) | It holds the RunTask ClientToken store `spinifex-ecs-clienttokens` (:22). ADR-0003 S3 says an adapter does not own retry bookkeeping, so this belongs with `task`. |
-| `daemon/ecs_deps.go` | 63 | composition root (nearest class: cross-resource orchestration) | It binds IAM, images, `RunWorkerInstance` and the gateway URL/CA into `handlers_ecs.Deps` (:17-63). |
-| `daemon/daemon.go` (ECS sections only) | n/a | AWS protocol adapter transport + composition | Import at :74, fields at :168-169, 34 `ecs.<Action>` subscriptions at :1131-1167, construction at :2023-2043. |
-| `accountteardown/reapers_ecs.go` | 69 | cross-resource orchestration (account teardown) | It has one cluster reaper that calls the ECS AWS facade over NATS (:20-58). |
-| `gateway/operations.go` (:7, :30-32) | n/a | AWS protocol adapter | It registers the ECS operation inventory from `gateway_ecs.Actions` and `StubbedActionNames`. |
-| `gateway/gateway.go` (:301, :403, :523-524, :639) | n/a | AWS protocol adapter | Service routing to `ECS_Request` and the `ecs-tasks.amazonaws.com` PassRole principal constant live here. |
-
-Other ECS-aware code outside scope, with no import of ECS:
-`handlers/sts/assume_role.go:47-55,350-355` duplicates the `ecsInstanceRole` name and the `ecs-tasks.amazonaws.com` principal, and documents that it cannot import `handlers_ecs`.
-`foundation/aws/tags/tags.go:24` defines `ManagedByECS` for AMI resolution.
-`operator/imagecatalog/images.go:611-637` defines the `spinifex-ecs-node` and `spinifex-ecs-node-gpu` system AMIs.
-`domains/acm/renewal.go:169` is a comment-only reference.
-`../internal/ecsgw/client.go` is the agent's SigV4 gateway client, with target prefix at :29 and :101.
-
-Guest agent counterparty (`agents/ecs/agent/`), not inventoried file by file:
-It reaches the control plane only through signed gateway HTTP (`controlplane.go:18-35`), not NATS.
-It imports both `handlers/ecs/bus` and the `handlers_ecs` implementation package (`registration.go:37-38`, `controlplane.go:128,142,150`, `taskrun.go:416-419`).
-That is the ADR-0004 S5 debt: the agent imports a domain implementation package to get wire types.
-
-## 2. Served AWS actions and protocol
-
-Protocol: AWS JSON 1.1, with the action taken from the `X-Amz-Target` suffix (`gateway/ecs.go:15-20`, any prefix accepted).
-Requests use the canonical prefix `AmazonEC2ContainerServiceV20141113` (`gateway/ecs/handler.go:26`).
-Responses are encoded with `jsonutil` (epoch-second times), except `PollAssignments`, which uses `encoding/json` (RFC3339 times) (`handler.go:131-157`).
-Errors are `awserrors` codes rendered by the shared ErrorHandler.
-An action missing from the table returns `InvalidAction` (`gateway/ecs.go:32-36`), and a stub returns `NotImplemented` (501) (`handler.go:43-45`).
-Request order is: account check (:40-46), bounded body (:48), `ResourceARNs` scope derivation (:54), `checkPolicyResources(r, "ecs", action, …)` (:58), handler with PassRole closure (:69), then encode.
-Gateway to daemon is a NATS request on `ecs.<Action>` with a 30 s timeout (`handlers/ecs/service_nats.go:12,29-177`), served by queue group `spinifex-workers` (`daemon/daemon.go:1131-1167`).
-`gateway/operations.go:30-32` publishes Registered = all 39 table keys and Stubbed = 5.
-`ecsScopes` (`authz.go:57-124`) has an entry for every table action, and `ResourceARNs` fails closed if an entry is missing (:140-144).
-
-| Action | Status | Resource | Handler (daemon) |
-|---|---|---|---|
-| CreateCluster | registered | cluster | `service.go:231` |
-| DescribeClusters | registered | cluster (+counts from instance/task/service) | `service.go:268` |
-| ListClusters | registered | cluster | `service.go:338` |
-| DeleteCluster | registered | cluster (cascade orchestration) | `cluster_teardown.go:24` |
-| UpdateCluster | stubbed | cluster | — |
-| PutClusterCapacityProviders | registered | cluster (references capacityprovider) | `capacity_providers.go:16` |
-| CreateCapacityProvider | registered | capacityprovider | `capacity_providers.go:44` |
-| DescribeCapacityProviders | registered | capacityprovider | `capacity_providers.go:80` |
-| DeleteCapacityProvider | registered | capacityprovider | `capacity_providers.go:125` |
-| RegisterTaskDefinition | registered (PassRole on task/execution role, `actions.go:99`) | taskdefinition | `service.go:416` |
-| DeregisterTaskDefinition | registered | taskdefinition | `service.go:546` |
-| DescribeTaskDefinition | registered | taskdefinition | `service.go:571` |
-| ListTaskDefinitions | registered | taskdefinition | `service.go:586` |
-| ListTaskDefinitionFamilies | stubbed | taskdefinition | — |
-| RunTask | registered (PassRole via taskdef lookup, ClientToken) | task (+placement, containerinstance) | `tasks.go:28` |
-| StartTask | registered (PassRole via taskdef lookup) | task (+containerinstance) | `task_stop.go:53` |
-| StopTask | registered | task | `task_stop.go:28` |
-| DescribeTasks | registered | task | `tasks.go:273` |
-| ListTasks | registered | task | `tasks.go:299` |
-| SubmitTaskStateChange | registered (AWS action; used by agent) | task | `instances.go:445` |
-| PollAssignments | registered (non-AWS, agent-only, raw JSON) | task inbox (wire) | `poll.go:37` |
-| ReportTaskGPU | registered (non-AWS, agent-only) | task | `gpu_report.go:31` |
-| CreateService | registered (PassRole via taskdef lookup; AZ-rebalancing gate) | service | `services.go:128` |
-| UpdateService | registered (same) | service | `services.go:209` |
-| DeleteService | registered | service | `services.go:267` |
-| DescribeServices | registered | service | `services.go:302` |
-| ListServices | registered | service | `services.go:326` |
-| ListServicesByNamespace | stubbed | service | — |
-| RegisterContainerInstance | registered (agent boot + 30 s liveness) | containerinstance | `instances.go:65` |
-| DeregisterContainerInstance | registered | containerinstance | `cluster_teardown.go:60` |
-| UpdateContainerInstancesState | registered | containerinstance | `cluster_teardown.go:103` |
-| DescribeContainerInstances | registered | containerinstance | `instances.go:127` |
-| ListContainerInstances | registered | containerinstance | `instances.go:167` |
-| ProvisionCapacity | registered (non-AWS) | containerinstance capacity (EC2 launch orchestration) | `capacity.go:70` |
-| PutAccountSetting | stubbed | account setting | — |
-| ListAccountSettings | stubbed | account setting | — |
-| TagResource | registered | cross-resource (by ARN) | `tags.go:227` |
-| UntagResource | registered | cross-resource (by ARN) | `tags.go:254` |
-| ListTagsForResource | registered | cross-resource (by ARN) | `tags.go:207` |
-
-Unsupported actions are any ECS action absent from the table, and they return `InvalidAction`.
-Examples include task sets, `ExecuteCommand`, `UpdateCapacityProvider`, `DeleteTaskDefinitions`, attribute APIs and service deployments/revisions.
-Three agent-facing actions (`PollAssignments`, `ReportTaskGPU`, `ProvisionCapacity`) are non-AWS shapes on the AWS ECS signing surface.
-`SubmitTaskStateChange` is an AWS action that our agent uses as its state-report channel.
-
-## 3. Durable records / KV buckets
-
-| Bucket | Key shape | Record type | Owning resource | Writers | Readers |
-|---|---|---|---|---|---|
-| `ecs-account-<accountID>` (history 1, schema v1, created lazily, `store.go:27-29,207-217`) | `clusters/<name>/meta` | `ClusterRecord` (`records.go:155`) | cluster | CreateCluster `service.go:259`; PutClusterCapacityProviders `capacity_providers.go:36`; tags `tags.go:152`; DeleteCluster removes the subtree `cluster_teardown.go:47` | cluster CRUD; RunTask/StartTask/CreateService existence checks (`tasks.go:36`, `task_stop.go:60`, `services.go:148`) |
-| same | `clusters/<c>/instances/<id>` | `InstanceRecord` (`records.go:333`) | containerinstance | `upsertInstance` plain Put `instances.go:215`; `recordHeartbeat` `:356`; UpdateContainerInstancesState `cluster_teardown.go:131`; reaper `scheduler.go:390`; tags `tags.go:200`; CAS reserve/release `tasks.go:212`, `task_stop.go:148`, `instances.go:527`; Deregister deletes `cluster_teardown.go:90` | placement (`tasks.go:182`), cluster counts (`service.go:301`), reaper, describe/list |
-| same | `clusters/<c>/tasks/<taskID>` | `TaskRecord` (`records.go:389`) | task | RunTask `tasks.go:84`; StartTask `task_stop.go:104`; stop/force-stop `task_stop.go:171,199`; recordTaskState `instances.go:409,433`; EIP assign `task_stop.go:284`; ENI rollback `tasks.go:168`; GPU report `gpu_report.go:58`; sweep `tasks_gc.go:98,119`; tags `tags.go:189` | service reconcile (`services.go:589-622`), cluster/instance counts, reaper, sweep |
-| same | `clusters/<c>/assignments/<ciID>/<taskID>` | `bus.Assign` | task (agent inbox) | `publishAssign` `tasks.go:269` | agent via `PollAssignments` (`poll.go:58`); acked and reclaimed via `poll.go:49,97` |
-| same | `clusters/<c>/stops/<ciID>/<taskID>` | `bus.StopDirective` | task (agent inbox) | `requestStopTask` `task_stop.go:176` | agent via `PollAssignments` (`poll.go:74`); acked and reclaimed via `poll.go:55,106` |
-| same | `clusters/<c>/services/<name>` | `ServiceRecord` (`records.go:454`) incl. `Deployments`, `Events` | service | Create/Update/Delete `services.go:198,256,295`; reconcile `services.go:433`; deployment-failure count `instances.go:496`; tags `tags.go:178` | task paths read it for LB and EIP config (`task_stop.go:241,273`) |
-| same | `taskdef-families/<family>/latest-rev`, `taskdef-families/<family>/revs/<rev>` | `int`, `TaskDefRecord` (`records.go:280`) | taskdefinition | Register `service.go:452,455`; Deregister `:564`; tags `tags.go:167` | `resolveTaskDef` (`service.go:622`) for RunTask/StartTask/Create/UpdateService; gateway PassRole via NATS `DescribeTaskDefinition` |
-| same | `capacity-providers/<name>` | `CapacityProviderRecord` (`records.go:200`) | capacityprovider | Create `capacity_providers.go:72`; Delete `:142` | Describe only |
-| `spinifex-ecs-leader` (TTL 60 s, `store.go:31-33,221`) | `scheduler` (`scheduler.go:24`) | kvlease lease | scheduler (orchestration) | `kvlease` in `Scheduler` (`scheduler.go:96-109`) | the same |
-| `spinifex-ecs-clienttokens` (TTL 15 min, `gateway/ecs/clienttoken.go:22-26`) | `<accountID>.<token>` (`foundation/lifecycle/idempotency/store.go:91-93`, no namespace) | `idempotency` entry with `ecs.RunTaskOutput` | task (currently held by the adapter) | gateway `runTaskWithClientToken` (`clienttoken.go:71-86`) | the same |
-
-`store.go:23-25`, `:134-138` and `LeaderLeaseKey` describe per-cluster `<accountID>/<clusterName>` lease keys, but the code uses one global key `scheduler`, and `LeaderLeaseKey` has no production caller.
-No reader outside ECS opens any of these buckets.
-A Go grep for bucket names and helpers outside `handlers/ecs` and `gateway/ecs` found only RDS's own `AccountBucketName`.
-`accountteardown` and the agent reach ECS state only through the ECS API (NATS facade or gateway), not through KV.
-No KV migration is registered for `ecs-account-*`; it runs `migrate.DefaultRegistry.RunKV` at version 1 (`store.go:213`).
-No production code was found that deletes an `ecs-account-*` bucket.
-
-## 4. Identity and scope
-
-The partition is hard-coded to `aws`.
-The Region is the daemon `config.Region` captured at construction (`daemon/daemon.go:2029`).
-The account is the SigV4 caller's account, carried on every NATS request.
-All records live in the caller's per-account bucket, so the account scope is structural.
-
-| Resource | Identifier | ARN (`records.go:107-135`) | Scope |
-|---|---|---|---|
-| cluster | name; omitted means `default` (`conv.go:27-36`, `service.go:233-235`) | `arn:aws:ecs:<region>:<acct>:cluster/<name>` | account + Region; no AZ |
-| taskdefinition | `family:revision`; revision is `latest-rev + 1` (`service.go:531-541`) | `…:task-definition/<family>:<rev>` | account + Region |
-| service | name unique per cluster | `…:service/<cluster>/<name>` | account + Region; subnets/SGs stored, not validated |
-| task | UUIDv4 (`tasks.go:65`, `task_stop.go:85`) | `…:task/<cluster>/<taskID>` | account + Region; AZ is implicit through its container instance and is not stored on the task |
-| containerinstance | the `InstanceIdentityDocument` string supplied by the agent (in practice the EC2 instance ID), with fallback `ci-<UTC timestamp>` (`instances.go:67-70`) | `…:container-instance/<cluster>/<id>` | account + Region; AZ from the registration attribute (`instances.go:100-101,331`) |
-| capacityprovider | name | `…:capacity-provider/<name>` | account + Region |
-
-AWS parity note (verify): AWS mints a UUID container-instance ID that is distinct from `ec2InstanceId`, and AWS expects `instanceIdentityDocument` to be the signed IMDS JSON document.
-Ours uses the supplied string as both values (`instances.go:276`).
-Identifier reuse: DeleteCluster removes every record (`cluster_teardown.go:47`), so a cluster name and its child names are reusable immediately.
-There is no tombstone or identifier-reuse rule (Q-43/Q-44).
-Task-definition revisions are not reserved against reuse after a concurrent collision (see gap G9).
-Container instances authenticate as the account's `ecsInstanceRole` (`instancerole.go:12-30`).
-`PollAssignments`, `SubmitTaskStateChange` and `ReportTaskGPU` check only account scope plus IAM policy on the named instance or task ARN (`authz.go:83-95`).
-The handlers do not bind the caller to the specific container instance (`poll.go:37-56`, `instances.go:445-475`, `gpu_report.go:31`).
-
-## 5. Lifecycle paths per resource
-
-There are no desired or observed generation fields on any ECS record (`records.go:155-493`).
-The only fencing mechanisms are:
-- one global scheduler lease (`scheduler.go:96-109`), checked only at the start of each pass (`:170`, `:189`);
-- CAS (`kv.Update` with revision) on container-instance reservations (`tasks.go:212`, `task_stop.go:148`, `instances.go:527`).
-
-Every other write is a read-modify-`Put` without revision (`service.go:179-188`).
-ClientToken idempotency exists only for RunTask (`gateway/ecs/actions.go:203-238`).
-
-### cluster
-Create is `CreateCluster` (`service.go:231-264`): get, then a plain Put if absent.
-A repeat create by name returns the existing record without comparing parameters; AWS also treats it as idempotent (verify).
-Readiness: the cluster is `ACTIVE` on create (`:250`), synchronously.
-There is no reconcile loop.
-Delete is `DeleteCluster` (`cluster_teardown.go:24-53`): it force-stops every task (:43-45), deletes the whole `clusters/<n>/` subtree (:47), and returns a synthetic `INACTIVE` (:51).
-No durable deletion intent is written.
-On teardown it is driven by `accountteardown/reapers_ecs.go:53-58`.
-
-### taskdefinition
-Create is `RegisterTaskDefinition` (`service.go:416-459`): it validates (`:472-511`), computes `rev = latest + 1` without CAS (`:530-541`), Puts the revision (`:452`), then Puts `latest-rev` (`:455`).
-There is no client token, and AWS has none for this action.
-Readiness: the definition is `ACTIVE` synchronously.
-Delete is `DeregisterTaskDefinition` (`:546-568`), which sets `INACTIVE` and retains the record.
-There is no hard delete (`DeleteTaskDefinitions` is not served).
-`resolveTaskDef` (`:622-647`) does not check status, so RunTask, StartTask and CreateService accept an `INACTIVE` revision.
-AWS refuses new runs and services on an INACTIVE revision after a short window (verify).
-
-### service
-Create is `CreateService` (`services.go:128-206`): an `ACTIVE` service with the same name is returned as-is (:159-160), and `ClientToken` is ignored.
-It Puts the record with a PRIMARY deployment (:196-198), then runs `reconcileService` synchronously on the serving worker (:202).
-Readiness: the service record is `ACTIVE` immediately.
-Rollout state moves `IN_PROGRESS` → `COMPLETED` when the primary running count reaches the desired count and no old-deployment tasks remain (`deployments.go:170-196`).
-The circuit breaker sets `FAILED` after 3 failed starts (`deployments.go:222-243`, `records.go:54`), with optional rollback (`:247-267`).
-Launch backoff is 15 s to 5 min (`deployments.go:206-216`).
-Reconcile has two triggers:
-- the leader loop `ecs/services` (`scheduler.go:143-148`) runs `reconcileAllServices` (`services.go:445-476`) on a wake from `ecs.bus.<acct>.<cluster>.service-reconcile` (`instances.go:472`), a 10 s revisit while unsettled, and a 5 min resync;
-- `CreateService` and `UpdateService` also reconcile inline on the API worker (`services.go:202`, `:259`).
-The reconcile launches through `s.RunTask` without a client token (`services.go:495-525`) and stops surplus tasks (`:531-581`).
-Delete is `DeleteService` (`:267-299`): it requests a stop for each task (:288-289), then sets `INACTIVE` immediately (:291).
-`ServiceStatusDraining` (`records.go:30`) is never written.
-There is no `force` or desired-count precondition check, and the INACTIVE record is never pruned.
-
-### task
-Create is `RunTask` (`tasks.go:28-93`) or `StartTask` (`task_stop.go:53-113`), in this order:
-1. Reserve capacity by CAS on the container instance (`tasks.go:180-218` / `task_stop.go:118-153`).
-2. Build the record in memory (`tasks.go:221-243`; desired `RUNNING`, last `PENDING`).
-3. For awsvpc, create and attach the ENI through EC2 (`tasks.go:131-176`).
-4. Make the first durable task write (`tasks.go:84` / `task_stop.go:104`).
-5. Write the assignment inbox (`tasks.go:249-270`); a failure there is only logged (`:87-89`).
-Readiness comes from the agent's `SubmitTaskStateChange`, which goes through `recordTaskState` (`instances.go:361-439`).
-`RUNNING` sets `StartedAt`, registers ELBv2 targets and assigns an EIP when the service asks for it (:415-418).
-`STOPPED` releases the target, EIP, inboxes, ENI and reservation (:422-436).
-Stop has two paths:
-- `StopTask` calls `requestStopTask` (`task_stop.go:162-179`), which sets desired `STOPPED` and posts a stop directive for the agent.
-- `forceStopTask` (`:186-209`) is used by the cluster delete, deregister, drain and reaper paths; it marks the task `STOPPED` without telling the agent.
-Repair: the leader `ecs/timers` loop runs `sweepStoppedTasks` (`tasks_gc.go:21-105`).
-That sweep retries owed ENI releases (at most 20 per pass, 30 s to 15 min backoff) and deletes STOPPED tasks after 1 h.
-Idempotency: RunTask honours ClientToken through `idempotency.Do` (`gateway/ecs/clienttoken.go:71-86`); a partial placement is finalized and replayed.
-StartTask has no token, and AWS has none for it (verify).
-Internal launch from the service reconcile is not idempotent per logical launch.
-
-### containerinstance
-Create is `RegisterContainerInstance` (`instances.go:65-124`), which is an upsert with a plain Put (`:195-219`).
-The agent calls it at boot and on every 30 s heartbeat (`agents/ecs/agent/controlplane.go:22-25`).
-Readiness: the instance is `ACTIVE` on first registration (`:207`).
-A zero reported capacity drains it (`Reaped=true`), and returning capacity restores it (`:44-60`).
-`AgentConnected` is derived from `Status == ACTIVE` (`:278`), not from liveness.
-Repair: the leader reaper (`scheduler.go:340-396`) marks an instance `DRAINING` with `Reaped=true` after 90 s without `LastSeen`, and force-stops its tasks (`:400-416`).
-`recordRegister` and `recordHeartbeat` (`instances.go:325-357`) are fed only by bus subjects that nothing publishes.
-Operator state: `UpdateContainerInstancesState` (`cluster_teardown.go:103-140`) sets the status.
-`DRAINING` force-stops service tasks immediately (`:160-171`).
-Delete is `DeregisterContainerInstance` (`:60-97`): it refuses if tasks are active unless `force`, deletes the assignments inbox and the record (the stops inbox is left), and does not touch the EC2 instance.
-Capacity is provisioned by `ProvisionCapacity` (`capacity.go:70-170`), which calls `RunInstances` directly with no ECS-side intent record and no idempotency.
-
-### capacityprovider
-Create is `CreateCapacityProvider` (`capacity_providers.go:44-76`): an existing name returns the existing record without comparing parameters, and the ASG ARN is stored without validation.
-Readiness: it is `ACTIVE` immediately and inert, with no scheduler coupling (`service.go:74-76`).
-Delete is a hard key delete (`:125-146`), with no check that a cluster references the provider.
-
-## 6. NATS subjects and guest/controller messages
-
-| Subject | Direction / type | Payload | Retry / redelivery |
-|---|---|---|---|
-| `ecs.<Action>` ×34 (`service_nats.go:29-177`; subs `daemon.go:1134-1167`) | gateway (and `accountteardown`, gateway PassRole lookup) → daemon, request/reply, queue `spinifex-workers` | aws-sdk-go ECS input/output or `PollAssignments*`, `ReportTaskGPU*`, `ProvisionCapacity*` structs | 30 s timeout. There is no retry in `NATSECSService`, and `natsmsg.NATSRequest` (`foundation/messaging/nats/nats.go:269`) retry behaviour was not verified here. |
-| `ecs.bus.<acct>.<cluster>.service-reconcile` (`bus/subjects.go:43`) | any worker → leader, core publish | none | Fire-and-forget (`instances.go:472-474`); loss is covered by the 10 s revisit and 5 min resync (`scheduler.go:30-58`). |
-| `ecs.bus.*.*.instance-register.*`, `…instance-heartbeat.*`, `…task-state.*` (`scheduler.go:236-245`) | subscribed by leader | `bus.RegisterInstance`, `bus.Heartbeat`, `bus.TaskState` | No production publisher (residue). Core NATS with no ack or redelivery, and messages are dropped while there is no leader. |
-| `ecs.bus.<acct>.<cluster>.assign.<id>` (`bus/subjects.go:30`) | unused | — | Residue; assignment moved to the KV inbox. |
-| `ec2.CreateNetworkInterface`, `ec2.DeleteNetworkInterface` (`eni.go:64,119`) | ECS → EC2 AWS-action subjects | aws-sdk-go EC2 types | 30 s timeout (`eni.go:19`). NotFound counts as converged (`:148-155`). A failed release is retried by the sweep. |
-| `ec2v1.InstanceCommandSubject(<instanceID>)` (`eni.go:126-128`) | ECS → owning node, `contracts/ec2/v1` | `EC2InstanceCommand{AttachENI or DetachENI}` | Versioned contract; detach is forced (`:109`). |
-| `ec2.AllocateAddress`, `ec2.AssociateAddress`, `ec2.DescribeAddresses`, `ec2.DisassociateAddress`, `ec2.ReleaseAddress` via `ec2eip.NewNATSEIPService` (`services.go:81-121`; subjects in `domains/ec2/eip/service_nats.go:26-34`) | ECS → EC2 EIP AWS adapter | aws-sdk-go EC2 types | A failed associate releases immediately (`services.go:91-96`). A failed release is logged, and the fields are cleared anyway (`task_stop.go:296-300`). |
-| ELBv2 `RegisterTargets` / `DeregisterTargets` via `handlers_elbv2.NewNATSELBv2Service` (`services.go:45-58`) | ECS → ELBv2 AWS adapter | aws-sdk-go ELBv2 types | Best effort and log-only (`task_stop.go:252-255`), with no retry. The exact subject string was not verified. |
-| `ec2.RunInstances.<type>.<node>` | not used by ECS | — | ECS passes no node, so `RunWorkerInstance` runs the local in-process `instanceService.RunInstances` (`daemon/eks_worker_launch.go:29-51`). |
-
-Guest protocol: agent ↔ gateway HTTP, SigV4-signed with the instance role through `internal/ecsgw`; it is not NATS.
-- `RegisterContainerInstance`: boot, then every 30 s as liveness.
-- `PollAssignments`: delivery is at-least-once. The inboxes are KV keys, and an entry is redelivered on every poll until the agent acks it in `AckTaskIDs` or `AckStopIDs` on the next poll (`poll.go:45-56`), or until the task stops (`instances.go:430-431`, `task_stop.go:204-205`).
-- `SubmitTaskStateChange`: duplicate reports are tolerated only through edge detection on `prev != status` (`instances.go:415,422`). There is no ordering or monotonicity guard and no check of the reporting instance (`:372-373`).
-- `ReportTaskGPU`.
-Bootstrap contract: cloud-config writes `/etc/spinifex-ecs/agent.env` and the gateway CA (`userdata.go:31-58`).
-
-## 7. Boundary-crossing imports
-
-ECS → other packages (non-stdlib, non-foundation first):
-
-| From | Imports | Symbol(s) | Flag |
-|---|---|---|---|
-| `handlers/ecs/service.go:19`, `instancerole.go:6` | `handlers/iam` | `SystemInstanceRoleEnsurer` (`service.go:92`), `EnsureSystemInstanceProfile` (`instancerole.go:38`), `ConvergeSystemRolePolicy` (`:46`) | IAM |
-| `handlers/ecs/services.go:20` | `handlers/elbv2` | `NewNATSELBv2Service().RegisterTargets/DeregisterTargets` | other domain's AWS adapter used as an internal API |
-| `handlers/ecs/services.go:18` | `domains/ec2/eip` | `NewNATSEIPService` (Allocate/Associate/Describe/Disassociate/Release) | EC2 / network; AWS adapter as internal API |
-| `handlers/ecs/eni.go:13` | `contracts/ec2/v1` | `EC2InstanceCommand`, `EC2CommandAttributes`, `AttachENIData`, `DetachENIData`, `InstanceCommandSubject` | EC2 instance and network (versioned contract) |
-| `handlers/ecs/eni.go:64,119` | string subjects `ec2.CreateNetworkInterface` / `ec2.DeleteNetworkInterface` | — | network; EC2 AWS-action subject used as an internal API |
-| `handlers/ecs/capacity.go:11-12` | `domains/ec2/ebs/metadata`, `domains/ec2/instancetypes` | `RootDeviceName`; `IsGPUTypeName`, `GPUVendorForType` | EC2 |
-| `handlers/ecs/placement.go:9` | `domains/ec2/instancetypes` | `HotPlugENISlotsForType` | EC2 |
-| `handlers/ecs/service.go:16`, `capacity.go:10`, `eni.go:12`, `services.go:15-17` | aws-sdk-go `service/ec2`, `service/elbv2` | `RunInstancesInput`, `DescribeImages*`, ENI/EIP/target types | EC2 / ELBv2 types |
-| `handlers/ecs/*` | foundation | `messaging/nats.NATSRequest`, `state/kvlease`, `state/kvutil`, `state/migrate`, `lifecycle/reconciler`, `aws/errors`, `aws/arn`, `aws/ami`, `aws/tags` | — |
-| `gateway/ecs/authz.go:11-12` | `ingress/aws/bodyscope`, `ingress/aws/query` | `bodyscope.Parse`, `query.MaxSliceLen` | ingress |
-| `gateway/ecs/clienttoken.go:13` | `foundation/lifecycle/idempotency` | `OpenStore`, `Do`, `ParamHash`, `ErrParamMismatch`, `ErrUnavailable` | — |
-| `daemon/ecs_deps.go:8` | `github.com/mulgadc/bluebottle/pkg/masterkey` | `ReadShared` (:39) | Bluebottle (daemon wiring only; `handlers/ecs` itself does not import Bluebottle) |
-| `daemon/ecs_deps.go:10,50-52` | `handlers/iam` | `NewIAMServiceImpl`, `*IAMServiceImpl` | IAM |
-| `daemon/ecs_deps.go:29,32` | EKS-named daemon helpers | `resolveGatewayBaseURL` (`daemon/eks_deps.go:164`), `RunWorkerInstance` (`daemon/eks_worker_launch.go:29`) | EC2 instance launch shared with EKS |
-
-Packages that import ECS:
-
-| Importer | Imports | Symbol(s) |
-|---|---|---|
-| `gateway/ecs/actions.go:12` | `handlers/ecs` | `NewNATSECSService`, `ECSService`, `ProvisionCapacityInput`, `PollAssignmentsInput`, `ReportTaskGPUInput` |
-| `gateway/ecs/authz.go:10` | `handlers/ecs` | `ClusterShortName`, `ServiceShortName`, `TaskShortID`, `ContainerInstanceShortID`, `CapacityProviderShortName`, `ParseTaskDefRef`, `ResourceARNSegment`, `ClusterARN`, `ServiceARN`, `TaskARN`, `ContainerInstanceARN`, `CapacityProviderARN`, `TaskDefARN`, `TaskDefRefARN` |
-| `gateway/ecs.go:10`, `gateway/operations.go:7` | `gateway/ecs` | `Actions`, `ResourceARNs`, `RawJSONActions`, `WriteJSONResponse`, `WriteRawJSONResponse`, `StubbedActionNames` |
-| `daemon/daemon.go:74`, `daemon/ecs_deps.go:9` | `handlers/ecs` | `Service`, `Scheduler`, `NewService`, `WithDeps`, `InitLeaderBucket`, `NewScheduler`, `Deps` |
-| `accountteardown/reapers_ecs.go:9` | `handlers/ecs` | `NewNATSECSService`, `ECSService` (`ListClusters`, `DeleteCluster`) |
-| `agents/ecs/agent/{registration,controlplane,taskrun}.go` | `handlers/ecs` (implementation package) | `AttrInstanceType`, `AttrAvailabilityZone`, `PollAssignmentsInput/Output`, `ReportTaskGPUInput`, `ContainerGPUReport` |
-| `agents/ecs/agent/{agent,controlplane,reconcile,registration,taskrun}.go` | `handlers/ecs/bus` | `Assign`, `StopDirective`, `TaskState`, `ContainerStatus`, `InstanceCapacity`, status constants |
-
-Explicit flags:
-- **IAM / PassRole.** PassRole is enforced only in the gateway (`gateway/ecs.go:65-67`, through `gw.checkPassRole` at `gateway/gateway.go:645-648` with `iam:PassedToService=ecs-tasks.amazonaws.com`).
-  RegisterTaskDefinition checks the role ARNs in the body (`actions.go:99`).
-  RunTask, StartTask, CreateService and UpdateService first fetch the task definition through the ECS NATS facade (`actions.go:44-56`).
-  `ProvisionCapacity` attaches the system `ecsInstanceRole` instance profile (`capacity.go:103,141`) and launches through `RunWorkerInstance`, which goes straight to `d.instanceService.RunInstances` (`daemon/eks_worker_launch.go:51`).
-  That bypasses the EC2 gateway's PassRole closure (`gateway/ec2.go:199`).
-  `handlers/sts/assume_role.go:52-55` duplicates `ecsInstanceRole`, which is implicit coupling with no import.
-- **Quota.** `handlers/ecs` has no quota import.
-  `ProvisionCapacity` appears to bypass `gw.Quota.EnforceLaunch` and `ChargeLaunch` (`gateway/ec2.go:196-205`), because it calls the in-process instance service (verify whether `instanceService.RunInstances` enforces quota itself).
-  ECS resources themselves have no quota.
-- **EC2 instance.** `RunInstances` (via `Deps.RunInstances`), `DescribeImages` (via `Deps.Images` = `d.imageService`), `instancetypes`, EBS root-device metadata, and the `contracts/ec2/v1` instance command for ENI hot-plug.
-- **Network.** ENI create/delete through EC2 AWS subjects, EIP through the `domains/ec2/eip` NATS adapter, subnets and SGs passed through unvalidated.
-- **Bluebottle.** Only `masterkey.ReadShared` in `daemon/ecs_deps.go:39`, used to build the IAM service.
-  Without it, `ProvisionCapacity` is disabled and the scheduler refuses leadership (`scheduler.go:226-231`).
-
-## 8. Known gaps against Q-78 / ADR-0003 (observed)
-
-- **G1. No generations.** No ECS record has a desired or observed generation (`records.go:155-493`). Reconcilers act on whole records with plain Put (`service.go:179-188`).
-- **G2. Realization before intent.** RunTask and StartTask reserve capacity and create/attach an ENI before the task record exists (`tasks.go:66-84`, `task_stop.go:86-104`). A crash in that window leaves a reservation (`PlacedTasks` naming an unknown task) and possibly an ENI with no owning record. No reconciler targets orphan reservations; `tasks_gc.go` handles only owed ENI release on STOPPED tasks.
-- **G3. Lost-update window on instance reservations.** Reservations use CAS (`tasks.go:212`, `task_stop.go:148`, `instances.go:527`), but the same key is rewritten with a plain read-modify-Put every 30 s by agent registration (`instances.go:195-215`). The same happens in UpdateContainerInstancesState (`cluster_teardown.go:131`), the reaper (`scheduler.go:390`) and tags (`tags.go:200`), so a CAS commit between their Get and Put can be overwritten. `handlers/ecs/task_accounting_test.go` (`TestDescribeContainerInstances_StalePlacedTasksDoNotInflateCount`) acknowledges stale `PlacedTasks`.
-- **G4. Unfenced concurrent writers.** The leader lease is checked only at pass start (`scheduler.go:170,189`). KV writes are not fenced by the lease. API workers also run `reconcileService` inline (`services.go:202,259`) while the leader runs `reconcileAllServices` (`scheduler.go:177`), so two writers can launch through `RunTask` for the same service. This is the observed code shape; no test was found covering it.
-- **G5. Cascade delete removes cleanup ownership.** `DeleteCluster` (`cluster_teardown.go:43-47`) deletes the whole subtree, including STOPPED tasks whose `ENIReleased=false` or whose EIP or target cleanup failed. That removes the only record of owed cleanup, which Q-78 forbids. There is no deletion intent or incomplete-cleanup state. AWS refuses DeleteCluster while services, tasks or registered instances exist (verify against AWS docs and a test).
-- **G6. Force-stop does effects before persisting.** `forceStopTask` releases the EIP and ENI before persisting STOPPED (`task_stop.go:197-199`). A failed EIP release is logged and its allocation ID cleared (`:296-300`), so it cannot be retried. Target deregistration is best effort with no retry (`:252-255`). `forceStopTask` also posts no stop directive, so whether a live agent stops its containers is unverified.
-- **G7. EIP before record.** `assignTaskPublicIP` allocates and associates the EIP before persisting its allocation ID (`task_stop.go:277-286`).
-- **G8. Silent reservation leak.** `releaseReservation` gives up silently after 5 CAS conflicts and returns nil (`instances.go:506-531`), which leaks capacity with no reconciler.
-- **G9. Non-atomic revision allocation.** `RegisterTaskDefinition` computes the revision as read + 1 with no CAS (`service.go:430,531-541`). It writes the revision before `latest-rev` (`:452-455`), so concurrent registers can produce the same revision and overwrite it.
-- **G10. No guard on task-state reports.** `recordTaskState` sets `LastStatus` unconditionally (`instances.go:372-373`), so a late RUNNING after STOPPED regresses the state. The reporting `InstanceID` is ignored.
-- **G11. DeleteService is immediate.** It sets INACTIVE directly, with no DRAINING (`services.go:291`; `records.go:30` is unused). It has no `force` or desired-count check, AWS requires one (verify), and the INACTIVE record is never pruned.
-- **G12. Create idempotency without comparison.** CreateService ignores `ClientToken` and returns an existing ACTIVE service without comparing parameters (`services.go:159-160`). CreateCapacityProvider (`capacity_providers.go:60-61`) and CreateCluster (`service.go:246-262`) behave the same way.
-- **G13. Retry bookkeeping in the adapter.** The RunTask idempotency store lives in the AWS adapter (`gateway/ecs/clienttoken.go:22-56`). ADR-0003 S3 says retry bookkeeping is not the adapter's.
-- **G14. ProvisionCapacity.** It has no durable intent and no idempotency, and it silently caps count at 10 (`capacity.go:89-91,158`). Launched instances are linked only by the tag `spinifex:ecs-cluster` (`:150`).
-- **G15. Liveness derived from status.** Container-instance `AgentConnected` is derived from Status, not liveness (`instances.go:278`). Registration makes an instance ACTIVE immediately (`:207`).
-- **G16. Residue subscriptions.** The scheduler subscribes to three bus subjects that nothing publishes (`scheduler.go:236-245`).
-- **G17. AWS adapters as internal APIs.** Account teardown uses the ECS facade (`accountteardown/reapers_ecs.go:21`), and gateway PassRole resolution calls `DescribeTaskDefinition` over NATS (`actions.go:48`). ECS calls the ELBv2 and EIP adapters and the `ec2.*NetworkInterface` subjects (`services.go:46,82`, `eni.go:64,119`). ADR-0003 S4 and ADR-0004 S4 forbid an AWS adapter used as an internal service API.
-- **G18. Incomplete account teardown.** It reaps only clusters (`reapers_ecs.go:20-22`). Task definitions, capacity providers and the `ecs-account-*` bucket remain; no deleting code was found.
-- **G19. No pagination.** List APIs ignore `nextToken` and `maxResults` (`service.go:338,586`, `tasks.go:299`, `services.go:326`, `instances.go:167`). Q-89 requires listing tests that exceed one page.
-- **G20. Status not checked on launch.** RunTask, StartTask and CreateService accept an INACTIVE task-definition revision (`service.go:622-647`), but AWS refuses (verify).
-- **G21. Capacity providers not taggable.** Capacity-provider ARNs are rejected by the tag parser (`tags.go:52-71`), although create accepts tags.
-- **G22. Delete leaves stop inboxes.** `DeregisterContainerInstance` deletes the assignments inbox but not the stops inbox (`cluster_teardown.go:87-90`).
-
-## 9. First-resource candidates (ranked)
-
-1. **taskdefinition.**
-   Size: about 300 lines in `service.go:416-706` plus about 180 in `conv.go:70-265`, plus `TaskDefRecord` and `ContainerDef` (`records.go:211-328`).
-   Private state: `taskdef-families/*` only.
-   Dependencies: none at the handler. PassRole is checked in the gateway on role ARN strings, so extraction does not require deciding IAM authority. It has no EC2, network or Bluebottle dependency.
-   Consumers needing a narrow query: task launch, service create/update (`resolveTaskDef`), gateway PassRole lookup, and tags.
-   Tests: `handlers/ecs/taskdef_validation_test.go`, `service_test.go` (`RegisterTaskDefinition_*`, `ListTaskDefinitions_*`, `DescribeTaskDefinition_Unknown`), `cluster_teardown_test.go` (`DeregisterTaskDefinition_*`), `conv_test.go`, `records_test.go`, `tags_test.go` (`TaskDefinitionRoundTrip`).
-   Known gaps: G9 and G20.
-   Assessment: the best first slice. Its `resolveTaskDef` edge also gives the ADR-0004 rollout item 3 capability seam.
-2. **capacityprovider.**
-   Size: about 220 lines in `capacity_providers.go` plus records.
-   Private state: `capacity-providers/*`.
-   Dependencies: none; the provider is inert, and the ASG ARN is opaque.
-   Prerequisite split: `PutClusterCapacityProviders` must move to `cluster`.
-   Tests: `handlers/ecs/capacity_providers_test.go`.
-   Gaps: G12, G21, and an unchecked delete.
-   Assessment: the smallest and safest slice, with no IAM, EC2 or Bluebottle dependency. It has almost no consumers, so it demonstrates little about cross-resource capabilities.
-3. **cluster.**
-   Size: CRUD of about 180 lines (`service.go:231-411`) plus `PutClusterCapacityProviders`.
-   Private state: `clusters/<n>/meta`.
-   Dependencies: counts read instance, task and service records (`service.go:300-335`), and its existence is checked by task and service launch. The delete cascade is orchestration over every child (G5).
-   Tests: `service_test.go` (`CreateCluster_Idempotent`, `DescribeClusters_*`, `ListClusters_*`), `cluster_teardown_test.go` (`DeleteCluster_*`), `capacity_providers_test.go`, `tags_test.go`, `accountteardown/reapers_ecs_test.go`.
-   Assessment: a CRUD-only slice is feasible without IAM or Bluebottle. DeleteCluster has to stay as domain-root orchestration until the children have owners.
-4. **containerinstance.**
-   Size: about 450 lines across `instances.go`, part of `cluster_teardown.go` and `placement.go`.
-   Private state: `instances/*` plus the reservation fields, which placement and task CAS-mutate.
-   Dependencies: the agent wire contract, EC2 (`instancetypes`; `RunInstances` via `ProvisionCapacity`) and IAM (`ecsInstanceRole`).
-   Tests: `instance_recovery_test.go`, `instance_list_status_test.go`, `placement_test.go`, `eni_placement_test.go`, `capacity_test.go`, `scheduler_test.go`, `task_accounting_test.go`, `service_test.go` (GPU/register/heartbeat/reaper).
-   Assessment: G3 must be characterized first, and reservation ownership needs a decision. It is not a first slice.
-5. **service.**
-   Size: about 1,050 lines (`services.go`, `deployments.go`).
-   Dependencies: task launch and stop, taskdefinition, ELBv2 targets and EC2 EIP. Its reconcile runs on both the leader and the API workers (G4).
-   Tests: `services_test.go`, `deployments_test.go`, `revisit_test.go`, `eip_test.go`, `services_eip_nats_test.go`, gateway `actions_test.go` and `azrebalancing_test.go`.
-   Assessment: it depends on the task and taskdefinition owners existing.
-6. **task.**
-   Size: about 1,300 lines (`tasks.go`, `task_stop.go`, `tasks_gc.go`, `poll.go`, `eni.go`, `netcfg.go`, `gpu_report.go`, part of `instances.go`).
-   Dependencies: EC2 ENI and EIP, ELBv2, container-instance reservations, the agent contract, the ClientToken store and gateway PassRole.
-   Tests: `eni_test.go`, `eni_placement_test.go`, `tasks_gc_test.go`, `task_accounting_test.go`, `poll_test.go`, `gpu_report_test.go`, `service_test.go` (`RunTask_*`, `RecordTaskState_*`), gateway `clienttoken_test.go`, `actions_test.go`, and e2e `tests/e2e/ecs/ecs_test.go` and `tests/e2e/gpu/ecs_gpu_test.go` in the mulga root.
-   Assessment: it carries most of G2, G3, G6, G7, G8 and G10, so it should be extracted last.

docs/package-boundary/adr-0004-inventory-eks.md

@@ -1,348 +0,0 @@
-# ADR-0004 source inventory — EKS
-
-Code-read inventory of `refactor/spinifex-package-boundaries` at `6c50349bd`.
-It is not verified against a live system, and the branch is 59 commits behind `dev` at the time of writing, so recheck any file it cites before acting on it.
-Items marked "verify" or "uncertain" are unconfirmed.
-Gaps record observed behaviour, not target behaviour.
-
-Add-on update: every add-on row, the add-on boundary edges (Section 7.1) and the add-on gaps were re-read from the code at `18794e017`, after the add-on owner and guest wire contract were extracted.
-That update is a code read only; it is not verified against a live system.
-Everything else still describes `6c50349bd`, and line numbers outside the add-on rows may have drifted.
-The add-on lifecycle contract and its gap list are in `docs/package-boundary/eks-addon-lifecycle.md`; the proposed lifecycle correction is in `docs/package-boundary/eks-addon-lifecycle-design.md`.
-
-
-Repo: `spinifex` submodule, branch `refactor/spinifex-package-boundaries`; all paths are relative to `spinifex/spinifex/` unless stated.
-Binding inputs: ADR-0001, ADR-0002, ADR-0003, ADR-0004 and Q-78 as on `origin/main` of mulga.
-Proposed resource vocabulary (ADR-0004 S1): `cluster`, `nodegroup`, `addon`, `access`.
-Line numbers are from the working tree at time of writing; "uncertain" marks inferences not verified against a test or live system.
-
-## 1. File table
-
-Production `.go` files only; `_test.go` excluded.
-Line counts are `wc -l`.
-
-### 1.1 `handlers/eks/` (single package `handlers_eks`, 37 files, 12,310 lines at `6c50349bd`)
-
-The add-on files `addon_catalog.go`, `addon_installer.go` and `addon_state.go` have left this package; Section 1.1a lists where they went.
-The add-on rows below are at `18794e017`.
-
-| File | Lines | Class | Reason |
-|---|---|---|---|
-| `access_entry.go` | 255 | resource owner (`access`) | AccessEntry record CRUD/CAS, supported access-policy catalogue, scope validation, ARN/record projection. |
-| `addon_status_reconciler.go` | 32 | temporary residue (report relay hosted on `cluster`) | `applyAddonStatusReport` (`:22`) converts a `contracts/eks/v1` `AddonStatusReport` into `addon.Report` and calls the reconciler-declared `addonReports` capability (`:13-15`, bound to `*addon.Owner` at `:17`); `report.Version` and `report.TS` are dropped. Removal: add-on delivery observation gets its own host (Section 7.1, edge E3). |
-| `addons.go` | 287 | AWS protocol adapter (`addon`) + internal guest route | Six AWS add-on actions validate against `addon.Lookup`, check cluster existence (`acctKVForCluster`), call `*addon.Owner` and project `addon.Record` to SDK shapes (`addonRecordToAWS` `:222`); `UpdateAddon` builds the `Successful` update with `Id = rec.Arn` (`:192-197`). Internal `ListStagedAddonManifests` (`:117`) copies `addon.Manifest` into `eksv1.StagedAddonManifest`. `addons()` (`:265`) builds a new `Owner` per call; `addonBucket` (`:278`) opens the account bucket for the staging installer. |
-| `awsgw_endpoint.go` | 42 | resource owner (`cluster`) | Builds the per-cluster OIDC issuer URL. |
-| `claim.go` | 31 | resource owner (`nodegroup`) | CAS-create helper; only caller is `nodegroup.go:123`. |
-| `clienttoken.go` | 57 | resource owner (`cluster`) | CreateCluster `clientRequestToken` store (`spinifex-eks-clustertokens`). |
-| `cluster_reconciler.go` | 1209 | resource owner (`cluster`), mixed | Cluster readiness/health, CP restart, member replacement, etcd reset; also hosts the add-on status subscription for `addon` (`addonReports` field `:171-176`, `WithAddonStatusSource` `:264-269`, plain subscribe `:447-458`), wired in `service_impl.go:2160-2163` (add-on rows at `18794e017`). |
-| `cluster_state.go` | 515 | resource owner (`cluster`) | `ClusterMeta` record, status enum, CAS helpers, delete-reap bookkeeping, `DeleteClusterPrefix` (`:493`, which also erases nodegroup, access and add-on record and manifest keys by prefix without calling the add-on owner). |
-| `cp_vpc.go` | 128 | resource owner (`cluster`) | EKS-specific managed CP VPC realization via `domains/network/systemvpc`. |
-| `eks_billable_reaper.go` | 133 | resource owner (`cluster`) | Node-local GC backstop for CP VMs whose meta is gone; cites `ADR-0003:S2` as the general lifecycle envelope, while the resource's own contract remains incomplete. |
-| `eks_deleting_reaper.go` | 159 | resource owner (`cluster`) | Re-drives DELETING clusters with backoff and exhaustion. |
-| `igw.go` | 32 | resource owner (`cluster`) | Legacy-topology IGW ensure/delete via `systemvpc`. |
-| `k3s_ha_control_plane.go` | 628 | resource owner (`cluster`), mixed | CP placement, spread, replacement, fresh CP; also contains `natsHostScheduler` (`:461-628`), a NATS fan-out over `contracts/cluster/v1` that is a capability implementation and should move to the composition root or the scheduler owner. |
-| `k3s_server_role.go` | 51 | resource owner (`cluster`) | Ensures the system CP instance profile through `handlers_iam.EnsureSystemInstanceProfile` (identity dependency). |
-| `k3s_server_vm.go` | 700 | resource owner (`cluster`) + guest wire contract | CP VM/ENI launch/terminate and AMI lookup; `buildK3sUserData` (`:524+`) is the CP guest boot contract; `lookupEKSGPUNodeAMI` is used by `nodegroup`. |
-| `nats_bootstrap.go` | 347 | resource owner (`cluster`) + guest wire contract | Bootstrap subject vocabulary and envelope (`:47-57`) are a guest contract; persistence of token/kubeconfig/JWKS/CA is cluster state. |
-| `nlb.go` | 591 | resource owner (`cluster`) | EKS-specific NLB realization via an ELBv2-shaped capability; also `ReapLBCLoadBalancers` (`:517`) for LBC-created ALBs in the customer VPC. |
-| `nodegroup.go` | 1409 | resource owner (`nodegroup`), mixed | Nodegroup CRUD, scaling CAS, worker launch/terminate; also creates customer IAM roles/instance profiles (`:768-838`), stages a GPU add-on (`stageGPUDeviceAddon` `:405-416`, called from `:473`, through `addons().EnsureGPUDevicePlugin`; at `18794e017`), and decrypts the cluster join token (`:1160`, reads `cluster` secrets). |
-| `nodegroup_userdata.go` | 171 | resource owner (`nodegroup`) + guest wire contract | Worker cloud-init (K3S_TOKEN, registry mirrors) is the worker guest boot contract. |
-| `oidc_keypair.go` | 202 | resource owner (`cluster`) | Per-cluster OIDC signing key (encrypted) and JWKS. |
-| `private_endpoint.go` | 88 | resource owner (`cluster`) | Customer-VPC private-endpoint ENI. |
-| `reconciler_registry.go` | 183 | resource owner (`cluster`) | In-process registry of per-cluster reconciler goroutines. |
-| `recovery_directive.go` | 144 | resource owner (`cluster`) + guest wire contract | Per-member etcd recovery directive; `RecoveryDirective` is read by the on-VM recovery agent. |
-| `restore_snapshot.go` | 355 | resource owner (`cluster`) | Operator-only total-loss DR path; reads etcd snapshots through `providers/objectstore`. |
-| `security_groups.go` | 505 | resource owner (`cluster`), mixed | Cluster SG realization in system and customer VPCs; `EnsureNodegroupSGRules` (`:351`) is nodegroup realization. |
-| `service.go` | 73 | temporary residue | `EKSService`, one method per AWS action plus internal methods; it is the NATS RPC surface between gateway and daemon and is also consumed by `accountteardown` and the operator CLI; removal condition: per-resource commands behind `domains/eks/awsapi` registration (ADR-0004 S3) with any remaining process boundary versioned under `contracts/eks/v1`. |
-| `service_impl.go` | 2518 | mixed: cross-resource orchestration + resource owners | Split: CreateCluster launch sequence and `purgeClusterInfra` (`:504-1013`, `:1268-1452`) are cluster realization orchestration; cluster read/delete (`:1041-1162`) is `cluster`; access-entry actions (`:1527-1816`) are `access`; nodegroup wrappers (`:1465-1525`) are `nodegroup`; Tag actions (`:1842-1992`) span cluster and nodegroup; IdP stubs (`:1818-1832`); `DesiredDNSChanges` (`:2143`) is a DNS projection; `EKSServiceDeps` and capability interfaces (`:39-245`). |
-| `service_nats.go` | 190 | temporary residue | Gateway-side NATS client implementing `EKSService` over `eks.<Action>` subjects; removal condition as for `service.go`. |
-| `state_report.go` | 75 | guest/controller wire contract (`cluster`) | `eks.state.*` subject and `ServerStateReport`; the add-on subject and report moved to `contracts/eks/v1`, leaving only `unmarshalAddonStatusReport` (`:69`), which decodes into `eksv1.AddonStatusReport` (add-on rows at `18794e017`). |
-| `store.go` | 255 | resource owner (`cluster`), mixed | Bucket names, leader bucket, migrations, key helpers for every resource (the add-on key helpers have moved to `domains/eks/addon/store.go`; 217 lines at `18794e017`); split per-resource key helpers into each owner; `AccountWatchBuckets` (`:215`) is effectively a projection consumed by DNS. |
-| `token_review.go` | 128 | resource owner (`access`) | Authenticates a `get-token` bearer against STS and maps to AccessEntry groups; `ResolveTokenReview` is called from the gateway process and opens EKS KV there. |
-| `token_webhook.go` | 57 | guest/controller wire contract | `eks.VerifyToken` subject and request/response consumed by `runtime/roles/awsgw`. |
-| `types.go` | 131 | mixed: `nodegroup` + `access` + residue | `NodegroupRecord` (nodegroup), `AccessEntryRecord`/`AccessScope` (access); `ClusterRecord` (`:19`) and `OIDCProviderConfigRecord` (`:125`) have no production readers or writers (temporary residue; removal: delete). |
-| `volume_reclaim.go` | 81 | resource owner (`cluster`) | Surfaces (never deletes) CSI-created EBS volumes at cluster delete. |
-
-### 1.1a Add-on owner and wire contract (at `18794e017`)
-
-| File | Lines | Class | Reason |
-|---|---|---|---|
-| `domains/eks/addon/record.go` | 52 | resource owner (`addon`) | `Record` and `Manifest` (persisted JSON fields unchanged from the former `AddonRecord`), `Status` (six values; `UPDATE_FAILED` and `DELETE_FAILED` absent), `ErrNotFound`, `ErrExists`. Moved from `addon_state.go`. |
-| `domains/eks/addon/store.go` | 237 | resource owner (`addon`) | `Prefix`, `Key`, `ManifestKey`, `Get`, `List`, `ListManifests` (exported) and `put`, `putManifest`, `deleteManifest`, `deleteRecord`, `casUpdate` (private). `put` is unconditional, so create is get-then-put. Moved from `addon_state.go` and the add-on key helpers in `store.go`. |
-| `domains/eks/addon/owner.go` | 205 | resource owner (`addon`) | `Owner` is the sole writer: `Create` (`:48`), `EnsureGPUDevicePlugin` (`:79`), `Update` (`:94`), `Delete` (`:122`), `ApplyReport` (`:140`), `markFailed` (`:171`, always `CREATE_FAILED`), `nextStatus` (`:186`). `Desired` and `Change` are the validated inputs. Moved from `addons.go`, `addon_status_reconciler.go` and the former `markAddonFailed`/`nextAddonStatus`. |
-| `domains/eks/addon/delivery.go` | 77 | resource owner (`addon`) | `Installer` interface, `StagingInstaller` (writes or deletes the manifest sub-key), `Bucket`, and the owner's own `Phase`/`Report` (no version or timestamp). Moved from `addon_installer.go`. |
-| `domains/eks/addon/catalog.go` | 122 | resource owner (`addon`), read-only catalogue | Static bundles `aws-load-balancer-controller` 2.11.0, `argocd` 3.0.23, `aws-ebs-csi-driver` 1.40.1, hidden `nvidia-device-plugin` 0.17.4 and hidden `spinifex-noop` 0.1.0; `Lookup`, `Specs`, `ValidateCatalog`. Moved from `addon_catalog.go`. |
-| `contracts/eks/v1/addon_status.go` (repo root) | 37 | guest/controller wire contract | `AddonStatusSubject`, `AddonDeliveryPhase` (`applied`, `ready`, `failed`), `AddonStatusReport` (`addon`, `version`, `phase`, `message`, `ts`). Moved from `state_report.go`. |
-| `contracts/eks/v1/addon_manifest.go` (repo root) | 18 | guest/controller wire contract | `StagedAddonManifest` and `InternalAddonsResponse`; the deployed HTTP body uses Go field names because the REST-JSON marshaller ignores the `json` tags, which `addon_manifest_test.go` pins. Moved from `addon_installer.go`, `gateway/eks` (`internalAddonsOutput`) and `agents/eks/gatewayfetch` (`stagedAddon`, `internalAddonsResponse`). |
-| `contracts/eks/v1/README.md` | n/a | contract record | Compatibility boundary: a generation-aware protocol needs a successor version, not a v1 edit. |
-
-Guest counterparty: `scripts/images/eks-node/mulga-eks-addon-sync.sh` (417 lines) runs on the primary server only, renders per add-on name, GCs rendered files whose name is no longer staged with `kubectl delete --wait=false`, ignores `configurationValues`, and builds the report with `printf`.
-
-Additional dead code observed in `store.go`: `OIDCProviderKey` (`:95`) and `EventKey` (`:132`) have no production callers.
-
-### 1.2 `gateway/eks.go` and `gateway/eks/` (15 files, 1,473 lines)
-
-| File | Lines | Class | Reason |
-|---|---|---|---|
-| `gateway/eks.go` | 379 | AWS protocol adapter | REST-JSON route table (`:27-219`), dispatch, authorization ordering, `iam:PassRole` check (`:319-347`), caller identity extraction. |
-| `gateway/eks/access.go` | 89 | AWS protocol adapter | Unmarshal and NATS call for access actions. |
-| `gateway/eks/addons.go` | 60 | AWS protocol adapter | Same for add-on actions. |
-| `gateway/eks/authz.go` | 310 | AWS protocol adapter | Per-action resource ARN derivation (`eksScopes` `:45-109`); imports `handlers_eks.PrincipalARNHash`. |
-| `gateway/eks/cluster.go` | 66 | AWS protocol adapter | Cluster action wrappers. |
-| `gateway/eks/handler.go` | 47 | AWS protocol adapter | JSON response/error writers. |
-| `gateway/eks/internal_addons.go` | 33 | guest/controller wire contract | `GET /clusters/{c}/internal-addons/{acct}` for the addon-sync agent; calls `handlers_eks.NewNATSEKSService(...).ListStagedAddonManifests` and wraps the result as `eksv1.InternalAddonsResponse` (at `18794e017`). |
-| `gateway/eks/internal_authz.go` | 129 | AWS protocol adapter + temporary residue | Principal gate for internal routes; `lookupClusterMeta` (`:102-129`) reads `eks-account-*` and `ClusterMeta` directly from the gateway process; removal: owner-published membership query. |
-| `gateway/eks/internal_publish.go` | 91 | guest/controller wire contract | HTTP-to-NATS relay of bootstrap/state/addon channels; the `addon` channel relays the raw payload to `eksv1.AddonStatusSubject` (`:75-76`, at `18794e017`). |
-| `gateway/eks/internal_recovery.go` | 37 | guest/controller wire contract | `GET .../internal-recovery/{acct}/{instance}`. |
-| `gateway/eks/nodegroup.go` | 65 | AWS protocol adapter | Nodegroup wrappers. |
-| `gateway/eks/oidc.go` | 45 | AWS protocol adapter | IdP-config wrappers (handlers are stubs). |
-| `gateway/eks/passrole.go` | 43 | AWS protocol adapter | Extracts passed role ARNs for CreateCluster, CreateNodegroup, CreateAddon and UpdateAddon. |
-| `gateway/eks/tags.go` | 35 | AWS protocol adapter | Tag wrappers. |
-| `gateway/eks/token_review.go` | 51 | guest/controller wire contract | `POST /clusters/{c}/token-review` for the in-VM token webhook. |
-
-### 1.3 Other production files wiring or reading EKS
-
-| File | Lines | Class | Reason |
-|---|---|---|---|
-| `daemon/eks_deps.go` | 220 | cross-resource orchestration (composition-root binding) | Builds `EKSServiceDeps`, binding VPC, ELBv2, image, EIP, IGW, NAT-GW, route-table, placement-group, IAM, EBS and object-store owners. |
-| `daemon/eks_adapter.go` | 58 | cross-resource orchestration (composition-root binding) | `SubnetVPCResolver` over the VPC service. |
-| `daemon/eks_cp_control.go` | 67 | temporary residue | CP describe/start/stop calls `domains/ec2/awsapi/instance.{DescribeInstances,StartInstances,StopInstances}`, i.e. an EC2 AWS adapter used as an internal API (ADR-0004 S3/S4); removal: EC2-owned instance-control capability. |
-| `daemon/eks_worker_launch.go` | 175 | cross-resource orchestration (composition-root binding) | `WorkerLauncher` via `instanceService.RunInstances`, `ec2.RunInstances.{type}.{node}` and `contracts/ec2/v1` instance commands. |
-| `daemon/daemon.go` (EKS parts: `:75`, `:167`, `:1085-1128`, `:2018-2019`, `:2175`, `:2302-2304`, `:3017-3018`) | n/a | temporary residue | NATS subscription table for 38 `eks.*` subjects, service init, reconciler spawn, reaper registration, shutdown; comment at `:1085-1087` says every handler returns NotImplemented, which is stale. |
-| `daemon/dns_reconcile.go` (`:27`, `:64-68`, `:97-98`) | n/a | temporary residue | Watches every `eks-account-*` bucket via `AccountWatchBuckets` and pulls `DesiredDNSChanges`; removal: EKS-published DNS projection. |
-| `gateway/oidc_discovery.go` | 117 | temporary residue | Public OIDC discovery/JWKS endpoints read `eks-account-*`, `GetClusterMeta` and `OIDCJWKSKey` directly; removal: cluster-owned OIDC projection. |
-| `handlers/sts/oidc_jwks.go` | 126 | temporary residue (STS-owned file) | `FetchClusterJWKS` reads `eks-account-*`/`OIDCJWKSKey` (`:101-110`); ADR-0001 step 7 names this STS-to-EKS import for removal via an identity-owned contract or EKS projection. |
-| `runtime/roles/awsgw/eks_token_verify.go` | 75 | guest/controller wire contract (server) | Queue-subscribes `eks.VerifyToken` (`:30`) and verifies presigned STS URLs. |
-| `accountteardown/reapers_eks.go` | 138 | cross-resource orchestration | Account teardown deletes nodegroups then clusters through the `EKSService` NATS client. |
-| `operator/cli/admin_eks.go` | 77 | guest/controller wire contract (operator) | `spx admin eks restore-snapshot` over `eks.RestoreSnapshot`. |
-| `gateway/gateway.go` (`:300`, `:345-346`, `:403`, `:455`, `:513`), `gateway/auth.go` (`:278-279`), `gateway/operations.go` (`:34-36`) | n/a | AWS protocol adapter (legacy registration residue) | Legacy dispatch, OIDC public routes and inventory; removal per ADR-0002 when `domains/eks/awsapi` registers. |
-| `bootstrap/config/config.go:44` | n/a | AWS protocol adapter (config) | `eks` listed in `AWSGWServiceNames`. |
-
-Guest counterparties (not inventoried individually): `agents/eks/{gatewaypublish,gatewayfetch,tokenwebhook,credentialprovider,konnectivitycert,webhookcert}` and `scripts/images/eks-node/*`.
-`agents/eks/gatewayfetch/fetch.go` decodes the internal-addons body through `contracts/eks/v1` `InternalAddonsResponse` (at `18794e017`).
-`agents/eks/tokenwebhook/webhook.go` imports `handlers_eks.WebhookTokenReviewResult`, so a guest binary imports the domain monolith (ADR-0004 S5 debt).
-
-## 2. Served AWS actions and protocol
-
-Protocol: REST-JSON, method+path routed by `rest.NewRouter("eks", eksRoutes)` (`gateway/eks.go:234`); unknown routes return `InvalidAction` (`:241`).
-Authorization order (`gateway/eks.go:237-312`): route lookup, account from auth context, internal-route principal gate, body read, query folding, `ResourceARNs`, `checkPolicyResources`, `checkEKSPassRole`, handler over NATS.
-Gateway-to-daemon transport: NATS request/reply `eks.<Action>`, queue group `spinifex-workers` (`daemon/daemon.go:1090-1127`).
-`gateway/operations.go:34-36` declares EKS as `Registered: eksActionNames()` with no `Stubbed` or `Unsupported` list, so stubs and internal routes are indistinguishable in the inventory.
-
-| Action | Route | Status | Owner |
-|---|---|---|---|
-| CreateCluster | POST /clusters | implemented (async) | cluster |
-| ListClusters | GET /clusters | implemented | cluster |
-| DescribeCluster | GET /clusters/{c} | implemented | cluster |
-| DeleteCluster | DELETE /clusters/{c} | implemented (synchronous purge) | cluster |
-| UpdateClusterConfig | POST /clusters/{c}/update-config | stub NotImplemented (`service_impl.go:1455`) | cluster |
-| UpdateClusterVersion | POST /clusters/{c}/updates | stub NotImplemented (`service_impl.go:1459`) | cluster |
-| ListUpdates | GET /clusters/{c}/updates | stub at gateway (`gateway/eks.go:49-52`) | cluster (update records do not exist) |
-| DescribeUpdate | GET /clusters/{c}/updates/{id} | stub at gateway (`gateway/eks.go:53-56`) | cluster/nodegroup/addon |
-| CreateNodegroup | POST /clusters/{c}/node-groups | implemented (async) | nodegroup |
-| ListNodegroups | GET /clusters/{c}/node-groups | implemented | nodegroup |
-| DescribeNodegroup | GET .../node-groups/{n} | implemented | nodegroup |
-| UpdateNodegroupConfig | POST .../node-groups/{n}/update-config | implemented (synchronous, returns `Successful`) | nodegroup |
-| UpdateNodegroupVersion | POST .../node-groups/{n}/update-version | stub NotImplemented (`nodegroup.go:1120-1127`) | nodegroup |
-| DeleteNodegroup | DELETE .../node-groups/{n} | implemented (synchronous) | nodegroup |
-| CreateAccessEntry | POST /clusters/{c}/access-entries | implemented (STANDARD type only, `service_impl.go:1561-1564`) | access |
-| ListAccessEntries | GET /clusters/{c}/access-entries | implemented | access |
-| DescribeAccessEntry | GET .../access-entries/{p} | implemented | access |
-| UpdateAccessEntry | POST .../access-entries/{p} | implemented | access |
-| DeleteAccessEntry | DELETE .../access-entries/{p} | implemented | access |
-| AssociateAccessPolicy | POST .../access-entries/{p}/access-policies | implemented (cluster scope only, `service_impl.go:1692`) | access |
-| DisassociateAccessPolicy | DELETE .../access-policies/{policy} | implemented | access |
-| ListAssociatedAccessPolicies | GET .../access-entries/{p}/access-policies | implemented | access |
-| ListAccessPolicies | GET /access-policies | implemented (static catalogue) | access |
-| DescribeAddonVersions | GET /addons/supported-versions | implemented (static catalogue) | addon |
-| ListAddons | GET /clusters/{c}/addons | implemented | addon |
-| CreateAddon | POST /clusters/{c}/addons | implemented (async via guest) | addon |
-| DescribeAddon | GET .../addons/{a} | implemented | addon |
-| UpdateAddon | POST .../addons/{a}/update | implemented | addon |
-| DeleteAddon | DELETE .../addons/{a} | implemented | addon |
-| AssociateIdentityProviderConfig | POST .../identity-provider-configs/associate | stub NotImplemented (`service_impl.go:1818`) | cluster (no IdP resource) |
-| DescribeIdentityProviderConfig | POST .../identity-provider-configs/describe | stub | cluster |
-| DisassociateIdentityProviderConfig | POST .../identity-provider-configs/disassociate | stub | cluster |
-| ListIdentityProviderConfigs | GET .../identity-provider-configs | stub | cluster |
-| TagResource / UntagResource / ListTagsForResource | POST/DELETE/GET /tags/* | implemented for cluster and nodegroup ARNs; NotImplemented for addon/access-entry ARNs (`service_impl.go:1882,1925,1958`) | cluster, nodegroup |
-| PublishInternal | POST /clusters/{c}/internal-publish | internal (not AWS) | cluster guest contract |
-| WebhookTokenReview | POST /clusters/{c}/token-review | internal (not AWS) | access guest contract |
-| ListInternalAddons | GET /clusters/{c}/internal-addons/{acct} | internal (not AWS) | addon guest contract |
-| GetRecoveryDirective | GET /clusters/{c}/internal-recovery/{acct}/{id} | internal (not AWS) | cluster guest contract |
-
-Internal NATS-only methods (no HTTP route): `SetRecoveryDirective`, `RestoreSnapshot`, `ListStagedAddonManifests` (backs ListInternalAddons).
-Unregistered AWS EKS operations (InvalidAction today) include Fargate profiles, pod identity associations, encryption config, insights, `RegisterCluster`/`DeregisterCluster`, `DescribeAddonConfiguration`, `DescribeClusterVersions` and EKS Anywhere subscriptions (list from memory of the EKS API, not verified against the pinned model).
-`iam:PassRole` is enforced for CreateCluster `roleArn`, CreateNodegroup `nodeRole`, and CreateAddon/UpdateAddon `serviceAccountRoleArn` (`gateway/eks/passrole.go`).
-
-## 3. Durable records and KV buckets
-
-| Bucket | Key | Record | Owner | Writers | Readers outside EKS |
-|---|---|---|---|---|---|
-| `eks-account-{acct}` (history 1, migration v1, `store.go:30-38,235-245`) | `clusters/{c}/meta` | `ClusterMeta` (`cluster_state.go:65`) | cluster | `service_impl.go` create/launch/delete/tags, `cluster_state.go` CAS helpers, `cluster_reconciler.go`, `restore_snapshot.go:150`, `nats_bootstrap.go:297` (CA) | **`gateway/eks/internal_authz.go:110-124`**, **`gateway/oidc_discovery.go:38,60`** |
-| same | `clusters/{c}/nodegroups/{ng}` | `NodegroupRecord` (`types.go:33`) | nodegroup | `nodegroup.go` (claim, CAS, put, delete), `service_impl.go:1964` (tags) | none observed |
-| same | `clusters/{c}/access-entries/{sha256(principalARN)}` | `AccessEntryRecord` (`types.go:93`) | access | `service_impl.go:1548-1785`, creator-admin seed `service_impl.go:994-1001` (cluster launch) | **gateway process via `ResolveTokenReview` (`token_review.go:113`)** |
-| same | `clusters/{c}/addons/{a}` | `addon.Record` (`domains/eks/addon/record.go`) | addon | `domains/eks/addon/owner.go` only, reached from `addons.go`, `addon_status_reconciler.go` and `nodegroup.go:405-416` (GPU add-on); plus the `DeleteClusterPrefix` sweep (at `18794e017`) | none observed |
-| same | `clusters/{c}/addons/{a}/manifest` | `addon.Manifest` (`domains/eks/addon/record.go`) | addon | `domains/eks/addon/delivery.go` `StagingInstaller`, `Owner.Delete`, the `DeleteClusterPrefix` sweep | guest via internal-addons route, as `contracts/eks/v1` `StagedAddonManifest` |
-| same | `clusters/{c}/oidc-signing-key.pem.enc` | encrypted ECDSA key | cluster | `oidc_keypair.go:86`, zeroized `service_impl.go:1281` | none |
-| same | `clusters/{c}/oidc-jwks.json` | JWKS | cluster | `oidc_keypair.go:94` | **`handlers/sts/oidc_jwks.go:110`**, **`gateway/oidc_discovery.go:93`** |
-| same | `clusters/{c}/oidc-jwks-verified` | marker | cluster | `nats_bootstrap.go:291` | none |
-| same | `clusters/{c}/admin-kubeconfig.enc` | encrypted kubeconfig | cluster | `nats_bootstrap.go:269` | none observed in Go (uncertain whether a CLI reads it) |
-| same | `clusters/{c}/k3s-node-token.enc` | encrypted join token | cluster | `nats_bootstrap.go:251` | read by `nodegroup.go:1164` (cross-resource) |
-| same | `clusters/{c}/recovery/{instanceId}` | `RecoveryDirective` | cluster | `recovery_directive.go:102`, reconciler | guest via internal-recovery route |
-| same (whole bucket) | all keys | watch | cluster | n/a | **`daemon/dns_reconcile.go:68`** via `AccountWatchBuckets` |
-| `spinifex-eks-leader` (TTL 60s, `store.go:40-42,249`) | `{acct}/{c}` reconciler lease; `.../teardown` lease | kvlease | cluster | `cluster_reconciler.go:391-403`, `service_impl.go:1174-1197` | none |
-| `spinifex-eks-clustertokens` (TTL 15m, `clienttoken.go:16-21`) | idempotency store keyed by account+token | `idempotency.Store[string]` | cluster | `service_impl.go:529-558,612-619` | none |
-
-Dead key helpers: `OIDCProviderKey` (`store.go:95`), `EventKey` (`store.go:132`).
-Secret material in `ClusterMeta`: `ControlPlaneTemplate` is a copy of `K3sServerInput` that clears creds but not `OIDCPrivateKeyPEM` or `JoinToken` (`service_impl.go:945-959`; fields at `k3s_server_vm.go:146,171`), so the OIDC private key and k3s join token appear to be persisted unencrypted in the meta record that the gateway also reads; no scrubbing or custom marshaller was found (verify with a KV dump before acting).
-`ClusterMeta` combines desired intent (`Version`, `RoleArn`, `ResourcesVpcConfig`, `Logging`, `Tags`), realization refs (CP nodes, NLB, CP VPC, EIP, ENIs), observations (`HealthIssue`, `NodeCount`, `NodegroupNodeCounts`, `CertificateAuthorityB64`) and delete bookkeeping (`DeletingSince`, `DeleteReapAttempts`).
-Hidden realizations outside EKS stores: CP VMs (system account, `ManagedBy=eks`), CP VPC/subnets/NAT-GW/route tables/IGW, NLB and target groups (ELBv2), SGs, private-endpoint ENI, egress EIP, HA placement group, worker instances (customer account), IAM roles/instance profiles, CSI EBS volumes.
-
-## 4. Identity and scope
-
-| Resource | Identifier / ARN | Account | Region | AZ |
-|---|---|---|---|---|
-| cluster | name; `arn:aws:eks:{region}:{acct}:cluster/{name}` (`foundation/aws/arn/eks.go:21`) | tenant account owns the record; CP VMs, CP VPC and NLB are realized in the system account `awsidentifiers.GlobalAccountID` (`service_impl.go:738`, `:2240-2248`); customer-VPC SGs and the private-endpoint ENI in the tenant account | single Region from `deps.Region`; not stored as a field | Regional; HA CP spread over hosts by AZ (`k3s_ha_control_plane.go:528`) |
-| nodegroup | `arn:...:nodegroup/{c}/{ng}/{discriminator}` where the discriminator is derived from (account, cluster, name) (`nodegroup.go:333`), so a recreated nodegroup gets the same ARN, unlike AWS's per-incarnation UUID | tenant | deps.Region | AZs implied by subnets |
-| addon | `arn:...:addon/{c}/{a}` (`arn/eks.go` `FormatEKSAddon`, minted in `domains/eks/addon/owner.go` `Create`; the gateway authorizer derives the same string, `gateway/eks/authz.go` `addonARN`); no per-incarnation suffix, whereas the AWS API reference examples show `addon/{c}/{a}/{uuid}` | tenant | owner `region` from deps.Region | n/a |
-| access | `arn:...:access-entry/{c}/{sha256(principal)}` (`access_entry.go:244`); AWS's shape differs (verify) | tenant | deps.Region | n/a |
-
-Uniqueness is per account bucket and cluster-name prefix; there is no immutable internal UID distinct from the name for any EKS resource.
-
-## 5. Lifecycle paths per resource
-
-### cluster
-Create: `CreateCluster` `service_impl.go:504`; validates, optional token claim (`:529-558`), resolves VPC, builds `ClusterMeta{Status: CREATING}`, `claimClusterName` CAS-create with FAILED-to-CREATING reclaim (`:1200-1265`), finalizes token, then launches `launchClusterInfra` on an in-process goroutine with `context.Background()` (`:625-645`).
-Launch stages (`:718-1012`): CP VPC, SGs, CP ingress, customer SGs, private ENI, NLB, OIDC keypair, join token, CP placement/launch, persist CP refs, NLB target registration, persist final meta, creator-admin AccessEntry, `spawnBootstrap`, `spawnReconciler`; any failure calls `failClusterLaunch` (`:696`) which marks FAILED.
-Readiness: `ClusterReconciler.reconcileOnce` (`cluster_reconciler.go:594`) flips CREATING to ACTIVE when bootstrap artifacts exist (`:1173-1189`) and the CP state report is healthy; CREATING becomes FAILED after `createTimeout` (`:1151-1171`).
-Reconciliation/repair (ACTIVE): health recording, CP restart (`:708`), etcd reset via recovery directives (`:780`), member replacement (`:1010`), all under the per-cluster lease `spinifex-eks-leader/{acct}/{c}` (`:391-418`).
-Restart: `SpawnRegisteredReconcilers` (`service_impl.go:342-425`) rescans buckets, reclaims orphaned nodegroups, resubscribes missing bootstrap kinds and respawns reconcilers; it does not resume an interrupted `launchClusterInfra`.
-Delete: `DeleteCluster` (`service_impl.go:1104-1155`) returns success for an absent cluster, takes the teardown lease (losers return DELETING), sets DELETING, runs `purgeClusterInfra` synchronously in the request (`:1268-1452`), and only erases `clusters/{c}/` after all blocking steps succeed; `EKSDeletingReaper` re-drives with backoff and exhaustion (`eks_deleting_reaper.go:46-159`); `EKSBillableReaper` terminates CP VMs whose meta is gone (`eks_billable_reaper.go:46-100`).
-Fencing: leases only; no desired/observed generation (no `generation` field anywhere in `handlers/eks`).
-Idempotency: `clientRequestToken` honoured for CreateCluster only, 15-minute TTL, replays current meta by name.
-
-### nodegroup
-Create: `createNodegroup` (`nodegroup.go:205`) requires cluster ACTIVE (`:226-229`), `ClaimNodegroupRecord` CAS-create (`:112-126`, ResourceInUse at `:363`), then background `launchNodegroupInfra` (`:373`, `:449-535`).
-Readiness: `waitWorkersReady` (`:537-563`) polls the cluster's `NodegroupNodeCounts` observation until ready count reaches baseline plus desired, then sets ACTIVE (`:521-523`), else CREATE_FAILED.
-Update: `UpdateNodegroupConfig` runs `reconcileNodegroup` inline under revision CAS (`:914-979`, `launchWorkersCAS` `:985`, `recordLaunchedWorker` `:1038`) and returns `Update{Status: Successful}` (`:900-906`).
-Restart: `reclaimOrphanedNodegroups` (`:1189-1228`) terminates workers of CREATING/partial CREATE_FAILED records and marks them CREATE_FAILED; it does not resume.
-Delete: `deleteNodegroup` (`:1129-1158`) terminates workers then deletes the record synchronously and returns a DELETING projection of a record that no longer exists.
-Fencing: KV revision CAS; no generation, no lease.
-Idempotency: no `clientRequestToken`; duplicate create loses the claim and gets ResourceInUse.
-
-### addon
-Paths in this subsection are at `18794e017`.
-Create: `CreateAddon` (`addons.go:61`) validates against the catalogue and calls `Owner.Create` (`domains/eks/addon/owner.go:48`), which is get-then-put (not CAS-create) and then stages the manifest (`delivery.go` `StagingInstaller.Install`); a staging failure marks `CREATE_FAILED`; returns CREATING.
-Readiness: guest addon-sync agent publishes `eks.addon.{acct}.{c}.status` via internal-publish; `applyAddonStatusReport` (`addon_status_reconciler.go:22`) relays to `Owner.ApplyReport` (`owner.go:140`), whose `nextStatus` (`:186`) maps `ready` to ACTIVE from any status and `failed` to CREATE_FAILED/DEGRADED; `report.Version` and `report.TS` are dropped before the owner sees them.
-Update: `UpdateAddon` (`addons.go:163`) calls `Owner.Update` (`owner.go:94`), which CAS-sets UPDATING from any status and restages; the handler returns `Successful` with `Id = rec.Arn` (`addons.go:193-194`); a staging failure marks `CREATE_FAILED`.
-Delete: `DeleteAddon` (`addons.go:201`) calls `Owner.Delete` (`owner.go:122`), which unstages and deletes the record and manifest immediately; the DELETING status is only in the response.
-Fencing: revision CAS on the record; no generation or incarnation.
-Idempotency: none (`clientRequestToken` ignored).
-
-### access
-Create: `CreateAccessEntry` (`service_impl.go:1548-1581`) get-then-put (`:1570-1578`), synchronous; creator-admin entry seeded during cluster launch (`:994-1001`).
-Mutations: `casUpdateAccessEntry` (`access_entry.go:141`) for update/associate/disassociate; delete via `DeleteAccessEntryRecord`.
-Effect: read at authentication time by `Authenticate`/`ResolveTokenReview` (`token_review.go:31-128`); no in-cluster realization or readiness.
-Delete with cluster: swept by `DeleteClusterPrefix` (`cluster_state.go:493`).
-Idempotency: none (`clientRequestToken` ignored).
-
-## 6. NATS subjects and guest/controller messages
-
-| Subject / route | Direction | Delivery | Defined | Consumer |
-|---|---|---|---|---|
-| `eks.<Action>` (38 subjects) | gateway (or teardown/CLI) to daemon | core NATS request/reply, queue `spinifex-workers` | `service_nats.go:30-189`, `daemon/daemon.go:1090-1127` | `EKSServiceImpl` |
-| `eks.bus.{acct}.{c}.{k3s-bootstrap-token,k3s-admin-kubeconfig,k3s-oidc-jwks,k3s-ca}` | CP VM via gateway relay to daemon | core NATS publish/subscribe, at-most-once | `nats_bootstrap.go:47-57`, subscribe `:197` | `NATSBootstrap` persists to KV with `persistWithRetry` (`:32`) |
-| `eks.state.{acct}.{c}.server` | CP VM via relay | core NATS, periodic | `state_report.go:13` | `ClusterReconciler` (`cluster_reconciler.go:432`) |
-| `eks.addon.{acct}.{c}.status` | CP VM via relay | core NATS, level-triggered every 30 s per staged add-on | `contracts/eks/v1/addon_status.go:10` (at `18794e017`) | `ClusterReconciler` (`:448`), relayed to `addon.Owner.ApplyReport` |
-| `eks.VerifyToken` | EKS (gateway process) to awsgw STS | request/reply, queue `awsgw-eks-token-verify` | `token_webhook.go:13` | `runtime/roles/awsgw/eks_token_verify.go:30` |
-| `contracts/cluster/v1` `NodeStatusSubject`, `NodeVMsSubject` | EKS to every node | fan-out request | `k3s_ha_control_plane.go:502,579` | daemons |
-| `ec2.RunInstances.{type}.{node}` and `contracts/ec2/v1` instance commands | EKS (daemon binding) to EC2 | request/reply | `daemon/eks_worker_launch.go` | EC2 instance owner |
-| HTTP `internal-publish`, `token-review`, `internal-addons`, `internal-recovery` | guest to gateway (SigV4, system CP role) | HTTP | `gateway/eks.go:59-87` | gateway relays/queries |
-| HTTP `/oidc/eks/{region}/{acct}/{c}/.well-known/openid-configuration`, `/keys` | public | HTTP, unauthenticated | `gateway/gateway.go:345-346` | `gateway/oidc_discovery.go` |
-
-Retry and recovery paths: guest `eks-gateway-publish` retries until HTTP 2xx (`agents/eks/gatewaypublish/publish.go:26-31,97-114`), but the gateway returns 2xx after a core-NATS publish even with no subscriber (`gateway/eks/internal_publish.go:84-89`), so a bootstrap message published while the daemon subscriber is absent is lost.
-On restart, `BootstrapPendingKinds` resubscribes only missing kinds (`nats_bootstrap.go:115-139`, `service_impl.go:410-416`); a lost one-shot message is not replayed, so the cluster reaches FAILED at the create timeout.
-A bootstrap subscriber that exits with error marks the cluster FAILED (`service_impl.go:2192-2212`).
-SDK retries of `eks.<Action>` land on any node of the queue group; dedup relies on token store, claim CAS, teardown lease or nothing (addon/access create).
-
-## 7. Boundary-crossing imports
-
-### Outbound from `handlers/eks`
-- IAM/identity: `handlers/iam` `EncryptSecret`/`DecryptSecret` (`nats_bootstrap.go`, `oidc_keypair.go`, `nodegroup.go`), `EnsureSystemInstanceProfile` (`k3s_server_role.go`), `SystemInstanceRoleEnsurer` (`service_impl.go`); `instanceProfileEnsurer` capability with `GetRole/CreateRole/PutRolePolicy/CreateInstanceProfile/AddRoleToInstanceProfile` (`service_impl.go:214`) means EKS creates IAM roles and instance profiles.
-- Bluebottle: `bluebottle/pkg/auth` `ResolveRoleARN` (`nodegroup.go`) and `ParseRoleARN` (`service_impl.go`); daemon binding uses `bluebottle/pkg/masterkey`; gateway uses `bluebottle/pkg/auth` (`gateway/eks.go`).
-- STS: token verification over `eks.VerifyToken` (process contract, no import).
-- EC2 instance: `domains/ec2/systeminstance` (`SystemInstanceInput/Output`, `ExtraENIInput`, `BootAMI`), `domains/ec2/instancetypes`, `domains/ec2/placementgroup` (alias `ec2placementgroup`), `domains/ec2/ebs/metadata`, `runtime/compute/vm` (reapers), plus `WorkerLauncher`, `k3sInstanceLauncher`, `k3sAMIResolver`, `eipProvisioner`, `csiVolumeReclaimer` capabilities.
-- Network: `domains/network/systemvpc` (`Ensure`, `Delete`, `EnsureIGW`, `DeleteIGW`, provisioner interfaces); `sgProvisioner` and `k3sVPCProvisioner` capabilities (EC2 SDK shapes).
-- DNS: `domains/dns` `EKSName`, `EKSChanges`, `ResolveBaseDomain`, `Change`, `ActionUpsert` (`service_impl.go`).
-- ACM/CA: no import; the platform/NATS CA PEM is passed as `GatewayCACert` from `d.config.NATS.CACert` (`daemon/eks_deps.go:64-71`) and baked into CP user data; the k3s cluster CA is self-generated in the guest and returned on `k3s-ca`.
-- ELBv2: `nlbProvisioner` capability with ELBv2 SDK shapes (`nlb.go:19`), bound to `d.elbv2Service`.
-- Storage/object: `providers/objectstore` (etcd snapshots), Predastore system creds via deps.
-- Contracts: `contracts/cluster/v1`; `contracts/eks/v1` from `addons.go`, `addon_status_reconciler.go`, `state_report.go` and `service_impl.go` (at `18794e017`).
-- EKS add-on owner: `domains/eks/addon` from `addons.go`, `addon_status_reconciler.go`, `service_impl.go` and, through `addons()`, `nodegroup.go` (at `18794e017`; edges in Section 7.1).
-- Foundation: `kvutil`, `kvstore`, `kvlease`, `migrate`, `idempotency`, `reconciler`, `telemetry`, `aws/{arn,errors,tags,identifiers,ami}`, `messaging/nats`.
-- Bootstrap: `bootstrap/config.Config` in `EKSServiceDeps`.
-
-### Inbound to `handlers/eks`
-- `gateway/eks/*`: `NewNATSEKSService`, input/output types (including `ListStagedAddonManifestsInput` in `internal_addons.go`), `PrincipalARNHash`, `ClusterMeta`, `ClusterMetaKey`, `AccountBucketName`, `CPInstanceRoleName`, `ResolveTokenReview`, bootstrap/state subject builders; the add-on subject now comes from `contracts/eks/v1` (at `18794e017`).
-- `gateway/oidc_discovery.go`: `AccountBucketName`, `GetClusterMeta`, `OIDCJWKSKey`.
-- `handlers/sts/oidc_jwks.go`: `AccountBucketName`, `OIDCJWKSKey` (STS reads EKS private KV).
-- `runtime/roles/awsgw/eks_token_verify.go`: `TokenVerifySubject`, `TokenVerifyRequest`, `TokenVerifyResponse`.
-- `agents/eks/tokenwebhook/webhook.go`: `WebhookTokenReviewResult` (guest imports domain).
-- `daemon/*`: `EKSServiceImpl`, `NewEKSServiceImpl`, `EKSServiceDeps`, `NewNATSHostScheduler`, `SubnetVPCResolver`, `WorkerLauncher`, `AccountWatchBuckets`.
-- `accountteardown/reapers_eks.go`: `EKSService`, `NewNATSEKSService`.
-- `operator/cli/admin_eks.go`: `NewNATSEKSService`, `RestoreSnapshotInput`.
-
-### Reverse knowledge of EKS in other domains (comments or tag conventions, no import)
-`domains/ec2/instance/service_impl.go:394-396` (worker tag comment), `domains/ec2/systeminstance/launcher.go:4,25`, `handlers/elbv2/{launcher.go:5,service_impl.go:1232-1250,service_impl_nlb_sg.go:196}`, `domains/ec2/vpc/eni.go:54`, `domains/ec2/guestmetadata/resolver.go:37`, `domains/dns/{naming.go:118-124,reconcile.go:58,303}`, `handlers/iam/managed_policies.go` (EKS managed policies), `accountteardown/reapers_iam.go:26-33` (OIDC providers).
-
-### 7.1 Add-on boundary edges remaining at `18794e017`
-
-Each edge is temporary unless marked otherwise, and each clears only on its stated condition; moving a file does not clear any of them.
-E1 to E5 match the `handlers/eks` → `domains/eks/addon` debt entry in `docs/PACKAGE_BOUNDARY_MIGRATION.md`; E6 to E9 match the further-edges entry beside it.
-
-| ID | Edge | Kind | Where | Removal condition |
-|---|---|---|---|---|
-| E1 | Cluster delete erases add-on records and manifests | handler → owner state (legacy) | `handlers/eks/cluster_state.go:493` `DeleteClusterPrefix` sweeps `clusters/{c}/` without calling `addon.Owner` | Cluster delete invokes an add-on-owned teardown capability. |
-| E2 | GPU node-group launch stages `nvidia-device-plugin` | node group → add-on orchestration | `handlers/eks/nodegroup.go:405-416` `stageGPUDeviceAddon`, called at `:473`, through `Owner.EnsureGPUDevicePlugin`; best-effort, errors logged and swallowed, never undone on node-group delete, and any existing record (including `CREATE_FAILED`) blocks re-staging | The node-group owner is extracted and the dependency is designed. |
-| E3 | Add-on delivery reports are received by the cluster reconciler | cluster hosts add-on observation | `handlers/eks/cluster_reconciler.go:171-176,264-269,447-458` subscribes; `addon_status_reconciler.go:22` relays through the reconciler-declared `addonReports`; wired in `service_impl.go:2160-2163` | Add-on delivery observation has its own host under the lifecycle correction. |
-| E4 | Cluster existence is checked by the handler | handler admission | `addons.go` actions call `acctKVForCluster`, which reads `ClusterMeta` existence only; cluster status is ignored | Moves into the owner when cluster admission is defined. |
-| E5 | Guest wire types | versioned contract (not debt) | `contracts/eks/v1` imported directly by `handlers/eks` (`addons.go`, `addon_status_reconciler.go`, `state_report.go`, `service_impl.go`), `gateway/eks` (`internal_addons.go`, `internal_publish.go`) and `agents/eks/gatewayfetch` | Permanent while v1 guests exist; v1 is retired only through a successor version and an explicit retirement, never by editing v1. |
-| E6 | The persisted `Record` crosses to the AWS adapter | owner record exposed | `addons.go` `addonRecordToAWS` (`:222`) and `ListAddons`/`DescribeAddon` read `addon.Get`/`addon.List` directly | Replace with an owner-returned view when the add-on AWS adapter moves to `domains/eks/awsapi`. |
-| E7 | The guest route reaches the owner through the monolith RPC | gateway → `handlers/eks` | `gateway/eks/internal_addons.go` uses `handlers_eks.NewNATSEKSService` and `ListStagedAddonManifestsInput`; `handlers/eks/addons.go:117` maps `addon.Manifest` to `eksv1.StagedAddonManifest` | As for `service.go`/`service_nats.go`: per-resource commands behind `domains/eks/awsapi` registration, with the internal request versioned with the add-on contract. |
-| E8 | Owner and contract keep separate shapes | deliberate duplication (not debt) | `addon.Manifest`/`addon.Report` beside `eksv1.StagedAddonManifest`/`eksv1.AddonStatusReport`; the handler maps between them | None: the owner does not import the wire contract. Revisit only if a successor contract makes the mapping lossy. |
-| E9 | The add-on ARN is derived in two places | duplicated identity rule | `domains/eks/addon/owner.go` `Create` and `gateway/eks/authz.go` `addonARN` both call `arn.FormatEKSAddon` from names | Clears when authorization obtains the ARN from the owner, which becomes necessary if an incarnation suffix is adopted. |
-
-## 8. Known gaps against Q-78 / ADR-0003 (observed)
-
-1. No desired or observed generation for any EKS resource; fencing is leases (cluster) or record-revision CAS (nodegroup, addon, access).
-2. Intent and observation share `ClusterMeta` (`cluster_state.go:65-190`), and nodegroup readiness reads a cluster observation (`nodegroup.go:537-549`).
-3. Cluster realization runs in request-spawned goroutines (`service_impl.go:625-645`); restart does not resume the launch, so an interrupted create ends FAILED after timeout (`cluster_reconciler.go:1151-1171`) rather than converging the same intent.
-4. CP VMs are live before their IDs are persisted (`service_impl.go:960-970` comment); a crash in that window leaves VMs reachable only by the billable reaper after meta removal.
-5. Bootstrap artifacts travel on at-most-once core NATS (`nats_bootstrap.go:197`, `gateway/eks/internal_publish.go:84`); a lost message is not replayed.
-6. DeleteCluster purges synchronously inside the request (`service_impl.go:1141-1154`) rather than accepting asynchronously as AWS does.
-7. DeleteCluster does not reject a cluster with nodegroups (AWS returns ResourceInUseException) and `DeleteClusterPrefix` erases nodegroup records while their worker instances keep running; `accountteardown/reapers_eks.go:15-18` documents this; workers carry no `ManagedBy` tag (`nodegroup.go:669-670`) and no reaper reclaims them (uncertain whether the e2e suite exercises this order).
-8. DeleteNodegroup is synchronous and removes the record immediately (`nodegroup.go:1145-1157`), so the resource is absent before EC2 termination completes; no durable deletion intent.
-9. UpdateNodegroupConfig and UpdateAddon return `Successful` synchronously (`nodegroup.go:900-906`; `addons.go:192-197` at `18794e017`) and `DescribeUpdate`/`ListUpdates` are stubs, so there is no truthful pending update.
-10. CreateAddon and CreateAccessEntry use get-then-put (`domains/eks/addon/owner.go` `Owner.Create` at `18794e017`; `service_impl.go:1570-1578`), so concurrent creates can both succeed.
-11. Addon status reports are not fenced by version or incarnation (`addon_status_reconciler.go:22` drops `AddonStatusReport.Version` and `TS` at `18794e017`), so a stale report can flip UPDATING to ACTIVE, or a stale `failed` to CREATE_FAILED.
-12. DeleteAddon deletes the record and manifest before in-cluster removal is observed (`domains/eks/addon/owner.go` `Owner.Delete` at `18794e017`), and the guest sends no removal report.
-13. `clientRequestToken` is honoured only by CreateCluster; CreateNodegroup, CreateAddon, CreateAccessEntry, UpdateNodegroupConfig, UpdateAddon ignore it.
-14. Nodegroup and addon ARNs are deterministic by name, so identifiers are reused across incarnations (Q-78 references Q-43 identifier-reuse rules).
-15. ADR-0004 S3 inventory: EKS has no `Stubbed`/`Unsupported` classification (`gateway/operations.go:34-36`) although nine actions return NotImplemented, and four internal routes are mixed into the AWS action inventory.
-16. ADR-0003 S4 / ADR-0004 S4: three non-owner readers of EKS private KV (`gateway/eks/internal_authz.go`, `gateway/oidc_discovery.go`, `handlers/sts/oidc_jwks.go`) plus `daemon/dns_reconcile.go` watching all EKS buckets.
-17. ADR-0004 S3: `daemon/eks_cp_control.go` uses the EC2 AWS adapter package as an internal API.
-18. ADR-0004 S5: guest `agents/eks/tokenwebhook` imports `handlers/eks`.
-19. Observed security risk outside Q-78: plaintext OIDC private key and k3s join token in `ClusterMeta.ControlPlaneTemplate` (Section 3).
-20. Comment hygiene: EKS lifecycle evidence now cites `ADR-0003:S2`; `daemon/daemon.go:1085-1087` is stale.
-
-## 9. First-resource candidates
-
-| Rank | Resource | Approx. size | Private state | Dependencies | Tests | Extractable without EKS identity/IAM/Bluebottle? |
-|---|---|---|---|---|---|---|
-| 1 | access | about 670 lines (`access_entry.go` 255, `service_impl.go:1527-1816` about 290, `token_review.go` 128, parts of `types.go`) | `clusters/{c}/access-entries/*` only | cluster existence query (`acctKVForCluster` reads `ClusterMeta`), cluster launch seeds creator admin, gateway authz needs `PrincipalARNHash`, gateway token-review calls `ResolveTokenReview`, STS verify contract | `access_entry_test.go`, `service_impl_test.go` (access cases), `token_review_test.go`, `token_webhook_test.go`, `gateway/eks/authz_test.go`, `gateway/eks/token_review_test.go`, e2e `TestEKS/AccessEntry`, `GetToken` | The record CRUD and policy catalogue are extractable now with a narrow cluster-exists query; the token-review part touches STS identity (`eks.VerifyToken` contract) but not IAM or Bluebottle, so it can stay behind its existing NATS contract. |
-| 2 | addon (owner extracted) | at `18794e017`: owner 693 lines in `domains/eks/addon`, wire contract 55 lines in `contracts/eks/v1`, adapter and relay 319 lines left in `handlers/eks` (`addons.go`, `addon_status_reconciler.go`) | `clusters/{c}/addons/*` incl. manifests | the edges in Section 7.1 | `domains/eks/addon/*_test.go`, `contracts/eks/v1/*_test.go`, `handlers/eks/addon_lifecycle_test.go`, `addons_test.go`, `addon_status_reconciler_test.go`, `gateway/eks_addon_wire_test.go`, `agents/eks/gatewayfetch/addons_wire_test.go`, e2e `AddonDelivery`, `TestEKSGPUPodExposure` | Extracted without IAM or Bluebottle; the report-relay seam (E3) and the guest contract (E5) exist, and every lifecycle gap remains. |
-| 3 | nodegroup | about 1,650 lines (`nodegroup.go`, `nodegroup_userdata.go`, `claim.go`, part of `types.go`, nodegroup SG rules) | `clusters/{c}/nodegroups/*` | cluster meta (ACTIVE, CP ENI IP, VPC, readiness counts), cluster join token decrypt (`handlers_iam.DecryptSecret`), IAM role/profile creation, Bluebottle `ResolveRoleARN`, EC2 worker launch, AMI lookup, cluster SGs, GPU addon | `nodegroup_test.go` (39 tests), `nodegroup_recreate_test.go`, `nodegroup_instance_profile_test.go`, `nodegroup_userdata_test.go`, `daemon/eks_worker_launch_test.go`, e2e IRSA pod path | No; it needs IAM, Bluebottle and cluster secrets capabilities first. |
-| 4 | cluster | about 8,500 lines (remainder of `handlers/eks`) | meta, OIDC material, kubeconfig, join token, recovery directives, leader/token buckets | every external owner (EC2, VPC, ELBv2, DNS, IAM, objectstore, placement, scheduler) and all guest contracts | `create_cluster_test.go`, `delete_cluster_test.go`, `cluster_reconciler*_test.go` (5 files), `lifecycle_invariants_test.go` (RLC1-3), reapers, `nats_bootstrap_test.go`, `oidc_keypair_test.go`, `k3s_*_test.go`, `nlb_test.go`, `security_groups_test.go`, `cp_vpc_test.go`, `restore_snapshot_test.go`, e2e `TestEKS` | No; it is the orchestration hub and depends on identity, Bluebottle and the CA. |
-
-Recommendation: extract `access` first, defining a cluster-owned "cluster exists and is not deleting" query and moving `PrincipalARNHash` and the key helper into the access package.
-Its shortcomings (gap 10, no client token, no generation) are small and can be recorded in its lifecycle contract without first resolving cluster realization.

docs/package-boundary/adr-0004-inventory-rds.md

@@ -1,310 +0,0 @@
-# ADR-0004 source inventory — RDS
-
-Code-read inventory of `refactor/spinifex-package-boundaries` at `6c50349bd`.
-It is not verified against a live system, and the branch is 59 commits behind `dev` at the time of writing, so recheck any file it cites before acting on it.
-Items marked "verify" or "uncertain" are unconfirmed.
-Gaps record observed behaviour, not target behaviour.
-
-
-Repo: `/home/tf-user/Development/mulga/spinifex`, branch `refactor/spinifex-package-boundaries`, paths relative to `spinifex/`.
-Inputs read: ADR-0001, ADR-0003, ADR-0004 and Q-78 from `origin/main`.
-Method: static reading of production code only; nothing here was verified against a live cluster.
-Line references are `file:line` at the time of reading.
-"Uncertain" marks a claim inferred from code shape rather than traced end to end.
-
-## 0. Scope findings
-
-`daemon/instance_records.go` (217) and `daemon/instance_records_migrate.go` (361) are **not RDS**.
-They own the EC2 per-instance record space `i.<id>` in the EC2 instance-state bucket (`daemon/instance_records.go:25`) and its migrations (`daemon/instance_records_migrate.go:29-70`), and import only `runtime/compute/vm`, `kvstore`, `kvutil`, `migrate`.
-They are excluded from the RDS inventory and belong to the EC2 instance owner.
-
-Other production files that wire or consume RDS, found by import search:
-
-- `daemon/daemon.go` (RDS sections only: `:170-171`, `:1171-1206`, `:2046-2078`, `:2306-2312`, `:2374-2382`).
-- `daemon/rds_deps.go`.
-- `daemon/dns_reconcile.go:71-76` and `:99-104` (RDS bucket watch and RDS DNS desired set).
-- `daemon/bedrock_deps.go:43` (Ochre/Bedrock borrows `handlers_rds.NewNATSVolumeAttacher`).
-- `handlers/quota/live.go:82-93` (`EnforceRDSInstances`).
-- `accountteardown/reapers_rds.go` (instance, subnet-group, parameter-group reapers).
-- `gateway/rds.go`, `gateway/operations.go:43-45`, `gateway/gateway.go:302,525-526` (service map and dispatch).
-- `bootstrap/config/config.go:338` (`RDSConfig`).
-- Test-support (non-`_test.go` but not production): `tests/integration/service_daemon_lite.go:51` reflects every `Service` method onto `rds.<Method>`; `tests/e2e/harness/rds*.go`.
-- Counterparty only: `agents/rds/agent/*.go` and `cmd/rds-agent` import `handlers/rds` for wire types.
-
-## 1. File table
-
-Total in-scope production lines: 16,468 (handlers/rds 14,873; gateway/rds 853; gateway/rds.go 95; daemon/rds_deps.go 98; daemon/instance_records* 578, not RDS).
-
-### handlers/rds
-
-| File | Lines | Class | Reason |
-|---|---|---|---|
-| `store.go` | 451 | temporary residue (prerequisite: per-resource packages exist; removal: each resource owns its own key helpers and bucket view) | One file holds every resource's key space, bucket configs, migrations hook, leader bucket and cross-tenant enumeration (`:20-47`, `:39-50` key list). **Mixed**: split into instance (`db-instances/`, `bootstrap-payloads/`, `instance-index/`, leader bucket), snapshot (`db-snapshots/`, `backups/`, `retained-volumes/`), subnetgroup (`db-subnet-groups/`), parametergroup (`db-parameter-groups/`), events (`events/`), plus a domain-root bucket factory. |
-| `records.go` | 501 | temporary residue (same prerequisite) | All durable record structs in one file. **Mixed**: `DBInstanceRecord`/`PendingModifiedValues`/`ModifyLease`/`BootstrapState`/`AgentState`/`SnapshotOperation` → instance; `DBSnapshotRecord`/`AutomatedBackupRecord`/`RetainedVolumeRecord` → snapshot; `DBSubnetGroupRecord` → subnetgroup; `DBParameterGroupRecord`/`DBParameterRecord` → parametergroup; `Parameter`, `EngineHealth`, `Heartbeat*` → guest/controller wire contract. |
-| `service.go` | 314 | temporary residue (prerequisite: resource owners; removal: each owner holds its own deps, domain root holds composition) | Single `Service` with one `Deps` bag for every resource (`:19-45` in stripped view), per-account bucket cache and generic JSON CAS helpers. **Mixed**: `Deps` splits by resource; CAS helpers could move to `foundation/state/kvstore` (already exists) rather than a new common package. |
-| `service_nats.go` | 189 | temporary residue (prerequisite: awsapi calls owners in-process or via a domain contract; removal: no AWS-SDK-typed NATS client used as internal API) | AWS-SDK-shaped NATS client used by the gateway, quota (`handlers/quota/live.go:89`) and account teardown (`accountteardown/reapers_rds.go:19`). It is currently the de facto internal API, which ADR-0004 S3/S4 forbid for quota. |
-| `subjects.go` | 90 | **Mixed**: guest/controller wire contract (`rds.bus.*`, `CommandQueueGroup`, `:6-19`, `:65-90`) + temporary residue (Layer-1 `rds.<Action>` gateway→daemon subjects, `:21-63`) | Bus subjects are the real agent-path process boundary; the per-action subjects exist only because the gateway and daemon are separate dispatch hops. |
-| `arn.go` | 92 | AWS protocol adapter (with per-resource kinds) | `FormatARN`/`ParseARN`/`ResourceKind` (`:15-40`) are used for projection, tags and gateway authz scopes. Kinds could live with each resource; parsing belongs in awsapi. |
-| `filters.go` | 68 | AWS protocol adapter | Generic AWS `Filter` parsing (`ReadFilters`, `:38`), used by gateway catalogs and describes. |
-| `paging.go` | 74 | AWS protocol adapter | AWS `MaxRecords`/`Marker` paging (`:23`) and per-type page keys (`:52-74`). |
-| `validate.go` | 271 | resource owner (instance) — partly AWS protocol adapter | `CreateDBInstance` request validation (`:69`), identifier rules (`:196`), unimplemented-parameter rejection (`:225`). **Mixed**: SDK input parsing → awsapi; identifier/engine/class invariants → instance. |
-| `status.go` | 69 | resource owner (instance) | DB instance status enum and transition table (`:7-60`). |
-| `create.go` | 305 | resource owner (instance) | `CreateDBInstance` orchestration, record reservation, rollback (`:29-147`, `:195-305`). |
-| `describe.go` | 298 | **Mixed**: resource owner (instance) query + AWS protocol adapter (projection to `rds.DBInstance`, `:145-298`) | Projection is awsapi; record read/filter is the owner's query. |
-| `lifecycle.go` | 454 | resource owner (instance) — **contains an EC2 adapter** | Reboot/Stop/Start (`:57-162`), transition helpers (`:276-405`). `natsInstanceCommander` (`:407-454`) sends `ec2v1` instance commands and `ec2.start`; that is an EC2-owned capability implemented inside RDS (residue). |
-| `launch.go` | 629 | cross-resource orchestration (instance realization over EC2/VPC/EBS/IAM) — **contains an EC2 adapter** | `LaunchDBInstanceVM` (`:178-366`) composes system instance, two ENIs, data volume, AMI lookup. `natsVolumeAttacher` (`:560-629`) is an EC2 volume-attach NATS client that Bedrock also imports (`daemon/bedrock_deps.go:43`); it is EC2/EBS residue living in RDS. |
-| `network.go` | 200 | resource owner (instance) with cross-resource read of subnetgroup | Endpoint placement (`:36-75`) reads the subnet-group record directly (`:43`) and EC2 VPC/subnet/SG via `networkResolver` (`:16-20`). |
-| `vpc.go` | 163 | cross-resource orchestration (system VPC/SG realization for instances) | `EnsureSystemVPC`/`EnsureSystemSecurityGroup` (`:78`, `:104`) over `domains/network/systemvpc`. |
-| `storage.go` | 155 | resource owner (instance) | Storage grow via `volumeResizer` (EC2 `DescribeVolumes`/`ModifyVolume`, `:23-26`). |
-| `modify.go` | 528 | resource owner (instance) | `ModifyDBInstance` plan/immediate/pending paths (`:76-198`, `:199-528`). |
-| `modify_lease.go` | 173 | resource owner (instance) | Per-instance modify lease stored on the instance record (`:24-173`). |
-| `pending.go` | 256 | resource owner (instance) — with parametergroup orchestration | Applies `PendingModifiedValues` (`:28`), parameter-group apply to an instance (`:149`), filesystem grow finish (`:240`). `applyParameterGroup` is the instance side of parameter-group propagation. |
-| `replace.go` | 252 | resource owner (instance) — replacement | VM replace for class/storage change (`:43-166`), index rewrite (`:200`). |
-| `recovery.go` | 220 | resource owner (instance) — failure detection | Health classification (`:54`, `:76`); reports failure, does not repair (`:19-21`). |
-| `reconciler.go` | 896 | **Mixed**: resource owner (instance reconcile, `:238-660`, `:797-866`) + snapshot reconcile (`:674-796`) + foundation wiring (leader lease, `:83-188`) | Split: instance reconciler, snapshot creating-resolver, domain-root leader/loop. `describeInstanceState` (`:867-896`) is an EC2 projection adapter. |
-| `window.go` | 356 | resource owner (instance) — maintenance/backup windows | Pure window arithmetic for an instance's backup and maintenance windows. |
-| `backup.go` | 597 | **Mixed**: resource owner (instance: retention, windows, `runBackupWindow` `:206`, `runMaintenanceWindow` `:340`) + snapshot (automated snapshot identity `:385-394`) + AWS adapter (`DescribeDBInstanceAutomatedBackups` `:396-536`) | Automated backup is an instance obligation that produces snapshot records. |
-| `backup_sweep.go` | 340 | resource owner (snapshot) — retention reaper | `BackupRetentionReaper` (`:24-112`) deletes automated snapshots and reclaims retained volumes (`:310`); implements `vm.Reaper` from `runtime/compute/vm` (`:31`). |
-| `snapshot.go` | 768 | resource owner (snapshot) — tightly coupled to instance | Create/Describe/Delete (`:49`, `:224`, `:299`); create CASes the instance into `backing-up` (`:672-723`) and quiesces the engine. |
-| `restore.go` | 411 | cross-resource orchestration (snapshot → new instance) | `RestoreDBInstanceFromDBSnapshot` (`:24-162`) reads a snapshot record, creates a volume from it and runs the instance create path. |
-| `delete.go` | 548 | resource owner (instance) — with snapshot orchestration | `DeleteDBInstance` (`:34`), teardown (`:178-227`), final snapshot (`:247-447`), retained data volume (`:449-535`). |
-| `agent.go` | 667 | **Mixed**: guest/controller wire contract (IO types, `Command`, `CommandReply`, `InstanceIndexEntry`, `:30-160`, `:628-636`) + resource owner (instance agent-path handlers `:161-627`) | Split: `contracts/rds/v1` vocabulary vs instance owner handlers. |
-| `commands.go` | 239 | **Mixed**: guest/controller wire contract (command types and params, `:12-40`) + resource owner (instance: `issueCommand` issuer, `:86-230`) | Issuer is instance realization; constants are wire vocabulary. |
-| `bootstrap.go` | 280 | resource owner (instance) — bootstrap secret | Encrypted master-password payload (`:31-280`), encrypted via `handlers_iam.EncryptSecret` (`:105`, `:199`). |
-| `servingcert.go` | 109 | resource owner (instance) — realization helper | Per-instance serving cert minted from cluster CA (`:46`). Uncertain whether this should sit with a PKI owner instead. |
-| `userdata.go` | 68 | guest/controller wire contract | Agent user-data rendering (`:36`), the boot-time contract with the DB VM. |
-| `iam.go` | 83 | **Mixed**: resource owner (instance — instance profile ensure, `:56-83`) + guest/controller wire contract (`InternalAgentActions`, `:24-30`) | The internal-action list is shared by the gateway gate (`gateway/rds/authz.go:52-63`) and the role grant. |
-| `dns.go` | 72 | resource owner (instance) — publishes a projection | `DesiredDNSChanges` (`:25-51`) is an RDS-owned projection consumed by the daemon DNS reconcile. |
-| `events.go` | 312 | cross-resource orchestration (event journal) — uncertain final home | Per-source event rings for instance and snapshot (`:88-141`) plus `DescribeEvents` (`:159`). Not in the ADR-0004 resource table; candidate for the domain root, not a `common` package. |
-| `tags.go` | 280 | cross-resource orchestration + AWS adapter | One tag implementation dispatching on ARN kind over all four record types (`:51-73`, `:202-230`). Split: each owner exposes a tag mutation on its own record; awsapi routes by ARN. |
-| `engine.go` | 287 | resource owner (engine) — read-only catalogue | Engine registry, version/username/dbname/password rules (`:15-287`); consumed by the agent (`LookupEngine`). |
-| `engine_postgres.go` | 505 | resource owner (engine) | Postgres engine catalogue and parameter specs/validators. |
-| `engine_mariadb.go` | 503 | resource owner (engine) | MariaDB engine catalogue and parameter specs/validators. |
-| `paramcatalog.go` | 449 | resource owner (engine) | Parameter spec model, size-derived defaults, `ResolveEffectiveParameters` (`:354`); imports `domains/ec2/instancetypes` for class memory (`:219`). |
-| `catalog.go` | 214 | **Mixed**: resource owner (engine — `EngineVersions`, `OrderableOptions` `:102-141`) + AWS protocol adapter (filter value sets, SDK projection `:142-214`) | |
-| `sizing.go` | 57 | resource owner (engine/instance-class facade) | `db.*` → EC2 instance type map (`:14-21`), validated against `domains/ec2/instancetypes` (`:30`). |
-| `subnetgroup.go` | 340 | resource owner (subnetgroup) | Create/Describe/Delete (`:39`, `:94`, `:138`), subnet resolution via EC2 (`:176`). Reads instance records for in-use check (`:258-277`). |
-| `parametergroup.go` | 599 | resource owner (parametergroup) — with instance orchestration | CRUD + DescribeDBParameters (`:29-380`); `propagateParameterGroup` (`:210-236`) pushes changes into instances; `resolveGroupParameters` (`:549`) is called by instance create/modify/restore. |
-
-### gateway/rds and gateway/rds.go
-
-| File | Lines | Class | Reason |
-|---|---|---|---|
-| `gateway/rds.go` | 95 | AWS protocol adapter (ingress glue) | Query parse, action check, caller derivation, authz order, quota injection and `ExpectedNodes` injection (`:16-75`). Imports `bluebottle/pkg/auth.ParseRoleARN` (`:87`). |
-| `gateway/rds/handler.go` | 220 | AWS protocol adapter | Action inventory (registered/internal/unsupported, `:96-169`), typed Query→XML adapter (`:47-79`), `Env{ExpectedNodes, QuotaCheck}` (`:33-43`). |
-| `gateway/rds/authz.go` | 112 | AWS protocol adapter | Principal-class gate and per-action resource ARN scopes (`:20-112`). |
-| `gateway/rds/instance.go` | 50 | AWS protocol adapter | Instance actions + `DescribeEvents`; runs `QuotaCheck` before forwarding create (`:13-19`). |
-| `gateway/rds/snapshot.go` | 35 | AWS protocol adapter | Forwards snapshot/restore/automated-backup actions. |
-| `gateway/rds/subnetgroup.go` | 23 | AWS protocol adapter | Forwards subnet-group actions. |
-| `gateway/rds/parametergroup.go` | 31 | AWS protocol adapter | Forwards parameter-group actions. |
-| `gateway/rds/tags.go` | 23 | AWS protocol adapter | Forwards tag actions. |
-| `gateway/rds/enginecatalog.go` | 184 | **Mixed**: AWS protocol adapter (engine catalogue actions) + temporary residue (prerequisite: EC2-owned instance-type-availability projection; removal: `clusterRunnableTypes` replaced by that projection) | `clusterRunnableTypes` (`:146-174`) calls the EC2 AWS adapter `ec2instanceapi.DescribeInstanceTypes` with `env.ExpectedNodes`. |
-| `gateway/rds/agent.go` | 287 | **Mixed**: guest/controller wire contract (agent-facing internal actions, long poll, reply relay `:146-287`) + AWS protocol adapter (agent identity gate `:36-111`) | Reads the `rds-system` bucket directly via JetStream (`:72-100`), i.e. the gateway reads RDS private state. |
-
-### daemon and other wiring
-
-| File | Lines | Class | Reason |
-|---|---|---|---|
-| `daemon/rds_deps.go` | 98 | cross-resource orchestration (composition root) — contains residue | Binds EC2/VPC/IGW/RT/NGW/EIP/image/volume/snapshot services, IAM ensurer, master key and config into `handlers_rds.Deps` (`:20-85`). `describeInstancesFanOut` (`:90-97`) calls the EC2 AWS adapter with `len(clusterConfig.Nodes)` as expected-node count (residue; same blocker family as `ExpectedNodes`). Loads `bluebottle/pkg/masterkey` (`:52`). |
-| `daemon/daemon.go` (RDS sections) | n/a | cross-resource orchestration (composition root) | Subscriptions (`:1171-1206`), service init (`:2046-2066`), reconciler start (`:2070-2078`), backup reaper registration (`:2310-2312`), and the RDS reconciler lease reused as the cluster-wide GC election (`:2374-2382`). The last is residue: an RDS lease elects unrelated reapers. |
-| `daemon/dns_reconcile.go` (`:71-76`, `:99-104`) | n/a | cross-resource orchestration | Watches RDS per-account buckets via `handlers_rds.AccountWatchBuckets` and consumes `DesiredDNSChanges`. Bucket watch is a read of RDS private storage by the daemon. |
-| `daemon/bedrock_deps.go:43` | n/a | temporary residue (prerequisite: EC2/EBS-owned attach capability; removal: Bedrock and RDS both bind the EC2 one) | Another domain imports RDS for an EC2 volume-attach client. |
-| `handlers/quota/live.go:82-93` | n/a | temporary residue (prerequisite: RDS-owned count projection per ADR-0004 S4; removal: quota no longer calls `NATSService.DescribeDBInstances`) | Quota counts instances through the AWS-shaped RDS API. |
-| `accountteardown/reapers_rds.go` | n/a | temporary residue / cross-domain consumer | Drives instance/subnet-group/parameter-group deletion through `NATSService` AWS-shaped calls (`:41`, `:76`, `:87`, `:109`, `:125`, `:139`, `:159`). Account teardown is explicitly out of ADR-0004 scope. |
-| `daemon/instance_records.go`, `daemon/instance_records_migrate.go` | 217 / 361 | not RDS | EC2 instance record space; excluded. |
-
-## 2. Served AWS actions and protocol
-
-Protocol: AWS Query in (form `Action=`), XML out using the IAM-style `<ActionResponse><ActionResult>` envelope (`gateway/rds.go:14-26`, `gateway/rds/handler.go:59-75`).
-Order: unknown action → `InvalidAction` before policy (`gateway/rds.go:29-32`); principal-class gate before policy (`:41-43`); resource ARN derivation and policy (`:45-51`); dispatch (`:57-64`).
-Unsupported actions return `OperationNotSupported` from `Dispatch` (`gateway/rds/handler.go:182-185`).
-Path: gateway adapter → NATS request `rds.<Action>` (queue group `spinifex-workers`) → daemon `Service` method (`daemon/daemon.go:1173-1206`), except where noted.
-
-| Action | Status | Owning resource | Notes |
-|---|---|---|---|
-| CreateDBInstance | registered | instance | `QuotaCheck` in gateway before forward (`gateway/rds/instance.go:13-19`); 5 min budget (`service_nats.go:16`). |
-| DescribeDBInstances | registered | instance | |
-| ModifyDBInstance | registered, scoped | instance | Also binds parametergroup. |
-| DeleteDBInstance | registered, scoped | instance | Orchestrates final snapshot. |
-| RebootDBInstance / StartDBInstance / StopDBInstance | registered, scoped | instance | |
-| CreateDBSnapshot | registered, scopes instance + snapshot | snapshot (orchestrates instance) | |
-| DescribeDBSnapshots | registered | snapshot | |
-| DeleteDBSnapshot | registered, scoped | snapshot | |
-| RestoreDBInstanceFromDBSnapshot | registered, scopes snapshot + instance | cross-resource (snapshot → instance) | |
-| DescribeDBInstanceAutomatedBackups | registered | instance (automated backup) / snapshot | |
-| CreateDBSubnetGroup / DescribeDBSubnetGroups | registered | subnetgroup | |
-| DeleteDBSubnetGroup | registered, scoped | subnetgroup | |
-| CreateDBParameterGroup / DescribeDBParameterGroups | registered | parametergroup | |
-| ModifyDBParameterGroup / DescribeDBParameters / DeleteDBParameterGroup | registered, scoped | parametergroup | Modify propagates to instances. |
-| AddTagsToResource / RemoveTagsFromResource / ListTagsForResource | registered, ARN scope | per ARN kind (all four) | `tags.go:51-73`. |
-| DescribeEvents | registered | cross-resource (events) | |
-| DescribeDBEngineVersions | registered | engine | Served **in the gateway process** with no NATS hop (`gateway/rds/enginecatalog.go:41-73`). |
-| DescribeOrderableDBInstanceOptions | registered (`typedEnv`) | engine (+ EC2 availability) | Served in the gateway; NATS fan-out to EC2 `DescribeInstanceTypes` using `ExpectedNodes` (`gateway/rds/enginecatalog.go:146-174`). |
-| RegisterDBInstance, SubmitDBStateChange | registered, internal | instance (agent path) | Gateway → bus `rds.bus.{acct}.{db}.register/health` (`service_nats.go:49-57`). |
-| GetDBBootstrapConfig, AcknowledgeDBBootstrap | registered, internal | instance (bootstrap) | Gateway → `rds.GetDBBootstrapConfig` / `rds.AcknowledgeDBBootstrap`. |
-| PollDBCommands | registered, internal | instance (command channel) | Served **in the gateway**: queue-subscribes the bus command subject; no daemon handler (`gateway/rds/agent.go:182-247`). |
-| CreateDBInstanceReadReplica, PromoteReadReplica, CreateDBCluster, ModifyDBCluster, DeleteDBCluster, DescribeDBClusters, FailoverDBCluster, CreateOptionGroup, ModifyOptionGroup, DeleteOptionGroup, DescribeOptionGroups, RestoreDBInstanceToPointInTime | unsupported (12) | n/a | `gateway/rds/handler.go:157-168`. |
-
-Totals: 26 customer actions registered, 5 internal agent actions, 12 unsupported, 43 in the inventory (`ActionNames`/`UnsupportedActionNames` feed `gateway/operations.go:43-45`).
-Daemon subscribes 29 `rds.*` subjects (`daemon/daemon.go:1175-1205`); the two catalogue actions and `PollDBCommands` have no daemon subject.
-Any action outside the table and not in the map is `InvalidAction`.
-
-## 3. Durable records and KV buckets
-
-| Bucket | Key | Record | Owning resource (proposed) | Writers | Readers (flag = outside RDS owner) |
-|---|---|---|---|---|---|
-| `rds-account-{accountID}` (history 1, version 1, lazily created; `store.go:20-38`, `:201-213`) | `db-instances/{id}` | `DBInstanceRecord` | instance | create, restore, lifecycle, modify, pending, replace, agent, delete, reconciler, recovery, backup, snapshot (`backing-up` CAS), tags | subnetgroup (`subnetgroup.go:258-277`), parametergroup (`parametergroup.go:211`, `:342`), dns.go, **daemon DNS reconcile via bucket watch** (`daemon/dns_reconcile.go:75`), **quota** indirectly via `DescribeDBInstances` (`handlers/quota/live.go:89`), **account teardown** via `NATSService` |
-| same | `bootstrap-payloads/{id}` | `BootstrapPayloadEnvelope` (encrypted) | instance | create, restore, replace (via bootstrap.go) | agent.go (`GetDBBootstrapConfig`, `AcknowledgeDBBootstrap`), reconciler (stale pending report) |
-| same | `db-snapshots/{id}` (colon → `.`, `store.go:84-97`) | `DBSnapshotRecord` (manual and automated) | snapshot | snapshot.go, delete.go (final snapshot), backup.go (automated) | restore.go, backup_sweep.go, reconciler snapshot pass, describe, tags |
-| same | `backups/{dbId}/automated/{ts}` | `AutomatedBackupRecord` | snapshot (automated-backup index) — instance obligation | backup.go | backup_sweep.go, DescribeDBInstanceAutomatedBackups |
-| same | `retained-volumes/{volumeID}` | `RetainedVolumeRecord` | snapshot (cleanup obligation for a former instance's volume) | delete.go `retainDataVolume` (`:484`) | snapshot.go `releaseRetainedVolume`/`reclaimRetainedVolume` (`:414`, `:430`), backup_sweep `reclaimOrphanedVolumes` (`:310`) |
-| same | `db-subnet-groups/{name}` | `DBSubnetGroupRecord` | subnetgroup | subnetgroup.go, tags.go | network.go `resolvePlacement` (`:43`, instance create/restore), describe, **account teardown** via `NATSService` |
-| same | `db-parameter-groups/{name}/meta` | `DBParameterGroupRecord` | parametergroup | parametergroup.go, tags.go | instance create/modify/restore via `resolveGroupParameters` (`parametergroup.go:549`), pending.go |
-| same | `db-parameter-groups/{name}/params/{key}` | `DBParameterRecord` | parametergroup | parametergroup.go (`:188`, delete `:361`) | `ListDBParameterOverrides` (`store.go`), instance apply paths |
-| same | `events/{sourceType}/{sourceId}` | event ring (bounded, 14-day trim) | events (cross-resource) | `RecordEvent` from every resource (`events.go:88`) | `DescribeEvents` |
-| `rds-system` (history 1, version 1; `store.go:24-26`, `:184-195`) | `instance-index/{ec2InstanceID}` | `InstanceIndexEntry{AccountID, DBInstanceIdentifier, VMGeneration}` (`agent.go:628-636`) | instance (agent identity binding) | create (`create.go:124`), restore (`restore.go:142`), replace (`replace.go:200-215`), delete teardown | **gateway reads JetStream directly** (`gateway/rds/agent.go:72-100`); `Service.LookupInstanceIndex` (`agent.go:637`) |
-| `spinifex-rds-leader` (TTL 60s; `store.go:28-33`, `:216-223`) | `reconciler` | kvlease record | domain-root reconciler leadership | `kvlease` via `Reconciler` (`reconciler.go:104-118`) | **daemon GC uses it as the cluster-wide reaper election** (`daemon/daemon.go:2374-2382`) |
-
-Non-durable state that affects decisions: in-memory heartbeat liveness map per daemon (`service.go` `liveness`, `agent.go:284-321`), persisted to KV only on change or every `HeartbeatPersistFloor` (5 beats, `records.go` constants).
-The reconciler leader reads this node-local map (`recovery.go:191`), so a beat answered by another node's queue-group member is seen only via the KV floor (uncertain impact; observation, not intent).
-
-## 4. Identity and scope per resource
-
-| Resource | Identifier | Immutable internal ID | ARN | Account | Region | AZ |
-|---|---|---|---|---|---|---|
-| instance | `DBInstanceIdentifier`, account-unique via `createJSONRevision` (`create.go:62`), reusable after delete | `DbiResourceID` `db-…` (`create.go:153`) | `arn:aws:rds:{region}:{acct}:db:{id}` (`arn.go:15-27`) | customer account in bucket name and record; VM, system ENI and IAM profile live in the system account `GlobalAccountID` (`create.go:51-55`, `delete.go:544-546`) | not stored; taken from `Service.region` at projection (`service.go`) | not recorded on the instance; derived only via subnet (uncertain whether projected at all; no `AvailabilityZone` reference in `describe.go`) |
-| snapshot | `DBSnapshotIdentifier`; automated ones `rds:{db}-{ts}` (`backup.go:385`) | none beyond backing EC2 `SnapshotID` | `…:snapshot:{id}` | customer | from service | none |
-| subnetgroup | `DBSubnetGroupName` (case-sensitive key; `default` rejected, `subnetgroup.go:318`) | none | `…:subgrp:{name}` | customer | from service | each member subnet carries its AZ (`records.go` `DBSubnetGroupSubnet`) |
-| parametergroup | `DBParameterGroupName`; `default.*` synthesized per engine, not stored (`parametergroup.go:24`, `:456-482`) | none | `…:pg:{name}` | customer | from service | n/a |
-| engine | engine name + version | n/a (static) | none | global | global | n/a |
-| agent identity | EC2 instance ID as `RoleSessionName` of the system instance role (`gateway/rds/agent.go:36-69`) | `VMGeneration` in index | n/a | system account principal mapped to customer account via index | n/a | n/a |
-
-## 5. Lifecycle paths per resource
-
-### instance
-
-- Create: validate (`validate.go:69`) → placement reads subnet group / default VPC (`network.go:36`) → resolve parameter group (`create.go:47`) → ensure IAM instance profile in system account (`create.go:51`) → reserve record at `creating` with `createJSONRevision` (`create.go:62`) → stage encrypted bootstrap payload (`create.go:83`) → `LaunchDBInstanceVM` inline in the request (`create.go:87`) → `recordLaunch` CAS (`create.go:115`, `:195-244`) → write instance index (`create.go:124`) → event, return `creating`.
-- Create failure: deferred rollback deletes payload and CAS-deletes the record if `DbiResourceID` unchanged (`create.go:246-295`); post-launch failures run `Unwind` (`create.go:297-305`).
-- Readiness: reconciler `reconcileCreating` moves `creating → available` on first healthy agent beat (`reconciler.go:448-464`); `failed` after `BootstrapTimeout` (default 20 min, `reconciler.go:30`).
-- Steady state: `reconcileHealth` (`recovery.go:76-101`) records/clears failure; `StatusRecovering` exists in the table (`status.go:22`, `:34`, `:43`, `:47`) but **no code transitions into it** (only `status.go` references it), so there is no automatic repair.
-- Reboot/Start: `beginTransition` CAS (`lifecycle.go:65`, `:143`) then reconciler `reconcileRestarting` (`reconciler.go:479`).
-- Stop: CAS to `stopping`, inline engine stop + VM stop, inline `completeTransition` to `stopped` before responding (`lifecycle.go:103-133`); reconciler resumes a dead stop (`reconciler.go:584-605`).
-- Modify: immediate non-disruptive in-request; disruptive writes `PendingModifiedValues` (`modify.go:153`), then either waits for the maintenance window (`backup.go:340`) or runs under the modify lease (`modify.go:176`); reconciler resumes under the same lease (`reconciler.go:510-583`); class/storage change replaces the VM (`pending.go:80`, `replace.go:43-166`).
-- Replace: launch new VM (`replace.go:113`), rewrite index (`:125`), CAS record with `VMGeneration++` (`:137`), rollback on error (`:218`).
-- Delete: validate, deletion protection, reserve final snapshot (`delete.go:68`), CAS to `deleting` (`delete.go:73-91`), inline teardown (`delete.go:178-227`): stop engine, terminate VM, final snapshot, purge automated backups, release or retain data volume, delete ENIs (best-effort), delete index, delete payload, delete record last; reconciler replays teardown (`reconciler.go:611-628`) and marks `failed` after `transitionTimeout` 10 min.
-- Fencing: KV revision CAS on the record everywhere; per-instance `ModifyLease` on the record (`modify_lease.go:24-173`); cluster `kvlease` for the reconciler (`reconciler.go:104-118`); `VMGeneration` binds bootstrap payloads and acknowledgements (`agent.go:345`, `:470`, `:552-553`) and the index.
-- No desired generation exists on `DBInstanceRecord` (`records.go`); `VMGeneration` is a realization generation, not desired-intent generation.
-- Idempotency: AWS `CreateDBInstance` has no client token, and a repeat returns `DBInstanceAlreadyExists` (`create.go:64-66`); delete retry re-runs teardown and must repeat the snapshot choice (`delete.go:57-63`); register is idempotent (`agent.go:159-160`).
-
-### snapshot
-
-- Create: check name free (`snapshot.go:92`) → CAS instance to `backing-up` with `SnapshotOperation` (`:672-705`) → create record `creating` (`:109-115`) → quiesce + EC2 snapshot inline (`:144-188`) → `putJSON` record `available` (`:126-130`) → restore instance status (`:706-723`).
-- Readiness: synchronous; the API returns `available`.
-- Repair: reconciler resolves records stuck `creating` past 15 min against EC2 (`reconciler.go:674-751`, `findEC2SnapshotFor` `:752`); returns instances stuck in `backing-up` (`reconciler.go:630-652`).
-- Delete: `removeDBSnapshot` deletes EC2 snapshot then record and releases a retained volume when last holder (`snapshot.go:330-470`).
-- Automated: window-driven `takeAutomatedBackup` (`backup.go:270`), stamped on instance record so a leader change cannot double-fire (`reconciler.go:654-660` comment); retention sweep `BackupRetentionReaper` via the daemon GC backstop (`backup_sweep.go:45-112`, `daemon/daemon.go:2310`).
-- Fencing: instance-record CAS into `backing-up`; `createJSON` on snapshot key; no snapshot generation.
-
-### subnetgroup
-
-- Create: validate, resolve subnets through EC2 `DescribeSubnets` (`subnetgroup.go:176-239`), `createJSON` (`:78`); synchronous; status always `Complete` (`:31`, `:287`).
-- Describe: KV read and list (`:94-133`).
-- Delete: read, scan all instance records for `DBSubnetGroupName == name` (`:154-163`), plain `kv.Delete` (`:165`).
-- No modify action is served (`ModifyDBSubnetGroup` is neither registered nor unsupported, so it returns `InvalidAction`).
-- No reconciliation; no generation; tags via `tags.go` CAS.
-
-### parametergroup
-
-- Create: validate family against engine registry, `createJSON` meta (`parametergroup.go:29-84`); synchronous.
-- Modify: validate whole batch, then `putJSON` one key per parameter (`:186-196`), then synchronously push to every attached instance via agent `apply-params` (`:198`, `:210-236`, `pending.go:149`).
-- Delete: refuse `default.*`, scan instances for current or pending use (`:342-352`), delete params then meta (`:354-367`).
-- `default.*` groups are synthesized from the engine catalogue (`:456-482`).
-- No generation; no reconciliation of a failed propagation beyond `ParameterApplyFailed` on the instance (`pending.go:214`).
-
-### engine
-
-- Static, process-local catalogue validated at daemon boot (`engine.go:69`, `daemon/daemon.go:2055`); no lifecycle; ADR-0003 S1 excludes pure read-only catalogues from the lifecycle contract.
-
-## 6. NATS subjects and controller-to-agent messages
-
-Gateway → daemon (request/reply, queue group `spinifex-workers`): `rds.<Action>` for the 24 customer actions listed in `subjects.go:21-63`, plus `rds.GetDBBootstrapConfig` and `rds.AcknowledgeDBBootstrap` (`subjects.go:17-19`).
-Budgets: default 30 s, create 5 min, lifecycle 4 min, delete 6 min, modify 6 min, snapshot 6 min, snapshot delete 4 min, restore 8 min (`service_nats.go:12-36`).
-A gateway timeout does not cancel the daemon's inline orchestration (uncertain; inferred from request/reply shape), so the client may see an error while the create continues.
-
-Agent path (agent speaks SigV4 Query to awsgw over the mgmt bridge, `daemon/rds_deps.go:71-73`):
-
-- `RegisterDBInstance` → gateway → `rds.bus.{acct}.{db}.register` → daemon `SubjectRegisterWildcard` (`subjects.go:12-13`, `:65-67`).
-- `SubmitDBStateChange` (heartbeat folded into state, 30 s interval, stale after 90 s; `records.go` constants) → `rds.bus.{acct}.{db}.health`.
-- `GetDBBootstrapConfig` / `AcknowledgeDBBootstrap` → Layer-1 subjects; replay is allowed until acknowledged for the bound `VMGeneration` (`gateway/rds/agent.go:150-180`).
-- `PollDBCommands` long poll (1-20 s) queue-subscribes `rds.bus.{acct}.{db}.command` with group `spinifex-rds-agents` and publishes carried replies to `rds.bus.{acct}.{db}.command-reply` before subscribing (`gateway/rds/agent.go:198-287`).
-- Command types: `set-password`, `apply-params`, `stop-engine`, `grow-filesystem`, `quiesce`, `unquiesce` (`commands.go:12-23`); params `MasterUsername`, `MasterUserPassword`, `QuiesceLabel`, `QuiesceDeadlineSeconds` (`commands.go:27-35`).
-- Issuer: `issueCommand` subscribes the reply subject, then republishes the same `CommandID` every 2 s until reply or budget (`commands.go:144-199`, `:72`); core NATS, not durable, by design for the password (`subjects.go:77-80`).
-- Agent retry: results are retained and re-carried until a poll succeeds (`agents/rds/agent/command.go:100-133`).
-- Agent dedupe: the agent executes every delivered command and has **no CommandID dedupe** (`agents/rds/agent/command.go:128-131`, `:140-157`); a republish delivered after execution but before the issuer sees the reply can execute twice (uncertain; depends on timing between reply relay and the 2 s republish).
-- Commands carry no `VMGeneration` (`agent.go:134-139`), so the fence against a superseded VM is the bus subject plus the gateway's index lookup rather than a generation in the message.
-- `LookupEngine`, `Engine`, `ParamType*`, `ApplyType*`, `Parameter`, `EngineHealth*`, `HeartbeatInterval`, `BootstrapMode*`, `ParameterRollbackMessage`, `SupportedInstanceClasses`, `SmallestInstanceClass` are imported by the agent from `handlers/rds` (full list from import scan); this is the S5 "accidental import of a domain monolith".
-
-RDS → EC2 subjects used as capabilities: `ec2v1.InstanceCommandSubject(id)` for stop/reboot/attach (`lifecycle.go:434`, `launch.go:583`), `ec2.start` (`lifecycle.go:451`), `ec2.DescribeStoppedInstances` (`launch.go:616`), plus EC2 `DescribeInstances` fan-out (`daemon/rds_deps.go:90-97`) and `DescribeInstanceTypes` fan-out (`gateway/rds/enginecatalog.go:147`).
-
-Recovery/redelivery: there is no durable work queue; recovery relies on the reconciler re-reading record status (`reconciler.go:238-282`) under `kvlease`, a 5 min resync and KV watch wakeups (`reconciler.go:120-145`).
-
-## 7. Boundary-crossing imports
-
-### Outbound from RDS (handlers/rds, gateway/rds, gateway/rds.go, daemon/rds_deps.go)
-
-- **Quota**: none from `handlers/rds`; the gateway injects `gw.Quota.EnforceRDSInstances` as `Env.QuotaCheck` (`gateway/rds.go:59-62`).
-- **ExpectedNodes / EC2 instance availability**: `gateway/rds/enginecatalog.go` → `domains/ec2/awsapi/instance.DescribeInstanceTypes` with `env.ExpectedNodes` (`:147`); `daemon/rds_deps.go` → `domains/ec2/awsapi/instance.DescribeInstances` with node count (`:96`).
-- **EC2 instance**: `domains/ec2/systeminstance` (`launch.go:99-100`, `delete.go:236`), `domains/ec2/instance` (`lifecycle.go`, `StartStoppedInstanceOutput`), `contracts/ec2/v1` (`launch.go`, `lifecycle.go`), `runtime/compute/vm` (`agent.go`, `create.go` `vm.VolumeSerial`, `backup_sweep.go` `vm.Reaper`), `foundation/aws/ami`, `domains/ec2/instancetypes` (`sizing.go:30`, `paramcatalog.go:219`).
-- **VPC/network**: `domains/network/systemvpc` (`vpc.go`, `launch.go`, `daemon/rds_deps.go`), EC2 SDK VPC/subnet/SG/ENI calls through narrow interfaces `networkResolver` (`network.go:16`) and `launchVPCProvisioner` (`launch.go:76-96`); `foundation/aws/tags` (`ManagedByRDS`).
-- **Storage/EBS**: EC2 SDK `CreateVolume`/`DeleteVolume` (`launch.go:107-111`), `DescribeVolumes`/`ModifyVolume` (`storage.go:23-26`), EC2 snapshot provider (`snapshot.go:25`), volume attach over EC2 NATS (`launch.go:560-629`).
-- **IAM**: `handlers/iam` for `SystemInstanceRoleEnsurer`/`EnsureSystemInstanceProfile` (`iam.go:56-77`) and `EncryptSecret`/`DecryptSecret` for bootstrap payloads (`bootstrap.go:105`, `:199`).
-- **DNS**: `domains/dns` (`dns.go:17`, `dns.RDSName`, `dns.Change`).
-- **Bluebottle**: `bluebottle/pkg/auth.ParseRoleARN` (`gateway/rds.go:87`), `bluebottle/pkg/masterkey.ReadShared` (`daemon/rds_deps.go:52`); `handlers/rds` imports no Bluebottle package.
-- **Ingress/foundation**: `ingress/aws/query`, `foundation/aws/xml`, `foundation/aws/errors`, `foundation/aws/identifiers`, `foundation/state/{kvstore,kvlease,kvutil,migrate}`, `foundation/lifecycle/reconciler`, `foundation/messaging/nats`, `foundation/pki`, `bootstrap/config`.
-
-### Inbound into RDS (who imports `handlers/rds` or `gateway/rds`)
-
-- `handlers/quota/live.go:14,89` → `NewNATSService(...).DescribeDBInstances` (quota reads RDS through the AWS-shaped API).
-- `accountteardown/reapers_rds.go:9` → `NATSService` describe/modify/delete for instances, subnet groups, parameter groups.
-- `daemon/daemon.go:78` → `Service`, `Reconciler`, all `Subject*`, `ValidateEngineRegistry`, `NewService`, `NewReconciler`, `NewBackupRetentionReaper`.
-- `daemon/rds_deps.go` → `Deps`, `LaunchDeps`, `BackupPolicy`, `NewNATSVolumeAttacher`, `NewDescribeInstanceState`, `NewNATSInstanceCommander`.
-- `daemon/dns_reconcile.go:16,75` → `AccountWatchBuckets` (raw bucket handles).
-- `daemon/bedrock_deps.go:10,43` → `NewNATSVolumeAttacher` (Bedrock/Ochre depends on RDS).
-- `gateway/operations.go:8,44-45` → `gateway_rds.ActionNames`, `UnsupportedActionNames`.
-- `gateway/rds/*` → `KVBucketRDSSystem`, `InstanceIndexKey`, `InstanceIndexEntry`, `InstanceRoleName`, `InternalAgentActions`, `ResourceKind*`, `FormatARN`, `ParseARN`, catalogue functions, `NATSService`, wire types.
-- `agents/rds/agent/*` → wire types and engine catalogue (section 6).
-- `tests/integration/service_daemon_lite.go:51`, `tests/e2e/harness/rds.go` (test support).
-
-## 8. Known gaps against Q-78 / ADR-0003 (observed in code, not live-verified)
-
-1. No desired generation on any RDS record (`records.go`); mutations advance only the KV revision, and `PendingModifiedValues` is overwritten by a later modify (`modify.go:153`), so Q-78 "atomically commits a monotonically increasing desired generation" is not met.
-2. Desired and observed state share one record: `Status`, `Agent`, `UnhealthySince`, `FailureReason` sit beside requested class/storage on `DBInstanceRecord` (`records.go`).
-3. Create performs all external realization inline in the request (`create.go:87`); if the daemon dies after `LaunchDBInstanceVM` returns and before `recordLaunch` (`create.go:115`), the record stays `creating` with no `InstanceID`, times out to `failed` (`reconciler.go:458-462`), and a later delete has no IDs to clean (`delete.go:229-233`, `:537-547`); the VM, ENIs and volume are tagged `ManagedBy=rds` (`launch.go:260`, `:314`, `:400`) but no RDS orphan reaper for them was found.
-4. The same interruption window exists for restore (`restore.go:111` → `:136`) and replace (`replace.go:113` → `:137`); replace's rollback runs only on error return, not on process death.
-5. Delete removes the only ownership record (`delete.go:217`) after ENI deletion that is best-effort and only logs failures (`delete.go:203`, `launch.go:510-520`), contrary to Q-78 "a handler never removes the only reconstructible ownership of required cleanup".
-6. `deleting` that exceeds 10 min becomes `failed` (`reconciler.go:621-624`); a `failed` instance is then handled by `reconcileHealth` rather than continuing teardown, so incomplete cleanup is not shown as a distinct state (uncertain; inferred from the `switch` at `reconciler.go:256-272`).
-7. `StatusRecovering` is declared and reachable in the transition table but never entered, so there is no repair path (`status.go:22`; `recovery.go:19-21` says v1.0 reports rather than repairs).
-8. Synchronous completion where AWS is asynchronous: AWS `CreateDBSnapshot` returns `Status=creating`, here it returns `available` after the snapshot is taken inline (`snapshot.go:126-130`); AWS `StopDBInstance` returns `stopping`, here the handler completes to `stopped` inline (`lifecycle.go:126`); AWS `DeleteDBInstance` leaves the instance describable as `deleting`, here the record is deleted before the response when teardown succeeds inline (`delete.go:97-104`, `:217`).
-9. DB subnet group and parameter group deletes are check-then-delete across keys with no fence against a concurrent `CreateDBInstance` that has already read the group (`subnetgroup.go:151-165`, `parametergroup.go:339-367`; `network.go:43`, `create.go:47`), so an instance can reference a deleted group.
-10. `ModifyDBParameterGroup` writes one key per parameter with no batch atomicity (`parametergroup.go:186-196`); a KV failure mid-loop leaves a half-applied batch despite the comment at `:140-143`, and propagation to instances is synchronous best-effort (`:210-236`).
-11. Quota admission is a non-atomic count before create (`gateway/rds/instance.go:13-19`, `handlers/quota/live.go:85-93`), so concurrent creates can exceed the cap; it counts `len(DescribeDBInstances)` which pages at 100 by default (`paging.go:14`, `describe.go:71`), so an account above 100 instances is under-counted (uncertain whether quota limits ever exceed 100).
-12. Agent commands are republished every 2 s with no agent-side `CommandID` dedupe and no generation (section 6), so internal redelivery is not proven idempotent for `grow-filesystem`, `apply-params` or `set-password`.
-13. Heartbeat liveness lives partly in per-node process memory (`agent.go:284-321`) and the leader judges health from its own map plus a KV floor (`recovery.go:191-209`), which Q-78 discourages for recovery decisions (observation-side, lower severity).
-14. The RDS reconciler lease doubles as the cluster-wide GC election for unrelated reapers (`daemon/daemon.go:2374-2382`); without RDS service init every cluster-wide reaper is skipped.
-15. No lifecycle contract record exists for any RDS resource (ADR-0003 S1); nothing in `handlers/rds` states readiness predicates or deletion obligations as a contract.
-
-## 9. First-resource candidates
-
-| Rank | Resource | Size (prod lines) | Private state | Dependencies | Tests | Extractable without quota / ExpectedNodes / IAM / Bluebottle? |
-|---|---|---|---|---|---|---|
-| 1 | **subnetgroup** | ~340 (`subnetgroup.go`) + its slice of `store.go`, `records.go`, `tags.go`, `arn.go`; gateway 23 | `db-subnet-groups/{name}` only | EC2 `DescribeSubnets` through a narrow interface already (`network.go:16-20`, `subnetgroup.go:189`); reverse dependency: needs an instance-owned query "instances referencing subnet group X" to replace `instancesUsingGroup` (`subnetgroup.go:154`); forward dependency: instance placement reads the group record directly (`network.go:43`), which must become a subnetgroup query | `handlers/rds/subnetgroup_test.go` (338), `tags_test.go`, `gateway/rds/handler_test.go`/`authz_test.go`, e2e `tests/e2e/rds/groups_test.go` (`TestSubnetAndParameterGroups`), `isolation_test.go`, `api_matrix_test.go` (uncertain coverage depth for the last two) | **Yes.** No quota, ExpectedNodes, IAM, Bluebottle, agent or EC2-launch dependency. Synchronous AWS semantics (status `Complete`) match the code. Gap 9 (delete race) should be recorded, not fixed, during the move. |
-| 2 | **parametergroup** | ~600 (`parametergroup.go`) + slice of store/records/tags; gateway 31 | `db-parameter-groups/{name}/meta`, `…/params/{key}` | Engine catalogue (`engineForFamily`, `LookupParameter`, `ResolveEffectiveParameters`) — owned by **engine**; `SmallestInstanceClass` (EC2 `instancetypes`); cluster CA availability (`checkTLSEnforceable`, `parametergroup.go:581`); instance records for in-use check; **instance orchestration** in Modify (`propagateParameterGroup` → `applyParameterGroup` → agent `apply-params`) | `parametergroup_test.go` (806), `paramcatalog_test.go`, `enginefamily_test.go`, `tlsenforcement_test.go` (uncertain scope), `pending_test.go` (propagation), e2e `groups_test.go` | **Mostly.** No quota/ExpectedNodes/IAM/Bluebottle, but Create/Describe/Delete are synchronous while Modify crosses into instance realization and the agent wire contract; extraction requires engine first (or a narrow engine capability) and moving propagation to explicit cross-resource orchestration. |
-| 3 | **engine** | ~1,960 (`engine.go`, `engine_postgres.go`, `engine_mariadb.go`, `paramcatalog.go`, `catalog.go`, `sizing.go`) | none (static catalogue) | `domains/ec2/instancetypes`; consumed by the agent, parametergroup, instance | `engine_test.go`, `engine_mariadb_test.go`, `enginefamily_test.go`, `paramcatalog_test.go`, `catalog_test.go`, `sizing_test.go`, `gateway/rds/enginecatalog_test.go` | **Package: yes**; it is the natural S5 agent capability. **awsapi action `DescribeOrderableDBInstanceOptions`: no**, blocked on the EC2 instance-type-availability projection (`gateway/rds/enginecatalog.go:146-174`). Not a durable resource under ADR-0003 S1, so it is weak evidence for ADR-0004 rollout item 2. |
-| 4 | **snapshot** | ~1,700 (`snapshot.go`, `backup_sweep.go`, share of `backup.go`, `restore.go`, `reconciler.go:674-796`) | `db-snapshots/`, `backups/…/automated/`, `retained-volumes/` | Instance record CAS into `backing-up`, engine quiesce over agent commands, EC2 snapshot/volume services, instance delete (final snapshot, retained volumes), restore runs the instance create path | `snapshot_test.go` (879), `restore_test.go`, `backup_test.go` (1013), `backup_sweep_test.go`, `delete_test.go`; e2e `snapshot_restore_test.go`, `automated_backup_test.go` | No quota/ExpectedNodes/IAM/Bluebottle on snapshot itself, but restore inherits instance create (IAM, launch); snapshot creation needs instance-owned quiesce/hold capabilities first. Not a first candidate. |
-| 5 | **instance** | ~8,000+ (create, describe, lifecycle, launch, network, vpc, storage, modify, modify_lease, pending, replace, recovery, reconciler, window, backup, delete, agent, commands, bootstrap, servingcert, userdata, iam, dns, validate, status) | `db-instances/`, `bootstrap-payloads/`, `rds-system` `instance-index/`, `spinifex-rds-leader` | Quota admission, ExpectedNodes-style EC2 fan-out (`daemon/rds_deps.go:90-97`), IAM ensurer and secret crypto, Bluebottle master key (via composition), EC2 system instance / ENI / volume / snapshot / attach, system VPC, DNS, agent wire contract | `create_test.go`, `launch_test.go`, `lifecycle_test.go`, `delete_test.go`, `modify_test.go`, `modify_lease_test.go`, `pending_test.go`, `replace_test.go`, `reconciler_test.go`, `recovery_test.go`, `revisit_test.go`, `agent_test.go`, `bootstrap_test.go`, `commands_test.go`, `storage_test.go`, `network_test.go`, `vpc_test.go`, `describe_test.go`, `status_test.go`, `iam_test.go`, `userdata_test.go`, `watch_source_test.go`, `gateway/rds/agent_test.go`, `authz_test.go`; e2e `create_describe`, `lifecycle`, `modify`, `delete`, `failure`, `graceful_reboot`, `connect`, `cross_subnet`, `isolation`, `authz`, `mariadb*` | **No.** Every listed blocker applies. Extract last. |
-
-Cross-resource files to settle alongside the first extraction: `tags.go` (per-ARN dispatch), `events.go` (journal), `arn.go`/`filters.go`/`paging.go` (awsapi), `store.go`/`records.go`/`service.go` (split per owner).
-
-Recommendation: extract **subnetgroup** first, with two narrow capabilities defined by the consumer: an instance-owned "references to subnet group" query for delete, and a subnetgroup-owned "resolve placement subnets" query for instance create.
-Then do **engine** (package and agent capability, leaving the orderable action on the legacy path until the EC2 projection exists), then **parametergroup** with propagation made explicit orchestration.
-Test coverage for subnetgroup is unit plus one e2e test; ADR-0003 S5 failure/recovery evidence for its delete race does not exist yet.

docs/package-boundary/eks-addon-bundle-inventory.md

@@ -1,416 +0,0 @@
-# EKS managed add-on: per-bundle readiness and removal inventory
-
-Code read at 3a55e2ee1; not verified live.
-
-This is a discovery record for the add-on lifecycle work: for every bundle in the catalogue it lists what is rendered, how identity is wired, what "ready" currently proves, what removal deletes and leaves, and where the bundle collides with other objects.
-It proposes no design; gaps are stated per bundle and collected at the end.
-It complements `docs/package-boundary/eks-addon-lifecycle.md` (present contract) and `docs/package-boundary/eks-addon-lifecycle-design.md` (draft correction, which refines open proposal Q-146: delete holds until owned objects and identity associations are observably removed, and `ACTIVE` means workloads applied and ready).
-Paths starting `spinifex/` are Go sources; `scripts/` and `tests/` are relative to the spinifex repo root; bundle files are under `scripts/images/eks-node/addons/<addon>/<version>/` and are abbreviated to `<addon>/<version>/<file>`.
-The sync agent `scripts/images/eks-node/mulga-eks-addon-sync.sh` is abbreviated to `sync.sh`.
-
-## 1. Catalogue
-
-| Name | Version | `RequiresIRSA` | Hidden | Evidence |
-|---|---|---|---|---|
-| `aws-load-balancer-controller` | 2.11.0 | true | no | `spinifex/domains/eks/addon/catalog.go:31-33` |
-| `argocd` | 3.0.23 | false | no | `catalog.go:34-36` |
-| `aws-ebs-csi-driver` | 1.40.1 | true | no | `catalog.go:37-39` |
-| `nvidia-device-plugin` | 0.17.4 | false | yes (auto-staged by GPU node groups) | `catalog.go:42-44`, `spinifex/domains/eks/addon/owner.go:79-89`, `spinifex/handlers/eks/nodegroup.go:408-416` |
-| `spinifex-noop` | 0.1.0 | false | yes (e2e fixture) | `catalog.go:48-50` |
-
-`RequiresIRSA` is read only by `DescribeAddonVersions` as `requiresIamPermissions` (`spinifex/handlers/eks/addons.go:255`); `CreateAddon` neither requires a role for the two `true` bundles nor rejects one for the three `false` bundles (`addons.go:61-99`).
-The bundle manifests are baked into the AMI (`scripts/images/eks-node/manifest.conf:58-67`), but container images are not: every image below is pulled from a public registry at pod start.
-
-## 2. AWS reference semantics
-
-Checked against the API reference on 2026-10-09.
-
-| Item | AWS documents | Source |
-|---|---|---|
-| `status` | Enum `CREATING`, `ACTIVE`, `CREATE_FAILED`, `UPDATING`, `DELETING`, `DELETE_FAILED`, `DEGRADED`, `UPDATE_FAILED`; the API reference gives no per-value definition of `ACTIVE` or `DEGRADED`. | API_Addon |
-| `health` | `AddonHealth.issues`, a list of `AddonIssue`. | API_AddonHealth |
-| `AddonIssue` | `code` one of `AccessDenied`, `InternalFailure`, `ClusterUnreachable`, `InsufficientNumberOfReplicas`, `ConfigurationConflict`, `AdmissionRequestDenied`, `UnsupportedAddonModification`, `K8sResourceNotFound`, `AddonSubscriptionNeeded`, `AddonPermissionFailure`; plus `message` and `resourceIds`. | API_AddonIssue |
-| `serviceAccountRoleArn` | "The ARN of the IAM role that's bound to the Kubernetes `ServiceAccount` object that the add-on uses"; on create, "If you don't specify an existing IAM role, then the add-on uses the permissions assigned to the node IAM role", and an IAM OIDC provider must exist for the cluster. | API_Addon, API_CreateAddon |
-| `configurationValues` | "validated against the schema returned by `DescribeAddonConfiguration`". | API_CreateAddon |
-| `resolveConflicts` | `NONE`/`PRESERVE` leave a self-managed install's values and may fail creation; `OVERWRITE` replaces them; with no prior install all three behave alike. | API_CreateAddon |
-| `DeleteAddon` | "When you remove an add-on, it's deleted from the cluster"; `preserve` "preserves the add-on software on your cluster but Amazon EKS stops managing any settings for the add-on. If an IAM account is associated with the add-on, it isn't removed." The example response is `DELETING`. | API_DeleteAddon |
-
-The issue codes imply AWS distinguishes replica shortfall (`InsufficientNumberOfReplicas`), missing objects (`K8sResourceNotFound`), field conflicts (`ConfigurationConflict`) and admission denials; how EKS computes them, and whether `DEGRADED` is driven by them, is not stated in the API reference and was not verified.
-Whether AWS offers each of these five names as an EKS add-on was not checked against a live `DescribeAddonVersions`.
-
-## 3. Shared mechanics (apply to every bundle)
-
-### 3.1 Render
-
-| Step | Behaviour | Evidence |
-|---|---|---|
-| Input | One TSV line per staged add-on: `name`, `version`, `roleArn`, `base64(configurationValues)`. | `spinifex/agents/eks/gatewayfetch/fetch.go` `emitAddonsTSV`, `sync.sh:357` |
-| Source | Every `*.yaml` in `/usr/share/spinifex-eks/addons/<name>/<version>/`, in glob order, concatenated with `---`. A missing directory reports `failed`. | `sync.sh:41`, `sync.sh:82-89`, `sync.sh:219-242` |
-| Output | One file per add-on name, `/var/lib/rancher/k3s/server/manifests/spinifex-addon-<name>.yaml`; the name carries no version. Replaced only when content changes. | `sync.sh:42-43`, `sync.sh:83`, `sync.sh:248-253` |
-| Block markers | `{{IRSA_ENV}}`, `{{IRSA_VOLUME}}`, `{{IRSA_VOLUME_MOUNT}}`, `{{ELB_INGRESS_PARAMS_SPEC}}` replaced by awk when alone on a line. | `sync.sh:224-230` |
-| Scalar placeholders | `{{SERVICE_ACCOUNT_ROLE_ARN}}`, `{{CLUSTER_NAME}}`, `{{AWS_REGION}}`, `{{AWS_VPC_ID}}`, `{{GATEWAY_ENDPOINT}}`, `{{GATEWAY_CA_PEM_B64}}`, `{{WEBHOOK_CA_B64}}`, `{{WEBHOOK_TLS_CRT_B64}}`, `{{WEBHOOK_TLS_KEY_B64}}` via `sed s|..|..|g`. The role ARN is inserted raw: it is not escaped for sed (`|`, `&`, `\`) or for YAML. | `sync.sh:231-240` |
-| Webhook cert | Opt-in by `webhook.conf`; minted once per add-on under `/var/lib/spinifex-eks/webhook-certs/<name>`, 10-year validity, reused thereafter. | `sync.sh:98-120`, `spinifex/agents/eks/webhookcert/cert.go:109-110` |
-| `configurationValues` | Read from the TSV and discarded; no bundle has a hook. | `sync.sh:359` |
-| Executor | Primary server only; server-join nodes do not run the agent. | `scripts/images/eks-node/eks-node-role.sh:138-149` |
-
-### 3.2 Identity (IRSA) wiring
-
-| Element | Present behaviour | Evidence |
-|---|---|---|
-| Role carrier | The role ARN lives in the add-on record and the staged manifest (`serviceAccountRoleArn`), and after render in the rendered file only. No pod identity association, no EKS-side binding object. | `spinifex/domains/eks/addon/record.go:24-45`, `sync.sh:144-160` |
-| Credential env | With a role: `AWS_ROLE_ARN`, `AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_STS_REGIONAL_ENDPOINTS`, region, `AWS_ENDPOINT_URL_STS` (gateway), `AWS_CA_BUNDLE` and `SSL_CERT_FILE`. Without one: region and CA only, so the SDK falls through to node instance-profile credentials over IMDS. | `sync.sh:144-195` |
-| Token | A projected ServiceAccount token, audience `sts.amazonaws.com`, 86400 s, mounted at `/var/run/secrets/eks.amazonaws.com/serviceaccount`, injected statically by the render. There is no pod-identity mutating webhook in the image, so the `eks.amazonaws.com/role-arn` ServiceAccount annotation is descriptive only; the env and volume are what take effect. | `sync.sh:161-178`; no `pod-identity-webhook` in the tree |
-| Issuer | The apiserver's `/openid/v1/jwks` is published at first boot; STS `AssumeRoleWithWebIdentity` resolves `iss` to a registered IAM OIDC provider in the role's account, fetches the cluster JWKS, and evaluates the role trust policy (`{iss}:sub`, `{iss}:aud`). | `scripts/images/eks-node/k3s-first-boot.sh:141-162`, `spinifex/handlers/sts/assume_role_with_web_identity.go:33-213`, `spinifex/handlers/sts/web_identity_trust_policy.go:137-170` |
-| Objects outside the bundle | The IAM role, its trust policy (naming `system:serviceaccount:<ns>:<sa>`) and the IAM OIDC provider are user-owned IAM objects; nothing in the add-on path creates, modifies or deletes them. | `tests/e2e/eks/eks_test.go:658-700` creates them in the test |
-| Authorisation | `iam:PassRole` on the role at `CreateAddon` and `UpdateAddon`. | `spinifex/gateway/eks/passrole.go:31-37` |
-| `eks-token-webhook` | Not part of IRSA: it is the apiserver TokenReview authenticator for `aws eks get-token` bearer tokens. | `scripts/images/eks-node/eks-token-webhook.initd:1-8` |
-
-Because identity association lives only in IAM (user-owned) and in the rendered pod spec, "identity associations observably removed" has no Spinifex-side object to observe for any bundle today: the role is not detached from anything when the add-on is deleted, matching AWS only in that AWS also does not delete the IAM role.
-
-### 3.3 Readiness
-
-| Path | Condition | Evidence |
-|---|---|---|
-| Primary | The K3s `Addon` CR in `kube-system` whose `.spec.source` is the rendered file carries a non-empty `addon.k3s.cattle.io/gvks` annotation. Reported `ready` on every tick this holds. No pod, replica or CRD condition is consulted. | `sync.sh:266-273`, `sync.sh:361-365` |
-| Fallback | Only after `ADDON_READY_GRACE` (90 s) with the annotation still empty: `kubectl get -f <rendered> --ignore-not-found`, then require at least one Deployment, StatefulSet or DaemonSet and that every one returned is rolled out (Deployment `availableReplicas >= spec.replicas` and generation observed; StatefulSet `readyReplicas >= spec.replicas`; DaemonSet `desiredNumberScheduled > 0` and `numberReady >= desiredNumberScheduled`). | `sync.sh:56`, `sync.sh:301-320`, `sync.sh:367-372` |
-| Otherwise | `applied`, which changes no status. | `sync.sh:374-376`, `spinifex/domains/eks/addon/owner.go:186-205` |
-| `failed` | Emitted only for a missing bundle directory or webhook cert failure; never for apply errors, crash-looping pods or lost replicas. | `sync.sh:87`, `sync.sh:112` |
-| Status mapping | `ready` lifts any non-`ACTIVE` status (including `CREATE_FAILED`, `DEGRADED`, `UPDATING`) to `ACTIVE` and clears `health`; `failed` maps `ACTIVE` to `DEGRADED`. Reports carry no generation. | `owner.go:140-168`, `owner.go:186-205` |
-
-Consequences common to all bundles:
-
-1. On the primary path `ready` means "K3s recorded the GVKs it applied", not "workloads running"; a bundle whose pods never start (image pull failure, unschedulable, crash loop) is `ACTIVE` and stays so.
-2. `ACTIVE` never degrades on workload loss: the annotation persists, so `ready` is re-sent every tick and no `failed` is ever produced for runtime health.
-3. `--ignore-not-found` drops absent objects from the fallback set, so a workload that failed to apply is simply not checked; one rolled-out workload is enough.
-4. After `UpdateAddon` the same tick that rewrites the file reads the annotation from the previous apply; whether K3s clears it before that read is not established, so `UPDATING` may settle `ACTIVE` on the old apply (the agent comment at `sync.sh:244-247` says a re-apply clears it, which suggests a window exists).
-5. The agent never deletes the K3s `Addon` CR; if K3s keeps that CR after the file is removed, a later re-create would match it by `.spec.source` and could report `ready` from a stale annotation before the new apply (K3s behaviour not verified).
-
-### 3.4 Removal
-
-| Step | Behaviour | Evidence |
-|---|---|---|
-| Trigger | A rendered `spinifex-addon-<name>.yaml` whose name is absent from a successful fetch. A failed fetch skips GC. | `sync.sh:344-348`, `sync.sh:386-391` |
-| Selection | The object list is the rendered file itself (`kubectl delete -f`), not labels or a GVK inventory. Objects created at runtime by the add-on, or by users through it, are never selected. | `sync.sh:397-401` |
-| Order | File order, which is glob order of the bundle files then document order. | `sync.sh:219` |
-| Waiting | `--wait=false`; errors go to syslog; the copy is deleted; nothing is reported. | `sync.sh:400-402` |
-| Leftovers on the node | `/var/lib/spinifex-eks/webhook-certs/<name>` and the K3s `Addon` CR. The `/tmp` sentinels are cleared. | `sync.sh:47`, `sync.sh:393` |
-| Control plane | Record and manifest keys are erased before any guest action. | `owner.go:122-137` |
-| Namespace completion | Namespace deletion needs every aggregated API available; the e2e asserts `v1beta1.metrics.k8s.io` is `Available` first because otherwise the namespace stays `Terminating`. | `tests/e2e/eks/eks_test.go:197-231` |
-
-`preserve` is ignored (record `eks-addon-lifecycle.md` row Desired state); for every bundle a preserve would need to keep the whole rendered object set and stop only the render and GC, so the per-bundle `preserve` rows below list what in addition would break or dangle.
-
-## 4. `spinifex-noop` 0.1.0
-
-### Objects
-
-| Kind | Name | Namespace | Source |
-|---|---|---|---|
-| Namespace | `spinifex-noop` | | `spinifex-noop/0.1.0/noop.yaml:5-10` |
-| ConfigMap | `spinifex-noop` | `spinifex-noop` | `noop.yaml:12-20` |
-
-Images: none.
-Both objects carry `app.kubernetes.io/managed-by: spinifex-addon` (`noop.yaml:9-10`, `noop.yaml:17-18`); nothing reads the label.
-
-### Identity
-
-No IRSA markers, no ServiceAccount; a supplied `serviceAccountRoleArn` is stored and inert.
-
-### Readiness
-
-| Question | Answer |
-|---|---|
-| Primary path | Ready once K3s records the two GVKs. This is the path the e2e exercises (`tests/e2e/eks/eks_test.go:158-195`). |
-| Fallback | Never succeeds: no workload kinds, so `length > 0` is false (`sync.sh:308`). |
-| Meaning of ready | Both objects applied; there is nothing to serve. |
-| False ready | None material. |
-
-### Removal
-
-| Question | Answer |
-|---|---|
-| Deleted | Namespace (cascading the ConfigMap) and ConfigMap. |
-| Left behind | The K3s `Addon` CR. |
-| Preserve | Keep both; nothing dangles. |
-| Completion signal | `kubectl get ns spinifex-noop` NotFound; this is exactly what the e2e asserts (`eks_test.go:221-231`). Blocked indefinitely by an unavailable aggregated API. |
-
-### Conflicts and configuration
-
-Namespace name could collide with a user namespace of the same name, which GC would then delete.
-No configuration hook.
-
-### Gaps
-
-1. The only bundle with removal proven end-to-end; its completion signal (namespace NotFound) does not generalise to bundles in `kube-system`.
-
-## 5. `argocd` 3.0.23
-
-### Objects
-
-Single file `argocd/3.0.23/argocd.yaml` (26,819 lines), upstream `install.yaml` passed through kustomize with `namespace: argocd` (`argocd.yaml:1-11`).
-
-| Kind | Count | Names | Namespace | Evidence |
-|---|---|---|---|---|
-| Namespace | 1 | `argocd` (first document) | | `argocd.yaml:12-17` |
-| CustomResourceDefinition | 3 | `applications.argoproj.io`, `applicationsets.argoproj.io`, `appprojects.argoproj.io` | | `argocd.yaml:25`, `:5904`, `:23563` |
-| ServiceAccount | 7 | `argocd-application-controller`, `-applicationset-controller`, `-dex-server`, `-notifications-controller`, `-redis`, `-repo-server`, `-server` | `argocd` | parsed |
-| Role / RoleBinding | 6 / 6 | per component | `argocd` | parsed |
-| ClusterRole / ClusterRoleBinding | 3 / 3 | `argocd-application-controller`, `argocd-applicationset-controller`, `argocd-server` | | `argocd.yaml:24243-24554` |
-| ConfigMap | 7 | `argocd-cm`, `-cmd-params-cm`, `-gpg-keys-cm`, `-notifications-cm`, `-rbac-cm`, `-ssh-known-hosts-cm`, `-tls-certs-cm` | `argocd` | parsed |
-| Secret | 2 | `argocd-notifications-secret`, `argocd-secret` | `argocd` | `argocd.yaml:24740`, `:24751` |
-| Service | 8 | applicationset-controller, dex-server, metrics, notifications-controller-metrics, redis, repo-server, server, server-metrics | `argocd` | parsed |
-| Deployment | 6 | `argocd-applicationset-controller`, `-dex-server`, `-notifications-controller`, `-redis`, `-repo-server`, `-server` (default 1 replica) | `argocd` | `argocd.yaml:24925-25887` |
-| StatefulSet | 1 | `argocd-application-controller`, 1 replica | `argocd` | `argocd.yaml:26301-26310` |
-| NetworkPolicy | 7 | one per component | `argocd` | parsed |
-
-Images: `quay.io/argoproj/argocd:v3.0.23` (8 containers), `ghcr.io/dexidp/dex:v2.41.1` (`argocd.yaml:25243`), `redis:7.2.11-alpine` (`argocd.yaml:25478`).
-Only the Namespace carries `app.kubernetes.io/managed-by: spinifex-addon`.
-
-### Identity
-
-No IRSA markers (`argocd.yaml:11`); a supplied role is stored and inert.
-All seven ServiceAccounts are created by the bundle.
-
-### Readiness
-
-| Question | Answer |
-|---|---|
-| Primary path | Per the agent comment, K3s never records the `gvks` annotation for this bundle, so the primary path does not fire (`sync.sh:49-55`); why is not established. |
-| Fallback | After 90 s: all 6 Deployments available at 1 replica and the StatefulSet at 1 ready replica, among those `kubectl get -f` finds. |
-| Meaning of ready | Every returned workload rolled out once. CRDs are not checked for `Established`; an absent CRD or workload is dropped by `--ignore-not-found` rather than failing the check. |
-| False ready | A partial apply that left any document unapplied still reads ready if the remaining workloads roll out; if the missing `gvks` is caused by an apply error on one object, this is the case in practice (hypothesis, not verified). Dex and notifications are counted though unused by default. |
-| Runtime | Images pulled from three public registries; on an air-gapped site the bundle never rolls out and stays `CREATING`. |
-
-### Removal
-
-| Question | Answer |
-|---|---|
-| Deleted | Everything in the file, Namespace first. CRD deletion removes every `Application`, `ApplicationSet` and `AppProject` in the cluster, including the runtime `default` AppProject. |
-| Left behind | Workloads Argo CD deployed into other namespaces (user applications) unless their `Application` carries the cascade finalizer and the controller survives long enough to process it; runtime Secrets in `argocd` go with the namespace. |
-| Stuck risk | `Application` objects carrying `resources-finalizer.argocd.argoproj.io` (user-set, upstream behaviour) need the application controller to clear them; it is deleted in the same pass, so the CRs, the `applications` CRD and the `argocd` namespace can stay `Terminating` indefinitely (not verified live). |
-| Preserve | Keep all objects; the cluster-scoped ClusterRoles and CRDs then remain unowned. |
-| Completion signal | `argocd` namespace NotFound and the three CRDs NotFound and the three ClusterRoles and ClusterRoleBindings NotFound; namespace and CRD finalizers must be watched explicitly, since NotFound on the namespace alone does not cover cluster-scoped objects. |
-
-### Conflicts and configuration
-
-A user-installed Argo CD (same CRDs, same ClusterRole names, namespace `argocd`) would be overwritten on create and wholly deleted on DeleteAddon; this is the `resolveConflicts` case.
-No configuration hook, although `argocd-cm`/`argocd-rbac-cm` are the natural targets.
-
-### Gaps
-
-1. Readiness depends entirely on the fallback, which neither checks CRDs nor detects missing objects.
-2. Removal deletes user-owned `Application` data cluster-wide and can wedge on finalizers; nothing reports it.
-3. No e2e coverage at all.
-
-## 6. `aws-ebs-csi-driver` 1.40.1
-
-### Objects
-
-| Kind | Name | Namespace | Source |
-|---|---|---|---|
-| ServiceAccount | `ebs-csi-controller-sa` (role annotation) | `kube-system` | `aws-ebs-csi-driver/1.40.1/00-rbac.yaml:5-14` |
-| ServiceAccount | `ebs-csi-node-sa` | `kube-system` | `00-rbac.yaml:16-23` |
-| ClusterRole | `ebs-external-attacher-role`, `ebs-csi-node-role`, `ebs-external-provisioner-role`, `ebs-external-resizer-role`, `ebs-external-snapshotter-role` | | `00-rbac.yaml:25`, `:65`, `:94`, `:184`, `:241` |
-| ClusterRoleBinding | `ebs-csi-attacher-binding`, `ebs-csi-node-getter-binding`, `ebs-csi-provisioner-binding`, `ebs-csi-resizer-binding`, `ebs-csi-snapshotter-binding` | | `00-rbac.yaml:309-381` |
-| Role / RoleBinding | `ebs-csi-leases-role` / `ebs-csi-leases-rolebinding` | `kube-system` | `00-rbac.yaml:384-417` |
-| CSIDriver | `ebs.csi.aws.com` | | `00-rbac.yaml:421-430` |
-| PodDisruptionBudget | `ebs-csi-controller` (maxUnavailable 1) | `kube-system` | `00-rbac.yaml:432-443` |
-| Secret | `spinifex-gateway-ca` (shared name, see Conflicts) | `kube-system` | `10-controller.yaml:8-17` |
-| Deployment | `ebs-csi-controller`, 2 replicas, preferred anti-affinity, tolerates `CriticalAddonsOnly` and `nvidia.com/gpu` | `kube-system` | `10-controller.yaml:19-82` |
-| DaemonSet | `ebs-csi-node`, tolerates `nvidia.com/gpu` only | `kube-system` | `20-node.yaml:5-58` |
-| StorageClass | `ebs-gp3`, default class, `WaitForFirstConsumer` | | `30-storageclass.yaml:5-18` |
-
-Images: `public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.40.1` (`10-controller.yaml:89`, `20-node.yaml:67`), `public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v5.2.0-eks-1-32-7` (`:147`), `external-attacher:v4.8.1-eks-1-32-7` (`:180`), `external-snapshotter/csi-snapshotter:v8.2.1-eks-1-32-7` (`:209`), `external-resizer:v1.13.2-eks-1-32-7` (`:238`), `livenessprobe:v2.14.0-eks-1-32-7` (`:270`, `20-node.yaml:155`), `node-driver-registrar:v2.13.0-eks-1-32-7` (`20-node.yaml:118`).
-The bundle header says it is vendored from upstream v1.41.0 while the catalogue version is 1.40.1 (`00-rbac.yaml:1-3`); the driver image is v1.40.1.
-
-### Identity
-
-| Question | Answer |
-|---|---|
-| Takes `serviceAccountRoleArn` | Yes (`RequiresIRSA`), not required at create. |
-| Annotated SA | `ebs-csi-controller-sa` (`00-rbac.yaml:12-13`, quoted, so an empty role renders `''`). |
-| Injected pod | Controller Deployment only: `{{IRSA_ENV}}` (`10-controller.yaml:110`), `{{IRSA_VOLUME_MOUNT}}` (`:114`), `{{IRSA_VOLUME}}` (`:289`). The node DaemonSet gets no credentials and makes no EC2 calls (`20-node.yaml:1-3`). |
-| SA owner | The bundle creates both ServiceAccounts. |
-| Without a role | Controller uses node instance-profile credentials over IMDS. |
-| Evidence | `tests/e2e/eks/eks_test.go:658-800` installs it with a role, creates the OIDC provider and role, and drives a PVC through CreateVolume/AttachVolume. |
-
-### Readiness
-
-| Question | Answer |
-|---|---|
-| Primary path | `ready` as soon as K3s records GVKs; neither replica need be running. The e2e waits for `ACTIVE` and then separately proves the data path (`eks_test.go:767-800`). |
-| Fallback | Controller Deployment 2 of 2 available and node DaemonSet `desired > 0` and all ready. |
-| Fallback false negative | A cluster with no worker node that tolerates scheduling (only the control plane, which carries `CriticalAddonsOnly` the DaemonSet does not tolerate) has `desired = 0`, so the fallback never passes. |
-| False ready | Controller pending or crash-looping (including STS web-identity failures from a bad trust policy), or sidecars failing, still `ACTIVE` via the primary path. The bundle ships no `VolumeSnapshot` CRDs while running `csi-snapshotter`; the sidecar's behaviour without them was not verified and nothing in readiness would show it. |
-| AWS comparison | `InsufficientNumberOfReplicas` and `AccessDenied` are the codes that would correspond to these states; neither is ever produced. |
-
-### Removal
-
-| Question | Answer |
-|---|---|
-| Deleted | All objects above, including the default StorageClass and the CSIDriver, in file order (RBAC and CSIDriver first, then Secret and controller, then node DaemonSet, then StorageClass). |
-| Left behind (in cluster) | PersistentVolumes and PVCs it provisioned; `VolumeAttachment` objects; runtime leader-election Leases in `kube-system`; `CSINode` driver entries until node plugins unregister. |
-| Left behind (outside cluster) | Spinifex EBS volumes backing those PVs, still allocated and possibly attached. |
-| Stuck risk | PVs with `reclaimPolicy: Delete` and VolumeAttachments carry CSI sidecar finalizers that only the deleted controller clears, so later PVC deletion or node drain hangs and the backing volume is never deleted (upstream sidecar behaviour, not verified on this build). |
-| Preserve | Keep the whole set; existing mounts and new provisioning keep working with the last-rendered role. |
-| Completion signal | All 20 rendered objects NotFound (all in `kube-system` or cluster-scoped, so there is no namespace to watch); a meaningful "removed" additionally needs zero `VolumeAttachment`s and PVs with `spec.csi.driver == ebs.csi.aws.com`, or an explicit statement that those are user data that outlive the add-on. |
-
-### Conflicts and configuration
-
-| Object | Collides with |
-|---|---|
-| `kube-system/spinifex-gateway-ca` | The identical name in `aws-load-balancer-controller` (`10-aws-load-balancer-controller.yaml:33-41`). Deleting either add-on deletes the Secret the other mounts; K3s re-applies the survivor's file only when it changes, so the survivor's next pod start fails until then (re-apply behaviour not verified). A removal check by NotFound on this Secret also never completes while the other add-on re-creates it. |
-| `StorageClass/ebs-gp3` with `is-default-class` | Any user default StorageClass, giving two defaults; K3s `local-storage` is disabled (`scripts/images/eks-node/setup.sh:98-101`). |
-| `CSIDriver/ebs.csi.aws.com`, the five ClusterRoles, `ebs-csi-*` names | A self-managed (Helm) EBS CSI install, the `resolveConflicts` case; DeleteAddon would remove the user's driver too. |
-
-No configuration hook; upstream exposes replica count, tolerations and StorageClass parameters through Helm values.
-
-### Gaps
-
-1. `ACTIVE` does not imply a running controller or working credentials.
-2. Removal orphans PVs, VolumeAttachments and Spinifex volumes, and can wedge them behind finalizers.
-3. Shares `spinifex-gateway-ca` with the load balancer controller, so removal of either is not isolated.
-4. Bundle provenance comment (v1.41.0) disagrees with the catalogue version (1.40.1).
-
-## 7. `aws-load-balancer-controller` 2.11.0
-
-### Objects
-
-| Kind | Name | Namespace | Source |
-|---|---|---|---|
-| CustomResourceDefinition | `ingressclassparams.elbv2.k8s.aws` | | `aws-load-balancer-controller/2.11.0/00-crds.yaml:1-8` |
-| CustomResourceDefinition | `targetgroupbindings.elbv2.k8s.aws` | | `00-crds.yaml:256-263` |
-| ServiceAccount | `aws-load-balancer-controller` (role annotation, unquoted) | `kube-system` | `10-aws-load-balancer-controller.yaml:2-13` |
-| Secret | `aws-load-balancer-webhook-tls` (node-minted cert) | `kube-system` | `10-…yaml:17-28` |
-| Secret | `spinifex-gateway-ca` (shared name) | `kube-system` | `10-…yaml:32-41` |
-| Role / RoleBinding | `aws-load-balancer-controller-leader-election-role` / `-rolebinding` | `kube-system` | `10-…yaml:44-48`, `:242-254` |
-| ClusterRole / ClusterRoleBinding | `aws-load-balancer-controller-role` / `-rolebinding` | | `10-…yaml:85-89`, `:258-269` |
-| Service | `aws-load-balancer-webhook-service` | `kube-system` | `10-…yaml:273-277` |
-| Deployment | `aws-load-balancer-controller`, 1 replica, tolerates `CriticalAddonsOnly` and `nvidia.com/gpu` | `kube-system` | `10-…yaml:288-372` |
-| List → IngressClassParams | `alb` (subnets injected from `EKS_ELB_SUBNET_IDS`) | | `10-…yaml:375-386`, `sync.sh:197-215` |
-| List → IngressClass | `alb` | | `10-…yaml:388-400` |
-| MutatingWebhookConfiguration | `aws-load-balancer-webhook`: `mservice` (Services, all namespaces), `mpod` (Pods in namespaces labelled `elbv2.k8s.aws/pod-readiness-gate-inject`), `mtargetgroupbinding`; all `failurePolicy: Fail` | | `10-…yaml:402-488` |
-| ValidatingWebhookConfiguration | `aws-load-balancer-webhook`: `vingressclassparams`, `vtargetgroupbinding`, `vingress`; all `failurePolicy: Fail` | | `10-…yaml:489-564` |
-
-Image: `public.ecr.aws/eks/aws-load-balancer-controller:v2.11.0` (`10-…yaml:320`).
-Webhook SANs come from `webhook.conf` (`aws-load-balancer-controller/2.11.0/webhook.conf`).
-
-### Identity
-
-| Question | Answer |
-|---|---|
-| Takes `serviceAccountRoleArn` | Yes (`RequiresIRSA`), not required at create. |
-| Annotated SA | `aws-load-balancer-controller` (`10-…yaml:11-12`); the placeholder is unquoted, so the raw ARN becomes a YAML scalar and an empty role renders a null annotation value. |
-| Injected pod | Controller Deployment: `{{IRSA_ENV}}` (`:319`), `{{IRSA_VOLUME_MOUNT}}` (`:349`), `{{IRSA_VOLUME}}` (`:372`); STS, EC2, ELBv2 and ACM endpoints pinned to the gateway by args (`:309-314`). |
-| SA owner | The bundle. |
-| Without a role | Node instance-profile credentials over IMDS. |
-| Evidence | No e2e installs this add-on; IRSA is proven only for EBS CSI and a hand-written pod (`tests/e2e/eks/eks_test.go:234-247`). |
-
-### Readiness
-
-| Question | Answer |
-|---|---|
-| Primary path | `ready` when GVKs are recorded; the controller need not be running. |
-| Fallback | The one Deployment at 1 available replica. |
-| False ready | Controller pending, crash-looping, or failing STS/ELBv2 calls is still `ACTIVE`. Worse than a passive false ready: with the controller absent, the six `failurePolicy: Fail` webhooks reject every Service create or update cluster-wide (`mservice` matches all namespaces except LBC's own objects) and every Ingress, so an `ACTIVE` add-on can break unrelated workloads. CRD `Established` is not checked. |
-| AWS comparison | `AdmissionRequestDenied` and `InsufficientNumberOfReplicas` are the matching issue codes; neither is produced. |
-
-### Removal
-
-| Question | Answer |
-|---|---|
-| Deleted | CRDs first (00 file sorts first), then ServiceAccount, Secrets, RBAC, Service, Deployment, IngressClassParams, IngressClass, and the webhook configurations last. |
-| Left behind (in cluster) | User Ingresses and `LoadBalancer` Services it reconciled, with the controller's finalizers still set; the runtime leader Lease. |
-| Left behind (outside cluster) | Spinifex ELBv2 load balancers, target groups, listeners and any security groups the controller created for those Ingresses and Services. |
-| Stuck risk | Deleting the `targetgroupbindings` CRD waits on TargetGroupBindings whose finalizer only the controller clears; user Ingresses and Services keep finalizers no one removes, so they cannot be deleted (upstream controller behaviour, not verified on this build). Between the Deployment deletion and the webhook deletion, Service and Ingress writes fail; if the webhook delete errors, that state persists. |
-| Preserve | Keep the whole set; the webhook cert and gateway CA must stay valid (cert is 10-year). |
-| Completion signal | All rendered objects NotFound plus both CRDs gone (no `Terminating`); a meaningful "removed" also needs no remaining object carrying the controller's finalizers and no remaining Spinifex ELBv2 resources tagged for this cluster, which only the host side can observe. |
-
-### Conflicts and configuration
-
-| Object | Collides with |
-|---|---|
-| `kube-system/spinifex-gateway-ca` | `aws-ebs-csi-driver` (Section 6). |
-| `IngressClass/alb`, `IngressClassParams/alb` | Any user IngressClass named `alb`, including a self-managed LBC. |
-| CRDs, ClusterRole, webhook configuration names | A self-managed LBC (Helm), the `resolveConflicts` case. |
-
-No configuration hook; `EKS_ELB_SUBNET_IDS` is the only per-cluster input and it comes from the node environment, not from `configurationValues`.
-The design draft names this bundle as the first configuration target.
-
-### Gaps
-
-1. `ACTIVE` can coexist with fail-closed webhooks that have no backend, which breaks Service and Ingress writes cluster-wide.
-2. Removal orphans user Ingresses/Services behind finalizers and Spinifex ELBv2 resources outside the cluster; nothing in the guest can observe the latter.
-3. Shares `spinifex-gateway-ca` with EBS CSI.
-4. Unquoted role placeholder takes the raw ARN as YAML.
-5. No e2e coverage.
-
-## 8. `nvidia-device-plugin` 0.17.4
-
-### Objects
-
-| Kind | Name | Namespace | Source |
-|---|---|---|---|
-| DaemonSet | `nvidia-device-plugin-daemonset`; nodeSelector `nvidia.com/gpu.present=true`; tolerates `nvidia.com/gpu`; `runtimeClassName: nvidia`; hostPath `/` read-only, `/var/lib/kubelet/device-plugins`, `/var/run/cdi` | `kube-system` | `nvidia-device-plugin/0.17.4/10-daemonset.yaml:14-75` |
-
-Image: `nvcr.io/nvidia/k8s-device-plugin:v0.17.4` (`10-daemonset.yaml:41`).
-Depends on a `RuntimeClass` named `nvidia` and on the node label written by `mulga-eks-provider-id` (`10-daemonset.yaml:1-12`); neither is in the bundle.
-
-### Identity
-
-No IRSA markers and no ServiceAccount object: pods run as the `kube-system/default` ServiceAccount that Kubernetes creates.
-`EnsureGPUDevicePlugin` stages it with no role (`owner.go:79-89`); a user-created one may carry an inert role.
-
-### Readiness
-
-| Question | Answer |
-|---|---|
-| Primary path | `ready` when K3s records the DaemonSet GVK, regardless of GPU nodes. |
-| Fallback | Requires `desiredNumberScheduled > 0`: never passes on a cluster with no GPU node. |
-| False ready | `ACTIVE` with zero GPU nodes (nothing serving); `ACTIVE` after every GPU node group is deleted; `FAIL_ON_INIT_ERROR=false` (`10-daemonset.yaml:46-47`) and no readiness probe mean a pod whose NVML init failed is Ready while advertising no `nvidia.com/gpu`. The e2e proves the real signal separately: DaemonSet `1=1` and node allocatable `nvidia.com/gpu=1` (`tests/e2e/gpu/eks_gpu_test.go:260-275`). |
-
-### Removal
-
-| Question | Answer |
-|---|---|
-| Deleted | The DaemonSet. |
-| Left behind | CDI spec files the plugin wrote to host `/var/run/cdi`; node `nvidia.com/gpu` capacity until kubelet reconciles the unregistered plugin (kubelet behaviour not verified). |
-| Re-staging | Deleting it while GPU node groups exist leaves GPUs unschedulable until the next GPU node group create re-stages it; deleting the last GPU node group never unstages it (`eks-addon-lifecycle.md` row Cross-resource dependency). |
-| Preserve | Keep the DaemonSet; nothing dangles. |
-| Completion signal | DaemonSet NotFound; pods gone from GPU nodes. |
-
-### Conflicts and configuration
-
-A user-installed NVIDIA device plugin or GPU Operator would run a second plugin on the same nodes and contend for the kubelet device-plugin socket; the DaemonSet name matches the upstream static manifest, so a self-managed upstream install is overwritten.
-No configuration hook (device-list strategy, time-slicing and MIG are upstream options).
-
-### Gaps
-
-1. Ready is indifferent to whether any GPU is exposed; the workload fallback is unreachable on non-GPU clusters.
-2. Ownership is split between the auto-stager and the user, with no record of which.
-
-## 9. Cross-bundle findings
-
-| # | Finding | Bundles | Evidence |
-|---|---|---|---|
-| X1 | Readiness is "applied" on the primary path; workload state is never consulted while `gvks` is present, and runtime failure never yields `failed`. | all | `sync.sh:361-365`, `sync.sh:87`, `sync.sh:112` |
-| X2 | The fallback silently drops absent objects (`--ignore-not-found`) and never checks CRD `Established`. | argocd, LBC, EBS | `sync.sh:304-319` |
-| X3 | DaemonSet `desired > 0` makes the fallback unreachable for node-dependent bundles on clusters without matching nodes, while the primary path reports ready for the same state. | EBS, nvidia | `sync.sh:317` |
-| X4 | Removal selects only objects in the rendered file; runtime-created and user-created dependants (PVs, VolumeAttachments, Applications, Ingresses with finalizers, Leases) are not inventoried, and out-of-cluster resources (EBS volumes, ELBv2) are invisible to the guest. | EBS, LBC, argocd | `sync.sh:397-401` |
-| X5 | `kube-system/spinifex-gateway-ca` is rendered by two bundles under one name; either removal deletes the other's dependency, and a NotFound completion check on it is ill-defined while both exist. | LBC, EBS | `10-aws-load-balancer-controller.yaml:32-41`, `aws-ebs-csi-driver/1.40.1/10-controller.yaml:8-17` |
-| X6 | No bundle has a stable ownership label set: only noop and the argocd Namespace carry `managed-by: spinifex-addon`; LBC, EBS and nvidia carry only upstream `app.kubernetes.io/name` labels, which a self-managed install of the same software carries too. A label-based inventory or conflict check has nothing reliable to select on. | all | per-bundle sources |
-| X7 | Identity association is only the rendered pod env plus user-owned IAM objects; there is nothing Spinifex-owned to remove or observe, and the SA annotation is not acted on by any webhook. | LBC, EBS | Section 3.2 |
-| X8 | `RequiresIRSA` is advisory: no create-time check either way, so a role on argocd, nvidia or noop is stored and inert. | all | `addons.go:61-99`, `addons.go:255` |
-| X9 | The role ARN is substituted raw into sed and YAML; only `iam:PassRole` (which requires the role to exist) constrains it. | LBC, EBS | `sync.sh:231`, `passrole.go:31-37` |
-| X10 | `configurationValues` has no schema, no `DescribeAddonConfiguration`, and no render hook for any bundle. | all | `sync.sh:359` |
-| X11 | Removal runs only on the primary server and only from the local rendered file; a replaced primary has no file and removes nothing. | all | `eks-node-role.sh:138-142`, `sync.sh:386` |
-| X12 | E2E covers create, ready and delete only for noop, create and data path for EBS (delete only in cleanup, not asserted), and DaemonSet rollout for nvidia; argocd and LBC have none. | all | `tests/e2e/eks/eks_test.go:158-247`, `tests/e2e/gpu/eks_gpu_test.go:260-275` |
-
-## 10. Per-bundle observable removal signal (summary)
-
-| Bundle | Objects that must be NotFound | Finalizer or dependant to watch | Out-of-cluster residue |
-|---|---|---|---|
-| `spinifex-noop` | Namespace `spinifex-noop` | Namespace `kubernetes` finalizer (aggregated API availability) | none |
-| `argocd` | Namespace `argocd`, 3 CRDs, 3 ClusterRoles, 3 ClusterRoleBindings | `Application` resources finalizer; CRD and namespace `Terminating` | none from the bundle; user workloads it deployed |
-| `aws-ebs-csi-driver` | 20 objects, all `kube-system` or cluster-scoped | PV and VolumeAttachment CSI finalizers | Spinifex EBS volumes |
-| `aws-load-balancer-controller` | 2 CRDs plus 13 objects, webhooks included | Ingress, Service and TargetGroupBinding controller finalizers; CRD `Terminating` | Spinifex ELBv2 load balancers, target groups, security groups |
-| `nvidia-device-plugin` | 1 DaemonSet | none | host CDI spec files |

docs/package-boundary/eks-addon-executor-inventory.md

@@ -1,141 +0,0 @@
-# EKS managed add-on: delivery executor and fencing inventory
-
-Code read at 3a55e2ee1; not verified live.
-
-This inventory supports the "Delivery executor" section (Section 10) and review decision R6 of `docs/package-boundary/eks-addon-lifecycle-design.md`, and the draft proposal PROP-SPINIFEXARCH-007 (`docs/development/proposals/spinifex-architecture/0007-contract-guest-and-persisted-state-evolution.md` in the mulga repo), which is not accepted.
-It records what the code does today and which evidence exists for choosing and fencing a guest-side delivery executor; it proposes no design.
-Paths are relative to `spinifex/` unless they start with `scripts/`, `images/`, `contracts/` or `tests/`, which are relative to the spinifex repo root.
-
-## 1. Which guest process delivers add-ons
-
-| Item | Present behaviour | Evidence |
-|---|---|---|
-| Delivery process | `mulga-eks-addon-sync` (shell loop): fetch staged set, render each bundle into `/var/lib/rancher/k3s/server/manifests/spinifex-addon-<name>.yaml`, report a phase per add-on, GC rendered files whose add-on is no longer staged with `kubectl delete --wait=false`. | `scripts/images/eks-node/mulga-eks-addon-sync.sh:341-405` (sync), `:80-254` (render), `:383-404` (GC) |
-| Loop and cadence | Every `ADDON_SYNC_INTERVAL` (30 s) forever; one-shot when unset. | `mulga-eks-addon-sync.sh:409-417`, `mulga-eks-addon-sync.initd:18`, `images/mkosi.profiles/eks-server/mkosi.extra/etc/systemd/system/mulga-eks-addon-sync.service:13` |
-| Fetch failure | Logs and keeps the current render; no GC runs that tick, so already-rendered files stay in the auto-deploy dir. | `mulga-eks-addon-sync.sh:344-348` |
-| Report | `printf` JSON `{addon,version,phase,message,ts}` piped to `eks-gateway-publish -channel addon`; failure ignored. No executor identity, instance ID or epoch in the payload. | `mulga-eks-addon-sync.sh:68-73`, `contracts/eks/v1/addon_status.go:31-37` |
-| Local state | Per-add-on sentinels in `/tmp` (`.addon-firstapply-*`, `.addon-diag-*`) and webhook certs in `/var/lib/spinifex-eks/webhook-certs`; no record of which generation or executor rendered a file. | `mulga-eks-addon-sync.sh:47`, `:279-293`, `:326-328` |
-| Credentials | Sources `/etc/spinifex-eks/first-boot.env`; signs with `EKS_ACCESS_KEY`/`EKS_SECRET_KEY` when present, else the SDK chain reads IMDS instance-role credentials. | `mulga-eks-addon-sync.sh:19-33`, `handlers/eks/k3s_server_vm.go:549-557` |
-| OpenRC unit | `/etc/init.d/mulga-eks-addon-sync`, `after k3s`, refuses to start without `first-boot.env`. | `scripts/images/eks-node/mulga-eks-addon-sync.initd:1-30`, installed by `scripts/images/eks-node/manifest.conf:43-44` |
-| systemd unit | `mulga-eks-addon-sync.service`, `After=k3s.service`, `WantedBy=multi-user.target`; baked disabled by preset. | `images/mkosi.profiles/eks-server/mkosi.extra/etc/systemd/system/mulga-eks-addon-sync.service:1-17`, `images/mkosi.profiles/eks-server/mkosi.extra/usr/lib/systemd/system-preset/40-mulga-eks-role.preset:73` |
-| Bake-time enablement | OpenRC image enables only `crond eks-node-role mulga-ebs-byid mulga-eks-provider-id mulga-vpc-mtu`; systemd image enables only `eks-node-role` and the snapshot/otel units. | `scripts/images/eks-node/manifest.conf:83`, `40-mulga-eks-role.preset:59-75` |
-
-## 2. How a VM decides it is the primary
-
-| Item | Present behaviour | Evidence |
-|---|---|---|
-| Role source | Host renders `SPINIFEX_K3S_ROLE=server` when `ServerURL` is empty and `server-join` otherwise; workers get `agent`. | `handlers/eks/k3s_server_vm.go:527-545`, `handlers/eks/nodegroup_userdata.go:64` |
-| Who gets `server` | Create: the first VM of the spread (`nodes[0]`) or the single CP; every other spread member gets `ServerURL` and so `server-join`. | `handlers/eks/k3s_ha_control_plane.go:290-323`, `:272-282`; test `k3s_ha_control_plane_test.go` `TestPlaceControlPlane_SpreadFirstInitsRestJoin` |
-| Selector | `eks-node-role.sh` runs once at first boot, writes `/etc/spinifex-eks/role`, enables and starts role services, then disables itself; a later boot exits early if the role file exists. | `scripts/images/eks-node/eks-node-role.sh:90-96`, `:118-122`, `:177` |
-| Primary-only service | Only the `server` branch enables and starts `mulga-eks-addon-sync` (and `k3s-first-boot`); `server-join` does not. | `eks-node-role.sh:124-150` vs `:151-166`; `scripts/images/eks-node/eks-node-role_test.sh:129`, `:162` |
-| Primacy is static | "Primary" is a first-boot, per-VM, irreversible choice; nothing in the guest re-evaluates it, and no kube Lease, k3s leader or etcd leader is consulted. | `eks-node-role.sh:90-96`; no lease/leader reference in `scripts/images/eks-node/*.sh` |
-| Can a non-primary run it | Not by any code path read: `server-join` never enables it, the preset and OpenRC runlevel bake it disabled. Only manual enablement inside the VM would start a second instance, and the gateway would admit it (Section 4). | `eks-node-role.sh:151-166`, `40-mulga-eks-role.preset:73`, `manifest.conf:83` |
-| K3s auto-deploy in HA | Each K3s server watches its own local `server/manifests` dir; whether the auto-deploy controller applies on every server or only on the elected K3s leader is unverified. If it runs only on the leader, a primary that is not the K3s leader would not apply. | not verified; `mulga-eks-addon-sync.sh:261-273` relies on the `Addon` CR the controller creates |
-
-## 3. Control-plane membership on the host
-
-| Item | Present behaviour | Evidence |
-|---|---|---|
-| Record | `ClusterMeta` at `clusters/{name}/meta` in `eks-account-{account}` (history 1). | `handlers/eks/store.go:28-40`, `handlers/eks/cluster_state.go:64-201` |
-| Member list | `ControlPlaneNodes []ControlPlaneNode{NodeID, InstanceID, ENIID, ENIIP, MgmtIP}`; scalar `ControlPlaneInstanceID` etc. mirror `[0]`. No role, boot incarnation, executor or epoch field. | `cluster_state.go:109-125`, `:203-212` |
-| "Primary" in meta | Index `[0]` by comment and by the create path; the scalars mirror it. | `cluster_state.go:119-124`, `handlers/eks/service_impl.go:941-949` |
-| Membership predicate | `IsControlPlaneMember(instanceID)`: any entry in `ControlPlaneNodes` or the scalar. | `cluster_state.go:238-251` |
-| Template | `ControlPlaneTemplate` persisted with `ServerURL`, creds and per-node fields cleared, replayed for replacements. | `cluster_state.go:128-133`, `service_impl.go:951-963` |
-| Meta revision | Written by every `casUpdateMeta` caller, including health and node-count changes, so its KV revision is not a membership counter. | `cluster_state.go:276`, `:416-436`, `cluster_reconciler.go:620`, `:635` |
-
-## 4. Gateway authentication of the internal routes
-
-| Item | Present behaviour | Evidence |
-|---|---|---|
-| CP principal | Assumed-role session of role `spinifex-eks-server` in the system account; IMDS mints it with `RoleSessionName` = instance ID. | `handlers/eks/k3s_server_role.go:14`, `handlers/sts/assume_role.go:136-161`, `domains/ec2/guestmetadata/metadata.go:476-511` |
-| Role policy | `eks:PublishInternal`, `eks:ListInternalAddons`, `eks:WebhookTokenReview`, `eks:GetRecoveryDirective` on `Resource: "*"`; one role shared by every cluster's CP VMs. | `k3s_server_role.go:27`, `:42-43` |
-| Caller mapping | `eksCaller`: `SessionName` from `ctxIdentity`, role name from the underlying role ARN, not the session name. | `gateway/eks.go:307-320` |
-| `AuthorizeInternal` | Requires the CP principal class, then `meta.IsControlPlaneMember(caller.SessionName)` for the named cluster; `GetRecoveryDirective` additionally requires the path instance ID to equal the caller's. | `gateway/eks/internal_authz.go:44-96`; tests `gateway/eks_authz_test.go` `TestEKSRequest_InternalRoutesAdmitTheCPAgentPrincipal`, `gateway/eks/internal_authz_test.go` `TestAuthorizeInternal_RejectsClusterTheCallerDoesNotServe` |
-| Which routes it gates | All four internal routes. `ListInternalAddons` and `GetRecoveryDirective` are scoped `sourceInternalCluster`; `PublishInternal` and `WebhookTokenReview` are scoped `sourceInternalBodyCluster`, binding the URL cluster and the body account (read by the same decoder the handler uses) to the caller's membership before policy evaluation. Before that correction both were policy-only. | `gateway/eks/authz.go`, `gateway/eks/internal_authz.go`; tests `gateway/eks_authz_test.go` `TestEKSRequest_PublishInternal*`, `TestEKSRequest_WebhookTokenReview*` |
-| Member vs primary | The gateway can tell members apart (instance ID) and bind a fetch to a cluster, but cannot identify the delivery executor: meta has no role field, and `[0]` stops being the `server`-role VM after `[0]` is replaced (Section 6). | `internal_authz.go:74`, `cluster_state.go:463-490` |
-| Static-cred fallback | When the instance profile cannot be ensured, the VM gets the shared system access key; it has no instance identity, so every internal route, including `PublishInternal` and `WebhookTokenReview`, is denied for the VM's life; this fallback is unsupported. | `k3s_server_role.go:30-50`, `k3s_ha_control_plane.go:161-166`, `k3s_server_vm.go:549-557` |
-| Handler-side identity | `ListStagedAddonManifests` receives only the cluster name; no caller identity reaches the daemon. | `gateway/eks/internal_addons.go:20-33`, `handlers/eks/addons.go:117-128`, `daemon/daemon.go:1116` |
-
-## 5. Status relay path
-
-| Hop | Present behaviour | Evidence |
-|---|---|---|
-| Agent | `eks-gateway-publish` wraps `{accountId, channel, kind, payload}` and POSTs `/clusters/{cluster}/internal-publish`, 30 attempts. | `agents/eks/gatewaypublish/publish.go:29`, `:57-114` |
-| Gateway authz | IAM policy against `arn:aws:eks:{region}:{bodyAccount}:cluster/{name}`; with the role's `Resource: "*"`, any CP-role session of any cluster can publish to any cluster's add-on subject. | `gateway/eks/authz.go:155-160`, `k3s_server_role.go:27` |
-| Relay | `PublishInternal` publishes the raw payload to core NATS `eks.addon.{account}.{cluster}.status`; only a trace header is added. No caller instance ID, session or epoch is attached. | `gateway/eks/internal_publish.go:48-91`, `contracts/eks/v1/addon_status.go:10-12` |
-| Subscriber | Lease holder's `ClusterReconciler.Run` subscribes plainly; decodes the report and calls `Owner.ApplyReport` with addon, phase and message only (version and ts dropped). | `handlers/eks/cluster_reconciler.go:447-461`, `handlers/eks/addon_status_reconciler.go:22-31` |
-| Apply | Revision CAS on the add-on record; no check of sender, membership, version or epoch. | `domains/eks/addon/owner.go:140-168`, `domains/eks/addon/store.go:198` |
-| State reports | `mulga-eks-state-report` runs on both server roles and uses the same unbound relay; the report carries no member identity and the reconciler keeps the newest. | `eks-node-role.sh:137`, `:160`, `handlers/eks/state_report.go:24-42`, `cluster_reconciler.go:535` |
-
-## 6. Primary failure, replacement and rebuild today
-
-| Event | What happens to add-on delivery | Evidence |
-|---|---|---|
-| Primary stopped or errored | Reconciler restarts it in place after `restartGrace` (same instance ID, same root volume); role file and enabled units persist, so `mulga-eks-addon-sync` resumes with its rendered files. | `cluster_reconciler.go:708-771`, `:927-937` |
-| Etcd reform (HA, all VMs running) | `members[0]` gets `cluster-reset`, others `wipe-rejoin`; members are stopped and restarted. Wipe removes only `server/db/etcd`, so rendered manifests survive. | `cluster_reconciler.go:780-880`, `scripts/images/eks-node/mulga-eks-k3s-recovery.sh:38`, `:183-186` |
-| HA member lost (terminated, gone, or describe error) | `ProvisionReplacementCP` launches with `ServerURL` set, so the replacement is `server-join` and never runs `mulga-eks-addon-sync`; `SwapControlPlaneMember` puts it at the dead member's index. | `cluster_reconciler.go:1010-1096`, `k3s_ha_control_plane.go:142-187`, `cluster_state.go:463-490` |
-| HA primary (`[0]`) lost | Same path: after the swap, `[0]` is a `server-join` VM and no member runs delivery. Staged manifests stay in KV, but nothing fetches, renders, GCs or reports until an operator intervenes; add-ons keep their last status. | as above; no other enable path for the service |
-| Describe error treated as lost | A member whose state query fails counts as lost and can be replaced after `replaceGrace` while still running; if it was the primary, it keeps rendering and reporting (Section 7). | `cluster_reconciler.go:982-1007`; test `cluster_reconciler_replace_test.go` `TestMaybeReplaceCP_DescribeErrorMemberTreatedAsLost` |
-| Single-CP total loss | Operator `RestoreSnapshot` launches a fresh `server` VM (`ServerURL` empty), so delivery resumes on the new VM from the KV-staged set; meta is committed before the old CP is terminated, and termination is retried 3 times. | `handlers/eks/restore_snapshot.go:217-353`, `k3s_ha_control_plane.go:206-246`, `restore_snapshot.go:28`, `:185-212` |
-| Cluster create timeout or bootstrap failure | Marked `FAILED` without purging; CP VMs keep running `mulga-eks-addon-sync`, `ListInternalAddons` still admits them (membership only), and the reconciler has exited, so reports are dropped. | `cluster_reconciler.go:1151-1167`, `service_impl.go:2138`, `cluster_reconciler.go:645-648` |
-| Async launch failure | `failClusterLaunch` purges infra (terminates CP VMs) and marks `FAILED`, keeping meta. | `service_impl.go:701-717` |
-| Reclaim of a `FAILED` name | CAS `FAILED`→`CREATING`, purge old infra, overwrite meta; add-on keys are not purged, so the new primary renders the inherited staged set. | `service_impl.go:1203-1261` |
-| Delete | `purgeClusterInfra` stops the reconciler and terminates the CP VMs. | `service_impl.go:1271-1272` |
-| Resume from durable state | A fresh `server` VM resumes because the staged set is durable KV and the listing has no per-executor cursor; a `server-join` replacement cannot resume because it never runs the agent. | `handlers/eks/addons.go:117-128`, `eks-node-role.sh:151-166` |
-
-## 7. Existing leases, epochs and generations
-
-| Mechanism | Scope | What it proves | Evidence |
-|---|---|---|---|
-| ClusterReconciler lease | Host daemon per cluster; key `{account}/{cluster}` in `spinifex-eks-leader`, value = daemon `HolderID`, CAS create, renew by revision CAS every 20 s, bucket TTL 60 s, `Lost()` ends `Run`. | One daemon reconciles at a time. The revision is private (no accessor) and is not passed to report apply or guest routes. Unrelated to the guest executor. | `cluster_reconciler.go:26`, `:391-418`, `:463-470`; `foundation/state/kvlease/lease.go:96-178`, `:201-225`; `store.go:38-40` |
-| Lease takeover | `SpawnRegisteredReconcilers` runs only at daemon start; a lease loser is not retried elsewhere in the code read. | Reports for a cluster whose lease holder died are dropped until some daemon restarts. | `daemon/daemon.go:2209`, `service_impl.go:385-428`, `handlers/eks/reconciler_registry.go:12-14`, `:166-171` |
-| Recovery directive epoch | Per member instance, `epoch = prev+1` on each set; guest applies once and records the applied epoch locally. | At-most-once boot-time directive. Not a fence: it is pulled only at boot and has no `disable delivery` action. | `handlers/eks/recovery_directive.go:30-43`, `:90-107`; `mulga-eks-k3s-recovery.sh:131-197` |
-| Per-cluster epoch or generation | None in `ClusterMeta`; add-on records have no generation. | Nothing. | `cluster_state.go:64-201`, `docs/package-boundary/eks-addon-lifecycle.md` row "Generation" |
-| K3s server roles | `server` (cluster-init) and `server-join`; fixed at first boot. | Identifies the delivery VM only until `[0]` is replaced. | `k3s_server_vm.go:527-531`, `:585-589` |
-| Etcd / kube leader election | K3s embedded etcd and kube controllers elect internally; nothing in Spinifex reads them. | Not used. | no reference in `scripts/images/eks-node/*.sh` or `handlers/eks` |
-
-## 8. Candidate fencing evidence available today
-
-"Fences" means the old executor provably cannot render or apply, and cannot have its reports accepted.
-
-| Evidence | Available how | Proves old executor cannot render/apply? | Gap and what would be needed |
-|---|---|---|---|
-| EC2 state `terminated` or instance not found | `cpControlAdapter.InstanceState` (system account `DescribeInstances`); `TerminateK3sServerVM` treats not-found as success. | Yes for render and apply once the VM is gone, if the instance record is authoritative; host-partition behaviour of the state record is unverified. | Reports already queued or retried by `eks-gateway-publish` are still accepted. Needs: state proven authoritative under host loss, and report acceptance bound to the current executor. `service_impl.go:153-170`, `k3s_server_vm.go:292-330` |
-| EC2 state `stopped` / `error` | Same query. | Only momentarily: the reconciler's restart ladder starts stopped/error members automatically, and the VM resumes delivery with the same identity. | Needs a stop that nothing restarts, or a guest-side disable that survives restart. `cluster_reconciler.go:708-771`, `:927-937` |
-| Describe error | Treated as lost. | No: the VM may be running. | Must not count as fencing evidence. `cluster_reconciler.go:982-1007` |
-| Removal from `ControlPlaneNodes` | `SwapControlPlaneMember`, `replaceControlPlaneForRestore`. | Partly: `ListInternalAddons` is denied, but the agent keeps its current render on fetch failure, K3s keeps applying those files; its reports are now denied by the membership check, but reports already in flight are still applied. | Needs: publish gated by executor and identity attached on relay; a guest rule that a denied fetch stops delivery. `internal_authz.go:74`, `mulga-eks-addon-sync.sh:344-348` |
-| Instance ID / incarnation | Session name = instance ID; stable across stop/start. | No: there is no per-boot incarnation, so a restarted VM is indistinguishable from before. | Needs an executor epoch recorded host-side and checked per request; instance ID alone identifies, it does not fence. `assume_role.go:136-161` |
-| KV revisions | Meta revision, add-on record revision, lease revision. | No: meta revision churns with health writes, add-on CAS ignores sender, lease revision is host-only and private. | Needs a dedicated, monotonic executor epoch written only on reassignment. `cluster_state.go:416-436`, `kvlease/lease.go:44-52` |
-| IMDS credential expiry | Instance-role creds last 3600 s, re-minted 5 min early. | No: up to an hour of validity after the last mint, and a running VM keeps re-minting. | Not a fence on its own. `domains/ec2/guestmetadata/credentials.go:14-20` |
-| Credential revocation | `RevokeRoleSessions` deletes sessions by role (called on role delete); no per-instance revocation. | No per-executor effect: the role is shared by every CP VM of every cluster. | Needs a per-session-name revocation, and proof the gateway rejects a revoked session; a running VM would still re-mint via IMDS unless IMDS also refuses it. `handlers/sts/session_credentials.go:212-219`, `handlers/iam/roles.go:221` |
-| Static system keys | Shared system access key on fallback. | No: shared, unrevocable per VM, no identity. | Fallback VMs cannot be executors under any identity-based rule. `k3s_server_role.go:30-50` |
-| K3s node removal / etcd member prune | `EKS_ETCD_PRUNE_PEER_IP` handled in `k3s-first-boot.sh`. | Would stop apply through the shared datastore if the member is removed, but not render to the VM's own disk. The prune path appears unreachable: the variable is set only on replacements, which are `server-join`, and `k3s-first-boot` runs only for `server`. | Needs a reachable prune, verification that a removed member's apiserver cannot write, and still the report gate. `k3s-first-boot.sh:165-187`, `k3s_ha_control_plane.go:153`, `eks-node-role.sh:136`, `:151-166` |
-| Guest service disable | None remotely; the only host-to-guest instruction channel is the boot-time recovery directive. | No. | Needs an online directive the agent polls, plus an acknowledgement the host can verify; an acknowledgement from the old executor proves only that it was alive to send it. `recovery_directive.go:12-28` |
-| Restore fence | `confirmOldCPTerminated` after meta commit. | Yes for the old VM once terminate succeeds, but it runs after the new VM was launched and committed. | Ordering is successor-first; R6 requires fence-first. `restore_snapshot.go:262-336` |
-
-## 9. Evidence each design option would need
-
-No option is chosen here; each row lists only what it would rely on.
-
-| Option | Evidence it needs that does not exist today |
-|---|---|
-| Host-assigned executor `{instanceId, epoch}` on the cluster record | An executor field and epoch advanced by CAS; the gateway attaching the caller instance ID (and epoch) to the relayed message; report apply checking it; a reassignment trigger that does not fire on describe errors; fence evidence from Section 8 rows that say "Yes". |
-| Executor follows meta `[0]` | A guest that can become executor after first boot (today role is fixed and replacements are `server-join`); the same gateway and relay identity checks; `[0]` reassignment tied to fencing of the old `[0]`. |
-| Run the agent on every server with guest-side leader election (kube Lease) | Host-visible proof of the guest leader (the host cannot read the kube Lease today); verification of K3s auto-deploy behaviour across servers; reports still need gateway-attached identity, since a partitioned old leader can still publish. |
-| Disable-on-handover directive | An online directive channel and agent polling; a rule that a fetch denial or missing directive stops rendering; an acknowledgement or instance-state proof before the successor acts. |
-| Credential-based fence | Per-instance session revocation, IMDS refusing to re-mint for a fenced instance, and gateway rejection of revoked sessions; still insufficient for files already rendered to the old VM's disk. |
-
-## 10. Gaps
-
-1. Delivery runs only on the first-boot `server` VM; replacing HA `[0]` leaves no member running `mulga-eks-addon-sync`, with no detection or recovery.
-2. Corrected: `PublishInternal` and `WebhookTokenReview` were policy-only, so any CP-role session, any tenant holding `eks:*`, or static system credentials could publish reports or resolve tokens for any cluster and account. Both now require cluster membership (see Section 3); the binding is to a member, not to the delivery executor.
-3. The relay attaches no caller identity, and the payload carries none, so the owner cannot tell which member, executor or epoch produced a report; this conflicts with the direction in draft PROP-SPINIFEXARCH-007 S3, which is not accepted.
-4. `ClusterMeta` has no role, executor or epoch field; `[0]` as "primary" is a convention that diverges from the guest role after a swap.
-5. A member removed from meta is denied fetches but keeps applying its last render and keeps having its reports accepted.
-6. Describe errors count as member loss, so a running primary can be replaced without fencing.
-7. The restore path launches and commits the successor before fencing the old CP.
-8. No per-instance credential revocation; the CP role and instance profile are shared across all clusters.
-9. `FAILED` clusters (create timeout, bootstrap failure) keep running CP VMs whose agent is still admitted while reports are dropped.
-10. The etcd peer prune for replacements appears unreachable (`server-join` does not run `k3s-first-boot`); not verified live.
-11. Whether K3s auto-deploy applies from a non-leader server's manifests dir is unverified, which bears on any multi-server option.
-12. No lease takeover for a cluster whose reconciler holder died until a daemon restarts, so reports are dropped meanwhile.

docs/package-boundary/eks-addon-lifecycle-design.md

@@ -1,450 +0,0 @@
-# EKS managed add-on: lifecycle correction design
-
-**Status:** Draft for requirements review; no behaviour changes until approved.
-Reviewer decisions recorded (see "Review decisions"); the body is consistent with them.
-
-This is the Phase 2 design for correcting the EKS managed add-on lifecycle, written against the code at `18794e017`.
-The present contract and its gaps are in `docs/package-boundary/eks-addon-lifecycle.md`, and that record stays authoritative for what the code does today.
-Paths are relative to `spinifex/` unless they start with `tests/`, `scripts/` or `contracts/`, which are relative to the spinifex repo root.
-
-## Standing
-
-The design is resource-specific.
-It refines proposed requirement Q-146 and the recorded conflict C-82 (`docs/specs/requirements/managed-services.md` in the mulga repo); both are open and pending review, and this document closes neither.
-It does not create an architecture proposal, a universal lifecycle interface, a generic finalizer package or a shared repository; ADR-0003 S2 (accepted) rules those out without two real consumers.
-It applies ADR-0003 S1 to S5 and ADR-0004 (both accepted) to one resource.
-It is not a NATS or JetStream redesign: every transport stays as it is.
-Edge IDs E1 to E9 refer to Section 7.1 of `docs/package-boundary/adr-0004-inventory-eks.md`.
-
-Q-75 (contract evolution) and Q-76 (command, event and notification semantics) in `docs/specs/requirements/resource-lifecycle.md` are not accepted ADRs.
-They are Decided requirements: a named person stated them, and they have normative effect on this design.
-They are not open questions here; only the mechanics of implementing them are.
-Q-75 requires a safe compatibility path for older consumers, so a weaker substitute for a guarantee (such as a timing heuristic standing in for a fence) does not satisfy it.
-
-## Review decisions
-
-The reviewer decided the following; the sections below apply them.
-
-| # | Decision |
-|---|---|
-| R1 | Q-75 and Q-76 are Decided requirements with human provenance and normative effect, not accepted ADRs; only their implementation mechanics remain open. |
-| R2 | No v1 timing heuristics: neither "version match after a settle window" nor "delete complete after two listings" is used, because neither is a fence or proof of removal and both produce false success. |
-| R3 | `CREATE_FAILED` and `UPDATE_FAILED` are terminal for their generation. A late `ready` is recorded diagnostically and does not restore `ACTIVE`. A new `UpdateAddon` starts a new generation, and `DeleteAddon` stays available. `DEGRADED` recovers to `ACTIVE` on a valid report for the current generation. |
-| R4 | The external ARN stays name-only for this correction; the internal immutable `incarnationId` (UUID) is added now. The AWS-style per-incarnation ARN is a separate, explicitly migrated future change, because it affects IAM resource policies and must not break them silently. The divergence stays recorded. |
-| R5 | No v1 fallback. v1 stays byte-compatible and readable, but lifecycle mutations that need fencing or observed deletion require an authenticated, v2-capable active control-plane executor; otherwise they return `InvalidRequestException` telling the operator to upgrade the control-plane image. This is a deliberate compatibility gate, not a hard cutover. |
-| R6 | Delivery belongs to the EKS cluster, not to whichever server was primary. The cluster control plane assigns the active delivery executor, identified by executor identity and epoch; only that executor may fetch and report, a replaced executor is fenced before its successor acts, and the successor resumes from durable desired state without advancing the generation. The add-on side declares a narrow capability that the cluster owner provides. |
-| R7 | The v2 guest contract does not emit `InvalidParameterValue`; it uses structured internal reasons. The AWS adapter maps invalid customer input to `InvalidParameterException`, and a bad directive detected by the guest becomes a reported realisation failure. v1 is unchanged. |
-| R8 | `namespaceConfig` is rejected at the gateway while the pinned SDK cannot represent or apply it, and is never silently dropped; this is its own early slice. |
-| R9 | Non-empty `configurationValues` stay rejected until a bundle publishes a schema and actually renders the value. |
-| R10 | The `iam:PassRole` check on `serviceAccountRoleArn` stays; it is already implemented (`gateway/eks/passrole.go`). |
-| R11 | `InvalidRequestException` answers a mutation that conflicts with an in-progress lifecycle operation; `ResourceInUseException` is reserved for a conflicting resource on create. |
-| R12 | Deadlines are profile-configured and the chosen deadline is persisted when the operation is accepted; no per-bundle timing values yet. |
-| R13 | A delete that supersedes an in-flight update marks that update `Cancelled`. |
-| R14 | `serviceAccountRoleArn` is rejected for bundles with no IRSA marker. |
-| R15 | The GPU add-on is not re-created automatically after a user deliberately deleted it. |
-| R16 | Finalisation is an explicit, operator-visible add-on migration action with preflight evidence that legacy consumers are gone, as Q-75 requires. |
-
-## AWS behaviour relied on
-
-Verified against the AWS EKS API reference on 2026-10-08:
-
-- `Addon.status` is one of `CREATING`, `ACTIVE`, `CREATE_FAILED`, `UPDATING`, `DELETING`, `DELETE_FAILED`, `DEGRADED`, `UPDATE_FAILED` (API_Addon).
-- `CreateAddon` and `DeleteAddon` examples show `addonArn` as `arn:aws:eks:{region}:{account}:addon/{cluster}/{addon}/{uuid}`, so each incarnation has its own suffix (API_CreateAddon, API_DeleteAddon examples; the type reference does not state the format).
-- `UpdateAddon` returns an `Update` with its own `id`; the example returns `status: InProgress` and `type: AddonUpdate` with `params` such as `ServiceAccountRoleArn` and `ResolveConflicts` (API_UpdateAddon).
-- `Update.id` is "a UUID that is used to track the update"; `Update.status` is `InProgress`, `Failed`, `Cancelled` or `Successful`; `errors` carries details of a `Failed` update (API_Update).
-- `DescribeUpdate` takes `addonName` as a query parameter, "required if the update is an add-on update"; `Successful` means complete and `Failed` carries an error detail (API_DescribeUpdate).
-- `DeleteAddon` returns the add-on with `status: DELETING` (example), and `preserve` "preserves the add-on software on your cluster but Amazon EKS stops managing any settings for the add-on. If an IAM account is associated with the add-on, it isn't removed" (API_DeleteAddon).
-- `resolveConflicts` is `OVERWRITE`, `NONE` or `PRESERVE`; on create, with no self-managed version installed, "Amazon EKS sets all values to default values, regardless of the option that you specify"; on update, `NONE` leaves a changed value and "the update might fail", `OVERWRITE` resets it, `PRESERVE` keeps it (API_CreateAddon, API_UpdateAddon).
-- `configurationValues` "are validated against the schema returned by `DescribeAddonConfiguration`" (API_CreateAddon, API_UpdateAddon).
-- `podIdentityAssociations` on update: "If this value is left blank, no change. If an empty array is provided, existing associations owned by the add-on are deleted" (API_UpdateAddon).
-- The documented errors for `CreateAddon` and `UpdateAddon` are `ClientException`, `InvalidParameterException`, `InvalidRequestException` ("invalid given the state of the cluster"), `ResourceInUseException` (409), `ResourceNotFoundException` (404) and `ServerException`; `DeleteAddon` lists the same set without `ResourceInUseException`.
-  `IdempotentParameterMismatch` is not among them.
-- `CreateAddon` accepts `namespaceConfig`, which the pinned `aws-sdk-go` v1.55.8 model does not carry.
-
-Not verified, and marked where used: which error AWS returns for an update or delete issued while another operation is in progress (R11 decides ours); whether `ready` after `CREATE_FAILED` or `UPDATE_FAILED` returns the add-on to `ACTIVE` (R3 decides ours); how long AWS honours a `clientRequestToken`; what a token reused with different parameters returns; whether `DescribeUpdate` finds an update of a deleted predecessor; the `AddonIssue` code set; whether a role passed to an add-on that needs no IAM permissions is rejected (R14 decides ours); and the Terraform provider's waiters.
-
-## 1. Identity and generation
-
-| Item | Design |
-|---|---|
-| Incarnation | `incarnationId`: a random UUID minted when `CreateAddon` is accepted and never changed. Delete followed by create of the same name yields a new incarnation. |
-| Desired generation | `desired.generation`: an unsigned integer, 1 on create, incremented by exactly one in the same CAS that accepts each `UpdateAddon` and each `DeleteAddon`. Tag changes do not advance it, because tags are not delivered. |
-| Where stored | In the add-on record `clusters/{cluster}/addons/{addon}` in `eks-account-{account}`, next to the existing fields, so one revision CAS covers intent, generation and status. No new bucket or key family. |
-| Lookup key | Still cluster plus add-on name; the incarnation is a fence, not an address. |
-
-The external ARN stays `addon/{cluster}/{addon}`, minted by the owner and independently derived by the gateway authorizer from names (inventory edge E9), and `incarnationId` is internal (R4).
-This keeps authorization and existing IAM policies unchanged.
-The divergence from AWS remains recorded as contract gap 10: AWS's `CreateAddon` reference examples carry a per-incarnation suffix, and a recreated Spinifex add-on reuses its predecessor's ARN.
-Adopting the suffix is a separate change with its own explicit migration, because policies naming today's ARN would stop matching new add-ons and the authorizer would have to obtain the ARN from the owner before its policy check.
-Shaping `incarnationId` as a UUID keeps that change possible.
-
-## 2. Directives, reports and the stale-report rule
-
-A directive is what the controller wants the guest to realise for one add-on; a report is what the guest observed.
-In the successor contract (Section 9) both carry `incarnationId` and `generation`, and both carry the delivery executor's epoch (Section 10).
-
-Directive listing (v2): the response carries `executorEpoch` once, then one entry per add-on:
-
-| Field | Meaning |
-|---|---|
-| `addonName`, `incarnationId`, `generation` | Identity and the desired generation this directive represents. |
-| `mode` | `install`, `remove` or `relinquish`. |
-| `addonVersion`, `serviceAccountRoleArn`, `configurationValues` | The desired spec; for `remove` and `relinquish`, the last installed spec, so a guest without local state can still render the object list it must remove. |
-
-Report (v2): `addon`, `incarnationId`, `generation`, `executorEpoch`, `phase` (`applied`, `ready`, `failed`, `removing`, `removed`, `relinquished`), `reason` (a structured code, required when `phase` is `failed`), `message`, `ts`.
-The gateway attaches the executor identity it derived from the authenticated caller; no executor identity is read from the payload.
-
-Exact comparison rule, applied by the owner to record `R` and report `P`, where `X` is the cluster's current active executor (identity and epoch):
-
-1. If `P`'s gateway-attached executor identity is not `X`'s identity, or `P.executorEpoch` is not `X`'s epoch, discard `P` as a stale executor.
-2. If `R` is absent, discard `P`.
-3. If `P.incarnationId != R.incarnationId`, discard `P` as a stale incarnation.
-   This rejects every report from before a delete and recreate.
-4. If `P.generation < R.desired.generation`, discard `P` as stale.
-   This rejects reports produced for a superseded update and reports produced before a delete was accepted.
-5. If `P.generation > R.desired.generation`, discard `P` and log a protocol error; a guest cannot learn a generation the controller has not issued.
-6. If `P.phase` does not fit `R.desired.mode` (`ready` while removing, `removed` while installing), discard `P` and log a protocol error.
-7. Otherwise apply the transition in Section 6 through a revision CAS on `R`; on a CAS conflict, re-read `R` and start again from step 1.
-
-Within one generation, reports apply in arrival order.
-A reordered pair from the same generation describes the same desired state, and the level-triggered agent corrects it on its next tick.
-Discards never write the record; they are counted in a metric, labelled by rule, so a stuck or fenced guest is visible.
-
-## 3. Desired intent separated from observed state
-
-Record shape sketch (JSON field names; existing fields unchanged):
-
-```json
-{
-  "schemaVersion": 2,
-  "addonName": "aws-load-balancer-controller",
-  "arn": "arn:aws:eks:...:addon/prod/aws-load-balancer-controller",
-  "incarnationId": "6f1c...",
-  "createdAt": "...", "modifiedAt": "...", "tags": {},
-  "origin": "user",
-  "createToken": {"token": "...", "paramHash": "..."},
-  "desired": {
-    "generation": 3, "acceptedAt": "...", "mode": "install",
-    "addonVersion": "2.11.0", "serviceAccountRoleArn": "...", "configurationValues": "",
-    "resolveConflicts": "OVERWRITE",
-    "deletion": null
-  },
-  "observed": {"generation": 2, "phase": "ready", "reason": "", "message": "", "receivedAt": "...",
-               "contract": "v2", "executorEpoch": 4},
-  "status": "UPDATING",
-  "health": [{"code": "...", "message": "..."}],
-  "deadline": {"at": "...", "duration": "...", "profileKey": "..."},
-  "updates": [{"id": "...", "type": "AddonUpdate", "status": "InProgress", "generation": 3,
-               "params": [{"type": "AddonVersion", "value": "2.11.0"}], "createdAt": "...",
-               "token": "...", "paramHash": "...", "errors": []}],
-  "addonVersion": "2.11.0", "serviceAccountRoleArn": "...", "configurationValues": ""
-}
-```
-
-`desired` is written only by API commands; `observed` only by accepted reports (and by diagnostic late reports, Section 6); `status`, `health` and `deadline` are owner-derived from both and stored so reads stay a single get.
-`origin` is `user` or `auto-staged` (Section 13).
-The trailing top-level `addonVersion`, `serviceAccountRoleArn` and `configurationValues` are the legacy fields, kept in step with `desired` during the transition so an older binary can still read the record.
-Updates are kept inline, bounded to the most recent N per incarnation, so accepting an update and advancing the generation is one CAS rather than a two-key write.
-
-The staged manifest sub-key becomes a projection of `desired`.
-The internal listing is built from the records, and the manifest key is still written for rollback until finalisation.
-That removes today's crash window between the record put and staging (contract row "Restart and interruption"), because there is no longer a second write that can be missed.
-
-Migration of existing records, as an idempotent and resumable expand step:
-
-| Today's status | `incarnationId` | `desired.generation` | `observed` | Status after |
-|---|---|---|---|---|
-| `ACTIVE`, `DEGRADED` | newly minted | 1 | generation 1, phase from status, contract `v1` | unchanged |
-| `CREATING`, `UPDATING` | newly minted | 1 | none | unchanged; a deadline from the profile starts at migration |
-| `CREATE_FAILED` | newly minted | 1 | none | unchanged, with the stored `health` as an issue |
-
-Minting at migration is safe because no deployed report carries an incarnation.
-Existing non-empty `configurationValues` are preserved, never silently dropped, and gain a health issue stating they are not applied (Section 7).
-An older binary that rewrites a migrated record decodes it into today's `Record` and drops the new fields, so the new lifecycle semantics activate only once every daemon in the Region is known to run the new release (Section 9).
-Activation does not end rollback; the separate finalisation action does (Section 9, R16).
-
-## 4. Deletion
-
-On a cluster whose active executor has proven v2, `DeleteAddon` creates a durable obligation and keeps ownership until guest-side removal is observed:
-
-1. One CAS sets `status: DELETING`, advances `desired.generation` to `D`, sets `desired.mode` to `remove` (or `relinquish` when `preserve=true`), records `desired.deletion.requestedAt`, persists the delete deadline and, if an update is in flight, marks it `Cancelled` (R13).
-   The response is the record in that state.
-2. The listing now carries a `remove` or `relinquish` directive for generation `D` instead of omitting the add-on.
-3. The record stays until a report for (`incarnationId`, `D`) with phase `removed` or `relinquished` arrives; the owner then deletes the record and the manifest key.
-4. While the record exists, `CreateAddon` of the same name returns `ResourceInUseException`, and `DescribeAddon` and `ListAddons` show it as `DELETING` or `DELETE_FAILED`.
-   Recreate is therefore serialised behind removal, so a new incarnation never overlaps its predecessor's objects in the guest.
-
-On a cluster without a v2 executor, `DeleteAddon` returns `InvalidRequestException` (Section 9).
-
-What "observed" means for a v2 guest:
-today's GC loop deletes the rendered file, runs `kubectl delete --wait=false` on a temporary copy and forgets it.
-The v2 agent keeps the removed manifest in a durable per-add-on location instead of `mktemp`, and on every tick runs `kubectl get -f <copy> --ignore-not-found`.
-It reports `removing` while any listed object remains and `removed` only when every object in the rendered manifest returns NotFound, including namespaces held by finalizers.
-If the apiserver is unreachable, it sends nothing for that add-on rather than `removed`.
-If it has no local copy (state lost, never rendered, or a newly assigned executor), it renders the bundle from the directive's last spec into a scratch file and uses that object list.
-It keeps reporting `removed` on every tick while the directive is listed, so a lost report is repeated.
-
-For `relinquish`, the v2 agent deletes only the rendered file, leaving the objects in place (K3s does not delete applied objects when the auto-deploy file disappears, per the agent's own comment), and reports `relinquished` once the file is gone.
-What K3s does with its `Addon` object in that case is unverified.
-
-Unavailable guest: the record stays `DELETING` and the directive stays listed.
-When the persisted delete deadline passes, the owner sets `DELETE_FAILED` with a health issue, keeps the record and keeps listing the directive.
-`DELETE_FAILED` is recoverable: a later `removed` report for (`incarnationId`, `D`) completes the deletion, and a repeated `DeleteAddon` re-arms the deadline without advancing the generation, since the intent is unchanged.
-A guest that recovers after hours therefore converges the retained directive with no operator action.
-
-Cluster deletion destroys the control-plane VMs and with them every in-cluster object, so add-on removal is implied there.
-That path should call an add-on-owned teardown capability that records removal-by-cluster-teardown instead of the blanket `DeleteClusterPrefix` sweep (inventory edge E1); it is not subject to the v2 gate.
-
-## 5. Update as a durable asynchronous operation
-
-`UpdateAddon` is accepted in one CAS that advances `desired.generation` to `G`, sets `status: UPDATING`, appends an update `{id: new UUID, type: AddonUpdate, status: InProgress, generation: G, params}` and persists the update deadline.
-The response is that update, not the add-on ARN marked `Successful`.
-
-| Event for generation `G` | Update status | Add-on status |
-|---|---|---|
-| `ready` report | `Successful` | `ACTIVE` |
-| `failed` report | `Failed`, with `reason` and `message` in `errors` | `UPDATE_FAILED` |
-| Deadline passes with no `ready` | `Failed` ("timed out") | `UPDATE_FAILED` |
-| `DeleteAddon` accepted first | `Cancelled` | `DELETING` |
-
-`DescribeUpdate` with `addonName` routes to the add-on owner and finds the update by `id` among the record's retained updates for the current incarnation; an unknown `id` is `ResourceNotFoundException`.
-Whether AWS still resolves an update of a deleted predecessor is unverified.
-`ListUpdates` with `addonName` lists the retained ids.
-Both are gateway stubs today, so this work also implements them for add-ons only; cluster and node-group updates stay stubs.
-
-`UpdateAddon` is accepted from `ACTIVE`, `DEGRADED`, `CREATE_FAILED` and `UPDATE_FAILED`.
-From `CREATING`, `UPDATING`, `DELETING` and `DELETE_FAILED` it conflicts with an in-progress lifecycle operation and returns `InvalidRequestException` (R11).
-On a cluster without a v2 executor it returns `InvalidRequestException` (Section 9).
-An update whose parameters equal the current desired spec is still accepted as a new generation, so a client gets a trackable update.
-
-## 6. Status set and transitions
-
-The owner adds `UPDATE_FAILED` and `DELETE_FAILED` to its `Status`; `DELETING` becomes a stored status.
-Every report row below assumes the report passed the rule in Section 2.
-
-| From | Event | To | Notes |
-|---|---|---|---|
-| absent | `CreateAddon` accepted | `CREATING` | generation 1; create deadline persisted |
-| `CREATING` | `ready` | `ACTIVE` | |
-| `CREATING` | `failed`, or create deadline | `CREATE_FAILED` | terminal for generation 1 |
-| `ACTIVE` | `failed` | `DEGRADED` | |
-| `DEGRADED` | `ready` | `ACTIVE` | health clears |
-| `ACTIVE`, `DEGRADED`, `CREATE_FAILED`, `UPDATE_FAILED` | `UpdateAddon` accepted | `UPDATING` | generation +1 |
-| `UPDATING` | `ready` | `ACTIVE` | update `Successful` |
-| `UPDATING` | `failed`, or update deadline | `UPDATE_FAILED` | update `Failed`; terminal for that generation |
-| `CREATING`, `ACTIVE`, `DEGRADED`, `CREATE_FAILED`, `UPDATE_FAILED` | `DeleteAddon` accepted | `DELETING` | generation +1 |
-| `UPDATING` | `DeleteAddon` accepted | `DELETING` | generation +1; in-flight update `Cancelled` |
-| `DELETING` | `DeleteAddon` | `DELETING` | idempotent; same generation; record returned |
-| `DELETE_FAILED` | `DeleteAddon` | `DELETING` | same generation; deadline re-armed |
-| `DELETING` | delete deadline | `DELETE_FAILED` | record and directive retained |
-| `DELETING`, `DELETE_FAILED` | `removed` or `relinquished` | absent | record and manifest deleted |
-| `CREATE_FAILED`, `UPDATE_FAILED` | `ready` for the same generation | unchanged | recorded diagnostically in `observed`; does not restore `ACTIVE` |
-
-Failure is terminal for its generation and recoverable only by a new generation (R3).
-Today `ready` lifts `CREATE_FAILED` to `ACTIVE`; that changes, because a failed update must keep a stable `Failed` outcome in `DescribeUpdate` and Terraform should not see a failed resource silently turn healthy.
-The cost is low: the agent reports `failed` only for deterministic causes (missing baked bundle, webhook certificate generation, a directive the guest cannot realise), and a slow rollout reports `applied`, not `failed`.
-`DEGRADED` is different: it describes a realised generation whose health dipped, so a valid `ready` for the current generation returns it to `ACTIVE`.
-Staging is no longer a failure source, because the listing is a projection of the record.
-
-## 7. Per-field decisions
-
-The catalogue holds five bundles, one version each: `aws-load-balancer-controller` 2.11.0 and `aws-ebs-csi-driver` 1.40.1 (both `RequiresIRSA`), `argocd` 3.0.23, and the hidden `nvidia-device-plugin` 0.17.4 and `spinifex-noop` 0.1.0.
-No accepted field may be stored and ignored.
-Every rejection of invalid customer input uses `InvalidParameterException` (`awserrors.ErrorEKSInvalidParameter`); today's add-on validation returns `InvalidParameterValue`, which is not an EKS error code, and correcting it is part of slice 2.
-State conflicts use `InvalidRequestException` and an existing resource on create uses `ResourceInUseException` (R11).
-
-| Field | Today | Decision |
-|---|---|---|
-| `addonVersion` | Validated against the catalogue; applied. | Keep. Each bundle supports its one version; an unknown version is rejected. A change advances the generation. |
-| `serviceAccountRoleArn` | Stored, `iam:PassRole` checked (already implemented, `gateway/eks/passrole.go`), rendered as web-identity env when the bundle has IRSA markers. | Implement for `aws-load-balancer-controller` and `aws-ebs-csi-driver`; keep the `iam:PassRole` check (R10). Reject for `argocd`, `nvidia-device-plugin` and `spinifex-noop`, whose bundles have no IRSA markers, so the role would be inert (R14). An update that changes it advances the generation. |
-| `configurationValues` | Stored, staged, transported, discarded by the agent. | Reject a non-empty value for every bundle until that bundle publishes a schema through `DescribeAddonConfiguration` and the agent renders it (R9). Implement per bundle afterwards, `aws-load-balancer-controller` first. Existing stored values are kept and flagged (Section 3). |
-| `resolveConflicts` | Accepted and dropped. | Create: accept all three values; with no pre-existing install AWS treats them alike. The v2 agent with `NONE` or `PRESERVE` checks the bundle's objects before the first render and reports `failed` on a conflict. Update: accept `OVERWRITE` (what K3s auto-deploy does); reject `NONE` and `PRESERVE` until field-level drift detection exists, because K3s will overwrite regardless. Persist the value and return it as an update param. |
-| `preserve` (delete) | Ignored. | Implement as `relinquish` (Section 4). Clusters without a v2 executor cannot delete at all (Section 9). |
-| `podIdentityAssociations` | Accepted and dropped. | Reject a non-empty list; EKS Pod Identity is not implemented. Accept an absent field and, on update, an empty list, which is a correct no-op because no associations exist. Return the field empty. |
-| `clientRequestToken` | Dropped. | Implement for create and update (Section 8). |
-| `tags` | Stored on create and returned; tag actions on the add-on ARN return `NotImplemented`. | Keep create-time tags. Implement `TagResource`, `UntagResource` and `ListTagsForResource` for add-on ARNs as an owner-provided tag mutation that does not advance the generation and is not gated, because a Terraform tag change on `aws_eks_addon` otherwise fails. |
-| `namespaceConfig` | Not in the pinned SDK model, so silently dropped when the body is decoded. | Reject at the gateway when present in the raw body, while the pinned SDK cannot represent or apply it; never drop it silently (R8, slice 1). |
-
-## 8. Client token idempotency
-
-The token is stored in the record it created, not in a TTL bucket, so create stays one atomic write.
-
-- `CreateAddon` becomes a CAS-create (`kv.Create`), fixing contract gap 1.
-  The record stores the token and a hash of the request without the token.
-- A create that loses to an existing record compares tokens: same token and same hash returns the existing add-on as it now stands; same token and a different hash is rejected; a different or absent token is `ResourceInUseException`.
-- `UpdateAddon` with a token already held by a retained update of the current incarnation returns that update if the hash matches and is rejected if it does not; the replayed update is not re-applied and does not advance the generation.
-- `DeleteAddon` has no token; a repeat while `DELETING` returns the record again, and a repeat once the record is gone is `ResourceNotFoundException`, as today.
-- The token lives as long as its record or retained update. A retry after the add-on was deleted creates a new incarnation.
-- A token reused with different parameters: recommend `InvalidParameterException`, because `IdempotentParameterMismatch` is not in the EKS error lists, although `CreateCluster` returns it today (open question 1).
-
-## 9. Guest contract evolution and the v1 compatibility gate
-
-`contracts/eks/v1` is frozen; its README already says a generation-aware protocol needs a successor.
-
-- `contracts/eks/v2` adds the directive and report fields of Section 2, the new phases and structured `reason` codes.
-  It is served on a new internal route (for example `GET /clusters/{c}/internal-addons/v2/{account}`) and reports go on a new subject (for example `eks.addon.v2.{account}.{cluster}.status`), so neither side has to sniff payloads.
-- Following Q-75, the producer never emits v2 to a guest that has not asked for it, and v1 routes, subject and bodies stay byte-identical, pinned by the existing contract tests.
-- The v2 internal routes do not return AWS error codes such as `InvalidParameterValue` (R7).
-  A malformed guest request gets a structured internal reason (for example `NotActiveExecutor`, `StaleExecutorEpoch`, `UnknownContractMajor`, `MalformedReport`).
-  Invalid customer input never reaches the guest: the AWS adapter rejects it with `InvalidParameterException`.
-  A directive the guest finds it cannot realise (for example a bundle missing from the image) is reported as `failed` with a `reason`, and becomes `CREATE_FAILED`, `UPDATE_FAILED` or `DEGRADED` through Section 6.
-- The v2 agent ships in a new EKS node image, so a cluster stays on v1 until its control-plane VM is replaced; the mixed window can be long.
-
-How v2 capability is established: the cluster owner records it per executor epoch, from the authenticated caller only.
-The gateway already authenticates the internal routes as a control-plane VM session and binds the session name (the VM's instance ID) to the named cluster (`AuthorizeInternal`).
-For the v2 route it additionally requires that instance ID to be the cluster's active executor for the current epoch (Section 10), and only then does the cluster owner mark that epoch v2-capable.
-No field in a guest payload can assert capability or executor identity.
-A new epoch starts without capability, so the gate re-closes on every executor change until the new executor fetches v2.
-
-Gated actions: on a cluster whose current epoch is not v2-capable, `CreateAddon`, `UpdateAddon` and `DeleteAddon` (with or without `preserve`) return `InvalidRequestException` with a message instructing the operator to upgrade the cluster's control-plane image.
-Each of these needs either a fenced report (create and update reach `ACTIVE` only on a report for the current incarnation and generation) or observed deletion, which v1 cannot provide.
-Not gated: `DescribeAddon`, `ListAddons`, `DescribeUpdate`, `ListUpdates`, `DescribeAddonVersions`, `DescribeAddonConfiguration`, tag actions, and cluster-teardown removal (Section 4).
-
-v1 clusters during the transition:
-
-| Concern | v1 handling |
-|---|---|
-| Reads | Records are readable and listed; the v1 route serves a v1 projection of the records. |
-| Reports | v1 reports carry no incarnation, generation or epoch, so they never complete a create, update or delete. They are recorded diagnostically in `observed` with contract `v1`. Whether they may still move an add-on with no operation in flight between `ACTIVE` and `DEGRADED` is open question 6. |
-| Operations in flight at activation | A migrated `CREATING` or `UPDATING` record cannot complete on v1 and reaches `CREATE_FAILED` or `UPDATE_FAILED` at its deadline; it converges after the image upgrade through a new `UpdateAddon` or a `DeleteAddon`. |
-| Conformance | Q-146 conformance is claimed only for clusters whose executor has proven v2. |
-
-This is a compatibility gate, not a hard cutover: v1 guests keep working, nothing is migrated destructively, and the gate opens per cluster once its image is upgraded.
-
-Activation and finalisation are separate steps:
-
-- **Activation** turns on the new lifecycle semantics and the gate once every daemon in the Region is known to run the new release, so no older binary can complete a delete by erasing the record; the transitional record stays readable by the previous release, so rollback is still possible.
-- **Finalisation** is an explicit, operator-visible add-on migration action (R16).
-  Its preflight shows evidence that legacy consumers are gone: no daemon older than the activating release in the Region, and no remaining reader of the manifest key or legacy record fields.
-  It then stops writing the manifest key and legacy fields, and ends rollback.
-  It never runs because a new binary started.
-
-Removing v1 requires a published retirement, detection that no cluster in the Region still fetches the v1 route, and evidence that every cluster has moved to a v2 image; until then both are served.
-
-## 10. Delivery executor
-
-Delivery belongs to the EKS cluster, not to whichever server VM was first booted as primary (R6).
-Today `eks-node-role.sh` enables `mulga-eks-addon-sync` only on the primary server, and any control-plane member that passes `AuthorizeInternal` may fetch and report.
-
-Ownership split:
-
-| Owner | Owns |
-|---|---|
-| Add-on owner (`domains/eks/addon`) | Desired state, generation, observed lifecycle and the comparison rule. |
-| Cluster owner | Assignment of the active delivery executor: `executor {instanceId, epoch}`, plus whether that epoch has proven v2. |
-
-The add-on package declares a narrow capability, for example `ActiveDeliveryExecutor(account, cluster) -> {instanceId, epoch, v2Capable}`, and the cluster owner provides it.
-The add-on package does not read `ClusterMeta` or other cluster internals.
-The gateway's internal-route check uses the same capability.
-
-Rules:
-
-- Only the active executor for the current epoch may fetch the v2 listing or post a v2 report; the gateway derives the executor identity from the authenticated caller and rejects any other control-plane member with `NotActiveExecutor`.
-- Every report carries the `executorEpoch` it was issued in its listing; the gateway rejects a mismatched epoch, and the owner re-checks it (Section 2 rule 1) because the epoch may advance between the gateway check and the CAS.
-- A replaced executor is fenced before its successor is assigned: the cluster owner advances the epoch only after the old executor is confirmed unable to render (VM stopped or terminated, or its delivery service disabled), not merely because it is unreachable.
-  There is never an autonomous second primary.
-- The new executor reads durable desired state from the listing and resumes delivery and reconciliation, including retained `remove` directives; nothing advances `desired.generation`.
-- The handover interval is guest unavailability: operations keep their persisted deadlines, and a deadline that passes during handover produces the same `CREATE_FAILED`, `UPDATE_FAILED` or `DELETE_FAILED` as any outage.
-  A cluster with no assigned executor is an unavailable guest.
-
-The cluster lease `spinifex-eks-leader/{account}/{cluster}` elects the host-side daemon that reconciles the cluster; it is unrelated to the executor epoch, which fences the guest side.
-
-## 11. Restart reconciliation
-
-Every obligation is in the record, so restart needs no request memory.
-An add-on reconcile pass runs under the cluster's existing lease (`spinifex-eks-leader/{account}/{cluster}`), at start and on a periodic tick, owned by `domains/eks/addon` and given its own host instead of the cluster reconciler's report callback (inventory edge E3).
-For each record it:
-
-- re-projects the listing and, until finalisation, rewrites a missing or stale manifest key, which re-drives a create or update interrupted after its CAS;
-- keeps the `remove` or `relinquish` directive listed for `DELETING` and `DELETE_FAILED` records, which re-drives an interrupted delete;
-- applies the persisted deadlines (create, update, delete); and
-- retries nothing that already completed, because each transition is a CAS keyed on generation.
-
-The only window left is a crash before the accepting CAS commits, in which case nothing was accepted and the client's retry with the same token is a new attempt.
-Which daemon re-drives: whichever holds the cluster lease after restart, through the existing `SpawnRegisteredReconcilers` respawn; no new process role is introduced.
-
-## 12. Status publications are notifications
-
-Under Q-76 a delivery report is a notification, a hint that observed state may have changed, and never proof of completion by itself: completion is the owner's CAS after the Section 2 rule accepts a report for the current executor epoch, incarnation and generation.
-Core NATS is therefore acceptable as the transport:
-
-- **Lost:** the agent re-sends the current phase for every listed directive on every tick, including `removed` for a retained deletion, and the directive stays listed until the owner records completion; a report lost during a restart or with no lease holder is replaced within one tick.
-- **Duplicated:** applying the same (`incarnationId`, `generation`, `phase`) twice changes nothing.
-- **Delayed or reordered:** a late report from an earlier generation, incarnation or executor epoch fails the comparison rule; within one generation the next tick corrects it.
-- **Never arriving:** deadlines turn silence into `CREATE_FAILED`, `UPDATE_FAILED` or a retained `DELETE_FAILED`, so liveness does not depend on delivery.
-
-Correctness rests on durable desired state, periodic guest reporting and restart reconciliation, not on the transport, so no JetStream stream, queue group or delivery change is proposed.
-v1 reports never drive a lifecycle completion (Section 9), so no weaker rule exists for v1.
-
-## 13. GPU node-group auto-staging
-
-`stageGPUDeviceAddon` (inventory edge E2) keeps calling the owner and gains the new create path: incarnation, generation 1, no token, no IRSA, `origin: auto-staged`.
-Interactions under this design:
-
-- An existing record in any status, including `DELETING`, remains a no-op for the node group, so the plugin is not re-staged while it is being removed.
-- A user's deliberate deletion is remembered (R15): when a user-initiated `DeleteAddon` of `nvidia-device-plugin` completes, the add-on owner keeps a cluster-scoped declined marker in its own key space, and auto-staging skips the add-on while the marker exists.
-  An explicit user `CreateAddon` of the add-on clears it, and cluster teardown removes it with the cluster.
-  The marker's key, and whether a deletion of an `origin: user` record also sets it, are open question 7.
-- On a cluster whose executor has not proven v2, the create path returns `InvalidRequestException` (Section 9), so auto-staging is logged and skipped, as its errors are today.
-- `CREATE_FAILED` now persists (Section 6), so a device plugin that failed because the baked bundle was missing stays failed until updated or deleted; node-group readiness does not wait on it, and the failure is visible in `DescribeAddon`.
-- Errors stay logged and swallowed, and node-group delete never unstages, until the node-group owner is extracted and the dependency is designed; this design does not change that edge.
-
-## 14. Required evidence
-
-Unit and owner tests (fault injection at the owner's store and installer seams), E2E and Terraform:
-
-1. Interruption before the accepting CAS (nothing accepted; retry with the same token creates once) and after it (reconcile re-projects the directive; no second incarnation).
-2. Repeated create (token replay returns the same incarnation), repeated update (same update id, generation advanced once) and repeated delete (same `DELETING` record, no second generation).
-3. Stale report after a newer update: a `ready` and a `failed` for generation `G-1` arriving after `G` is accepted leave the add-on `UPDATING`.
-4. Stale report after delete and recreate: a report carrying the old `incarnationId` never touches the new record.
-5. Terminal failure: a `ready` for the failed generation after `CREATE_FAILED` or `UPDATE_FAILED` is recorded in `observed` and leaves the status unchanged; `DEGRADED` returns to `ACTIVE` on a current-generation `ready`.
-6. Delete during `UPDATING` marks the update `Cancelled`.
-7. Controller restart while a create, an update and a delete are each outstanding: every one converges after respawn with no request memory.
-8. Unavailable guest during delete: the record stays `DELETING`, becomes `DELETE_FAILED` at the persisted deadline, and the directive stays listed.
-9. Eventual guest recovery: a `removed` report after `DELETE_FAILED` completes deletion; an `applied` then `ready` after a long outage completes a pending create or update if its deadline has not passed.
-10. Executor fencing: a fetch or report from a control-plane member that is not the active executor, or from a superseded epoch, is rejected at the gateway and at the owner; a handover resumes a pending create, update and delete without advancing the generation; the epoch does not advance while the old executor is merely unreachable.
-11. v1 compatibility gate: `CreateAddon`, `UpdateAddon` and `DeleteAddon` on a cluster without a v2-capable epoch return `InvalidRequestException`; reads and tag actions still work; v1 reports never complete an operation; v1 body and subject bytes are unchanged; capability cannot be asserted through a payload field.
-12. Field decisions: invalid customer input, including `namespaceConfig`, non-empty `configurationValues` and a role on a bundle without IRSA markers, returns `InvalidParameterException`; the role reaches the rendered bundle for both IRSA bundles.
-13. `DescribeUpdate` and `ListUpdates` for add-ons: `InProgress`, `Successful`, `Failed` with errors, and `Cancelled`.
-14. GPU auto-staging does not re-create `nvidia-device-plugin` after a user deleted it, and does after an explicit user create.
-15. Finalisation preflight refuses while a legacy consumer remains, and finalisation is visible to the operator.
-16. E2E in `tests/e2e/eks`: create, update (version and role), failed update, delete with observed removal, delete with `preserve`, guest outage across a delete, and executor replacement during an update.
-17. Terraform `aws_eks_addon`: repeated apply and destroy, an in-place update (version or role) and a tag change, alongside the unit and E2E evidence, as ADR-0003 S5 requires.
-    The Terraform provider's create, update and delete waiters are unverified and must be read before the fixture is written.
-
-## 15. Implementation slices
-
-Each slice is small and ordered; behaviour-changing slices are named as such.
-
-| # | Slice | Behaviour change |
-|---|---|---|
-| 0 | Expand the record: write `schemaVersion`, `incarnationId`, `origin`, `desired`, `observed` beside the legacy fields; migrate existing records; nothing reads the new fields yet. | None visible; persisted bytes grow. |
-| 1 | Reject `namespaceConfig` at the gateway when present in the raw body. | Yes: a previously dropped input is rejected. |
-| 2 | CAS-create, create token in the record, `InvalidParameterException` for add-on validation errors. | Yes: concurrent and retried creates; error code. |
-| 3 | Add-on reconcile pass with its own host under the cluster lease; listing built from records; manifest key kept for rollback. | None visible; clears the create-staging crash window. |
-| 4 | Remaining field decisions of Section 7, except `preserve` and `resolveConflicts` conflict detection. | Yes: previously accepted inputs are rejected. |
-| 5 | Cluster-owned delivery executor: executor identity and epoch, the declared capability, and the gateway's active-executor check on v2 routes. | None for v1 routes. |
-| 6 | `contracts/eks/v2`, v2 route and subject, structured reasons, v2 agent in a new node image, executor-epoch, incarnation and generation fencing, v2 capability recorded per epoch. | Yes, for v2 clusters only. |
-| 7 | Activation and the v1 compatibility gate for `CreateAddon`, `UpdateAddon` and `DeleteAddon`. | Yes: lifecycle mutations on v1 clusters return `InvalidRequestException`. |
-| 8 | Durable update operation, `UPDATE_FAILED`, persisted profile deadlines, `DescribeUpdate`/`ListUpdates` for add-ons, update token, terminal failure. | Yes: update response and status. |
-| 9 | Durable `DELETING`, `DELETE_FAILED`, delete deadline, recreate blocked while deleting, update `Cancelled` on delete. | Yes: delete response persists and describe no longer returns not found at once. |
-| 10 | `preserve` as `relinquish` and create-time conflict detection. | Yes. |
-| 11 | Tag actions on add-on ARNs. | Yes: `NotImplemented` becomes supported. |
-| 12 | GPU declined marker for deliberately deleted auto-staged add-ons. | Yes: a deleted device plugin stays deleted. |
-| 13 | Finalisation action with preflight; stop writing the manifest key and legacy fields. | None for current binaries; ends rollback. |
-
-The per-incarnation ARN suffix and v1 retirement are separate later changes with their own migrations, not slices of this correction.
-
-## 16. Open questions for the reviewer
-
-These are implementation mechanics; the decisions above are not reopened.
-
-1. Token reused with different parameters: `InvalidParameterException` (recommended) or `IdempotentParameterMismatch` to match `CreateCluster`, and should `CreateCluster` be revisited?
-2. Existing stored `configurationValues`: keep and flag (recommended), or reject further updates to such add-ons until cleared?
-3. How many updates to retain inline per incarnation?
-4. Is there an existing Region-wide release gate that can establish "every daemon runs the new release" for activation and supply finalisation's preflight evidence, or does slice 7 need one?
-5. Executor fencing mechanics: what evidence the cluster owner accepts that a replaced executor cannot render (instance state from the EC2 owner, a disabled service, an operator assertion), and who triggers reassignment.
-6. May a v1 report still move an add-on with no operation in flight between `ACTIVE` and `DEGRADED`, given it carries no generation or epoch?
-7. GPU declined marker: its key in the add-on owner's space, and whether deleting a user-created `nvidia-device-plugin` also sets it.
-8. Profile keys and default values for the create, update and delete deadlines.
-9. Rollback after activation: how the previous release presents records in `DELETING`, `DELETE_FAILED` or `UPDATE_FAILED`, which it does not model today.

docs/package-boundary/eks-addon-v1-compatibility.md

@@ -1,86 +0,0 @@
-# EKS add-on v1 compatibility: who would drop, erase or ignore added fields
-
-Code read at 3a55e2ee1; not verified live.
-
-Question: if a newer binary adds JSON fields (for example `incarnationId`, `generation`, `observedGeneration`, `origin`, `executorEpoch`) to the add-on record `clusters/{c}/addons/{a}`, the staged manifest `clusters/{c}/addons/{a}/manifest`, or a `contracts/eks/v1` payload, what does an older binary still running the code below do with them?
-
-Verdicts:
-
-- **DROP**: decodes into a Go struct and writes the re-encoded struct back, erasing undeclared fields (`encoding/json` keeps no unknown-field bag).
-- **OVERWRITE**: writes a freshly built value with no read of the stored bytes; whatever was stored is replaced.
-- **ERASE**: deletes the key; content is irrelevant.
-- **IGNORE**: read-only decode; unknown fields are skipped and nothing is written back.
-- **FAIL**: decode error when a known field's JSON type changes (not an added field).
-
-## Version fences that exist today
-
-| Fence | Where | Effect |
-|---|---|---|
-| Record schema/version field | none: `spinifex/domains/eks/addon/record.go:24` | No per-record version, so an old binary cannot tell a newer record from a v1 one. |
-| Manifest schema/version field | none: `spinifex/domains/eks/addon/record.go:40` | Same for the manifest. |
-| Per-account bucket schema stamp | `spinifex/handlers/eks/store.go:29` (`KVBucketEKSAccountVersion = 1`), checked at `store.go:203` | Bumping it makes every older binary refuse the whole `eks-account-{id}` bucket with `SchemaAheadError` (`spinifex/foundation/state/migrate/kv.go:87`), i.e. all EKS calls for that account, not just add-ons. |
-| Registered migrations for the bucket | none | `RegisterKV` is keyed by exact bucket name (`migrate/kv.go:48`, lookup at `kv.go:94`) while EKS buckets are named per account (`store.go:146`). |
-| Wire version | `contracts/eks/v1/addon_status.go:1-3` package doc | Says generation-aware delivery needs a successor version; nothing on the wire carries a version marker. |
-
-## Record and manifest writers
-
-| Writer | file:line | Record | Manifest |
-|---|---|---|---|
-| `Owner.Create` (via `CreateAddon` `handlers/eks/addons.go:84`) | `domains/eks/addon/owner.go:48`; `Get` then unconditional `put` `store.go:39,51` | OVERWRITE (the Get/Put pair is not `kv.Create`, so a record created between them is replaced wholesale) | OVERWRITE via `StagingInstaller.Install` `delivery.go:39` → `putManifest` `store.go:153` (built from the four v1 fields) |
-| `Owner.EnsureGPUDevicePlugin` (via `nodegroup.go:411`) | `owner.go:79` | IGNORE when a record exists (`ErrExists` swallowed); else as Create | as Create |
-| `Owner.Update` (via `UpdateAddon` `addons.go:181`) | `owner.go:94` → `casUpdate` `store.go:198` (Unmarshal `:209`, Marshal `:215`, `kv.Update` `:219`) | DROP | OVERWRITE via Install `owner.go:113` → `putManifest` |
-| `Owner.ApplyReport` (via reconciler `handlers/eks/addon_status_reconciler.go:23`) | `owner.go:140` → `casUpdate` | DROP when status or health changes; IGNORE when the report changes nothing (mutate returns false, no write) | untouched |
-| `Owner.markFailed` (staging failure in Create/Update) | `owner.go:171` → `casUpdate` | DROP | untouched |
-| `Owner.Delete` (via `DeleteAddon` `addons.go:211`) | `owner.go:122` → Uninstall `delivery.go:51`, `deleteRecord` `store.go:181,189` | ERASE | ERASE (`deleteManifest` `store.go:172`) |
-| `DeleteClusterPrefix` (DeleteCluster `service_impl.go:1153`, DELETING reaper `eks_deleting_reaper.go:141`, both via `purgeClusterInfra` `service_impl.go:1451`) | `handlers/eks/cluster_state.go:493` | ERASE (every key under `clusters/{c}/`) | ERASE |
-| `purgeClusterInfra(deleteMeta=false)` (`service_impl.go:712,1251`) | `service_impl.go:1450` | not touched | not touched |
-| Account teardown | `spinifex/accountteardown/reapers_eks.go:103` | ERASE, only indirectly through `DeleteCluster`; no add-on reaper and no EKS bucket delete | ERASE (same path) |
-| `TagResource` / `UntagResource` on an add-on ARN | `service_impl.go:1772,1812,1855` | not a writer: add-on ARNs return `NotImplemented`; create-time tags live only in `Record.Tags` | n/a |
-
-## Record and manifest readers
-
-| Reader | file:line | Verdict |
-|---|---|---|
-| `addon.Get` (DescribeAddon `addons.go:152`, Create existence check, Delete) | `store.go:58,70` | IGNORE; FAIL on a type change |
-| `addon.List` (ListAddons `addons.go:26`) | `store.go:78,108` | IGNORE; one undecodable record fails the whole listing |
-| `addon.ListManifests` (`ListStagedAddonManifests` `addons.go:125`) | `store.go:119,141` | IGNORE; one undecodable manifest fails delivery for every add-on in the cluster |
-| `addonRecordToAWS` | `addons.go:222` | IGNORE (projects v1 fields onto the SDK shape) |
-| `ListStagedAddonManifests` reply | `addons.go:129-137` | Projects the four v1 fields field-by-field, so unknown manifest fields never leave the daemon even if `addon.Manifest` gained them |
-
-## contracts/eks/v1 consumers
-
-| Consumer | file:line | Mechanism | Added field in a newer payload |
-|---|---|---|---|
-| Gateway NATS client for `eks.ListStagedAddonManifests` | `handlers/eks/service_nats.go:128`; `foundation/messaging/nats/nats.go:316` | `encoding/json` into `ListStagedAddonManifestsOutput` | DROP on relay: an older gateway re-renders only the fields its `StagedAddonManifest` declares |
-| Gateway `ListInternalAddons` response | `gateway/eks/internal_addons.go:20,32`; route `gateway/eks.go:55`; `gateway/eks/handler.go:21-22` | `jsonutil.BuildJSON` (Go field names, json tags ignored) | Emits whatever fields its build declares; nothing to ignore |
-| `eks-gateway-fetch -resource addons` | `agents/eks/gatewayfetch/fetch.go:115-123` | `encoding/json` (case-insensitive) then fixed four-column printf | IGNORE: extra top-level or per-add-on fields never reach the TSV |
-| `mulga-eks-addon-sync.sh` sync loop | `scripts/images/eks-node/mulga-eks-addon-sync.sh:357` | `IFS=tab read -r _addon _version _role _cfg_b64` | A fifth TSV column would land inside `_cfg_b64` (last variable takes the rest), which is unused today (`:359`); inserting a column before column 4 would shift positions. Fetch binary and script ship in the same image, so they move together. |
-| `mulga-eks-addon-sync.sh` report | `mulga-eks-addon-sync.sh:67-72` | printf of five fixed keys | N/A for added fields: an older guest never emits them |
-| Gateway `PublishInternal` relay | `gateway/eks/internal_publish.go:30,82-85` | `Payload json.RawMessage` published verbatim | Passes added report fields through untouched |
-| Reconciler report subscriber | `handlers/eks/cluster_reconciler.go:449-455`; `state_report.go:69-72`; `addon_status_reconciler.go:22-27` | `encoding/json` into `AddonStatusReport`, then only `Addon`, `Phase`, `Message` reach the owner | IGNORE: a report carrying `incarnationId`/`generation` is applied exactly as today, so an older reconciler cannot reject a stale incarnation's report |
-| Phase / status enums | `owner.go:186-204` | `nextStatus` default branch | An unknown phase changes nothing; an unknown stored status decodes, and a `failed` report moves it to `CREATE_FAILED` |
-
-## Consequences for an expand phase
-
-- Every path that writes the record back (Update, ApplyReport with a change, markFailed) DROPs added fields, so while any older binary can serve an add-on write, added record fields are best-effort and can silently revert to absent.
-- Every Install rebuilds the manifest from the record, so an older binary's Create/Update erases added manifest fields too.
-- Delete and cluster deletion ERASE regardless of content, so a newer `origin`/`declined` marker would not stop an older binary deleting the record.
-- Reads and all wire consumers tolerate added fields; changing the JSON type of an existing field instead fails whole listings on older binaries.
-- The only hard fence is the per-account bucket stamp, which is all-or-nothing for EKS in that account.
-
-## Characterisation tests pinning the above
-
-| Test | file |
-|---|---|
-| `TestOwnerWrites_DropUnknownRecordAndManifestFields` | `spinifex/domains/eks/addon/v1_compat_test.go` |
-| `TestOwnerApplyReport_NoOpKeepsUnknownFields` | `spinifex/domains/eks/addon/v1_compat_test.go` |
-| `TestOwnerDelete_ErasesRecordAndManifestWithUnknownFields` | `spinifex/domains/eks/addon/v1_compat_test.go` |
-| `TestReads_IgnoreUnknownFieldsWithoutRewriting` | `spinifex/domains/eks/addon/v1_compat_test.go` |
-| `TestList_OneUndecodableEntryFailsTheListing` | `spinifex/domains/eks/addon/v1_compat_test.go` |
-| `TestAddonStatusReport_IgnoresUnknownFields` | `contracts/eks/v1/v1_compat_test.go` |
-| `TestInternalAddonsResponse_IgnoresUnknownFields` | `contracts/eks/v1/v1_compat_test.go` |
-| `TestEmitAddonsTSV_IgnoresUnknownFields` | `spinifex/agents/eks/gatewayfetch/v1_compat_test.go` |
-| `TestAddonStatusReport_UnknownFieldsAreAppliedAsToday` | `spinifex/handlers/eks/addon_v1_compat_test.go` |
-| `TestListStagedAddonManifests_ProjectsOnlyV1Fields` | `spinifex/handlers/eks/addon_v1_compat_test.go` |
-
-Already covered elsewhere: cluster-prefix sweep of add-on keys (`handlers/eks/addon_lifecycle_test.go:306`), hand-written v1 record decode (`addon_lifecycle_test.go:85`), unknown phase is a no-op (`domains/eks/addon/owner_test.go:49`).

docs/package-boundary/eks-addon-v2-contract-design.md

@@ -1,541 +0,0 @@
-# EKS managed add-on: v2 guest contract design
-
-**Status:** Draft for review; contract design only, nothing implemented.
-
-This is the guest/controller wire contract that `docs/package-boundary/eks-addon-lifecycle-design.md` (Section 9, slice 6) calls `contracts/eks/v2`.
-It applies that design's review decisions R1 to R16 and does not reopen them.
-It defines payloads, routes, subjects, authorization binding, capability, executor identity and compatibility; the owner transitions, deadlines, record migration and agent implementation stay in the lifecycle design.
-Present-state claims cite code read on this branch; `docs/package-boundary/eks-addon-lifecycle.md` stays authoritative for what the code does today.
-Paths are relative to `spinifex/` unless they start with `scripts/`, `contracts/` or `tests/`, which are relative to the spinifex repo root.
-
-## Standing
-
-Q-75 (contract evolution) and Q-76 (asynchronous message semantics) in `docs/specs/requirements/resource-lifecycle.md` (mulga repo) are Decided requirements with human provenance and normative effect (R1); they are not ADRs.
-Q-146 and C-82 (`docs/specs/requirements/managed-services.md`) are open, and this document closes neither.
-ADR-0006 (`docs/adr/0006-contract-guest-and-persisted-state-evolution.md`, mulga repo) is accepted and gives Q-75 and Q-76 their normative shape: S2 for the N/N-1 window on a coordinated Mulga Regional release and the explicit compatibility matrix on an independently released component, S3 for closed majors and the ban on timing heuristics as capability proof, S4 for expand/migrate/contract on persisted formats, S5 for prepare/activate/migrate/finalise on release activation, and S6 for the guest legacy-mode lifecycle.
-This design applies ADR-0006 and cites its clauses as decisions wherever it relies on them.
-Where ADR-0006 leaves a mechanism to implementation (the release-membership record, the activation gate), it states the dependency in Section 3.
-Capability is demonstrated end-to-end only by an accepted, fenced v2 report; a `contractVersion` field, an image tag or a successful fetch alone never proves it (Section 2.3; V8).
-
-## 1. Purpose and non-goals
-
-Purpose: give the add-on owner reports it can fence by executor, incarnation and generation, and observed proof of removal, without changing a byte of v1.
-
-| Non-goal | Consequence |
-|---|---|
-| NATS or JetStream redesign | Reports stay on core NATS as notifications (Q-76); correctness rests on durable desired state, level-triggered re-sending and owner deadlines (lifecycle design Section 12). |
-| Changing v1 | `contracts/eks/v1` route, subject, both JSON forms and Go field names stay byte-identical and readable, pinned by `contracts/eks/v1/addon_status_test.go`, `addon_manifest_test.go` and `v1_compat_test.go`. |
-| Timing heuristics | No settle window, poll count, retry budget, credential expiry or deadline is used as a fence, capability proof or completion condition (R2, Q-75, ADR-0006 S3: negotiation is never by timing). |
-| Universal framework | No generic versioning, capability-negotiation or guest-protocol library; every type here is EKS add-on specific (ADR-0003 S2 rules out a shared one without two consumers). |
-| Lifecycle implementation | Owner state machine, deadlines, record migration, token idempotency and agent code are the lifecycle design's slices, not this contract. |
-| Per-incarnation ARN | The external ARN stays name-only (R4); `incarnationId` is internal. |
-
-## 2. Version and capability negotiation
-
-### 2.1 Versions
-
-| Item | Rule |
-|---|---|
-| Major | Every v2 body carries `"contractVersion": 2`. A receiver rejects any other value with reason `UnknownContractMajor` and changes no state (Q-75, ADR-0006 S3). |
-| Separation | v2 has its own routes and subject (Section 7); no receiver sniffs payloads to tell v1 from v2. |
-| Extensibility | Declared per body below: added optional fields may be ignored; an unknown value of a closed enum (`mode`, `phase`, `reason`, object `state`) is rejected for that entry (Q-75 "reject unknown required semantics or enum values"; ADR-0006 S3). |
-| Producer rule | The host never emits a v2 directive to a caller that has not called the v2 route as the authenticated active executor; v1 callers get v1 only (Q-75 "oldest active compatible representation"; ADR-0006 S3 "the producer emits the oldest active compatible representation unless an authenticated capability exchange proves support for the newer one"). |
-
-### 2.2 What N and N-1 mean here
-
-| Party | N-1 | N |
-|---|---|---|
-| Host release (daemon, awsgw, add-on owner, cluster owner) | Today's release: v1 route and subject only. | Serves v1 unchanged plus the v2 routes and subject, the executor record and the v1 compatibility gate. |
-| Guest image (control-plane VM agent) | v1 agent: `mulga-eks-addon-sync.sh` with `eks-gateway-fetch`/`eks-gateway-publish`. | v2 agent shipped in a new EKS node image. |
-
-The host release groups daemon, awsgw, the add-on owner and the cluster owner as one coordinated Mulga Regional release, not as independently released components (ADR-0006 S2): they ship from the same Spinifex build, and `cluster-deploy.yml`'s rolling restart runs the fleet mid-roll on mixed N and N-1 builds (release-and-version-inventory.md Section 2), which is exactly the N/N-1 coexistence S2 describes for a coordinated release.
-ADR-0006 S2's own worked example, that an AWSGW deployment or restart must not force a compatible Predastore deployment to restart, is about awsgw's cross-submodule relationship to Predastore, an independently released product with its own compatibility declaration; this contract does not touch Predastore, so that example does not reclassify awsgw here.
-The host window is N and N-1 in both rollout orders.
-The guest image is a long-lived consumer outside the Regional upgrade (a cluster keeps its image until its control-plane VM is replaced), so the v1 guest mode publishes its own support and retirement lifecycle as Q-75 and ADR-0006 S6 require; N+1 does not drop v1 merely because one release passed.
-New EKS images are picked newest-AMI-wins (`handlers/eks/k3s_server_vm.go:353-363`, release inventory Section 5), so an N-1 host can launch a v2 image; Section 8 covers that row.
-
-### 2.3 How capability is established
-
-Capability is never read from a guest payload, image tag, user-data value or AMI name.
-
-1. The caller authenticates with the control-plane instance role session that IMDS mints with `RoleSessionName` = instance ID (`handlers/sts/assume_role.go:136-148`); the role trusts only `ec2.amazonaws.com` (`handlers/iam/system_role.go:14`), and the gateway takes the role name from the underlying role ARN, never the session name (`gateway/eks.go:307-320`).
-2. The gateway runs the existing class and membership checks (`gateway/eks/internal_authz.go:44-96`), then the executor check (Section 4.3).
-3. The gateway relays the fetch to the daemon with the derived executor identity attached and lists v2 directives for it; a fetch alone marks no capability (V8).
-4. The gateway relays the first v2 report that passes checks 1 to 6 of Section 4.3; the cluster owner re-checks the epoch against the executor record and, in the same CAS that accepts that report, marks the epoch `addonContract: 2` if it is not already marked.
-5. The add-on owner reads capability only through the narrow interface in Section 4.2.
-
-A new epoch starts unmarked, so the compatibility gate re-closes on every executor change until the new executor's first valid, fenced v2 report is accepted (lifecycle design Section 9; V8).
-A fetch, or a directive the guest never acts on, may be retained as diagnostic evidence only; it is never sufficient on its own to prove capability (V8).
-
-## 3. Prerequisites: what must exist before v2 activates
-
-The release inventory found no release-membership record and no activation gate (`docs/package-boundary/release-and-version-inventory.md` Section 7).
-v2 can ship in the Prepare stage without them, but the v2 routes answer `NotActivated` and the gate stays open (old semantics for everyone) until each item below exists.
-
-| # | Prerequisite | Gap it closes | Why v2 needs it |
-|---|---|---|---|
-| P1 | An authoritative, cluster-visible record of the release each daemon and awsgw process runs | G1, G10 | Activation must prove no N-1 writer remains: N-1 `casUpdate` writers DROP added record fields and N-1 `Owner.Delete` ERASEs without observed removal (`eks-addon-v1-compatibility.md` "Record and manifest writers"). awsgw processes count, because an N-1 gateway serves v1 membership-only authorization and has no v2 route. |
-| P2 | An operator-visible activation record for the add-on lifecycle change, written only from P1 evidence | G2 | The v2 routes and the v1 compatibility gate key on it. It is EKS-add-on-specific for now (V9); the release-membership mechanism (P1) should be reusable by another resource, but this design does not build a platform-wide activation framework, unless a platform gate supersedes it first (lifecycle design open question 4). |
-| P3 | Record expand complete: `schemaVersion`, `incarnationId`, `desired`, `observed` written beside legacy fields by an idempotent, resumable pass | G7, G8 | Directives carry `incarnationId` and `generation`; a record without them cannot be listed in v2 (Section 5.4). |
-| P4 | The cluster-owned executor record and the bound v2 report route (Sections 4 and 7) | executor inventory gaps 2 to 4 | Without it no report can be bound to a caller, cluster or executor. |
-| P5 | A documented controlled-rollback procedure for the activated state | G4 | An N-1 binary that returns after activation must not complete or reinterpret v2 obligations (lifecycle design open question 9). |
-
-The per-account bucket stamp is not the fence: bumping `KVBucketEKSAccountVersion` (`handlers/eks/store.go:29`) makes every N-1 binary refuse the whole `eks-account-{id}` bucket with `SchemaAheadError` (`foundation/state/migrate/kv.go:55-67`), taking down every EKS action in the account, not just add-ons.
-The per-record `schemaVersion` plus P2 replaces it.
-
-## 4. Executor identity and epoch
-
-### 4.1 Shape and owner
-
-The cluster owner owns a new key `clusters/{cluster}/delivery-executor` in `eks-account-{account}`, beside `meta` rather than inside it.
-A separate key keeps the epoch off the meta revision, which churns on every health write (`handlers/eks/cluster_state.go:416-436`), and away from N-1 `casUpdateMeta` writers, which would DROP an added meta field.
-N-1 binaries ignore the key: cluster listing selects only keys ending `/meta` (`handlers/eks/service_impl.go:2422`), and `DeleteClusterPrefix` erases it with the cluster, which is correct.
-
-```json
-{
-  "schemaVersion": 1,
-  "instanceId": "i-0a1b2c3d4e5f60718",
-  "epoch": 5,
-  "assignedAt": "2026-10-09T10:00:00Z",
-  "addonContract": 2,
-  "addonContractProvenAt": "2026-10-09T10:00:31Z",
-  "fence": {
-    "previousInstanceId": "i-09f8e7d6c5b4a3921",
-    "previousEpoch": 4,
-    "evidence": "instance-terminated",
-    "observedAt": "2026-10-09T09:59:58Z"
-  }
-}
-```
-
-| Field | Rule |
-|---|---|
-| `instanceId` | The internal EC2 instance ID of a current control-plane member; empty means no executor is assigned (an unavailable guest). |
-| `epoch` | Unsigned, starts at 1, advanced by exactly one in the CAS that changes `instanceId`; never decreases while the key exists. |
-| `addonContract` | `0` (unproven) or `2`; reset to `0` in the CAS that advances `epoch`; set to `2` only in the CAS that accepts the first valid, fenced v2 report for the epoch, never on a fetch (Section 2.3; V8). |
-| `addonContractProvenAt` | RFC 3339 UTC; set in the same CAS as `addonContract: 2`; diagnostic only. |
-| `fence` | The evidence that let the epoch advance (Section 4.4); absent for epoch 1. |
-
-### 4.2 Interface the add-on side declares
-
-The add-on package declares, and the cluster owner implements, one read:
-
-```go
-// In domains/eks/addon.
-type DeliveryExecutor struct {
-    InstanceID    string
-    Epoch         uint64
-    AddonContract int // 0 or 2
-}
-
-type DeliveryExecutors interface {
-    ActiveDeliveryExecutor(ctx context.Context, account, cluster string) (DeliveryExecutor, error)
-}
-```
-
-The implementation carries the compile-time check (`var _ addon.DeliveryExecutors = (*Impl)(nil)`).
-The add-on package does not read `ClusterMeta`, the executor key or the gateway caller (R6).
-Assignment, reassignment and capability marking are cluster-owner methods the add-on side never calls.
-
-### 4.3 Gateway derivation and rejection rule
-
-The executor identity is `Caller.SessionName` after `requireCPAgent` passes (`gateway/eks/internal_authz.go:85-96`); nothing in the path, query or body names it.
-Static-credential fallback VMs have no session name and can never be executors (`handlers/eks/k3s_server_role.go:30-50`).
-
-| Order | Check | v2 reason | HTTP |
-|---|---|---|---|
-| 1 | SigV4, IAM policy, CP-agent class | existing gateway envelope (`AccessDenied`); these run before the v2 handler | 403 |
-| 2 | Activation record present (P2) | `NotActivated` if the record is absent; `ActivationStateUnavailable` if it is unreadable, invalid or unavailable | 503 |
-| 3 | Caller is a member of the named cluster in the named account (`IsControlPlaneMember`, `internal_authz.go:74`) | `NotClusterMember` | 403 |
-| 4 | Executor record names an instance | `NoActiveExecutor` | 503 |
-| 5 | `Caller.SessionName == executor.instanceId` | `NotActiveExecutor` | 403 |
-| 6 | Report only: body `executorEpoch == executor.epoch` | `StaleExecutorEpoch` | 409 |
-| 7 | Body decodes against v2 with `contractVersion == 2` | `UnknownContractMajor` or `MalformedReport` | 400 |
-
-The daemon re-runs checks 4 to 6 against the record it reads, and the owner re-checks the epoch inside its CAS loop (lifecycle design Section 2 rule 1), because the epoch can advance between the gateway check and the write.
-That same CAS marks `addonContract: 2` if unmarked, only for the first report it accepts under the current epoch (Section 4.1; V8).
-
-### 4.4 Fencing prerequisites and evidence options
-
-The epoch may advance only after the previous executor provably cannot render, apply, or have reports accepted (R6).
-Report acceptance is covered by checks 5 and 6 once the epoch advances; render and apply need evidence about the old VM.
-The options below come from the executor inventory (`docs/package-boundary/eks-addon-executor-inventory.md` Sections 8 and 9); the reviewer chooses.
-
-| Option | Evidence it requires | Exists today |
-|---|---|---|
-| A. Instance terminated or not found | EC2 instance record `terminated` or absent, proven authoritative under host partition; reassignment never triggered by a describe error (`cluster_reconciler.go:982-1007`) | Query exists; authority under host loss unverified; describe-error trigger must be excluded |
-| B. Instance stopped and pinned stopped | `stopped` plus a durable "do not restart" marker the reconciler's restart ladder honours (`cluster_reconciler.go:708-771`) | No |
-| C. Disable-on-handover directive | An online host-to-guest directive the agent polls, an acknowledgement, and a guest rule that a denied fetch stops rendering; an acknowledgement proves only that the old VM was alive to send it | No (only the boot-time recovery directive) |
-| D. Credential-based | Per-session revocation, IMDS refusing to re-mint for the fenced instance, gateway rejecting revoked sessions; still leaves files already rendered on the old disk | No |
-| E. Guest leader election (kube Lease) | Host-visible proof of the guest leader, verified K3s auto-deploy behaviour across servers; reports still need checks 5 and 6 | No |
-| F. Operator assertion | An explicit, audited operator action naming the old instance, recorded as `fence.evidence: "operator-asserted"` | No |
-
-**Approved (V10):** A is the only automatic evidence: an authoritative record that the former instance terminated or no longer exists.
-A lookup error, a network issue or a partition is never evidence of termination, and reassignment is never triggered by a describe error.
-F, an explicit, audited operator assertion, is the alternative when A cannot be obtained.
-The guest stand-by rule of Section 6.4 is defence in depth and is never counted as fence evidence.
-B to E each need new machinery and either prove less than A or prove only liveness.
-Under A, the restore path's successor-first ordering (`restore_snapshot.go:262-336`) and the unreachable etcd prune (executor inventory gap 10) must be resolved before the restore path may advance an epoch.
-
-## 5. Directive (fetch) payload v2
-
-### 5.1 Body
-
-Response to `GET /clusters/{clusterName}/internal-addons-v2/{accountId}` (Section 7), `Content-Type: application/json`:
-
-```json
-{
-  "contractVersion": 2,
-  "cluster": "prod",
-  "executor": {"instanceId": "i-0a1b2c3d4e5f60718", "epoch": 5},
-  "issuedAt": "2026-10-09T10:01:00Z",
-  "directives": [
-    {
-      "addonName": "argocd",
-      "incarnationId": "0b7e4f7a-2d55-4f0e-9a51-6f1fd1c0e2a4",
-      "generation": 4,
-      "mode": "remove",
-      "spec": {"addonVersion": "3.0.23"},
-      "deletion": {"requestedAt": "2026-10-09T09:58:00Z", "preserve": false},
-      "deadline": "2026-10-09T10:28:00Z"
-    },
-    {
-      "addonName": "aws-load-balancer-controller",
-      "incarnationId": "6f1c2a4e-8b0d-4c3e-a1f2-93b7d4e5c6a8",
-      "generation": 3,
-      "mode": "install",
-      "spec": {
-        "addonVersion": "2.11.0",
-        "serviceAccountRoleArn": "arn:aws:iam::123456789012:role/lbc-irsa"
-      },
-      "deadline": "2026-10-09T10:20:00Z"
-    }
-  ]
-}
-```
-
-| Field | Type | Rule |
-|---|---|---|
-| `contractVersion` | int | Always 2. |
-| `cluster` | string | The path cluster, echoed for logs. |
-| `executor.instanceId`, `executor.epoch` | string, uint64 | From the executor record, never from the request; the guest echoes `epoch` in every report. |
-| `issuedAt` | RFC 3339 UTC | Diagnostic only. |
-| `directives` | array | One entry per add-on record of the cluster, sorted by `addonName`; `[]`, never `null`, when empty. |
-| `addonName`, `incarnationId`, `generation` | string, UUID string, uint64 | Identity and the desired generation this entry represents. |
-| `mode` | enum `install`, `remove`, `relinquish` | Closed set. |
-| `spec.addonVersion` | string | Catalogue version; for `remove` and `relinquish`, the last installed spec, so a guest without local state can still render the object list. |
-| `spec.serviceAccountRoleArn` | string, omitted when unset | An IAM role ARN; see 5.3. |
-| `spec.configurationValues` | string, omitted | Never emitted until a bundle publishes a schema and a renderer (R9); a guest that receives it for a bundle without a renderer reports `failed`/`UnsupportedConfiguration`. |
-| `deletion` | object, omitted unless `mode` is `remove` or `relinquish` | `requestedAt` and `preserve`, the obligation being discharged. |
-| `deadline` | RFC 3339 UTC | The persisted operation deadline (R12), informational; the guest must not change what it reports, stop reporting or infer completion from it. The owner alone enforces it. |
-
-Extensibility: the guest ignores unknown optional fields at any level; it rejects an unknown `mode` for that entry by reporting `failed`/`InvalidDirective`.
-
-### 5.2 Rendering and the v1 quirk
-
-v1's body is written by `WriteJSONResponse`, which calls the SDK REST-JSON marshaller `jsonutil.BuildJSON` (`gateway/eks/handler.go:19-22`); it ignores `json` tags, so the deployed v1 body uses Go field names (`{"Addons":[{"AddonName",...}]}`) and the guest matches it only through `encoding/json`'s case-insensitive decoding (`contracts/eks/v1/README.md`).
-v2 avoids this by construction:
-
-- The v2 route has its own writer that calls `encoding/json` (`json.Marshal`, default HTML escaping) and sets `application/json`; it never goes through `WriteJSONResponse`.
-- v2 types carry only `json` tags, no `locationName`, and the literal-byte tests (Section 9) pin camelCase keys, so a regression to `BuildJSON` fails a test.
-- Optional fields use `omitempty`; arrays that must be present (`directives`, `reports`, `objects`) are initialised non-nil.
-- The guest decodes v2 strictly by key case (a Go decoder that rejects a case-mismatched key, for example by comparing a re-marshal), so the two forms can never be confused again.
-
-### 5.3 Validation the host applies before listing
-
-| Value | Rule | On violation |
-|---|---|---|
-| `serviceAccountRoleArn` | `arn:{partition}:iam::{12 digits}:role/{path}{name}` with IAM's role-name character set; already `iam:PassRole`-checked at the adapter (R10) and rejected for bundles with no IRSA marker (R14). | The AWS adapter returns `InvalidParameterException`, so it never reaches a record; a stored legacy value that fails is listed without the field and the owner records a health issue. |
-| `addonVersion` | In the catalogue. | Same. |
-| `configurationValues` | Never listed (R9). | Stored legacy values stay flagged on the record (lifecycle design Section 3). |
-
-The role-ARN format check at the adapter boundary is a standalone security correction, not v2 lifecycle work: it is being fixed directly on the existing v1 path and applies to v1 and v2 alike, independent of this design.
-v2 adds only the guest-side re-validation below; it is not what closes the adapter-side gap.
-The guest re-validates the role ARN with the same pattern before rendering, because the v1 renderer substitutes it raw into `sed` and YAML (`eks-addon-bundle-inventory.md` X9); a failure is `failed`/`InvalidDirective`.
-
-### 5.4 Records that cannot be listed
-
-A record without `incarnationId` or `desired.generation` (expand not yet run, or rewritten by an N-1 writer before activation) is omitted from the v2 listing and counted in a metric; the add-on reconcile pass re-expands it.
-After activation no N-1 writer exists (P1), so the omission is transient.
-
-## 6. Report payload v2
-
-### 6.1 Guest body
-
-`POST /clusters/{clusterName}/internal-addons-v2/{accountId}/reports`, one batch per tick:
-
-```json
-{
-  "contractVersion": 2,
-  "executorEpoch": 5,
-  "reports": [
-    {
-      "addonName": "aws-load-balancer-controller",
-      "incarnationId": "6f1c2a4e-8b0d-4c3e-a1f2-93b7d4e5c6a8",
-      "generation": 3,
-      "phase": "degraded",
-      "reason": "InsufficientReplicas",
-      "message": "deployment kube-system/aws-load-balancer-controller available 0/1",
-      "observedAt": "2026-10-09T10:05:00Z",
-      "objects": [
-        {"group": "apps", "kind": "Deployment", "namespace": "kube-system", "name": "aws-load-balancer-controller", "state": "present", "detail": "available=0/1"}
-      ]
-    },
-    {
-      "addonName": "argocd",
-      "incarnationId": "0b7e4f7a-2d55-4f0e-9a51-6f1fd1c0e2a4",
-      "generation": 4,
-      "phase": "removing",
-      "reason": "FinalizerBlocked",
-      "message": "namespace argocd Terminating",
-      "observedAt": "2026-10-09T10:05:00Z",
-      "objects": [
-        {"group": "", "kind": "Namespace", "namespace": "", "name": "argocd", "state": "terminating", "detail": "finalizers=kubernetes"},
-        {"group": "apiextensions.k8s.io", "kind": "CustomResourceDefinition", "namespace": "", "name": "applications.argoproj.io", "state": "absent", "detail": ""}
-      ]
-    }
-  ]
-}
-```
-
-| Field | Rule |
-|---|---|
-| `executorEpoch` | The epoch from the last successful directive fetch; one per batch. |
-| `reports[]` | At most one entry per `addonName`; a duplicate rejects the batch as `MalformedReport`. |
-| `phase` | Closed enum, Section 6.2. |
-| `reason` | Closed enum, Section 6.3; required for `failed`, `degraded` and `removing`, optional otherwise. |
-| `message` | Free text, at most 1024 bytes after truncation by the guest. |
-| `objects[]` | Per-object observations (Sections 6.6 and 6.7); required for `removing`, `removed` and `ready`, optional otherwise; bounded to the bundle's declared set. |
-| `ts` | Not present; `observedAt` replaces v1's unused `ts`. |
-
-The guest builds the body with a JSON encoder (a Go helper in the image, as `eks-gateway-publish` already is), never `printf`.
-v1's `printf` (`scripts/images/eks-node/mulga-eks-addon-sync.sh:67-72`) does not escape `message`, so a quote or backslash in a message produces invalid JSON; v2 removes that class of defect rather than escaping in shell.
-
-### 6.2 Phases
-
-| Phase | Valid in mode | Meaning |
-|---|---|---|
-| `applied` | `install` | Rendered for this generation; readiness (Section 6.6) not yet met. Never a failure. |
-| `ready` | `install` | Readiness definition for this bundle met for this generation. |
-| `degraded` | `install` | Was ready for this generation and no longer meets the definition. |
-| `failed` | `install` | A deterministic cause prevents realising this generation (`reason` required). |
-| `removing` | `remove` | Some object of the removal set still exists. |
-| `removed` | `remove` | Every object of the removal set observed absent. |
-| `relinquished` | `relinquish` | The rendered file is gone and the agent no longer manages the add-on. |
-
-`degraded` is an addition to the phase list in the lifecycle design Section 2: without it the guest must choose between `applied` (which cannot move `ACTIVE` to `DEGRADED`) and `failed` (which is terminal while `CREATING` or `UPDATING`, R3), and it cannot see the host status to choose.
-`degraded` is approved (V2) as an observed condition for an otherwise `ACTIVE` add-on: the owner maps `ACTIVE` plus `degraded` to `DEGRADED` and records it as a health issue.
-A `degraded` report during `CREATING` or `UPDATING` records the same health issue but does not by itself turn the operation into a terminal failure; the operation's deadline and an explicit terminal report (`failed`, `ready` or `removed`) decide that (V2).
-
-### 6.3 Reason codes
-
-Reasons are internal contract values, not AWS error codes (R7); the owner maps some onto `AddonIssue.code` (pinned SDK `AddonIssueCode*`, `aws-sdk-go` v1.55.8 `service/eks/api.go`) when it builds `health`.
-
-| Reason | Phases | `AddonIssue.code` | Meaning |
-|---|---|---|---|
-| `BundleMissing` | `failed` | `InternalFailure` | No baked bundle for name and version in this image. |
-| `InvalidDirective` | `failed` | `InternalFailure` | Unknown mode, invalid role ARN, or a field the guest cannot realise. |
-| `UnsupportedConfiguration` | `failed` | `InternalFailure` | `configurationValues` present for a bundle without a renderer. |
-| `RenderFailed` | `failed` | `InternalFailure` | Template or webhook certificate generation failed. |
-| `ApplyRejected` | `failed`, `degraded` | `AdmissionRequestDenied` | The apiserver or an admission webhook rejected an object of the bundle. |
-| `ObjectConflict` | `failed` | `ConfigurationConflict` | `resolveConflicts` `NONE`/`PRESERVE` and an object of the bundle exists without the add-on's ownership labels. |
-| `ObjectMissing` | `degraded` | `K8sResourceNotFound` | An object of the readiness set is absent after it was applied. |
-| `InsufficientReplicas` | `degraded` | `InsufficientNumberOfReplicas` | A workload of the readiness set is below its replica target. |
-| `WebhookUnavailable` | `degraded` | `AdmissionRequestDenied` | A fail-closed webhook of the bundle has no ready endpoint. |
-| `CredentialFailure` | `degraded` | `AccessDenied` | Only where a bundle's readiness definition observes it; none does yet. |
-| `FinalizerBlocked` | `removing` | none | An object of the removal set is `Terminating` behind a finalizer. |
-| `DeletePending` | `removing` | none | Delete issued; objects still present without a finalizer cause. |
-
-No reason exists for an unreachable apiserver: the guest sends nothing for that add-on, as the lifecycle design specifies, and silence is handled by deadlines.
-The gateway-side rejection reasons of Section 4.3 (`NotActivated`, `ActivationStateUnavailable`, `NotClusterMember`, `NoActiveExecutor`, `NotActiveExecutor`, `StaleExecutorEpoch`, `UnknownContractMajor`, `MalformedReport`) are response reasons, never report reasons.
-
-The reason table above is the closed set for `contractVersion: 2` (V4; Q-75, ADR-0006 S3).
-A genuinely new wire-level reason has new semantics and therefore requires a new major, never a silent addition to this enum; free-form detail that does not need enum-level routing belongs in `message`, not in an unbounded `reason` value.
-
-### 6.4 Gateway response and guest stand-by rule
-
-The v2 routes answer with `application/json` `{"contractVersion":2,"error":{"reason":"...","message":"..."}}`, rendered with `encoding/json`, instead of the AWS error envelope (R7); only check 1 of Section 4.3 keeps the gateway's existing envelope, because it runs before the route handler.
-A successful report POST returns `{"contractVersion":2,"accepted":N}` where `accepted` counts entries relayed, not entries applied: acceptance by the owner is a separate CAS that the guest never learns of (Q-76: a transport acknowledgement never implies domain completion).
-
-On `NotActiveExecutor`, `NotClusterMember` or `StaleExecutorEpoch`, the v2 agent stops delivery: it removes its rendered `spinifex-addon-*.yaml` files without `kubectl delete`, sends no reports and re-fetches on its next tick.
-This complements, but never substitutes for, the fence evidence of Section 4.4.
-
-### 6.5 Host-internal envelope and subject
-
-The gateway does not relay the guest bytes verbatim, unlike v1's `PublishInternal` (`gateway/eks/internal_publish.go:47-91`).
-It decodes the batch strictly, then publishes one host-internal envelope rendered with `encoding/json`:
-
-```json
-{
-  "contractVersion": 2,
-  "account": "123456789012",
-  "cluster": "prod",
-  "executor": {"instanceId": "i-0a1b2c3d4e5f60718", "epoch": 5},
-  "receivedAt": "2026-10-09T10:05:01Z",
-  "reports": [ ... the decoded entries, re-encoded ... ]
-}
-```
-
-`executor.instanceId` comes from the authenticated caller and `executor.epoch` is the value check 6 verified; no guest field reaches `executor`.
-Subject: `eks.addon.v2.{account}.{cluster}.status`, core NATS, no queue group, subscribed by the add-on reconcile pass under the cluster lease (lifecycle design Section 11).
-The account and cluster tokens come from the route captures after check 3, so they are bound to a cluster the caller serves; v1's subject is built from the body `accountId` (`internal_publish.go:63-76`).
-
-### 6.6 Readiness evidence per bundle
-
-v1 reports `ready` when K3s records the `addon.k3s.cattle.io/gvks` annotation, which proves only that K3s applied some GVKs, says nothing about workloads, and may reflect the previous generation (`eks-addon-bundle-inventory.md` Section 3.3, X1 to X3).
-v2 drops the annotation as a readiness source entirely and defines ready per bundle.
-
-Generation binding: the v2 renderer stamps every rendered object with annotations `eks.spinifex.io/addon-incarnation` and `eks.spinifex.io/addon-generation` and labels `app.kubernetes.io/managed-by: spinifex-addon` and `eks.spinifex.io/addon-name`.
-An object counts toward readiness only if it is present and carries the current incarnation and generation, so a report can never describe a previous apply; this also gives a stable ownership selector (X6).
-
-Common rule: every object of the bundle's readiness set is read individually (no `--ignore-not-found` aggregation); a missing one is `ObjectMissing`.
-
-| Bundle | Ready when, for the current generation |
-|---|---|
-| `spinifex-noop` | Namespace `spinifex-noop` and ConfigMap `spinifex-noop/spinifex-noop` present. |
-| `argocd` | Namespace `argocd` present; CRDs `applications`, `applicationsets`, `appprojects.argoproj.io` `Established`; all 6 Deployments `observedGeneration >= generation`, `updatedReplicas == availableReplicas == spec.replicas`; StatefulSet `argocd-application-controller` `updatedReplicas == readyReplicas == spec.replicas`. |
-| `aws-ebs-csi-driver` | All rendered objects present; Deployment `ebs-csi-controller` available equals `spec.replicas` (2); DaemonSet `ebs-csi-node` `numberReady == updatedNumberScheduled == desiredNumberScheduled`, where `desiredNumberScheduled == 0` is ready with a `message` saying no eligible node. |
-| `aws-load-balancer-controller` | CRDs `ingressclassparams` and `targetgroupbindings.elbv2.k8s.aws` `Established`; Deployment available equals `spec.replicas`; Service `aws-load-balancer-webhook-service` has at least one ready endpoint (EndpointSlice), because the bundle's six webhooks are `failurePolicy: Fail`; both webhook configurations present. |
-| `nvidia-device-plugin` | DaemonSet `nvidia-device-plugin-daemonset` `numberReady == updatedNumberScheduled == desiredNumberScheduled`; `desiredNumberScheduled == 0` is ready with a no-GPU-node message. |
-
-Not part of any ready definition: IRSA credential success, GPU allocatable exposure and data-path checks; open question 2.
-
-### 6.7 Removal evidence per bundle
-
-`removed` means every object of the bundle's removal set is observed NotFound by an individual `get` for each object (namespaced and cluster-scoped), not by listing a namespace.
-The removal set is scoped to add-on-owned Kubernetes objects only (V6): it is the rendered object list of the last installed spec, minus objects shared with another add-on that is still listed (below), and it never includes downstream workloads, PVs or other unrelated resources outside that set.
-`removing` reports carry `objects[]` with `present`, `terminating` or `absent` for every member of the set, so `DescribeAddon` health can show what blocks.
-
-| Bundle | Removal set (all NotFound) | Blockers reported as `FinalizerBlocked` | Residue outside the set |
-|---|---|---|---|
-| `spinifex-noop` | Namespace, ConfigMap | Namespace `kubernetes` finalizer (aggregated API availability) | none |
-| `argocd` | Namespace `argocd`, 3 CRDs, 3 ClusterRoles, 3 ClusterRoleBindings, and the namespaced objects | `Application` `resources-finalizer.argocd.argoproj.io`; CRD and namespace `Terminating` | user workloads Argo CD deployed |
-| `aws-ebs-csi-driver` | 20 rendered objects, `kube-system` or cluster-scoped | PV and VolumeAttachment CSI finalizers | PVs, VolumeAttachments, Spinifex EBS volumes |
-| `aws-load-balancer-controller` | 2 CRDs plus 13 rendered objects, webhook configurations included | Ingress, Service and TargetGroupBinding controller finalizers; CRD `Terminating` | Spinifex ELBv2 load balancers, target groups, security groups |
-| `nvidia-device-plugin` | 1 DaemonSet | none | host CDI files |
-
-Residue is reported in the final `removed` report's `objects[]` with `state: "residue"` where the guest can see it (PVs with `spec.csi.driver == ebs.csi.aws.com`); out-of-cluster residue is invisible to the guest.
-`removed` does not wait for in-cluster residue outside the removal set (V6): downstream workloads, unreleased PVs and other dependents are reported as residue, not waited on, and the record proceeds once the removal set itself is gone.
-Preserve-mode deletion (`deletion.preserve`) explicitly relinquishes the add-on's ownership of its objects rather than claiming their removal (V6); that obligation is discharged by the `relinquished` phase (Section 6.2), not by `removed`.
-
-Shared `kube-system/spinifex-gateway-ca` Secret (bundle inventory X5): v1 renders it under one shared name for both IRSA bundles, so either removal deletes the other's dependency and a NotFound check on it never completes while the other re-creates it.
-v2 renders a per-bundle name (`spinifex-gateway-ca-<addon>`) for every new install, so the overlap does not recur (V7).
-Compatibility path for a cluster migrating from v1 (V7): while a bundle's rendered objects still carry the old shared name, the shared-object rule applies unchanged, so an object rendered by more than one listed add-on is excluded from every removal set while more than one `install` directive renders it, and the guest deletes it only with the last one.
-A bundle moves to its per-bundle name only on its next v2 install or update, after which the shared-object rule no longer applies to it.
-
-## 7. Routes, subjects and authorization binding
-
-| Direction | v1 (unchanged) | v2 |
-|---|---|---|
-| Fetch route | `GET /clusters/{clusterName}/internal-addons/{accountId}`, action `ListInternalAddons` (`gateway/eks.go:55-57`) | `GET /clusters/{clusterName}/internal-addons-v2/{accountId}`, action `ListInternalAddonDirectives` |
-| Report route | `POST /clusters/{clusterName}/internal-publish`, channel `addon`, action `PublishInternal` (`gateway/eks.go:39-41`) | `POST /clusters/{clusterName}/internal-addons-v2/{accountId}/reports`, action `ReportInternalAddons` |
-| Gateway to daemon | NATS `eks.ListStagedAddonManifests` | NATS `eks.ListAddonDirectives`, request carries the gateway-derived executor identity |
-| Report subject | `eks.addon.{account}.{cluster}.status` | `eks.addon.v2.{account}.{cluster}.status` |
-| Response renderer | `jsonutil.BuildJSON` | `encoding/json` |
-| Scope | `ListInternalAddons` is `sourceInternalCluster`; `PublishInternal` is `sourceBodyAccountCluster` (`gateway/eks/authz.go:55,59`) | Both `sourceInternalCluster`, so `IsInternalAction` is true and `AuthorizeInternal` runs (`internal_authz.go:37-39`) |
-| IAM | Role policy `Resource: "*"` (`handlers/eks/k3s_server_role.go:27`) | The two new actions added to the same inline policy; the binding comes from `AuthorizeInternal` plus Section 4.3, not from the policy resource |
-
-Binding finding carried forward: today `PublishInternal` is not gated by `AuthorizeInternal`, so with the role's `Resource: "*"` any control-plane session of any cluster can publish add-on reports for any account and cluster, and the relay attaches no caller identity (executor inventory gaps 2 and 3).
-v2 does not reuse `PublishInternal`; its report route takes the cluster and account from the path, requires membership and the active executor, and attaches identity.
-Binding v1's `PublishInternal` to the caller's cluster is being fixed directly on the v1 path, as a standalone security correction independent of this design, not as v2 lifecycle work.
-v2 keeps its own binding regardless (Section 4.3, Section 6.5) and neither depends on, nor is credited for, that correction; post-activation v1 reports stay diagnostic-only either way (lifecycle design Section 9).
-
-After an executor is assigned for a cluster (P4), the N gateway binds the v1 fetch route to that assigned executor by applying check 5 of Section 4.3 to it, so a v2 agent on a non-executor member cannot fall back to v1 and become a second renderer (V5).
-v1 payload bytes are unchanged by this; before an executor is assigned, the v1 fetch route keeps its existing behaviour unchanged (V5).
-
-## 8. Compatibility matrix
-
-"Pre-activation" means P2 has not been recorded; "post-activation" means it has, which by P1 implies no N-1 host process remains.
-
-| Guest | Host | Fetch | Reports | Lifecycle mutations | Conformance |
-|---|---|---|---|---|---|
-| v1 | N-1 | v1 | v1, applied as today | Today's semantics | None claimed |
-| v1 | N, pre-activation | v1, byte-identical | v1, applied as today | Today's semantics | None claimed |
-| v1 | N, post-activation | v1 projection: records in `install` mode only | v1, recorded diagnostically in `observed`, never complete an operation | Gated (below) | None claimed |
-| v2 | N-1 | v2 answered `InvalidAction` (`gateway/eks.go:196-199`); agent uses legacy v1 mode that tick | v1 | Today's semantics | None claimed |
-| v2 | N, pre-activation | v2 answered `NotActivated`; legacy v1 mode that tick | v1 | Today's semantics | None claimed |
-| v2, active executor | N, post-activation | v2 | v2, fenced; the first report accepted for the epoch marks `addonContract: 2` if unmarked (V8) | Ungated once marked | Q-146 claimed for this cluster only |
-| v2, not executor | N, post-activation | `NotActiveExecutor`; stand-by (Section 6.4); v1 fetch also denied | none | n/a | n/a |
-
-Legacy-mode fallback is approved only for `InvalidAction` from the fetch route and `NotActivated` (V1).
-The guest never falls back on an authentication, membership, executor, epoch or other fencing failure, including `NotClusterMember`, `NoActiveExecutor`, `NotActiveExecutor` and `StaleExecutorEpoch`, because doing so would bypass the fence.
-This is a guest protocol fallback for an N-1 or not-yet-activated host, not a lifecycle fallback, and it never opens the gate; it does not conflict with R5.
-
-### 8.1 Gated mutations
-
-Per R5, the lifecycle design Section 9 and ADR-0006 S5 (migration follows activation, never the reverse), on a cluster whose current epoch is not marked `addonContract: 2`:
-
-| Action | Gated | Reason |
-|---|---|---|
-| `CreateAddon` (including GPU auto-staging through the owner) | Yes | `ACTIVE` needs a fenced report for the current incarnation and generation. |
-| `UpdateAddon` | Yes | Same. |
-| `DeleteAddon`, with or without `preserve` | Yes | Needs observed removal or relinquish. |
-| `DescribeAddon`, `ListAddons`, `DescribeUpdate`, `ListUpdates`, `DescribeAddonVersions`, `DescribeAddonConfiguration`, tag actions, cluster-teardown removal | No | No guest realisation required. |
-
-Response: `InvalidRequestException`, HTTP 400, message (proposed exact text):
-
-- epoch unmarked: `Add-on lifecycle operations on cluster {cluster} require an upgraded control-plane image. The cluster's add-on delivery agent has not established add-on contract v2. Upgrade the cluster's control-plane image and retry.`
-- no executor assigned: `Add-on lifecycle operations on cluster {cluster} are unavailable: the cluster has no active add-on delivery agent. Retry after the control plane recovers.`
-
-Pre-activation, nothing is gated; the gate exists only once P2 is recorded (lifecycle design slice 7).
-
-### 8.2 What old binaries do with new records
-
-From `docs/package-boundary/eks-addon-v1-compatibility.md`:
-
-| N-1 path | Effect on a v2 record | Why it forces ordering |
-|---|---|---|
-| `Owner.Update`, `Owner.ApplyReport` with a change, `markFailed` via `casUpdate` (`domains/eks/addon/store.go:198-219`) | DROP: re-encodes v1 `Record` and erases `incarnationId`, `desired`, `observed` | Expand must precede activation, and v2 semantics stay off while any N-1 writer can serve the cluster (P1, P2; ADR-0006 S4 expand/migrate/contract, S5 prepare/activate/migrate/finalise). |
-| `StagingInstaller.Install` | OVERWRITE the manifest from four v1 fields | The manifest key is a projection only until finalisation (R16). |
-| `Owner.Delete`, `DeleteClusterPrefix` | ERASE regardless of content | An N-1 delete would complete a v2 delete without observed removal; only activation prevents it. |
-| Readers (`Get`, `List`, `ListManifests`) | IGNORE added fields; FAIL the whole listing on a type change (`store.go:78,108`) | No existing field may change JSON type. |
-| Reconciler on the v1 subject | IGNORE added report fields; never subscribes to the v2 subject | v2 reports reach only an N lease holder; they are notifications, so a drop is repaired next tick. |
-| Bucket stamp | Bumping it makes N-1 refuse the whole account bucket | Use per-record `schemaVersion`, not the stamp. |
-
-Type-change hazard in the lifecycle design's record sketch: today `health` is a string (`domains/eks/addon/record.go:30`), and the sketch shows `health` as an array of issues.
-Writing that would make every N-1 `ListAddons` and `DescribeAddon` on the cluster fail to decode; `health` stays a string, and the issue list is carried in a new, optional `healthIssues` field for v2, never by changing `health`'s existing type (V3).
-
-## 9. Compatibility tests the v2 contract needs
-
-| # | Test | Pins |
-|---|---|---|
-| T1 | Literal-byte encode of a directive response and of a report batch, including empty `directives`, empty `objects`, omitted optional fields and HTML-escaped characters | camelCase keys, field order, `[]` not `null`, no `BuildJSON` |
-| T2 | Literal-byte decode of the same bytes into the v2 types, both directions (host encode to guest decode, guest encode to host decode) | Round trip without loss |
-| T3 | Unknown optional fields at top level, per directive, per report and per object are ignored | Declared extensibility |
-| T4 | Unknown `mode`, `phase`, `reason`, object `state`, and `contractVersion` 1 or 3 are rejected with the named reason and change no state | Q-75 closed enums and majors |
-| T5 | Case-mismatched keys (`AddonName`, `Directives`) are rejected by the guest decoder | The v1 quirk cannot recur |
-| T6 | Directives sorted by `addonName`; duplicate `addonName` in a report batch rejects the batch | Ordering and uniqueness |
-| T7 | A `message` containing quote, backslash, newline and non-ASCII round-trips through the guest encoder and the gateway envelope | No `printf` JSON |
-| T8 | The gateway envelope's `executor.instanceId` equals the authenticated session name even when the guest body carries `executor`, `instanceId` or `account` fields | No payload-asserted identity |
-| T9 | Each rejection row of Section 4.3, one test per row, including a member of another cluster and a non-executor member | Authorization binding |
-| T10 | A v1 body, v1 subject and v1 report remain byte-identical after v2 is added; the existing `contracts/eks/v1` tests run unchanged | v1 frozen |
-| T11 | A v2 report on the v1 subject and a v1 report on the v2 subject are both rejected or ignored without state change | No cross-major interpretation |
-| T12 | Guest fallback: `InvalidAction` and `NotActivated` select v1 mode for one tick; every other reason does not | Fallback cannot bypass the fence |
-| T13 | An N-1 decoder (`contracts/eks/v1` types) reading a v2 directive or report body fails or ignores safely, never yields a populated v1 value that would be applied | Mis-routing safety |
-| T14 | Readiness and removal fixtures per bundle: objects with a previous generation annotation never produce `ready`; a shared Secret is excluded while another add-on renders it | Section 6.6 and 6.7 |
-
-## 10. Review decisions
-
-| # | Decision |
-|---|---|
-| V1 | Legacy-mode fallback is approved only for `InvalidAction` and `NotActivated` (Section 8). The guest never falls back on an authentication, membership, executor, epoch or other fencing failure. |
-| V2 | `DEGRADED` (Section 6.2) is an observed condition for an otherwise `ACTIVE` add-on. A health loss reported during `CREATING` or `UPDATING` records a health issue but does not by itself turn the operation into a terminal failure; the operation's deadline and an explicit terminal report decide that. |
-| V3 | Legacy `health` stays a string (Section 8.2); v2 adds an optional `healthIssues` field. `health`'s type is never changed. |
-| V4 | v2 reason codes (Section 6.3) are a closed enum. A genuinely new wire-level reason has new semantics and requires v3; free-form detail goes in `message`, never in an unbounded `reason` value. |
-| V5 | The v1 fetch route (Section 7) is bound to the assigned executor once one exists, without changing v1 payload bytes; before assignment, existing v1 behaviour is preserved. |
-| V6 | Removal (Section 6.7) means removal of add-on-owned Kubernetes objects only; `removed` does not wait indefinitely for downstream workloads, PVs or unrelated resources. Preserve-mode deletion explicitly relinquishes ownership rather than claiming removal. |
-| V7 | Per-bundle CA-secret names for v2 (Section 6.7), with a deliberate compatibility path for existing v1 shared-secret bundles. |
-| V8 | An executor is not v2-capable merely because it fetched a directive. Capability is marked only after the first valid, fenced v2 report is accepted (Section 2.3, Section 4.1). A fetch may be retained as diagnostic evidence only. |
-| V9 | Activation (Section 3, P2) is EKS-add-on-specific for now. The release-membership mechanism should be reusable, but this design does not build a platform-wide activation framework. |
-| V10 | Automatic executor fencing (Section 4.4) uses only authoritative evidence that the former instance terminated or no longer exists. A lookup error, network issue or partition is never evidence of termination. The alternative is an audited operator assertion. |
-| V11 | `ActivationStateUnavailable` (HTTP 503, retryable) is defined in v2 from the outset for an unreadable, invalid or unavailable activation record. Only an absent record is `NotActivated`, and only `NotActivated` is eligible for the bounded fallback (V1); `ActivationStateUnavailable` never triggers fallback, because it is not evidence that v2 is inactive. |
-
-The v1 `PublishInternal` authorization-binding gap and the `serviceAccountRoleArn` format-validation gap (Sections 5.3 and 7) are being fixed directly on the v1 path as standalone security corrections, not as v2 lifecycle work; v2 keeps its own binding and its own guest-side validation regardless of either fix's timing.
-
-## 11. Open questions for the reviewer
-
-1. Route and subject names: `internal-addons-v2` and `eks.addon.v2.{account}.{cluster}.status` as proposed, or another spelling?
-2. Should readiness for the IRSA bundles include a credential check, and for `nvidia-device-plugin` a GPU allocatable check, or stay as defined in Section 6.6?
-3. Report batching: one batch per tick (proposed) or one POST per add-on as in v1?
-4. Should the directive carry `deadline` at all, given the guest must ignore it for behaviour, or only in diagnostics?
-5. v1 guest support lifecycle: what retirement notice and detection (no cluster fetching the v1 route in the Region) is published, and from which release is it counted?

docs/package-boundary/eks-release-membership-and-activation-design.md

@@ -1,205 +0,0 @@
-# EKS add-on v2: release membership and activation (case study)
-
-Code read at `25b3efdec`; not verified live.
-
-**Status:** Draft for review; design dependency only, nothing implemented.
-
-This is the design dependency for prerequisites P1 (release membership), P2 (activation record) and P5 (controlled rollback) of `docs/package-boundary/eks-addon-v2-contract-design.md` Section 3, plus the finalisation action of R16 (`docs/package-boundary/eks-addon-lifecycle-design.md`).
-It is written as an EKS case study: every finding is classed as **(a)** a generic compatibility or activation concept that may be reusable later, **(b)** an EKS add-on implementation need, or **(c)** out-of-scope platform-wide release orchestration.
-Paths are relative to `spinifex/` unless they start with `docs/`; `mulga/` paths are relative to the umbrella repo.
-
-## Standing
-
-ADR-0006 (`mulga/docs/adr/0006-contract-guest-and-persisted-state-evolution.md`) is accepted and cited by clause: S1 (boundary declaration), S2 (N/N-1 for a coordinated Regional release), S3 (no timing heuristic as capability proof; capability invalidated on replacement or rollback), S4 (an older writer that discards fields is not compatible), S5 (prepare, activate, migrate, finalise; scope limited to components that read, write or enforce the contract; controlled rollback), S6 (guest legacy mode) and S8 (evidence).
-ADR-0006's non-goals leave the membership data source and the `spx admin` command shape to implementation, and its Consequences say the add-on work must not decide the platform-wide release mechanism by itself.
-ADR-0003:S2 rules out a shared abstraction without two real consumers.
-Q-75 and Q-76 (`mulga/docs/specs/requirements/resource-lifecycle.md:261,275`) are Decided requirements; Q-183 and Q-184 are Open and used as context only.
-R1 to R16 and V1 to V10 are applied, not reopened; V9 keeps activation EKS-add-on-specific.
-
-## 1. Facts this design depends on
-
-| Fact | Evidence | Class |
-|---|---|---|
-| No cluster-visible record of the release each process runs; no activation gate | `docs/package-boundary/release-and-version-inventory.md` Section 7 (G1, G2) | a |
-| Only awsgw receives the ldflags build version; the daemon never does | `operator/cli/version.go:10-15`, `operator/cli/service.go:613`, `runtime/roles/awsgw/awsgw.go:59-71` | b |
-| Daemons rewrite `heartbeat.<node>` every 10 s in `spinifex-cluster-state` (1 h TTL), with no build or capability field | `daemon/heartbeat.go:9,41-72`, `daemon/jetstream.go:180-191,224-248` | b |
-| The only prefix lister of that bucket reads `heartbeat.`; other keys are read by exact name | `runtime/compute/cache/liveness.go:25,44-49,103-123`, `daemon/jetstream.go:327,362,408` | b |
-| Liveness already treats a missing or unreadable heartbeat as Unknown, never Stale | `runtime/compute/cache/liveness.go:32-41,74-95` | a |
-| Every daemon, N-1 included, answers a per-node health subject without any build field | `daemon/daemon.go:1030`, `daemon/daemon_handlers.go:337-360`, `contracts/cluster/v1/health.go:15-24` | b |
-| awsgw answers admin-only `GetVersion` for itself only; it writes nothing to cluster state | `gateway/spinifex.go:16-17,53-56,61-62`, `runtime/roles/awsgw/awsgw.go:584-586` | b |
-| Expected membership is the node config map with per-node services (`daemon`, `awsgw`, ...) | `bootstrap/config/config.go:20-28,186-197,544-562` | a |
-| A Region is one formed Spinifex cluster | Q-72 (`mulga/docs/specs/requirements/resource-lifecycle.md:215-225`) | a |
-| Every daemon serves every `eks.*` add-on subject in queue group `spinifex-workers` | `daemon/daemon.go:1089-1122` | b |
-| Reconciler lease holder and GC sweep holder are daemons identified by node name | `daemon/eks_deps.go:80`, `handlers/eks/cluster_reconciler.go:395,448`, `daemon/daemon.go:2336-2348,2411-2416` | b |
-| awsgw reads cluster meta for internal-route authz and relays fetch and report | `gateway/eks/internal_authz.go:48-80,143-170`, `gateway/eks.go:40-42,57` | b |
-| The v1 listing reads manifest keys, and a v1 guest removes any add-on absent from it | `handlers/eks/addons.go:132-156` | b |
-| Bumping a bucket stamp makes N-1 refuse the whole bucket | `foundation/state/migrate/kv.go:55-67,81-88`, `handlers/eks/store.go:28-29,197-208` | a |
-| No durable operator audit log; the account-deletion job is the one durable operator record (client token, `kv.Create` claim, replay) | `gateway/requestaudit.go:17-38`, `gateway/admin_deleteaccount.go:80-97,359-386` | a |
-| `spx admin eks` exists with one subcommand reaching the daemon over NATS | `operator/cli/admin_eks.go:14-77` | b |
-
-## 2. (a) Generic concepts this case study relies on
-
-These are concepts and invariants, not a protocol specification or a package.
-
-| Concept | Meaning |
-|---|---|
-| Contract scope | The roles that read, write or enforce one contract (ADR-0006:S5). Other roles are never consulted, so no global "everything upgraded" dependency arises. |
-| Slot | (Region, node, role), derived from configured membership, never from who reports. |
-| Incarnation | Random per process start; a restart is a new incarnation. |
-| Capability token | Compiled into the binary; never configurable; a build version string is diagnostic only. |
-| Self-report | Written only by the process it describes; additive to existing storage. |
-| Probe | One per-slot request at evaluation time proving the current occupant; a pre-protocol binary answers without the new fields, which is positive legacy evidence. |
-| Legacy signal | A record the pre-protocol binary already rewrites wholesale; a fresh rewrite without the protocol marker means a legacy occupant. |
-| Activation record | One per contract and scope, no TTL, single-CAS transitions, audit inside the record. |
-| Activation generation | Advanced on each activation; long-lived consumers' capability proofs are tagged with it, so rollback invalidates them (ADR-0006:S3). |
-
-| Invariant | Why |
-|---|---|
-| Unknown is never compatible: `stale`, `absent`, `unreachable` and `unexpected` slots block exactly as `legacy` does | Q-75; ADR-0006:S3 |
-| Time can only make the answer more conservative: staleness moves a slot from capable to stale, never the reverse | Ageing out removes evidence, never manufactures it; the staleness threshold is the only time constant and is never a fence |
-| Ageing out never removes an expected slot; a dead member blocks until a capable occupant reports or the node leaves configured membership | Slots come from configuration |
-| Prepare snapshots every slot's incarnation; activate requires the same incarnations | Replaces a settle window with identity (no timing heuristic) |
-| No automatic transition; only an operator action moves the state | Q-75: finalisation never follows a binary start |
-| Unreadable activation state selects neither old nor new semantics | Fail closed |
-| No bucket stamp bump as the fence | Stamp refusal is all-or-nothing (Section 1) |
-| Rollback is freeze, drain, deactivate, then downgrade; deactivation needs no binary evidence because it moves toward shared semantics | ADR-0006:S5 |
-| Long-lived consumers never gate activation; they gate per resource, and may gate finalisation | ADR-0006:S5 step 2, S6 |
-
-State machine (generic):
-
-| From | To | Evidence |
-|---|---|---|
-| absent or `PREPARED` | `PREPARED` | All in-scope slots capable; one config epoch; no unexpected slot; no unresolved legacy observation; contract readiness; snapshot stored |
-| `PREPARED` | `ACTIVE` | Same, plus incarnations equal the snapshot |
-| `ACTIVE` | `DEACTIVATING` | Authorisation only; freezes N-only mutations |
-| `DEACTIVATING` | `PREPARED` | Zero outstanding N-only obligations; every record in the N-1-readable form |
-| `DEACTIVATING` | `ACTIVE` | Activation evidence again (abort rollback) |
-| `ACTIVE` | `FINALISED` | Slots capable; no legacy observation since activation; migration complete; affected long-lived consumers proven; explicit confirmation |
-
-Concurrent operators: every transition is one CAS on the record revision with a client token; the loser gets the current state and changes nothing, and a repeat with the same token returns the recorded outcome.
-A crash before the CAS changes nothing; after it, the new state already carries its audit entry.
-
-## 3. (b) EKS add-on implementation needs
-
-### 3.1 Relevant participants
-
-| Participant | Why relevant | Must report |
-|---|---|---|
-| Every configured `daemon` slot | Any daemon may serve any add-on mutation; N-1 DROPs, OVERWRITEs and ERASEs (`docs/package-boundary/eks-addon-v1-compatibility.md`) | Preserves v2 record fields on every write; delete is an obligation, not an erase; honours the executor record; lease holder subscribes the v2 subject |
-| Lease holder and GC sweep holder | Daemons, covered by the daemon slot | as daemon |
-| Every configured `awsgw` slot | Enforcer and relay; an N-1 gateway answers v2 `InvalidAction`, letting a v2 guest fall back (V1) to an unbound v1 fetch (V5 lost) | v2 routes; v1 fetch bound to executor; v1 listing projected from records |
-| Control-plane VM agent | Long-lived consumer; capability per executor epoch (V8) | Not a slot; `addonContract: 2` tagged with the activation generation |
-| Not participants | predastore, viperblock, vpcd, nats, northstar, ui, non-EKS services; account teardown and OIDC readers write no add-on keys (`accountteardown/reapers_eks.go:102-107`, `handlers/sts/oidc_jwks.go:101`) | nothing |
-
-### 3.2 What the v2 design may depend on
-
-| Need | Choice | Reason |
-|---|---|---|
-| Daemon report | `release.daemon.<node>` in `spinifex-cluster-state` | TTL ages it out; invisible to N-1 readers (Section 1) |
-| Daemon legacy signal | Additive marker in `heartbeat.<node>` | N-1 rewrites the heartbeat without it every 10 s |
-| Daemon probe | Additive fields in the per-node health reply | N-1 answers without them |
-| awsgw report and probe | `release.awsgw.<node>`; additive fields in `GetVersion` to the node's configured gateway | N-1 answers `GetVersion` without them |
-| Activation record | EKS-owned, no-TTL storage keyed `eks-addon-lifecycle/2` | Not cluster-state (TTL), not per-account (scope is the Region), not the stamp |
-| Operator surface | `spx admin eks addon-lifecycle {status,prepare,activate,deactivate,finalise}`, with client token and reason; evaluated by a daemon on a subject only N daemons subscribe | Extends the existing group; an N-1 evaluator cannot exist |
-| Audit | Operator principal (or `local:<node>`), token, from, to, reason and evidence digest, in the same CAS, plus a log line | No durable audit facility exists |
-
-### 3.3 Activation preflight (EKS)
-
-| Check | Refusal (summary) |
-|---|---|
-| Every configured daemon and awsgw slot capable | `{role} on node {node} is {class}` |
-| One config epoch; no unexpected slot | `node {node} reports epoch {e}` or `is not in the Region configuration` |
-| No unresolved legacy observation | `an older {role} was observed on node {node}` |
-| Record expand complete (P3) over every `eks-account-*` bucket; enumeration fails closed (`handlers/eks/store.go:149-171`) | `{n} add-on records not yet expanded` or `buckets could not be enumerated` |
-| Executor record present per non-deleting cluster (P4); empty executor allowed and gated per cluster | `cluster {c} has no delivery-executor record` |
-| Incarnations unchanged since prepare | `{role} on node {node} restarted after prepare; run prepare again` |
-
-Clusters whose executor is not yet proven are reported, not refused: the per-cluster gate (v2 design Section 8.1) handles them.
-
-### 3.4 Rollback before finalisation (P5)
-
-| N-1 behaviour on activated state (compat doc) | Deactivation rule |
-|---|---|
-| DROPs `incarnationId`, `desired`, `observed` on update, report, failure | Allowed only with no obligation in flight; re-activation re-expands |
-| ERASEs on delete and cluster-prefix delete | No record in `DELETING` or `DELETE_FAILED`; each completes on an observed `removed` or `relinquished` report first |
-| Does not model `UPDATING`, `UPDATE_FAILED`, `DELETING`, `DELETE_FAILED` | None in those statuses; `UPDATE_FAILED` is resolved by `UpdateAddon` or `DeleteAddon` first |
-| OVERWRITEs the manifest from v1 fields | Harmless: manifest and legacy fields are still written until finalisation (R16) |
-| Readers IGNORE added fields | Safe |
-
-While `DEACTIVATING`, create, update and delete are refused with a rollback-in-progress message; reports, deadlines and reconcile continue so obligations drain under v2 semantics.
-After deactivation, v2 routes answer `NotActivated` and v2 guests fall back to v1 (V1).
-On re-activation, no earlier `addonContract: 2` marking counts until a fresh fenced v2 report under the new activation generation, because an N-1 reconciler may have swapped control-plane members without advancing any epoch.
-
-### 3.5 Finalisation (R16)
-
-| Preflight | Why |
-|---|---|
-| Every daemon and awsgw slot capable; no legacy observation since activation | No N-1 host remains |
-| Every record migrated; every awsgw and daemon projects the v1 listing from records | Manifest keys stop being written |
-| Every cluster explicitly activated onto v2 has current proof: its executor epoch proven v2 under the current activation generation; such a cluster with no executor, or `FAILED` with running VMs, refuses | A v1 guest served from manifest keys would remove every add-on (`handlers/eks/addons.go:132-134`) |
-| Every cluster still on v1 is enumerated as legacy in the finalisation output and audit entry | v1 clusters are visible, never silently treated as compatible |
-| Explicit `--confirm eks-addon-lifecycle/2` | Q-75, ADR-0006:S5 step 4 |
-
-Finalisation permits stopping, then removing, the manifest key and legacy top-level fields; it makes deactivation and any N-1 rejoin impossible; it ends rollback for the activated v2 track only, and does not retire the v1 routes or v1 guest mode or permit removing their support, which stay an ADR-0006:S6 retirement.
-
-### 3.6 Coverage
-
-| Prerequisite | Covered by | Not covered |
-|---|---|---|
-| P1 | 3.1, 3.2 | Post-activation awsgw legacy signal (gap 3) |
-| P2 | 3.2, 3.3 | none |
-| P5 | 3.4 | Repair after an N-1 writer runs post-activation (gap 2) |
-| P3, P4 | Consumed as evidence only | Expand pass; executor record and V10 fencing |
-
-### 3.7 Evidence required later
-
-| Test | Pins |
-|---|---|
-| Rolling upgrade in both orders (daemons first, gateways first); activation refused until the last slot is capable; v1 unchanged | ADR-0006:S2, S8 item 1 |
-| Activation refused with an N-1 daemon, an N-1 awsgw, a stopped node, a never-reported slot, an unexpected node, an unexpanded record, an unenumerable bucket | Unknown is never compatible |
-| Activation refused after a restart between prepare and activate | No settle window |
-| Rollback: freeze; pending delete completes on `removed`; deactivation refused while an obligation or unmodelled status exists; N-1 then serves every record with today's semantics | ADR-0006:S5, S8 items 5 and 6 |
-| Re-activation re-closes every per-cluster gate until a fresh v2 report | ADR-0006:S3 invalidation |
-| Finalisation refused while one v1 guest, one `FAILED` cluster with VMs, or one legacy observation remains, and without confirmation | R16, Q-75 |
-| Crash before and after each CAS; same-token retry returns the recorded outcome | Crash safety |
-| Two concurrent operators: one transition, one audit entry | CAS |
-| N-1 daemon started after activation is observed and freezes mutations; resolves only on a capable new incarnation | Detection |
-| Unreadable activation state: no semantics chosen | Fail closed |
-| N-1 binaries unaffected by the added keys and fields | ADR-0006:S2 |
-
-## 4. (c) Out of scope
-
-Global release orchestration, release managers and rollout drivers (ADR-0006:S5; Q-183 and Q-184 open); any "all Spinifex components upgraded" gate; Predastore, Viperblock, Bluebottle and non-EKS services; shared packages (ADR-0003:S2); NATS or JetStream redesign and stamp bumps; node enrolment, credential revocation and installer downgrade refusal (Q-71, Q-184).
-
-## 5. Other likely consumers (not enrolled)
-
-- EBS provider protocol: exact-match wire version today; would need provider slots and a newer-major activation.
-- RDS and ECS guest agents: versions recorded, never compared; would need per-instance capability under long-lived-consumer rules.
-- Predastore: independently released; would need a declared version matrix evaluated by capability, not N/N-1.
-- Viperblock formats: exact-match headers; would need expand and finalise for retained artefacts.
-- Bluebottle: a linked library, so the linking binary is the participant.
-- Spinifex KV crossings such as the instance-record migration: first-opener activation could become an operator action.
-
-Future service work can reuse or challenge the generic concepts in Section 2; this design does not decide that.
-
-## 6. Gaps and conflicts (flagged, not resolved)
-
-1. V9 limits activation to EKS; Section 2 states generic concepts, which V9 did not anticipate, though no framework or package is built.
-2. No fence stops an N-1 daemon started after activation; it can DROP or ERASE before detection, and the heartbeat starts after subscriptions (`daemon/daemon.go:2257,2309`). ADR-0006:S5's rule on automatic downgrade is met only on the controlled path.
-3. awsgw has no post-activation legacy signal; an N-1 gateway re-opens the V1 fallback and an unbound v1 fetch until its report goes stale.
-4. The daemon has no runtime build version (`operator/cli/service.go:613`).
-5. Finalisation requiring every cluster to prove v2 keeps the rollback window open as long as the slowest image upgrade; lifecycle design Section 9 frames finalisation around host consumers and v1 retirement separately.
-6. Section 3.3 and a fail-closed unreadable state may need a new v2 response reason, while v2 design Section 6.3 lists response reasons as fixed.
-7. Slots key on the configured node name, while Q-71 requires an enrolled identity (authority open under O-1).
-8. A node removed from configuration is not revoked (`operator/cli/get.go:80` uses a config-held token), so its record ageing out is paired with configuration evidence only.
-9. Not every reader of `spinifex-cluster-state` was confirmed (`operator/cli/admin_jsprobe.go:82` opens it).
-10. The restore path's successor-first ordering still blocks automatic epoch advance after re-activation (v2 design Section 4.4).
-
-## 7. Review decisions
-
-| # | Decision |
-|---|---|
-| M1 | Finalisation requires current v2 proof only for clusters explicitly activated onto v2. Clusters remaining on v1 are enumerated as legacy, never treated as compatible. Finalisation ends rollback for the activated v2 track; it does not retire v1 or permit removing its support (Section 3.5). |
-| M2 | An older relevant binary detected after activation freezes EKS add-on mutations Region-wide. Reads continue; create, update, delete, activation and executor reassignment fail closed until an operator upgrades or removes the incompatible writer or performs an explicit controlled rollback. The freeze covers the add-on contract only, not the platform. |
-| M3 | An unreadable, invalid or unavailable activation record returns the retryable v2 reason `ActivationStateUnavailable` (HTTP 503), defined in v2 from the outset. Only an absent record is `NotActivated`, the sole case eligible for the bounded v2-to-v1 fallback (v2 design V1, V11). |
-| M4 | The old-writer gap (Section 6 items 2 and 3), the unenrolled-node gap (items 7 and 8) and restore-before-fencing (item 10) are explicit prerequisites for eventual activation, recorded as design debt. They are not package-boundary work and do not extend it. |

docs/package-boundary/iam-enforcement-inventory.md

@@ -1,281 +0,0 @@
-# IAM enforcement inventory
-
-**Status:** Discovery record for ADR-0005 S6(1) and rollout evidence item 1; describes present code only and authorises no change.
-
-This inventory lists every direct consumer of IAM private state across Spinifex, Predastore, Goanna and Bluebottle, and records the current authentication flow at each entry point.
-The design that replaces these reads is `iam-enforcement-projection-design.md` in this directory; D-numbers below refer to its decision register (Section 14 of that document).
-AWSGW findings pinned by characterisation tests are listed in Section 7.
-
-## Sources and conventions
-
-| Repo | Revision read | Path prefix used below |
-|---|---|---|
-| Spinifex | `feat/adr-0005-iam-enforcement-projection` at `976df15d7` | relative to `spinifex/spinifex/` |
-| Predastore | `origin/dev` at `37ef410` (`predastore-wt-iamchar`) | `predastore:` |
-| Goanna | `origin/dev` at `b4c6b31` (`goanna-wt-iamchar`), code reading only; not characterised in Phase 0 (D6) | `goanna:` |
-| Bluebottle | `dev` at `f2f4149` | `bluebottle:` |
-
-"IAM private state" means the `spinifex-iam-*`, `spinifex-accounts` and `iam-account-<id>` KV buckets and their records, access-key and session secrets, the master key that encrypts them, stored policy documents, role and trust records, STS session records and tokens, OIDC provider records, and any type that encodes those records.
-Locations were found with the allyourbase index, then confirmed by direct reads, because the index was stale for `handlers/iam/service_impl.go` and does not cover the two worktrees.
-Line numbers are those at the revisions above.
-
-Process names: **awsgw** is the AWS gateway role (`runtime/roles/awsgw`), which hosts the IAM and STS implementations in-process; **daemon** is the Spinifex node daemon; **vpcd** hosts guest metadata; **spx** is the operator CLI; **s3d** is the Predastore server; **goannad** is the Goanna server.
-
-## 1. IAM private records today
-
-| Bucket | Key | Record (`handlers/iam/types.go` unless stated) | Secret fields |
-|---|---|---|---|
-| `spinifex-iam-access-keys` | `<AKID>` | `AccessKey` (68): `access_key_id`, `secret_access_key`, `user_name`, `account_id`, `status`, `created_at` | `secret_access_key` is AES-256-GCM under the master key, no associated data |
-| `spinifex-iam-users` | `<account>.<userName>` | `User` (36): `user_id`, `arn`, `access_keys`, `attached_policies`, `groups`, `inline_policies` | none |
-| `spinifex-iam-groups` | `<account>.<groupName>` | `Group` (53) | none |
-| `spinifex-iam-roles` | `<account>.<roleName>` | `Role` (111): `role_id`, `arn`, `assume_role_policy_document`, `max_session_duration`, `attached_policies`, `inline_policies` | none |
-| `spinifex-iam-policies` | `<account>.<policyName>` | `Policy` (84): `arn`, `policy_document`, versions | none |
-| `spinifex-iam-instance-profiles` | `<account>.<name>` | `InstanceProfile` (128) | none |
-| `spinifex-accounts` | `<account>` | `Account` (182): `status` ACTIVE, SUSPENDED or TERMINATING | none |
-| `iam-account-<account>` | `OIDCProviderKey(issuer)` (`handlers/iam/oidc_providers.go:42`) | OIDC provider record | none |
-| `spinifex-iam-session-credentials` | `<ASIA AKID>` | `SessionCredential` (`handlers/sts/session_credentials.go:40`): principal type, role/user IDs and ARNs, `expires_at` | `secret_encrypted` (AES-GCM, master key); `session_token_hmac` (HMAC-SHA256 keyed by the same master key, `handlers/sts/assume_role.go:613`) |
-
-Bucket constants are declared at `handlers/iam/service_impl.go:38-45` and `handlers/sts/session_credentials.go:23`.
-The master key is one 32-byte file, `<config>/master.key`, loaded group-readable with `masterkey.ReadShared` or `LoadShared` (`bluebottle:pkg/masterkey/masterkey.go:50-58`).
-It encrypts IAM and session secrets and keys the session-token HMAC, and the same key also seals ACM, ECR, RDS, EKS, Ochre and Bedrock secrets (Section 4).
-
-## 2. Consumer matrix
-
-One classification per row, from: authority, credential verifier, policy evaluator, role resolver, session consumer, bootstrap path, invalid private-store reader.
-Rows marked **CLI** or **test** are operator or test-support code.
-
-### 2.1 Spinifex
-
-| # | Location | Process | Reads / writes | Classification |
-|---|---|---|---|---|
-| S1 | `handlers/iam/service_impl.go:133` (`NewIAMServiceImpl`), all `IAMServiceImpl` methods | awsgw | Creates, migrates, reads and writes every IAM bucket in Section 1 except sessions; holds the master key (210) | authority |
-| S2 | `handlers/sts/session_credentials.go:61-230`, `handlers/sts/assume_role.go:543-600` (`mintSession`) | awsgw | Writes and reads `spinifex-iam-session-credentials`; encrypts the session secret and HMACs the token with the raw master key | authority. Borderline: ADR-0005 S2 names IAM the logical authority for session validity, but STS physically owns this store today. |
-| S3 | `handlers/iam/imds_rpc.go:34` (`SubscribeIMDSResponders`), `handlers/sts/imds_rpc.go:30` | awsgw | Serves instance-profile, `GetRole` and `AssumeRoleForInstance` over NATS to vpcd | authority |
-| S4 | `handlers/sts/assume_role_with_web_identity.go:197-215` (`verifyOIDCProviderRegistered`) | awsgw | Opens `iam-account-<roleAccount>` with its own JetStream handle and reads `OIDCProviderKey(issuer)`, bypassing `IAMService` | invalid private-store reader |
-| S5 | `handlers/sts/role_arn.go:27` (`ResolveRoleByARN`), used by `assume_role.go:167`, `assume_role_with_web_identity.go:83`, `gateway/sts.go:108` | awsgw | `IAMService.GetRole`: `RoleId`, `Arn`, `AssumeRolePolicyDocument`, `MaxSessionDuration` | role resolver |
-| S6 | `handlers/sts/assume_role.go:248-290`, `web_identity_trust_policy.go:42-61` | awsgw | Parses the role trust policy with `handlers_iam.ValidateTrustPolicyDocument` | policy evaluator |
-| S7 | `handlers/sts/get_session_token.go:73` | awsgw | `IAMService.GetUser` to bind the user ID at mint | role resolver. Borderline: it resolves a user, but the capability is the same canonical-principal resolution as roles. |
-| S8 | `handlers/sts/session_principal.go:61-125` (`VerifySessionPrincipal`) | awsgw | `GetUser`/`GetRole` by name, compares the live immutable ID with the one stored at mint | credential verifier |
-| S9 | `handlers/sts/presigned_url_verify.go:150-200` (`resolvePrincipalForVerify`), exposed over NATS by `runtime/roles/awsgw/eks_token_verify.go:29` | awsgw, for the EKS token webhook agent | `LookupAccessKey`, `DecryptSecret`, `GetUser`; session record via `LookupSessionCredential` and `handlers_iam.DecryptSecret(…, s.masterKey)` | credential verifier |
-| S10 | `handlers/sts/get_access_key_info.go:73` | awsgw | `LookupAccessKey` and session lookup to map an AKID to its account | credential verifier. Borderline: needs only the AKID-to-account fact. |
-| S11 | `gateway/auth.go:375-405` (`resolveLongLivedAKID`) | awsgw | `LookupAccessKey`, status, `DecryptSecret` | credential verifier |
-| S12 | `gateway/auth.go:409-472` (`resolveSessionAKID`), `483-520` (`checkSessionPrincipal`) | awsgw | `LookupSessionCredential`, expiry, `VerifySessionToken`, `DecryptSecret`, then S8 | session consumer |
-| S13 | `gateway/auth_accountstatus.go:67-97` (`checkAccountActive`) | awsgw | `GetAccount` status; ACTIVE cached 5 s (`accountActiveTTL`, line 16) | credential verifier. Borderline: account status gates every credential, so it is verification input rather than policy. |
-| S14 | `gateway/gateway.go:771` (`principalUser`) → `gateway/sts/GetCallerIdentity.go:61` (`ResolveCallerUser`) | awsgw | `GetUser`/role lookup for `aws:userid` and the user ARN | policy evaluator. Borderline: a principal read, but its only use is condition-key input. |
-| S15 | `gateway/gateway.go:809-885` (`evaluatePrincipalPolicyResources`) | awsgw | `GetUserPolicies` (`handlers/iam/service_impl.go:1997`) or `GetRolePolicies` (`handlers/iam/roles.go:579`): user, group, role and policy records; evaluates with `iampolicy.EvaluateWithKeys`; global root bypass at 827 | policy evaluator |
-| S16 | `gateway/gateway.go` `checkPassRole`, callers `gateway/ec2.go:194,212,221,477`, `gateway/ecs.go:66`, `gateway/eks.go:296` | awsgw | Evaluates `iam:PassRole` on an ARN; role existence comes from S5-style lookups in the handlers | role resolver. ADR-0005 S3 scope; listed for completeness, not a projection consumer. |
-| S17 | `gateway/ecr_principal.go:81-250` (`resolveECRPrincipal` and helpers) | awsgw (OCI registry path) | Rehydrates an ECR bearer token: `LookupAccessKey`, `GetUser`, `GetAccount`, `LookupSessionCredential`, S8 | credential verifier |
-| S18 | `gateway/admin_createaccount.go:112,135` | awsgw | Encrypts the new account admin's secret with the IAM master key into the admin idempotency store and decrypts it on replay | bootstrap path. Borderline: it keeps a copy of an IAM secret outside the IAM store. |
-| S19 | `runtime/roles/awsgw/awsgw.go:205-241, 272-286` | awsgw | Reads `master.key` raw, builds IAM and STS, wires the revoker, consumes `bootstrap.json` via `SeedBootstrap` (`handlers/iam/service_impl.go:855`) | bootstrap path |
-| S20 | `daemon/eks_deps.go:27-48` (`systemRoleEnsurer`), `daemon/ecs_deps.go:39-51` | daemon | Reads `master.key` and constructs a second full `IAMServiceImpl`, which opens and migrates every IAM bucket and writes roles, instance profiles and policies | invalid private-store reader. Borderline: it runs IAM's own code, but in a non-authority process with full store access and the master key. |
-| S21 | `domains/ec2/guestmetadata/nats_clients.go:42-95` | vpcd | Receives `handlers_iam.InstanceProfile` and `iam.GetRoleOutput` (including the trust document) over NATS from S3, cached 5 min (line 23); requests session credentials for guests | role resolver |
-| S22 | `operator/cli/admin.go:2876-2899` (`initIAMServiceFromConfig`), callers `admin_principal.go:43,116,246,313,476`, `admin_account_delete.go:76`, `admin.go:2918,3046` | spx (**CLI**) | Connects with the cluster NATS token, reads `master.key`, builds a full `IAMServiceImpl` | bootstrap path |
-| S23 | `operator/cli/admin.go:3449-3480` (`writeBootstrapFilesWithAdmin`) | spx (**CLI**) | Generates `master.key`, encrypts the system and admin secrets into `bootstrap.json`, writes plaintext `system-credentials.json` (3450) | bootstrap path |
-| S24 | `operator/cli/templates/predastore.toml:79-121` | spx (**CLI**) | Renders Predastore's `[iam]` section with the NATS token, `master.key` path and `spinifex-iam-access-keys`, plus `[[auth]]` system and Northstar keys | bootstrap path |
-| S25 | `accountteardown/reapers_iam.go:17-330`, used by `gateway/admin_deleteaccount.go:114` and `operator/cli/admin_account_delete.go:93` | awsgw, spx | Deletes IAM entities through `IAMService` methods only | authority. Borderline: a lifecycle caller through the authority's API, not a raw reader. |
-| S26 | `handlers/iam/mock/mock.go`, `handlers/sts/*_test.go`, `operator/cli/admin_test.go:1090` | **test** | Mock `IAMService`; tests open the session bucket and assert template bucket names | bootstrap path (test support) |
-
-`IAMService` lifecycle callers that use only the public interface and read no private record are not projection consumers: `handlers/{ecs,eks,rds,elbv2}` system-role ensurers, `domains/ec2/awsapi/{instance,spotinstance}` and the IAM AWS adapter `gateway/iam/*`.
-
-### 2.2 Predastore
-
-| # | Location | Process | Reads / writes | Classification |
-|---|---|---|---|---|
-| P1 | `predastore:internal/gate/auth/auth.go:84-152` | s3d | Duplicates `iamAccessKey`, `sessionCredential`, `iamUser`, `iamRole`, `iamGroup`, `iamPolicy` JSON shapes | invalid private-store reader |
-| P2 | `auth.go:299-356` (`NewNATSIAMProvider`) | s3d | `masterkey.LoadShared(master.key)` (311); NATS with the cluster token | invalid private-store reader |
-| P3 | `auth.go:624-725` (`LookupCredentials`) | s3d | Reads `spinifex-iam-access-keys/<AKID>`, checks `status`, decrypts `secret_access_key`; caches 60 s (217) with watchers (395-397) | invalid private-store reader. It is functionally a credential verifier, but every input is a private record. |
-| P4 | `auth.go:457-560` (`lookupSessionCredentials`) | s3d | Reads `spinifex-iam-session-credentials/<ASIA>`, expiry, decrypts `secret_encrypted`; never cached | invalid private-store reader |
-| P5 | `auth.go:764-998` (`resolveUserPolicies`, `resolveGroupPolicies`, `resolveRolePolicies`, `resolveManagedPolicies`) | s3d | Reads users, groups, roles, policies by `<account>.<name>`; reimplements group and managed-policy resolution | invalid private-store reader |
-| P6 | `internal/gate/middleware.go:263-290` | s3d | `iampolicy.EvaluateWithKeys` over P5's documents | policy evaluator |
-| P7 | `auth.go:158-180` (`ConfigProvider`), `auth.go:999-1029` (`ChainProvider`), `internal/gate/policy.go:19`, `middleware.go:311,326`, `internal/gate/handlers/list_buckets.go:90` | s3d | Config `[[auth]]` keys checked before IAM, returned with `SkipPolicyCheck: true` | bootstrap path |
-| P8 | `predastore:internal/storetest/keys.go:13` | **test** | Deterministic test master key for shard tests, not IAM | bootstrap path (test support) |
-
-### 2.3 Goanna
-
-Not characterised in Phase 0; deferred (D6).
-The rows below are inventory facts from code reading, not test evidence.
-Goanna is a future consumer of IAM state: it currently reads the IAM buckets directly, and it must adopt the final IAM enforcement projection, including IAM identity-policy enforcement, before production use.
-
-| # | Location | Process | Reads / writes | Classification |
-|---|---|---|---|---|
-| G1 | `goanna:internal/auth/provider.go:33-76` | goannad | Duplicates the access-key and session JSON shapes and bucket names | invalid private-store reader |
-| G2 | `provider.go:113-170` (`NewProvider`) | goannad | `masterkey.LoadShared(master.key)` (128) | invalid private-store reader |
-| G3 | `provider.go:174-226` (`lookupAccessKey`) | goannad | Reads `spinifex-iam-access-keys/<AKID>`, status, decrypts; caches 60 s (49) with a watcher | invalid private-store reader |
-| G4 | `provider.go:229-260` (`lookupSession`) | goannad | Reads `spinifex-iam-session-credentials/<ASIA>`, expiry, decrypts; uncached | invalid private-store reader |
-
-Goanna evaluates no IAM policy (no `iampolicy` import and no authorization code under `goanna:internal` or `goanna:cmd`), so it reads no policy, user, group or role record.
-
-### 2.4 Bluebottle
-
-Bluebottle reads no IAM state and declares no type that encodes an IAM record.
-
-| Package | Role | Note |
-|---|---|---|
-| `bluebottle:pkg/iampolicy` | Policy vocabulary and evaluator (`EvaluateWithKeys`, `evaluator.go:36`) | Shared contract, used by S15, P6 and STS; `door_test.go` already fixes the condition-key sets of the gateway and Predastore doors. |
-| `bluebottle:pkg/auth/arn.go` | ARN parsing (`ParseRoleARN` 39, `ResolvePolicyARN` 59) | Mechanism only. |
-| `bluebottle:pkg/sigv4` | SigV4 parse and verify | `Verify` (`verify.go:24`) takes the plaintext secret; there is no verify-by-derived-signing-key entry point. |
-| `bluebottle:pkg/masterkey` | AES-256-GCM key loading | The package comment describes shard encryption, but every IAM consumer uses it; `EncryptBase64`/`DecryptBase64` (135-160) pass no associated data. |
-
-## 3. Holders of the IAM master key that read no IAM record
-
-These are not IAM consumers, but each process holds the key that decrypts every IAM and session secret, and each uses the IAM package's helpers as its crypto boundary.
-
-| Location | Process | Use |
-|---|---|---|
-| `domains/acm/store.go:224,327` | awsgw | ACM private keys via `handlers_iam.EncryptSecret`/`DecryptSecret` |
-| `domains/ecr/auth/signingkey.go:179,206` | awsgw | ECR token signing key |
-| `gateway/bedrock/credentials.go:66,87` | awsgw | Bedrock credentials bucket |
-| `handlers/rds/bootstrap.go:105,199`, `daemon/rds_deps.go:56` | daemon | RDS bootstrap envelopes |
-| `handlers/eks/oidc_keypair.go:82,116`, `nats_bootstrap.go:247,265`, `nodegroup.go:1160`, `daemon/eks_deps.go:52` | daemon | EKS OIDC signing key, join tokens, kubeconfig |
-| `domains/ochre/vector/appliance.go:499,514`, `daemon/ochre_deps.go:63`, `operator/cli/admin_ochre.go:1005,1077` | daemon, spx | Ochre appliance passwords |
-| `daemon/daemon.go:2003,2138`, `daemon/ecs_deps.go:39` | daemon | Service dependencies |
-
-## 4. Current authentication flows
-
-### 4.1 AWSGW SigV4 (`gateway/auth.go:40-290`)
-
-| Step | Check | Failure response | Rate-limited |
-|---|---|---|---|
-| 1 | Per-IP lockout (51) | `RequestLimitExceeded` | n/a |
-| 2 | `sigv4.Parse`: envelope, scope, skew, payload (57-97) | `MissingAuthenticationToken`, `RequestEntityTooLarge`, `InternalError` (body read), `SignatureDoesNotMatch` (skew), `IncompleteSignature` | skew and malformed only |
-| 3 | Credential-scope service served (115-125) | Unknown-service error | no |
-| 4 | NATS connected (129) | 503 `ServiceUnavailable` | no |
-| 5a | AKIA: KV `Get` (S11) | unknown → `InvalidClientTokenId`; KV error → `InternalError` | unknown only |
-| 5b | AKIA: `status == Active` | `InvalidClientTokenId` | yes |
-| 5c | AKIA: decrypt secret | `InvalidClientTokenId` | yes |
-| 6a | ASIA: session `Get` (S12) | KV error → `InternalError`; absent → `InvalidClientTokenId` | absent only |
-| 6b | ASIA: expiry (424) | `ExpiredToken` | yes |
-| 6c | ASIA: `X-Amz-Security-Token` present and HMAC matches (431-442) | `InvalidClientTokenId` | yes |
-| 6d | ASIA: decrypt secret | `InvalidClientTokenId` | yes |
-| 7 | Other prefix (149-154) | `InvalidClientTokenId` | yes |
-| 8 | `sig.Verify` (163) | `SignatureDoesNotMatch` | yes |
-| 9 | Account ACTIVE (183, S13) | not active → `AccessDenied`; read error → `InternalError` | no |
-| 10 | Session principal continuity (191, S8) | replaced, gone or legacy → `InvalidClientTokenId`; fault → `InternalError` | verdicts except legacy |
-| 11 | Resolve `aws:userid` (198, S14) | fault → `InternalError`; missing user → key omitted | no |
-| 12 | Per action: policy resolution and evaluation (S15) | `AccessDenied`; resolution fault after 3 tries on transient NATS errors → `InternalError` | no |
-
-Caching: none for credentials, sessions or policies; only the ACTIVE account answer for 5 s.
-Backing state unavailable: the gateway's NATS connection closed fails fast with 503; a closed IAM/STS connection or a deleted access-key, session or accounts bucket gives `InternalError`, and no request proceeds (`TestCurrentBehaviour_Auth_BackingStateUnavailable`, in-process router over embedded JetStream; not exercised on a deployed cluster).
-
-Anti-enumeration:
-
-- An unknown AKIA, an inactive AKIA, an undecryptable secret and an unknown prefix all return `InvalidClientTokenId`, so status is not revealed.
-- A known, active key with a wrong signature returns `SignatureDoesNotMatch`, so key existence is distinguishable from an unknown key; whether AWS returns the same codes is unverified (Section 5).
-- Account status, principal continuity and policy reads all happen after the signature, so a caller without the secret cannot probe them (comments at 181 and 476-482).
-- An ASIA key past expiry returns `ExpiredToken` (HTTP 400) before the session token or signature is checked, so anyone holding only an expired session AKID learns that it existed (Section 7 F2, D10); records are kept for one hour past expiry (`janitorGracePeriod`, `handlers/sts/session_credentials.go:35`).
-- An AKIA key ignores `X-Amz-Security-Token`: the header is signed but never checked (Section 7 F1).
-- Timing: an unknown AKIA returns after one KV read, a known key additionally decrypts and HMACs; the error codes already differ, so the timing difference adds nothing for unknown versus bad-signature, and unknown versus inactive both stop after the same KV read. Not measured.
-
-### 4.2 AWSGW anonymous STS (`gateway/sts_anonymous.go:16-55`, `gateway/sts.go:81`)
-
-Only `AssumeRoleWithWebIdentity` is anonymous.
-The interceptor runs before the SigV4 middleware (`gateway/gateway.go:347`), so neither the per-IP auth lockout nor the post-auth throttle applies to it.
-Order in `handlers/sts/assume_role_with_web_identity.go`:
-
-1. Parameter validation and rejection of session policies (36-50), `ValidationError` or `MissingParameter`.
-2. Role ARN syntax to obtain the role account only (55-58).
-3. JWT parse, ES256 only, `exp` required (60-72); the key function (159-193) resolves `kid`, `iss`, issuer account and cluster, checks the OIDC provider registration by raw read (S4), and fetches the cluster JWKS through the EKS resolver. Every failure here, including a KV outage, returns `InvalidIdentityToken`.
-4. `iss`, `sub` non-empty and `aud` contains `sts.amazonaws.com` (74-81), `InvalidIdentityToken`.
-5. Role resolution (83-90): absent, foreign or non-canonical ARN → `AccessDenied`.
-6. Duration bound against the role's `MaxSessionDuration` (92-104), `ValidationError`.
-7. Trust policy evaluation (114), then mint (119).
-
-The token is verified before any role read, so an unauthenticated caller cannot probe role existence.
-A caller holding a valid token from a registered issuer can distinguish an existing role (step 6 `ValidationError` for an over-long duration) from an absent one (step 5 `AccessDenied`); whether AWS orders these checks the same way is unverified.
-No caching: every call reads the provider record and the JWKS.
-
-`GetAccessKeyInfo` (S10) is signed but needs no permission and returns the account for any AKIA or ASIA key it holds.
-
-### 4.3 Predastore S3 SigV4 (`predastore:internal/gate/middleware.go:214-330`)
-
-1. Unsigned request: allowed through only to a public bucket's permitted methods, else `AccessDenied`.
-2. `sigv4.Parse` errors map in `internal/gate/handlers/respond.go:20-100`.
-3. `ChainProvider.LookupCredentials` (P7): config `[[auth]]` keys first, then NATS IAM.
-4. AKIA (P3): cache hit returns at once; else KV read, status, empty-account check, decrypt, then user, group and policy resolution (P5) before the signature is verified.
-5. ASIA (P4): KV read, expiry, decrypt, then user or role policy resolution; no session-token check and no principal-continuity check.
-6. `sig.Verify` (250), then `EvaluateWithKeys` unless `SkipPolicyCheck` (263-290), then bucket ownership (299-322).
-
-| Condition | Response |
-|---|---|
-| Unknown AKID, or session absent or expired, or a session principal deleted | 403 `InvalidAccessKeyId` |
-| Inactive AKIA (685), undecryptable secret, empty account, deleted user behind an AKIA, any KV fault | 500 `InternalError` |
-| Malformed or foreign attached policy ARN (`ErrPrincipalConfig`) | 403 `AccessDenied` |
-| Wrong signature | 403 `SignatureDoesNotMatch` |
-| Policy deny or cross-account bucket | 403 `AccessDenied` |
-| IAM buckets not yet created | 403 `InvalidAccessKeyId` (bootstrap) |
-
-Caching: AKIA results for 60 s, invalidated by watchers on the access-key, user, policy and group buckets but not roles; a watcher that dies flushes once and then leaves the 60 s TTL as the only bound.
-Sessions are never cached.
-Backing state unavailable: KV faults give 500; config keys keep working because they are checked first.
-
-Anti-enumeration: an inactive key (500) is distinguishable from an unknown key (403) before any signature check, and policy resolution runs for any presented AKIA before the signature, so an unauthenticated caller who knows a key ID can drive user, group and policy reads.
-No account-status check exists in Predastore.
-
-### 4.4 Goanna CloudWatch SigV4 (`goanna:internal/auth/middleware.go:42-100`)
-
-From code reading only; not characterised in Phase 0; deferred (D6).
-
-1. `sigv4.Parse`; envelope errors map in `writeParseError` (104-128).
-2. Credential-scope service must be `monitoring` (59-66).
-3. `Provider.Lookup` (68): AKIA via G3 or ASIA via G4.
-4. `signed.Verify` (74), `SignatureDoesNotMatch` on failure.
-5. The account is attached to the context; no policy or account-status check follows.
-
-Unknown, inactive, empty-account and undecryptable keys all return 403 `InvalidClientTokenId`; an expired session returns 403 `ExpiredToken` before signature verification; a KV fault returns 503 `ServiceUnavailable`.
-Caching: AKIA 60 s with a watcher on the access-key bucket; sessions uncached.
-No session-token check and no principal-continuity check.
-
-## 5. AWS reference points used above
-
-Each claim cites an official AWS documentation page or is marked unverified; citations are listed in the design document, Section 12.
-
-- Unknown key versus wrong signature: the wire codes AWS returns (`InvalidClientTokenId` or `InvalidAccessKeyId` versus `SignatureDoesNotMatch`) are unverified; the STS *Common Error Types* page (https://docs.aws.amazon.com/STS/latest/APIReference/CommonErrors.html) lists neither code.
-- Key-ID existence: AWS documents `sts:GetAccessKeyInfo` as returning the account for an access key ID, and states that it does not indicate whether the key is active, inactive or deleted (*GetAccessKeyInfo*, https://docs.aws.amazon.com/STS/latest/APIReference/API_GetAccessKeyInfo.html).
-- Inactive key: unverified.
-- Expired session: the STS *Common Error Types* page lists `ExpiredTokenException` with HTTP 403; the Query-protocol code and status, and whether expiry is checked before or after the token and signature, are unverified.
-- An ASIA key used without its session token: unverified.
-- An AKIA key sent with `X-Amz-Security-Token`: unverified.
-- `aws:userid`: the account ID for the account root user and the unique ID for an IAM user (*IAM policy elements: Variables and tags*, "Principal key values", https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_variables.html#principaltable).
-
-## 6. Coverage gaps in this inventory
-
-- The AWSGW flow (4.1) is pinned by the tests in `gateway/auth_current_behaviour_test.go`, which drive the real router over real IAM and STS services on an embedded JetStream; no flow was exercised on a deployed cluster.
-- Predastore (4.3) and Goanna (4.4) flows are from code reading; Goanna is deferred (D6) and not characterised in Phase 0.
-- The ECR registry path (S17), STS presigned verify (S9) and `GetAccessKeyInfo` (S10) are not covered by the Phase 0 tests.
-- NATS subject ACLs: comments call the IMDS subjects "ACL-gated" (`handlers/iam/imds_rpc.go:13`), but the rendered `operator/cli/templates/nats.conf:41-44` defines one shared token with no per-user permissions; deployed NATS configuration was not inspected.
-- Northstar, Viperblock and `spinifex/spinifex/agents` were searched for the bucket names and record field names and none reads IAM state; `viperblock/internal/predastoretest/server.go:131` only renders a Predastore test config.
-- Viperblock loads its volume key with `masterkey.LoadShared` (`viperblock/viperblock/masterkey_load.go:25`) from a configured path; whether a deployment points that path at the IAM `master.key` was not checked.
-
-## 7. Characterisation findings (AWSGW, current behaviour)
-
-Each finding states what the named test in `gateway/auth_current_behaviour_test.go` proves about current behaviour, and nothing beyond it.
-
-**F1. AKIA credentials with a session-token header are accepted.**
-A long-lived AKIA key sent with an `X-Amz-Security-Token` header (here, another session's valid token) authenticates as the key's IAM user; the header is covered by the signature but never checked.
-Test: `TestCurrentBehaviour_Session_AssumeRoleCredentials/long_lived_with_session_token`.
-AWS behaviour for this request is unverified, so this is recorded as current behaviour only, with no change proposed.
-
-**F2. Session expiry is reported before token validation and SigV4 proof.**
-An expired ASIA session answers `ExpiredToken` (HTTP 400 today) whether the request carries the right token, no token, or a tampered token with a wrong secret; with no token, the door decrypts no secret before answering.
-A caller holding only the ASIA key ID therefore learns that an expired session existed.
-Tests: `TestCurrentBehaviour_Session_Expired` (`valid_token`, `no_token`, `wrong_token`) and `TestCurrentBehaviour_AntiEnumeration_RejectionShapes/session_expired_no_token`.
-Target ordering: D10.
-
-**F3. A tenant IAM user named `root` is presented with the account ID as its UserId.**
-`GetCallerIdentity` for a tenant user named `root` returns `UserId` (the `aws:userid` value) equal to the account ID instead of the AIDA ID stored on its user record; the ARN is the user's own ARN.
-AWS uses the account ID as `aws:userid` only for the account root user (Section 5).
-This is an identity-presentation issue: the same user with no policy is implicitly denied `iam:ListUsers`, so it does not take the global-root policy bypass for that action.
-Any effect on policies that test `aws:userid` in a condition is untested.
-Tests: `TestCurrentBehaviour_Auth_LongLivedKeyResolvesPrincipal/tenant_user_named_root`, `TestCurrentBehaviour_Policy_IdentityPolicyOutcomes/tenant_user_named_root_implicit_deny`.
-
-**F4. An orphaned access key authenticates as, and is authorized as, a same-name replacement user.**
-The tests construct the orphaned state directly by rewriting the user record so it no longer lists the key; the access-key record stays `Active`.
-They prove that the key still authenticates, that `DeleteUser` then succeeds, that after a same-name user is recreated the key authenticates as the new user (new `UserId`), and that the policy check applies the replacement user's policy: the key is allowed `iam:GetUser`, which only the replacement's policy grants, and denied `iam:ListUsers`, which only the original user's policy granted.
-How production reaches the orphaned state is INFERRED from code: `CreateAccessKey` and `DeleteAccessKey` each make two non-transactional writes (`handlers/iam/service_impl.go:665-683`, `757-770`), and an interruption between them would leave the key unlisted; no failure-injection test proves this.
-Tests: `TestCurrentBehaviour_Revocation_OrphanedKeyAuthenticatesAsRecreatedUser`, `TestCurrentBehaviour_Revocation_OrphanedKeyPermissionsAfterRecreation`.
-Design consequence: P23.
-
-**F5. A GetSessionToken session outlives its minting key.**
-Deactivating and then deleting the long-term key that minted a GetSessionToken session leaves the session authenticating as the IAM user.
-Test: `TestCurrentBehaviour_Session_GetSessionTokenOutlivesMintingKey`.
-Open question: D11.

docs/package-boundary/iam-enforcement-projection-design.md

@@ -1,374 +0,0 @@
-# IAM enforcement projection: v1 contract design
-
-**Status:** Draft for review; contract design only, nothing implemented.
-
-This is the minimum versioned Bluebottle IAM enforcement projection required by ADR-0005 S1 and S6(2), designed from the consumer inventory in `iam-enforcement-inventory.md` (cited below as S*n*, P*n* and G*n* rows).
-ADR-0005 and ADR-0006 are accepted and are cited as decisions; nothing here changes either.
-Clauses are numbered P1, P2… and are appended, never renumbered.
-Decisions are D1 to D12, each recorded once in the decision register (Section 14); a D-number anywhere else in either document means that register entry and nothing more.
-**Phase 1** means the first step of the code stack in Section 13; Phase 0 is the inventory and the AWSGW characterisation tests in `gateway/auth_current_behaviour_test.go`.
-Spinifex paths are relative to `spinifex/spinifex/`; other repos are prefixed as in the inventory.
-
-## 1. Purpose and standing
-
-P1. The projection is the set of signed, versioned facts a local enforcement point needs to verify a SigV4 credential and authorize a request without reading IAM persistence (ADR-0005 S1 "first required shared identity boundary").
-
-P2. Its semantic owner is the Spinifex IAM authority (today `handlers/iam` with `handlers/sts` for sessions; `domains/iam` once extracted).
-Bluebottle owns only its vocabulary, evaluation semantics and conformance fixtures (Section 10).
-
-P3. "Minimum" means every fact below has a named consumer in the matrix (Section 11), and no fact is included for a consumer that does not need it.
-
-## 2. Non-goals
-
-| Non-goal | Consequence |
-|---|---|
-| Moving `handlers/iam`, `handlers/sts` or IAM records | Records, keys, buckets and migrations stay where they are; the projection is produced from them. |
-| A broad `IAMService` replacement | Consumers get the projection or a narrow verification capability, never a wider service (ADR-0005 S6 last paragraph). |
-| Raw-KV, global-singleton or permissive fallback | No consumer may fall back to reading `spinifex-iam-*`, to a process-global credential table, or to allow-on-error while the projection is absent or stale. |
-| Predastore migration or production auth change | This document changes no code; Predastore's current flow (inventory 4.3) stays until its migration PR (Section 13). |
-| ECS/EKS `iam:PassRole` migration | ADR-0005 S3 and S6(6) work; role resolution is not a projection fact (P9). |
-| Transport, storage and key-custody design | Stated as responsibilities only (Section 10). |
-| EKS OIDC discovery | ADR-0005 S4, planned in `sts-oidc-discovery-projection-plan.md`. |
-
-## 3. Facts
-
-P4. Every fact carries a header: `contract` (`"mulga.iam.enforcement"`), `major` (`1`), `kind`, `key`, `generation` (per-key, strictly increasing), `rev` (authority-wide sequence at which it was emitted), `issuedAt` (authority clock), `critical` (list of required extension names, normally empty) and `sig` (Section 4).
-
-P5. **AccountFact**, key `accountID`.
-Fields: `status` (`ACTIVE`, `SUSPENDED`, `TERMINATING`).
-Consumers: AWSGW (S13, S17) and Predastore (D7); Goanna as a deferred future consumer (D6, D7).
-
-P6. **PrincipalFact**, key `(accountID, principalID)` where `principalID` is the immutable IAM unique ID (`AIDA…` user, `AROA…` role).
-Fields: `type` (`user`, `role`), `arn` (stored ARN including path), `name`, `status` (`present`, `deleted`), `policyGeneration`.
-A role-session principal is not a PrincipalFact; it is described by its CredentialFact and refers to its role's PrincipalFact.
-Consumers: AWSGW (S8, S14), Predastore (P4, P5), STS presigned verify (S9).
-
-P7. **CredentialFact**, key `accessKeyID`.
-Fields: `kind` (`long-term`, `session`), `accountID`, `principalID` (the user or role it was issued for), `status` (`Active`, `Inactive`, `Deleted`), `issuedAt`, `expiresAt` (sessions only), `sessionName` and `assumedRoleArn` (role sessions only, for `aws:userid` and caller ARN), `sessionTokenDigest` (sessions only, P14), `verifier` (Section 4.2), and optional `sourceCredentialID` (GetSessionToken sessions only: the long-term AKID that minted the session).
-`sourceCredentialID` is informational in major 1: no consumer enforces it, and it does not change revocation behaviour until D11 is decided.
-Consumers: AWSGW (S11, S12, S17), Predastore (P3, P4), STS (S9, S10); Goanna (G3, G4) as a deferred future consumer (D6).
-
-P8. **PolicySetFact**, key `(accountID, principalID)`.
-Fields: `policyGeneration` and `documents`, the ordered list of `iampolicy.PolicyDocument` that IAM resolves for that principal today (user inline, attached managed default versions, group inline and group managed; or role inline and attached managed), exactly the set `GetUserPolicies` and `GetRolePolicies` return (`handlers/iam/service_impl.go:1997`, `handlers/iam/roles.go:579`).
-Resolution, including the missing-group and unmodelled-AWS-managed-policy rules, happens once at the authority; consumers never resolve groups or managed ARNs (ADR-0005 S1 requires Predastore to stop reimplementing group resolution).
-A principal whose policies cannot be resolved is published with `documents` absent and `unresolvable: true`, and every consumer treats it as Malformed (P24), never as an empty grant.
-Consumers: AWSGW (S15), Predastore (P5, P6); Goanna once it adopts the projection, because it must then enforce IAM identity policy (D6).
-
-P9. Excluded facts and why:
-
-| Excluded | Reason |
-|---|---|
-| Role trust policies, `MaxSessionDuration` | Only STS issuance (S5, S6) needs them, and STS sits with the authority; a role resolver capability serves it (ADR-0005 S2). |
-| Instance profiles | Served to vpcd by the existing RPC (S3, S21); not an enforcement input. |
-| OIDC provider records | STS-only (S4); a narrow IAM capability, not a broadcast fact. |
-| Tags, paths beyond the ARN, creation times, policy versions other than default | No enforcement consumer reads them. |
-| Group facts | Folded into PolicySetFact (P8). |
-
-## 4. Integrity and secret handling
-
-### 4.1 Authenticity
-
-P10. Every fact is signed by the IAM authority with an Ed25519 signature (D2) over its canonical encoding, which includes every header field in P4.
-Consumers hold only the authority's public verification keys, identified by `keyID`, from their own configured trust bundle.
-A fact whose signature does not verify, whose `keyID` is unknown, or whose canonical encoding does not round-trip is Malformed (P24).
-
-P11. The signature, not transport access, is the trust root.
-Today any holder of the shared NATS token can write IAM buckets (inventory 6); a signed fact cannot be forged by a consumer that can write the transport.
-
-P12. Replay: a consumer accepts a fact for a key only if its `generation` is greater than the one it holds, so an old Active fact cannot be replayed over a later Inactive or Deleted one.
-Freshness (Section 5) bounds replay of a whole stale view.
-
-### 4.2 Credential verification material
-
-P13. SigV4 is a symmetric HMAC scheme, so verifying it needs key material derived from the secret.
-The projection never carries the long-term secret or the session secret, in plaintext or under the master key.
-Under D1 (Accepted in principle), the `verifier` carries SigV4 derived signing keys, `kSigning = HMAC chain(secret, date, region, service)`, one per (scope date, region, service) the consumer is entitled to, each encrypted to that consumer's own public key (audience-bound), and covered by the fact signature.
-A derived signing key is a sensitive signing capability, not a public verifier: its holder can sign any request within its date, Region and service scope.
-A Predastore view receives only `s3` keys for its Region (and a future Goanna view only `monitoring` keys), so a compromised consumer can forge requests only for its own service, Region and the dates it holds.
-Recipient-key rotation and scope issuance need implementation review before this material is used (D1 gate).
-Dates published: the current and next UTC date for header-signed requests; for presigned URLs, every date back to the consumer's maximum presign age (`bluebottle:pkg/sigv4/sigv4.go:23` sets 7 days) and never earlier than the credential's `issuedAt`.
-Bluebottle's `sigv4` needs a verify-by-signing-key entry point for this; today `Verify` takes only the plaintext secret (`bluebottle:pkg/sigv4/verify.go:24`).
-
-P14. Session tokens: `sessionTokenDigest` is SHA-256 of the wire token.
-Tokens are 32 random bytes (`handlers/sts/assume_role.go:68`), so an unkeyed digest cannot be inverted or brute-forced, and consumers need no shared HMAC key.
-The authority keeps its existing master-key HMAC in its private record; the digest is an additional projection field.
-
-P15. Must never appear in the projection: long-term or session secrets in any encoding; the master key or any key-encryption key; master-key ciphertexts (`secret_access_key`, `secret_encrypted`); session-token plaintext or the master-key HMAC; the authority's signing private key; IAM bucket names, KV keys or persistence record encodings; trust policies; bootstrap credentials.
-
-## 5. Freshness
-
-P16. The authority emits a signed **Watermark** fact at a fixed interval H = 5 s (D3) carrying `rev` (the highest `rev` emitted) and `issuedAt`.
-
-P17. A consumer's view is **current** when it has applied every fact up to the last watermark's `rev`, that watermark was received within S of the consumer's monotonic clock, and its `issuedAt` is within S plus the declared maximum clock skew of the consumer's wall clock.
-S is the staleness bound: 30 s normally, set per consumer profile, with a hard maximum of 60 s (D3; 60 s is today's Predastore AKIA cache TTL, inventory P3).
-
-P18. A view is built from a signed **Snapshot** (all live facts at a `rev`, terminated by a watermark at that `rev`) followed by incremental facts.
-Absence of a key in a current view means NotFound only if the view was built from a complete snapshot; a consumer that has not completed its first snapshot is Unavailable (P24).
-
-P19. When stale, a consumer refuses every request that depends on the projection (P26), and never extends a cached answer past S.
-
-## 6. Revocation
-
-P20. Access-key deactivation or deletion, principal deletion, account suspension and session revocation each publish a new fact generation (`Inactive`, `Deleted`, `deleted`, `SUSPENDED`) before the authority's API call returns success.
-Effect at every current consumer is bounded by S plus one watermark interval H.
-
-P21. Session revocation by principal deletion (`RevokeUserSessions`, `RevokeRoleSessions`, `handlers/sts/session_credentials.go:201,212`) publishes `Deleted` CredentialFacts for every affected session, and the principal's own `deleted` PrincipalFact independently invalidates any session the sweep misses (P23 check 3).
-This clause describes how today's delete-the-record mechanism would be projected; whether that mechanism stays or is replaced by an AWS-style deny on token issue time is D12 (Open), and the IAM authority is the sole publisher of session-validity facts (D9).
-A GetSessionToken session today outlives deletion of its source access key (`TestCurrentBehaviour_Session_GetSessionTokenOutlivesMintingKey`); whether it should is D11 (Open).
-
-P22. Deleted-key and deleted-session facts are retained as tombstones until the next snapshot; a consumer that misses a tombstone recovers by resynchronising to a snapshot in which the key is absent.
-
-## 7. Replacement and identity reuse
-
-P23. All facts bind to immutable IDs, never names.
-A consumer accepts a credential only if:
-
-1. the CredentialFact is `Active` and, for sessions, unexpired and its token digest matches;
-2. its `accountID` has an `ACTIVE` AccountFact;
-3. its `principalID` has a `present` PrincipalFact in the same account; and
-4. it evaluates the PolicySetFact keyed by that same `principalID`.
-
-A user deleted and recreated under the same name receives a new `principalID`, so the old key or session fails check 3 and can never reach the new user's PolicySetFact.
-AKIDs are never reused by the authority.
-
-Current-code evidence that this is a change, not a restatement:
-
-- The long-term `AccessKey` record holds only `user_name`, not the user ID (`handlers/iam/types.go:68-75`), and every verifier resolves the user by name (`gateway/auth.go:375-405`, S9, S17, P3).
-- `DeleteAccessKey` removes the key from the user record before deleting the access-key record (`handlers/iam/service_impl.go:757-770`), and `CreateAccessKey` writes the key before the user record (665-683); the two writes are not transactional.
-  That an interruption between them leaves an `Active` access-key record the user no longer lists is INFERRED from the code; no failure-injection test interrupts either call.
-- The resulting state is reproduced directly: `TestCurrentBehaviour_Revocation_OrphanedKeyAuthenticatesAsRecreatedUser` rewrites the user record so it no longer lists the key, and shows that the key still authenticates, that `DeleteUser` then succeeds because it checks only the user's list (585), and that after a same-name user is recreated the key authenticates as the new user (new `UserId`, same ARN).
-- `TestCurrentBehaviour_Revocation_OrphanedKeyPermissionsAfterRecreation` shows the policy consequence through the real router and dispatcher policy check: after recreation, the orphaned key is allowed an action only the replacement user's policy grants (`iam:GetUser`) and denied the action only the original user's policy granted (`iam:ListUsers`).
-  The orphaned key therefore receives the replacement user's permissions and loses the original user's.
-- Session records already bind `RoleID`/`UserID` and AWSGW re-checks them (S8, pinned by `TestCurrentBehaviour_Revocation_UserDeletedAndRecreated` and `TestCurrentBehaviour_Revocation_RoleDeletedAndRecreated`); Predastore does not (P4), from code reading.
-
-The producer PR therefore needs an expand migration (ADR-0006 S4) adding the user ID to every `AccessKey` record before it can publish P7's `principalID` for long-term keys.
-
-## 8. Failure classification
-
-P24. Each consumer classifies a lookup into exactly one class.
-The AWSGW column is pinned by the tests named in inventory Section 7; the Predastore column is from code reading (inventory 4.3).
-Goanna is not characterised in Phase 0 and has no column (D6).
-
-| Class | Meaning | Fail-closed | AWSGW today | Predastore today | Target wire response |
-|---|---|---|---|---|---|
-| NotFound | Current, complete view has no fact for the AKID | yes | 403 `InvalidClientTokenId` | 403 `InvalidAccessKeyId` | unchanged |
-| Invalid | Key `Inactive`/`Deleted`; principal deleted or replaced; account not ACTIVE before signature; token digest mismatch or missing | yes | `InvalidClientTokenId` (account: `AccessDenied` after signature) | inactive: 500 `InternalError`; no token or principal check | same code as NotFound; account check stays after the signature |
-| Expired | Session past `expiresAt` | yes | 400 `ExpiredToken`, before the session token and the signature are checked | 403 `InvalidAccessKeyId` | `ExpiredToken`, only after the token digest matches and the SigV4 proof verifies (D10, P25) |
-| Denied | Policy evaluation or ownership denies | yes | `AccessDenied` | 403 `AccessDenied` | unchanged |
-| Unavailable | No completed snapshot, or transport down | yes | 503 `ServiceUnavailable` (gateway NATS down) or `InternalError` | 500 `InternalError` | retryable 503 `ServiceUnavailable` (D5) |
-| Stale | View older than S (P17) | yes | n/a | n/a (60 s cache) | as Unavailable (D3, D5) |
-| Malformed | Bad signature, unknown `keyID`, unknown `major`, unknown `critical` entry, `unresolvable` policy set | yes, and alarms | `InternalError` | `ErrPrincipalConfig` → 403 `AccessDenied`, else 500 | 500 `InternalError`; never reported as NotFound |
-
-P25. Ordering preserved from AWSGW (inventory 4.1): NotFound and Invalid share one code, and checks that depend on mutable principal or account state run after the signature.
-Ordering changed from AWSGW (D10): Expired is reported only after both the session token digest matches and the SigV4 proof verifies, so a caller holding only an ASIA key ID no longer learns that an expired session existed.
-AWS's own ordering of these checks is unverified, so D10 is a target chosen on its own merits, not an AWS-parity claim.
-Policy facts are evaluated only after the signature verifies, so an unauthenticated caller cannot drive policy evaluation (Predastore today resolves policies before the signature, inventory 4.3).
-
-P26. No class other than a verified Allow lets a request proceed, and no consumer may answer from a fact older than S.
-
-## 9. Compatibility (ADR-0006)
-
-P27. `major` is explicit on every fact; a receiver rejects an unknown major as Malformed and changes no state (ADR-0006 S3).
-Additive optional fields stay on major 1 and are ignored by receivers that do not know them; a new field that changes enforcement meaning is listed in `critical`, and a receiver that does not know a listed name treats the fact as Malformed.
-
-P28. AWSGW, STS and the IAM authority are one coordinated Regional release: N and N-1 interoperate in both rollout orders (ADR-0006 S2).
-
-P29. Predastore, and Goanna once it adopts the projection (D6), are independently released: each declares the projection majors it supports in its release notes, and the authority publishes the oldest major any registered consumer has proved, through an authenticated capability declaration, not a version-like field or timing (ADR-0006 S3).
-An AWSGW restart or upgrade does not require a Predastore restart while their declared major is still published (ADR-0006 S2 example).
-
-P30. An incompatible change is a successor contract `major: 2` published alongside major 1 until every consumer has proved major 2; major 1 retires through ADR-0006 S5 finalisation with operator notice.
-
-## 10. Ownership boundary
-
-| Concern | Owner | Not owned by |
-|---|---|---|
-| Fact types, canonical encoding, signature verification, freshness predicate, failure classes, `iampolicy` evaluation semantics, verify-by-signing-key, conformance fixtures | Bluebottle (new package beside `pkg/iampolicy`) | — |
-| Producing facts, signing key custody and rotation, snapshot and watermark emission, revocation publication, the access-key ID migration | Spinifex IAM authority | Bluebottle |
-| Transport (stream or KV layout, subjects, NATS permissions), view storage and caching, per-consumer encryption keys | The IAM authority (publish side) and each consumer (subscribe side) | Bluebottle |
-| Request-specific authorization (S3 ownership, service conditions) | Each consumer | Bluebottle, IAM |
-
-P31. Bluebottle does not import Spinifex, hold deployment state, open NATS, name a bucket or subject, hold a credential or run a service.
-Fixture signing uses a test-only key generated in the fixtures; production signing lives only in the authority.
-
-P32. Transport must give consumers read-only access to the projection channel only, separate from the IAM private buckets; how is the authority's design, constrained by P11.
-
-## 11. Consumer matrix
-
-| Consumer | Needs from projection | Currently reads directly | Must stop reading | Failure behaviour required | Migration blockers |
-|---|---|---|---|---|---|
-| AWSGW SigV4 ingress | Account, Principal, Credential, PolicySet, or the in-process enforcement profile while co-located with the authority (D4) | `gateway/auth.go:375-520`, `gateway/auth_accountstatus.go:67`, `gateway/gateway.go:771,809` via `IAMService`/`STSService` | `LookupAccessKey`, `DecryptSecret`, `LookupSessionCredential`, `GetUserPolicies`, `GetRolePolicies`, `GetAccount` on the auth path | P24 table; NATS-down 503 preserved | AKID user-ID migration (P23); the audit and context-key boundary still in `gateway` (`PACKAGE_BOUNDARY_MIGRATION.md`, ingress pipeline debt) |
-| AWSGW ECR registry path | Account, Principal, Credential | `gateway/ecr_principal.go:97-250` | Same reads as above | 401 Bearer for Invalid, 503 for Unavailable or Stale (present split, `classifyIAMLookupErr` 67) | Same as AWSGW; not covered by the Phase 0 tests |
-| STS presigned verify and `GetAccessKeyInfo` | Credential (verifier for `sts`), Principal | `handlers/sts/presigned_url_verify.go:150-200`, `get_access_key_info.go:73` | Raw `DecryptSecret(…, s.masterKey)` and `LookupAccessKey` | `InvalidIdentityToken`/`ExpiredToken` as today; Unavailable is an error, never a token verdict | STS extraction (ADR-0005 S6(4)); session CredentialFacts are published by the IAM authority through its session-lifecycle capability (D9) |
-| Predastore | Account (D7), Principal, Credential (verifier for `s3`), PolicySet | `predastore:internal/gate/auth/auth.go:84-998` (P1-P5) | Every `spinifex-iam-*` bucket, `master.key` for IAM, duplicated record structs, group resolution | P24 table; inactive key → 403 not 500; token digest and principal checks added; policy only after signature | Independently released (P29); config `[[auth]]` keys (P7) may not override IAM or bypass ownership or policy (D8), so any bootstrap exception needs a separately designed, bounded bootstrap principal; Predastore characterisation tests |
-| Goanna (future, deferred: D6) | Credential (verifier for `monitoring`), Principal, Account (D7), PolicySet | `goanna:internal/auth/provider.go:33-260` (G1-G4), code reading only | Both IAM buckets and `master.key` | P24 table, plus IAM identity-policy enforcement it does not have today | Deferred (D6); not characterised in Phase 0; before production use it must adopt this projection and enforce IAM identity policy, not keep its account-only authentication |
-| Daemon system-role ensurer | None: it writes IAM, so it needs an IAM lifecycle capability, not the projection | `daemon/eks_deps.go:27-48`, `daemon/ecs_deps.go:39-51` | Second `IAMServiceImpl` and `master.key` | Fail closed instead of static-credential fallback (ADR-0005 S5, Q-25) | ADR-0005 S6(3) capability extraction |
-| vpcd guest metadata | None: role resolution over the existing RPC | `domains/ec2/guestmetadata/nats_clients.go:42-95` | `handlers_iam.InstanceProfile` type over NATS | unchanged | ADR-0005 S6(3) |
-
-## 12. AWS parity
-
-Each AWS claim cites an official AWS documentation page or is marked unverified.
-Citations:
-
-- [AWS1] *Troubleshoot IAM*, section "Changes that I make are not always immediately visible", https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency
-- [AWS2] *GetAccessKeyInfo*, AWS STS API Reference, https://docs.aws.amazon.com/STS/latest/APIReference/API_GetAccessKeyInfo.html
-- [AWS3] *Common Error Types*, AWS STS API Reference, https://docs.aws.amazon.com/STS/latest/APIReference/CommonErrors.html
-- [AWS4] *IAM identifiers*, section "Unique identifiers", https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
-- [AWS5] *Revoke IAM role temporary security credentials*, https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_revoke-sessions.html
-- [AWS6] *IAM policy elements: Variables and tags*, table "Principal key values", https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_variables.html#principaltable
-
-| Behaviour | What AWS does | This design | Status of the AWS claim |
-|---|---|---|---|
-| Propagation of IAM changes | IAM is eventually consistent; changes take time to become visible from all endpoints, and caching can add time [AWS1] | Bounded by S + H (P20); deliberately stricter | Documented [AWS1]; that page states no numeric bound |
-| Access-key deactivation or deletion | Unverified | Bounded by S + H (P20) | Unverified |
-| Unknown key vs wrong signature | Wire codes unverified ([AWS3] lists neither `InvalidClientTokenId` nor `SignatureDoesNotMatch`); `GetAccessKeyInfo` returns the account for an access key ID and does not indicate whether the key is active, inactive or deleted [AWS2] | Preserved (P25) | `GetAccessKeyInfo` documented [AWS2]; error codes unverified |
-| Inactive vs unknown key | Unverified | Same code (P24) | Unverified |
-| Expired session | [AWS3] lists `ExpiredTokenException` with HTTP 403; the Query-protocol code, its status and its ordering against token and signature checks are unverified | `ExpiredToken` only after token and SigV4 proof (D10, P25) | Partly documented [AWS3]; ordering unverified |
-| Role session after the role's policies change | Unverified | PolicySetFact is keyed by role `principalID` and updated in place | Unverified |
-| User deleted and recreated with the same name | A recreated user receives a different unique ID from the deleted one [AWS4] | Credentials and sessions of the old principal fail (P23) | Unique-ID rule documented [AWS4]; behaviour of an old key or session against a recreated user or role is unverified |
-| Revoking active role sessions | IAM attaches an inline deny policy (`AWSRevokeOlderSessions`) to the role, conditioned on `aws:TokenIssueTime` [AWS5] | Undecided: today sessions are deleted (P21); D12 is Open | Documented [AWS5] |
-| GetSessionToken session after its source key is deleted | Unverified | `sourceCredentialID` recorded, not enforced (P7, D11) | Unverified |
-| AKIA credentials sent with `X-Amz-Security-Token` | Unverified | Not addressed; current behaviour recorded in inventory Section 7 | Unverified |
-| `aws:userid` value | Account root user: the account ID; IAM user: the user's unique ID [AWS6] | PrincipalFact carries the immutable `principalID` (P6) | Documented [AWS6] |
-| Suspended or closed account | Unverified | AccountFact enforced by every production consumer (D7) | Unverified |
-| CloudWatch authorization | Unverified | Goanna must enforce identity policy before production use (D6) | Unverified |
-
-## 13. Later code stack
-
-Each step is its own PR, in order; none starts before ADR-0005 S6(1) characterisation tests for its consumer have landed.
-Phase 1 begins at step 1.
-
-| # | Change | Entry criteria | Exit criteria |
-|---|---|---|---|
-| 1 | Bluebottle: projection v1 types, canonical encoding, Ed25519 verify, freshness predicate, failure classes, `sigv4` verify-by-signing-key, conformance fixtures (valid, tampered, wrong key, unknown major, unknown critical, stale, replayed generation, replaced principal, expired, missing token, unresolvable policy set) | This design accepted | Fixtures pass in Bluebottle; AWSGW and Predastore test suites both run the fixtures without importing Spinifex (ADR-0005 evidence 2); no Spinifex import in Bluebottle |
-| 2 | Spinifex IAM authority: add user ID to `AccessKey` (expand, migrate per ADR-0006 S4); produce, sign and publish snapshot, incremental facts and watermarks; signing key custody; publication on every revocation path | PR 1 released; transport design reviewed; D1 gate met (recipient-key rotation and scope issuance reviewed); D12 decided before any change to session revocation | For the characterisation corpus, the projection's decision equals `IAMService`'s on every case; revocation visible within S + H in a test; no consumer changed |
-| 3a | AWSGW ingress and ECR path consume the projection, or the in-process enforcement profile (D4) | PR 2 deployed; AWSGW characterisation green | No `LookupAccessKey`, `DecryptSecret`, session lookup or policy resolution on the auth path; ADR-0005 evidence 3 tests pass |
-| 3b | STS presigned verify and `GetAccessKeyInfo` consume the projection | 3a merged | No raw master-key decrypt in `presigned_url_verify.go` |
-| 3c | Predastore consumes the projection | PR 2 published at a major Predastore declares; Predastore characterisation green; D8 bootstrap principal designed if any exception is kept | No `spinifex-iam-*` read, IAM struct copy or IAM `master.key` load; no `[[auth]]` override of IAM; ADR-0005 evidence 7 tests pass; compatibility matrix per P29 |
-| 3d | Goanna consumes the projection | D6 deferral lifted; Goanna characterisation tests written and green | No IAM bucket or IAM `master.key` access; IAM identity policy enforced on every action; account status enforced (D7) |
-
-Removing the daemon's second `IAMServiceImpl` (inventory S20) and the operator CLI's direct construction (S22) belongs to ADR-0005 S6(3) and S6(5) capability work, not this stack.
-
-## 14. Decision register
-
-Each entry stands alone.
-Status values: Accepted, Accepted in principle (direction fixed, implementation gated), Rejected, Deferred, Open (not decided).
-An implementation gate names what must exist or be reviewed before code that depends on the decision lands.
-
-### D1: SigV4 verification without the long-lived secret
-
-- **Question:** How does a consumer verify a SigV4 signature without holding the long-lived or session secret access key?
-- **Status:** Accepted in principle.
-- **Decision:** The CredentialFact `verifier` carries SigV4 derived signing keys, each scoped to one date, Region and service, and each encrypted to the receiving consumer's own public key (audience-encrypted).
-  Derived signing keys are sensitive signing capabilities: a holder can sign any request within the key's date, Region and service scope, so they are protected and handled as secrets, never as public verification material.
-- **Rationale:** SigV4 is a symmetric HMAC scheme, so verification needs key material derived from the secret; date, Region and service scoping limits a compromised consumer to forging requests for its own service, Region and held dates, and audience encryption stops one consumer reading another's keys.
-- **Implementation gate:** Recipient-key rotation and scope issuance (which dates, Regions and services each consumer receives) need implementation review before any derived key is issued or used.
-  Bluebottle `sigv4` needs a verify-by-signing-key entry point (`bluebottle:pkg/sigv4/verify.go:24` takes only the plaintext secret today).
-- **Links:** P13, P15, P7; inventory 2.4 (`bluebottle:pkg/sigv4`), S11, S12, P3, P4.
-
-### D2: Integrity of projection facts
-
-- **Question:** How are projection facts protected against forgery and tampering by any party that can write the transport?
-- **Status:** Accepted.
-- **Decision:** Every fact is signed with Ed25519 by the IAM authority over its canonical encoding, including every header field; consumers hold only the authority's public verification keys from their own trust bundle.
-- **Rationale:** Any holder of today's shared NATS token can write the IAM buckets (inventory Section 6), so transport access cannot be the trust root; an asymmetric signature means a consumer that can verify facts cannot forge them.
-- **Implementation gate:** Signing-key custody and rotation at the authority, and the trust-bundle distribution to consumers, are reviewed before the producer (Section 13 step 2) publishes.
-- **Links:** P4, P10, P11, P12, P31; inventory Section 6.
-
-### D3: Freshness bounds
-
-- **Question:** What heartbeat interval and staleness bounds define a current projection view?
-- **Status:** Accepted.
-- **Decision:** Heartbeat (watermark interval) H = 5 s; normal staleness bound S = 30 s; hard maximum S = 60 s, which no consumer profile may exceed.
-- **Rationale:** Revocation then takes effect at every current consumer within S + H; 60 s matches today's longest IAM cache in a consumer (Predastore's AKIA cache), so no consumer becomes weaker than today.
-- **Implementation gate:** The Bluebottle freshness predicate and its stale-view fixtures (Section 13 step 1) encode these values before any consumer relies on them.
-- **Links:** P16, P17, P19, P20, P24 (Stale); inventory P3.
-
-### D4: AWSGW in-process synchronous path
-
-- **Question:** May AWSGW, co-located with the IAM authority, verify and authorize through an in-process synchronous capability instead of consuming the projection?
-- **Status:** Accepted.
-- **Decision:** While AWSGW is co-located with the IAM authority, it may use an in-process synchronous enforcement profile that accepts the authority's availability dependency (ADR-0005 S1).
-  That profile uses the same narrow, versioned enforcement semantics as the projection; it must not use `IAMService`, raw IAM KV, master-key decryption or a gateway-owned cache.
-  Composition selects the profile at startup, and no request falls back between the in-process and projection profiles.
-  Authority-state failure stays fail-closed with a retryable 503 (D5).
-  If AWSGW is decoupled from the authority, it moves to the signed projection.
-- **Rationale:** Structural AWSGW cleanup does not wait for the full distributed projection, and AWSGW still enforces through one model shared with every other consumer.
-- **Implementation gate:** Before AWSGW adopts the in-process profile (Section 13 step 3a), tests show the profile is selected only at composition and that authority-state failure returns 503.
-- **Links:** Section 11 (AWSGW rows), Section 13 step 3a; inventory S11-S15, 4.1.
-
-### D5: Failure class when projection state is unavailable or stale
-
-- **Question:** What does a consumer return when it has no completed snapshot, its transport is down, or its view is older than S?
-- **Status:** Accepted.
-- **Decision:** Fail closed with a retryable HTTP 503 `ServiceUnavailable`.
-- **Rationale:** No request may proceed on missing or stale identity state, and a retryable 503 tells clients the failure is transient rather than a credential verdict; AWSGW already answers 503 when its NATS connection is down (`TestCurrentBehaviour_Auth_BackingStateUnavailable`).
-- **Implementation gate:** Each consumer's characterisation tests pin its current unavailable-state outcome before it migrates, so the change to 503 is deliberate and visible.
-- **Links:** P19, P24 (Unavailable, Stale), P26; inventory 4.1, 4.3.
-
-### D6: Goanna
-
-- **Question:** Is Goanna in scope for the Phase 0 characterisation and the first projection rollout?
-- **Status:** Deferred.
-- **Decision:** Goanna is not characterised in Phase 0 and is not part of the first rollout.
-  Any future production Goanna must consume the projection and enforce IAM identity policy; it must not keep its current account-only authentication model.
-- **Rationale:** Goanna reads the IAM buckets directly and evaluates no IAM policy today (inventory 2.3, 4.4, from code reading), so adopting the projection without adding policy enforcement would carry an authorization gap forward.
-- **Implementation gate:** Before Goanna is used in production: Goanna characterisation tests, adoption of the projection, and IAM identity-policy enforcement (Section 13 step 3d).
-- **Links:** P5, P7, P8, P29, Section 11 (Goanna row); inventory G1-G4, 4.4.
-
-### D7: Account-status enforcement
-
-- **Question:** Which consumers enforce account status (ACTIVE, SUSPENDED, TERMINATING)?
-- **Status:** Accepted.
-- **Decision:** Every production consumer enforces account status from the AccountFact.
-- **Rationale:** A suspended account must be refused everywhere; today only AWSGW checks it (inventory S13), Predastore does not (inventory 4.3).
-- **Implementation gate:** Each consumer's migration PR includes a test that a SUSPENDED account is refused after the signature.
-- **Links:** P5, P23 check 2, Section 11; inventory S13, 4.1 step 9, 4.3.
-
-### D8: Predastore config `[[auth]]` credentials
-
-- **Question:** May Predastore's config-file `[[auth]]` credentials keep overriding IAM?
-- **Status:** Rejected.
-- **Decision:** Config credentials may not override IAM or bypass bucket ownership or policy evaluation.
-  Any bootstrap exception needs a separately designed, bounded bootstrap principal.
-- **Rationale:** Today `[[auth]]` keys are checked before IAM and returned with `SkipPolicyCheck: true` (inventory P7), a permanent authority bypass that ADR-0005 S1 excludes from the target.
-- **Implementation gate:** If any bootstrap exception is kept, the bounded bootstrap principal is designed and reviewed before Predastore migrates (Section 13 step 3c).
-- **Links:** Section 11 (Predastore row), Section 13 step 3c; inventory P7, S24, 4.3.
-
-### D9: Owner and publisher of session-validity facts
-
-- **Question:** Which component owns and publishes session CredentialFacts: the IAM authority, or the STS session store that physically holds session records today?
-- **Status:** Accepted.
-- **Decision:** The IAM authority owns session-validity facts and is their sole publisher.
-  STS owns protocol handling and session-issuance orchestration; it is not a second authority for session validity or revocation.
-  The current STS session store may remain during transition, but every session creation, revocation and reconciliation flows through an IAM-owned session-lifecycle capability that produces the matching CredentialFact update.
-  This decision does not settle D11 or D12.
-- **Rationale:** ADR-0005 S2 makes IAM the logical authority for session-credential validity and revocation, while STS writes and reads `spinifex-iam-session-credentials` today (inventory S2); one publisher keeps a single source of revocation truth.
-- **Implementation gate:** Before the producer publishes session CredentialFacts (Section 13 step 2), STS session creation and revocation go through the IAM-owned session-lifecycle capability.
-- **Links:** P2, P7, P20, P21, Section 11 (STS row); inventory S2, S12.
-
-### D10: Error ordering for expired sessions
-
-- **Question:** At what point in request verification is `ExpiredToken` reported for an expired session?
-- **Status:** Accepted (target).
-- **Decision:** Emit `ExpiredToken` only after both session-token validation and SigV4 proof succeed.
-- **Rationale:** Today AWSGW reports expiry before checking the session token or the signature, so holding only an ASIA key ID reveals that an expired session existed (`TestCurrentBehaviour_Session_Expired`, `TestCurrentBehaviour_AntiEnumeration_RejectionShapes`).
-  AWS's actual ordering is unverified, so this is a chosen target, not a parity claim.
-- **Implementation gate:** A conformance fixture and an AWSGW test show an expired session with a missing or wrong token, or a bad signature, does not receive `ExpiredToken`; the current-behaviour tests are updated in the same change.
-- **Links:** P24 (Expired), P25, Section 12; inventory 4.1 steps 6b-6c, Section 7 finding F2.
-
-### D11: GetSessionToken session after its source key is deleted or deactivated
-
-- **Question:** Does a GetSessionToken session survive deletion or deactivation of the long-term access key that minted it?
-- **Status:** Open.
-- **Decision so far:** The future contract's session fact model gains an optional `sourceCredentialID`; revocation behaviour does not change now.
-- **Context:** Today the session survives both (`TestCurrentBehaviour_Session_GetSessionTokenOutlivesMintingKey`); AWS behaviour is unverified.
-- **Implementation gate:** Decided before any consumer enforces `sourceCredentialID`.
-- **Links:** P7, P21, Section 12; inventory Section 7 finding F5.
-
-### D12: Session revocation mechanism
-
-- **Question:** Is a session revoked by deleting its session record, as today, or by an AWS-style deny conditioned on token issue time?
-- **Status:** Open; not decided.
-- **Context:** AWS attaches an inline deny policy conditioned on `aws:TokenIssueTime` to the role [AWS5]; Spinifex deletes session records on principal deletion (P21).
-- **Implementation gate:** Decided before session revocation is implemented in the projection producer (Section 13 step 2).
-- **Links:** P20, P21, P22, Section 12; inventory S2.

docs/package-boundary/release-and-version-inventory.md

@@ -1,148 +0,0 @@
-# Release, Version and Upgrade Mechanism Inventory
-
-Code read at `3a55e2ee1`; not verified live.
-
-This is a read-only inventory of the release-membership, rolling-upgrade, rollback, schema-version and guest-image mechanisms that exist today, measured against what the draft Prepare → Activate → Migrate → Finalise sequence and N/N-1 negotiation would need.
-The draft proposal (`docs/development/proposals/spinifex-architecture/0007-…`, unaccepted) and Q-75/Q-76 in `docs/specs/requirements/resource-lifecycle.md` are the yardstick only; nothing here treats them as decided.
-The open question it answers is the EKS add-on design's question 4 (`eks-addon-lifecycle-design.md:445`): is there an existing Region-wide release gate that can establish "every daemon runs the new release"?
-**Short answer: no.**
-
-Path convention: paths without a prefix are relative to the `spinifex` repo root; `mulga/`, `predastore/`, `viperblock/` and `ansible/` paths are relative to the umbrella repo.
-Shell scripts, Ansible and docs were read, not run.
-
-## 1. How a node advertises its build version
-
-| Mechanism | Evidence | What it carries | Cluster-visible? |
-|---|---|---|---|
-| Build-time ldflags | `Makefile:60-62` | `git describe --tags --always --dirty` and short commit into `operator/cli.Version`/`Commit` | No; baked into the binary |
-| `spx version` | `spinifex/operator/cli/version.go:12-22` | Prints `Version (Commit) os/arch` of the local binary | No; local process only |
-| Gateway `GetVersion` | `spinifex/operator/cli/service.go:613`, `spinifex/runtime/roles/awsgw/awsgw.go:58-71,487-488`, `spinifex/gateway/spinifex.go:61-62`, `spinifex/gateway/spx/spx.go:27-39` | Version of whichever awsgw process answered the request | Only for the answering gateway; no per-node fan-out |
-| Daemon heartbeat (KV `spinifex-cluster-state`, key `heartbeat.<node>`) | `spinifex/daemon/jetstream.go:224-236,247`, `spinifex/daemon/heartbeat.go:56-72` | Node, config `Epoch`, services, capacity | Yes, but **no build version field** |
-| Node health (`spinifex.admin.<node>.health`, HTTPS `/health`) | `contracts/cluster/v1/health.go:15-24` | `ConfigHash`, `Epoch`, uptime, services | Yes, **no build version** |
-| Node status fan-out (`spinifex.node.status`, `spx get nodes`) | `contracts/cluster/v1/status.go:34-68`, `spinifex/gateway/spx/spx.go:47-60` | Capacity, roles, GPUs | Yes, **no build version** |
-| Node discovery (`spinifex.nodes.discover`) | `contracts/cluster/v1/discovery.go:14-16` | Node name only | Yes, no version |
-| Formation join | `spinifex/runtime/formation/formation.go:25-37,70-80` | Name, IPs, region/AZ, services | Join exchange has **no version or protocol field** |
-| `spinifex.toml` `version` / `epoch` | `spinifex/operator/cli/templates/spinifex.toml:2-3`, `spinifex/bootstrap/config/config.go:19-21`, `spinifex/daemon/daemon.go:2743-2757` | Config-format version ("4") and leader-bumped config epoch, hashed into `ConfigHash` | Indirectly via `ConfigHash`; this is config agreement, not binary version |
-| Deploy manifest file | `mulga/scripts/update-nodes.sh:165-189,320-321` | Five sub-repo describe strings, written to `/etc/spinifex/deploy-manifest.txt` and the operator's `~/.spinifex/deploy-manifests/` | No; a file on disk, only written by this dev script |
-
-**Finding:** there is no cluster-wide record of which release each member runs.
-Every version observation is either local (`spx version`, deploy manifest) or names one process (`GetVersion`).
-The only cross-node comparison is done by shell scripts over SSH (Section 2).
-
-## 2. How upgrades are performed today
-
-| Path | Evidence | Ordering | Drain | Health gate | Version gate |
-|---|---|---|---|---|---|
-| Online installer re-run (`curl … \| bash`) | `docs/admin/update/README.md:31-53`, `scripts/setup.sh:828-847,1187-1194` | Per node, operator-driven; no cluster coordination | Whatever `spinifex.target` stop does | None in the script beyond service restart | Optional `--version` pin, SHA-256 verified; no peer comparison |
-| Manual: install binary, `spx admin upgrade`, restart | `docs/admin/update/README.md:64-100`, `spinifex/operator/cli/admin_upgrade.go:55-120` | Per node | None | None | None; applies local config migrations and systemd unit reconcile only |
-| `mulga/scripts/update-nodes.sh` (dev/test) | `mulga/scripts/update-nodes.sh:29-38,218-243,329-348,359-377` | **Stop-the-world**: `spx admin cluster shutdown`, update all, start all | Coordinated GATE→DRAIN→STORAGE→PERSIST→INFRA with all-node ACKs | Retries `spx get nodes` until no `NotReady` | Prints `spx version` per host after the fact; header states mixed NATS/predastore/OVN versions are deliberately avoided |
-| Ansible `cluster-deploy.yml` (dev clusters) | `ansible/playbooks/cluster-deploy.yml:1-37`, `ansible/roles/deploy/tasks/main.yml:97-109,372,390,443-459,516-534` | **Rolling** `serial: 1`, `max_fail_percentage: 0` | `systemctl stop spinifex.target` (drain via `spinifex-shutdown.service`) | awsgw port, target active, predastore Raft quorum rejoin, stale-inode restart | None; no `spx admin upgrade` step, so the cluster runs mixed builds mid-roll with no compatibility check |
-| `spinifex/scripts/install-node.sh` (formation) | `scripts/install-node.sh:9-11,286-304,347-354` | n/a; forms a cluster | n/a | Smoke test | Refuses to form if `spx version` strings differ (exact match, SSH) |
-| Coordinated shutdown | `spinifex/operator/cli/admin.go:105-112`, `spinifex/daemon/shutdown.go:29-45` | Phase order with ACKs | Yes | ACK per phase | None |
-| Local drain | `spinifex/operator/cli/admin.go:120-133` | Local node | GATE+DRAIN only | n/a | None |
-| `spx admin preflight` | `spinifex/operator/cli/admin_preflight.go:15-28`, `docs/admin/update/README.md:156-183` | Local | n/a | Detects a unit still executing a replaced `spx` inode | Local only; not a peer comparison |
-
-The two multi-node paths disagree: `update-nodes.sh` avoids mixed versions by stopping the cluster, while `cluster-deploy.yml` rolls one node at a time and so runs N and N-1 together with nothing proving they interoperate.
-A known mixed-version hazard is already documented: an older daemon ignores the `Target` field of `ShutdownRequest` and treats a single-node drain as cluster-wide (`spinifex/daemon/shutdown.go:34-37`, `docs/admin/host-lifecycle/README.md:123`).
-RFC2-18 (`mulga/docs/specs/spinifex/rfc2-18_operator-lifecycle.md:34-53,88-104,165-177`, DRAFTING) records the same observations and states that after-the-fact version printing is not a compatibility gate.
-
-## 3. Feature flags, capability negotiation, minimum-version gates
-
-| Mechanism | Evidence | Scope | Shape |
-|---|---|---|---|
-| EBS provider capabilities | `spinifex/providers/ebs/provider.go:55-70`, `spinifex/providers/ebs/viperblock/provider_handlers.go:236-262`, `spinifex/daemon/daemon.go:2397` | Spinifex ↔ block provider | Boolean feature set fetched at runtime; the only real capability exchange found |
-| EBS provider wire version | `spinifex/providers/ebs/provider.go:13-25`, `spinifex/providers/ebs/errors.go:90-95`, `spinifex/providers/ebs/nats.go:212-316` | Every provider request/response | **Exact match** (`!= SchemaVersion` → `ErrUnsupportedVersion`); no N-1 window |
-| RDS data-volume contract image tag | `scripts/images/rds-postgres/manifest.conf:49-51`, `scripts/images/rds-mariadb/manifest.conf:59`, `spinifex/handlers/rds/launch.go:39-45,532`, `spinifex/operator/imagecatalog/images.go:666,686` | Control plane → guest image | AMI filter `rds-data-volume-contract=format-auth-v1` excludes older images; a static image-tag capability, not a runtime proof |
-| RDS bootstrap envelope | `spinifex/handlers/rds/bootstrap.go:23-36,190-192,212` | Host ↔ RDS guest | `envelopeVersion` exact match |
-| KV schema-ahead refusal | `spinifex/foundation/state/migrate/kv.go:55-67,87-89` | Any process opening a versioned bucket | Older build refuses to open a bucket stamped newer; fail-closed, not negotiation |
-| systemd unit marker | `spinifex/operator/host/systemd/reconcile.go:28-33,131-140`, `spinifex/operator/host/systemd/units_gen.go` | Host units | `# spinifex-unit-version: N`; replace only when older |
-
-Searches for `featuregate`, `feature_flag`, `minVersion` (non-TLS), `minimumVersion`, `semver` and version handshakes in Spinifex found nothing else.
-There is no "all members at version ≥ X" check, no release gate record and no cluster capability set.
-
-## 4. Persisted-state versions, migrations and rollback
-
-### 4.1 KV bucket schema versions
-
-| Item | Evidence | Notes |
-|---|---|---|
-| Version stamp | `spinifex/foundation/state/kvutil/state.go:3-4`, `spinifex/foundation/state/kvutil/kvutil.go:211-244` | `_version` key per bucket; `WriteVersion` is read-then-unconditional-put, not CAS |
-| Migration runner | `spinifex/foundation/state/migrate/kv.go:17-137` | Chain validation, run, stamp after each step; runs at bucket-open time in whichever process opens it |
-| Bucket-level versions | e.g. `spinifex/daemon/jetstream.go:53-55,139,154,187`, `spinifex/domains/ec2/vpc/service_impl.go:42-44,171-211`, `spinifex/handlers/rds/store.go:27-38,184,209` | About 40 constants, almost all `1`; IPAM, security groups and static pool are at `2` with **no registered migration** (`spinifex/domains/ec2/vpc/ipam.go:17`, `spinifex/domains/ec2/vpc/security_group.go:128`, `spinifex/domains/network/external/static_pool.go:22`) |
-| Registered migrations | `spinifex/daemon/instance_records_migrate.go:31-72`, `spinifex/migrate/VERSIONS.md` | Only `spinifex-instance-state` (1→5) and `spinifex-terminated-instances` (1→3) |
-| Catalogue | `spinifex/migrate/VERSIONS.md` | Lists three KV buckets; the other constants are not catalogued |
-
-A bucket stamped `1` whose target became `2` with no registered step fails `RunKV` with "no migrations registered" (`spinifex/foundation/state/migrate/kv.go:110-111`); `VERSIONS.md` says those installs required `spx admin init --force`, which is a hard cutover with no documented Q-75 exception record.
-
-### 4.2 Expand/contract precedent
-
-The instance-record crossing is the only expand–migrate–contract precedent found.
-`VERSIONS.md` states version 5 of `spinifex-instance-state` froze the legacy `instance.<id>` and `node.<id>` keys and left them in place so the crossing can be rolled back, and the tests cover idempotent re-run and not overwriting fresher records (`spinifex/daemon/instance_records_transition_test.go:221-315`).
-It is not a full Prepare→Activate→Finalise sequence: activation happens when the first upgraded process opens the bucket, the bump itself makes the older build refuse to start (`SchemaAheadError`), and there is no operator finalisation step that deletes the frozen keys.
-So "rollback" means a binary downgrade is refused rather than supported; it would need the version stamp to be reset by hand (not documented).
-
-### 4.3 Record- and file-level schema versions
-
-| Format | Evidence | Rule |
-|---|---|---|
-| EBS volume/snapshot/AMI metadata documents | `spinifex/domains/ec2/ebs/metadata/metadata.go:14-17,166-221` | `schema_version` = 2; decode rejects anything `!= 2` as corrupt, so N-1 cannot read N's documents and vice versa |
-| Node local state (disk + KV `node.<id>`) | `spinifex/daemon/local_state.go:14-34,131-133` | Exact match `1` |
-| Config files | `spinifex/migrate/VERSIONS.md`, `spinifex/migrate/migrate.go:191-246,327-332` | Per-file version; `RunConfig` backs up before each step; no config migrations registered |
-| Object-store migrations | `spinifex/migrate/migrate.go:126-161`, `spinifex/migrate/VERSIONS.md` | Stamp in a KV bucket, read-then-write; none registered |
-| Predastore cluster config | `predastore/internal/config/config.go:25-28,358` | `Version = 1`, exact match, "no migration between versions" |
-| Predastore placement record | `predastore/internal/gate/handlers/placement_record.go:17-48,85,125-139` | Readers accept v1–v3 and reject unknown; writer always emits v3 (not "oldest compatible") |
-| Viperblock WAL/chunk/checkpoint headers | `viperblock/viperblock/snapshot.go:346-349`, `viperblock/viperblock/viperblock.go:3622,4788,6532` | Magic + `Version` uint16, exact match |
-| Viperblock flat-section version byte | `viperblock/viperblock/snapshot.go:634,726-729` | Reserved, not checked |
-
-No resource record carries a per-record `schemaVersion` today; the EKS add-on design proposes adding one (`eks-addon-lifecycle-design.md:119,421`).
-
-### 4.4 Rollback
-
-No rollback command, procedure or test exists for the platform release.
-`spx admin upgrade` has no reverse mode (`spinifex/operator/cli/admin_upgrade.go:55-71`); a downgraded unit marker is reported as a conflict rather than restored (`spinifex/operator/host/systemd/reconcile.go:133-140`); config backups are files, not managed restore points (RFC2-18 C-127 at `mulga/docs/specs/spinifex/rfc2-18_operator-lifecycle.md:97-104`).
-The update runbook covers forward upgrade and stale-binary troubleshooting only (`docs/admin/update/README.md:31-213`).
-
-## 5. Guest images and agents
-
-| Guest | Image version source | Agent version | Control plane records | Upgrade path |
-|---|---|---|---|---|
-| EKS node (CP and worker) | `scripts/images/eks-node/manifest.conf:2,4,25,87-88`, `scripts/images/eks-node/setup.sh:16` (K3s `v1.32.5+k3s1`), catalogue `spinifex/operator/imagecatalog/images.go:576-590` (Version = Alpine `3.21.7`, mutable URL, sidecar SHA-256) | Guest binaries built without version ldflags; no guest reports one | `ClusterMeta.Version` = requested Kubernetes version, default `1.32`, not validated against the image (`spinifex/handlers/eks/service_impl.go:291,587`); `ControlPlaneInstanceID` (`spinifex/handlers/eks/cluster_state.go:109`); nodegroup `ReleaseVersion` is a fixed placeholder `1.0.0-spinifex-eks-node` (`spinifex/handlers/eks/nodegroup.go:43-46,296-299`); `PlatformVersion` is constant `eks.1` (`spinifex/handlers/eks/service_impl.go:2188-2191`) | None: `UpdateClusterVersion`, `UpdateNodegroupVersion`, `ListUpdates` and `DescribeUpdate` have no gateway route at this revision (`spinifex/gateway/eks.go`, 31 routes, none match); plan only in `mulga/docs/development/feature/eks-cluster-version-upgrade.md` |
-| EKS add-on bundles | Baked per version into the image under `addons/<name>/<version>` (`scripts/images/eks-node/mulga-eks-addon-sync.sh:38,81-87`) | Add-on status report carries add-on version only (`contracts/eks/v1/addon_status.go:31-37`) | Add-on record `addonVersion` | A new add-on version needs a new image; v1 guest protocol has no protocol version or capability field |
-| ECS container instance | `scripts/images/ecs-agent/manifest.conf:4,23,40-41` | `version = "dev"` unless ldflags set (`cmd/ecs-agent/main.go:18-24`); `Makefile:78` sets it but the image manifest build does not, so imaged agents likely report `dev` (not verified live) | `AgentVersion` stored and echoed in `VersionInfo` (`spinifex/handlers/ecs/records.go:237`, `spinifex/handlers/ecs/instances.go:283,333`); never compared | Relaunch capacity; no agent-update path |
-| RDS (postgres, mariadb) | `scripts/images/rds-postgres/manifest.conf:4,19,47-51`, `scripts/images/rds-mariadb/manifest.conf:4,21,55-59`, tags `engine`, `engine-version`, `rds-data-volume-contract` | `version = "dev"` default (`cmd/rds-agent/main.go:21-23,72`); image build sets no ldflags | `Agent.AgentVersion`, `EngineVersion` recorded (`spinifex/handlers/rds/records.go:405-415`, `spinifex/handlers/rds/agent.go:189`); never compared; AMI ID used at launch is logged, not stored on the DB record (`spinifex/handlers/rds/launch.go:189,267,353`) | `ModifyDBInstance EngineVersion` returns unimplemented (`spinifex/handlers/rds/modify.go:477-478`) |
-
-AMI selection for EKS, RDS and ECS takes the newest image matching tags (`spinifex/handlers/eks/k3s_server_vm.go:353-363`, `spinifex/handlers/rds/launch.go:552-558`, `spinifex/handlers/ecs/capacity.go:108-119`), so importing a new image silently changes what new resources receive while existing ones keep the old one.
-The control plane can find a guest's AMI only indirectly through the backing EC2 instance; it cannot learn the guest's protocol support from the guest.
-
-## 6. Offline artefacts and backups
-
-| Artefact | Evidence | Version field |
-|---|---|---|
-| EBS snapshot / AMI metadata | `spinifex/domains/ec2/ebs/metadata/metadata.go:57-58,79-80` | `schema_version` 2, exact match |
-| Viperblock snapshot checkpoint | `viperblock/viperblock/snapshot.go:346-349` | Header magic + version, exact match |
-| RDS DB snapshot record | `spinifex/handlers/rds/records.go:291-320` | `EngineVersion` only; no record schema version; data lives in an EC2 snapshot |
-| EKS etcd backups (`eks-backups-system` bucket) | `spinifex/handlers/eks/restore_snapshot.go:19-22,53-75` | Name-encoded tier and timestamp only; no K3s/etcd version recorded |
-| Account export | Not found | n/a |
-
-No artefact declares a support or retirement window, and none records the Spinifex release that wrote it.
-
-## 7. Gaps against Prepare → Activate → Migrate → Finalise and N/N-1
-
-Stated as gaps only; no design is implied.
-
-| # | Need | Current state | Gap |
-|---|---|---|---|
-| G1 | Authoritative record of each member's release | Heartbeat, health, status and formation carry no build version (Section 1) | No release-membership data source exists; activation evidence ("every writer can preserve the transitional form") cannot be produced |
-| G2 | Region-wide activation gate | None; only `install-node.sh` exact string match over SSH at formation | Nothing operator-visible records "activated"; the EKS add-on slice 7 would have to build it |
-| G3 | Finalisation step and preflight | Frozen instance keys are never retired; no finalise command | No operator-visible end of rollback for any change |
-| G4 | Controlled rollback | Schema-ahead refusal blocks downgrade; no reverse migration, no unit downgrade, no runbook | Rollback is unsupported once any bucket is bumped |
-| G5 | N/N-1 wire compatibility | Most contracts unversioned (C-9); versioned ones use exact match (EBS provider, RDS envelope, local state, EBS metadata, viperblock headers) | No contract implements a two-version window or "emit oldest compatible" |
-| G6 | Capability exchange | Only EBS provider booleans and an RDS image tag | No authenticated per-peer or per-guest capability record; no invalidation on replacement |
-| G7 | Fenced, owned migration | Migrations run on bucket open in any process; version stamp is read-then-put | No migration owner, fence or per-record progress; concurrent runners rely on idempotence |
-| G8 | Per-record schema version | Bucket-level only | Lazy per-record migration under a resource owner has no field to key on |
-| G9 | Coherent rolling procedure | Stop-the-world script vs `serial: 1` Ansible roll with no version or compatibility check; known mixed-version drain hazard | No supported rolling-upgrade contract with a stop-on-skew rule |
-| G10 | Guest version visibility | Agent versions recorded but default `dev` in images and never compared; EKS guests report none | Control plane cannot tell a v1 from a v2-capable guest except by a protocol it does not yet have |
-| G11 | Guest upgrade operations | EKS version/nodegroup updates unrouted; RDS engine upgrade unimplemented; newest-AMI-wins selection | No owned path to move a guest to a new image, and image choice is implicit |
-| G12 | Release manifest | `update-nodes.sh` writes a dev-only text manifest; installer pins one tarball | No immutable release manifest binding binary, contract versions and supported source versions (RFC2-18 Q-183 target) |
-| G13 | Artefact lifecycle | Snapshot/backup formats lack writer release and support window; etcd backups record no K3s version | Retained artefacts cannot be checked for usability before a format is retired |
-| G14 | Version catalogue accuracy | `VERSIONS.md` lists 3 of about 40 KV buckets | No single inventory of persisted formats to drive a finalisation preflight |

docs/package-boundary/sts-oidc-discovery-projection-plan.md

@@ -1,75 +0,0 @@
-# STS OIDC discovery projection plan
-
-**Status: Proposed** — scope is ADR-0005 S4 only; the Q-22 cut uses an EKS-owned in-process projection, not a wire contract.
-
-## Current state
-
-- `handlers/sts/oidc_jwks.go:14` imports `handlers/eks` for exactly two symbols: `AccountBucketName` (line 101) and `OIDCJWKSKey` (line 110).
-- No EKS store type, record or `ClusterMeta` is used by STS production code; the JetStream access is `s.js.KeyValue("eks-account-{issuerAccount}")` then `kv.Get("clusters/{cluster}/oidc-jwks.json")` (`oidc_jwks.go:93-126`).
-- The only caller is the `AssumeRoleWithWebIdentity` key function (`handlers/sts/assume_role_with_web_identity.go:159-193`): parse the `iss` claim (`ParseEKSIssuerURL`, line 173), check the IAM OIDC-provider registration in the role account (line 177), read the JWKS from the issuer account (line 180), select by `kid` (line 187).
-- Every failure in the key function becomes `InvalidIdentityToken` (HTTP 400) at line 71; there is no cache, so each request reads KV afresh.
-- The public `/oidc/eks/{region}/{accountID}/{clusterName}/.well-known/openid-configuration` and `/keys` routes are served by the root `gateway` package (`gateway/gateway.go:351-352`, `gateway/oidc_discovery.go`), not by STS, and are outside the Q-22 rule because the root gateway package has no domain.
-- The composition root is `runtime/roles/awsgw/awsgw.go:222` (`NewSTSServiceImpl`), which already wires one cross-service capability at line 228 (`SetSessionRevoker`).
-- The JWKS key is written once per cluster launch (`handlers/eks/service_impl.go:874` → `oidc_keypair.go:94`) as a single-key document, read by the EKS bootstrap cross-check (`nats_bootstrap.go:283`), and removed only by `DeleteClusterPrefix` when teardown completes (`service_impl.go:1442`); there is no rotation path.
-
-## Resolver (consumer-owned, in `handlers/sts`)
-
-```go
-// ClusterJWKSResolver returns the published JWKS document for an EKS cluster issuer.
-type ClusterJWKSResolver interface {
-	LookupClusterJWKS(ctx context.Context, accountID, clusterName string) (jwks []byte, found bool, err error)
-}
-```
-
-| Today's outcome | Resolver result | STS result (unchanged) |
-| --- | --- | --- |
-| Account bucket missing (`ErrBucketNotFound`) | `nil, false, nil` | `InvalidIdentityToken` |
-| Cluster missing, key absent, key deleted by teardown | `nil, false, nil` | `InvalidIdentityToken` |
-| Bucket open or key read fails (timeout, unavailable) | `nil, false, err` | `InvalidIdentityToken` |
-| Key present, any bytes | `raw, true, nil` | STS decodes; malformed or empty set is `InvalidIdentityToken` |
-
-- The resolver returns raw bytes so JSON decoding, the empty-set rule and `JWK`/`JWKS` types stay in STS, and the implementation never imports `handlers/sts`.
-- `FetchClusterJWKS` keeps its `(nil, nil)` / error contract, taking the resolver instead of `jetstream.JetStream`; `ParseEKSIssuerURL` and the registration-before-lookup ordering stay in STS.
-- `NewSTSServiceImpl` takes the resolver as a required argument and rejects nil, mirroring the nil-IAM check, so a missing binding fails at start-up rather than as a permissive path.
-
-## Placement: `handlers/eks`, not a new domain package
-
-- Rejected: a new `domains/eks/projection` package. `Domains_ImportNoAgentsIngressOrLegacy` (`architecture/layering/rules_test.go:130`) forbids any new-path `domains/*` package from importing a `handlers/*` legacy root, and the bucket/key layout lives only in `handlers/eks` today, so a projection package could not read it without that forbidden edge.
-- Instead: `handlers/eks` implements the resolver itself, beside the layout it already owns. `OIDCJWKSReader` (`handlers/eks/oidc_jwks_reader.go`), built with `NewOIDCJWKSReader(js jetstream.JetStream)`, does the same bucket-open-then-`Get`, the same timeouts and the same not-found classification `handlers/sts/oidc_jwks.go` used, next to `AccountBucketName` and `OIDCJWKSKey` in `store.go`.
-- This is transitional: `handlers/eks` is the current owner of the layout, not a permanent home for a cross-domain adapter. When `handlers/eks` is extracted to a `domains/eks` package, the resolver implementation moves with it.
-- `handlers/eks` must not import `handlers/sts` (that edge is still forbidden), so the compile-time interface check `var _ handlers_sts.ClusterJWKSResolver = (*handlers_eks.OIDCJWKSReader)(nil)` lives in the composition root, `runtime/roles/awsgw`, not beside the implementation.
-
-## Wiring
-
-- `runtime/roles/awsgw/awsgw.go:222` builds `handlers_eks.NewOIDCJWKSReader(js)` and passes it to `NewSTSServiceImpl` as a required, nil-checked argument.
-- `tests/integration/harness.go:150` is the only other production-shaped STS construction site and is wired identically.
-
-## Imports and allowlist
-
-- Disappears: `handlers/sts/oidc_jwks.go` `handlers_eks` import, the last non-test STS import of any EKS package.
-- Goes stale and is deleted: `{From: "handlers/sts", To: "handlers/eks"}` (`architecture/layering/allowlist_test.go:161`); the layering test fails on a stale entry, so it cannot be left in place.
-- Stays: `handlers/sts → handlers/iam` and `gateway/sts → handlers/iam`, because the IAM registration read, trust policy and session crypto are out of scope.
-- Test files that exercise the real KV layout move to `handlers/eks`, next to the layout they pin.
-- Every remaining `handlers/sts` test, including the web-identity fixtures and the `newTestSetup` constructor wiring, uses a test-owned `fakeJWKSResolver` and imports no EKS package.
-- The `Q22_STSImportsNoEKSImplementation` rule stays and now passes with no exception: STS imports no EKS package at all, not even through an allowlist entry, and that holds for test files too (`grep -rn 'handlers/eks' spinifex/handlers/sts/` is empty).
-
-## Test placement (post-cut)
-
-- `handlers/eks/oidc_jwks_reader_test.go`: the raw KV layout and literal bucket/key pin (`TestOIDCJWKSReader_UsesLiteralBucketAndKey`), missing bucket (`TestOIDCJWKSReader_AccountBucketMissingIsNotFound`), dependency failures and a tombstoned key (`TestOIDCJWKSReader_DependencyFailureSemantics`), input validation (`TestOIDCJWKSReader_NilJetStream`, `TestOIDCJWKSReader_EmptyArgs`), cluster status never consulted (`TestCurrentBehaviour_ClusterRecordStateIsNotConsulted`, moved from `handlers/sts`), and a deleted cluster prefix reading as not found (`TestCurrentBehaviour_ClusterPrefixDeletedIsNotFound`, moved from `handlers/sts`).
-- `handlers/sts/fake_jwks_resolver_test.go`: the `fakeJWKSResolver` test double — a map keyed by `(accountID, clusterName)` with seed/seedErr and call recording — used by every STS test in this package.
-- `handlers/sts/oidc_jwks_test.go`: `FetchClusterJWKS` unit tests against the fake, including `TestFetchClusterJWKS_PropagatesResolverErrorUnchanged` (renamed from `..._ResolverErrorIsWrapped`), which pins that a resolver error passes through unchanged via `assert.Same` and `errors.Is`.
-- `handlers/sts/oidc_jwks_current_behaviour_test.go`: the STS-owned outcomes — registration runs before any resolver call (`TestCurrentBehaviour_UnregisteredIssuerNeverCallsResolver`), the issuer's account is queried, not the role's (`TestCurrentBehaviour_CrossAccountIssuerQueriesIssuerAccount`), every resolver outcome (not-found, malformed JWKS, dependency error) maps to `InvalidIdentityToken` (`TestCurrentBehaviour_ClusterMissingIsInvalidIdentityToken`, `TestCurrentBehaviour_JWKSMalformedIsInvalidIdentityToken`, `TestCurrentBehaviour_ResolverErrorIsInvalidIdentityToken`), and rotation is read fresh each request (`TestCurrentBehaviour_JWKSRotationIsReadFreshEachRequest`).
-- `handlers/sts/service_impl_test.go` and `assume_role_with_web_identity_test.go`: `newTestSetup` wires a `*fakeJWKSResolver`, and the web-identity fixture seeds it directly instead of writing EKS KV records.
-- `tests/integration/sts_eks_oidc_issuer_test.go`: the one test that deliberately imports both `handlers/eks` and `handlers/sts` — `TestEKSClusterOIDCIssuer_RoundTripsThroughSTSParseEKSIssuerURL` proves a `ClusterOIDCIssuer` URL parses through `ParseEKSIssuerURL` to the same `(accountID, clusterName)`.
-- `gateway/oidc_discovery_current_behaviour_test.go` is unchanged: `TestCurrentBehaviour_WebIdentityJWKSMissingWireResponse` still builds a real `OIDCJWKSReader` plus a real STS service and pins the wire HTTP 400 with `<Code>InvalidIdentityToken</Code>`. The root `gateway` package has no domain classification and composes both packages intentionally, so this is not a package-boundary violation.
-- `gateway/oidc_discovery_current_behaviour_test.go` also pins the public discovery/JWKS endpoints: 404/500 bodies, verbatim JWKS bytes and `Cache-Control: public, max-age=300`, unaffected by this cut.
-
-## Non-goals
-
-- Moving IAM state, the OIDC-provider registry read or `handlers_iam` imports.
-- Redesigning web-identity verification, issuer parsing, audience or trust-policy handling, or fetching JWKS over HTTPS as AWS STS does.
-- Moving `handlers/sts` to `domains/sts`, or changing the cluster-status blindness, DELETING/FAILED key retention, or adding rotation.
-
-## Open questions
-
-- Should the root-gateway discovery handlers (`gateway/oidc_discovery.go:38,93`) consume an EKS-owned resolver in a later change, the same pattern as STS? The root gateway's `handlers/eks` import has no allowlist entry today because the root gateway has no domain classification, so it is tracked as debt rather than fixed here; S4 is about STS only.

docs/service-interfaces.yaml

@@ -51,9 +51,10 @@ services:
     # forwards to vpcd / viperblockd / predastore.
     #
     # Source layout: spinifex/runtime/roles/spinifex (entry), spinifex/daemon
-    # (subscribe registration), spinifex/domains/ec2/<resource> and spinifex/handlers/{ecs,elbv2,iam,eks} (per-
-    # subject handler bodies). Selector treats these paths as belonging
-    # to this service via the `additional_paths` field.
+    # (subscribe registration), spinifex/domains/<domain> and the legacy
+    # spinifex/handlers/{ecs,eks,elbv2,iam,rds} packages (per-subject handler
+    # bodies). Selector treats these paths as belonging to this service via
+    # the `additional_paths` field.
     additional_paths:
       - spinifex/daemon
       - spinifex/domains/ec2/account
@@ -148,7 +149,7 @@ services:
       - "elbv2.CreateListener"
       - "elbv2.DeleteListener"
       - "elbv2.DescribeListeners"
-      # ACM-alike certificate store (handlers/acm/service_nats.go topic map).
+      # ACM-alike certificate store (domains/acm; subscribed in daemon.go).
       - "acm.ImportCertificate"
       - "acm.DescribeCertificate"
       - "acm.ListCertificates"
@@ -426,7 +427,7 @@ suites:
   e2e-ecr:
     path: tests/e2e/ecr
     covers: [awsgw, spinifex_daemon, predastore]
-    # ECR control plane (handlers/ecr, gateway/ecr*) + OCI data plane: blobs and
+    # ECR control plane (domains/ecr, gateway/ecr*) + OCI data plane: blobs and
     # manifests in predastore, control + auth bridge in awsgw, KV via
     # spinifex_daemon. docker/crane/skopeo data-plane subtests skip when the
     # client binary or registry-host resolution is absent.

docs/terraform-workbooks/demo-app/Dockerfile

@@ -4,13 +4,13 @@
 # distroless base (no shell, no package manager) for a small, production-shaped
 # image suitable for pushing to ECR and pulling onto EKS workers.
 
-FROM golang:1.27-alpine AS build
+FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build
 WORKDIR /src
 COPY go.mod ./
 COPY main.go ./
 RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/spinifex-demo .
 
-FROM gcr.io/distroless/static:nonroot
+FROM gcr.io/distroless/static:nonroot@sha256:e2e927ec666bae08560abb3c55d0659eceabb657f56b6782ab500a9fc7f555e3
 COPY --from=build /out/spinifex-demo /spinifex-demo
 EXPOSE 8080
 USER nonroot:nonroot

docs/terraform-workbooks/demo-app/go.mod

@@ -1,3 +1,3 @@
 module github.com/mulgadc/spinifex-demo
 
-go 1.27.0
+go 1.27.2

docs/testing/e2e-coverage.md

@@ -28,7 +28,7 @@ The largest suite, 25 top-level tests. Notable entry points:
 - `TestWANEgressSMTPBlock` — egress port blocking.
 - `TestInstanceMetadata` — IMDS from inside a guest.
 - `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`.
-- `TestNegativeErrorPaths` — the `8a`–`8h` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, and so on).
+- `TestNegativeErrorPaths` — the `8a`–`8t` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, unknown or malformed instance IDs on the lifecycle verbs, and so on).
 - **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across all three host and network permutations.
 
 ### `natuplink` — routed NAT (`external_mode = "nat"`), one node or three

go.mod

@@ -1,81 +1,81 @@
 module github.com/mulgadc/spinifex
 
-go 1.27.0
+go 1.27.2
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.47.1
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20
-	github.com/aws/aws-sdk-go-v2/config v1.33.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.20.6
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1
+	github.com/aws/aws-sdk-go-v2 v1.47.3
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22
+	github.com/aws/aws-sdk-go-v2/config v1.33.9
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.9
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/containerd/containerd/v2 v2.4.1
+	github.com/containerd/containerd/v2 v2.4.2
 	github.com/distribution/reference v0.6.0
-	github.com/go-chi/chi/v5 v5.3.2
+	github.com/go-chi/chi/v5 v5.5.0
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
 	github.com/jackc/pgx/v5 v5.11.0
-	github.com/klauspost/compress v1.20.1
-	github.com/klauspost/cpuid/v2 v2.4.0
+	github.com/klauspost/compress v1.20.3
+	github.com/klauspost/cpuid/v2 v2.4.1
 	github.com/miekg/dns v1.1.73
 	github.com/minio/highwayhash v1.0.4
 	github.com/mulgadc/bluebottle v1.22.0
 	github.com/mulgadc/northstar v1.22.0
 	github.com/mulgadc/predastore v1.22.0
 	github.com/mulgadc/viperblock v1.22.0
-	github.com/nats-io/nats-server/v2 v2.15.0
+	github.com/nats-io/nats-server/v2 v2.15.1
 	github.com/nats-io/nats.go v1.54.0
 	github.com/opencontainers/runtime-spec v1.3.0
-	github.com/oracle/oci-go-sdk/v65 v65.126.1
+	github.com/oracle/oci-go-sdk/v65 v65.127.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
 	github.com/pelletier/go-toml/v2 v2.4.3
-	github.com/pterm/pterm v0.12.83
+	github.com/pterm/pterm v0.12.84
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
-	go.opentelemetry.io/otel v1.46.0
-	go.opentelemetry.io/otel/metric v1.46.0
-	go.opentelemetry.io/otel/sdk v1.46.0
-	go.opentelemetry.io/otel/sdk/metric v1.46.0
-	go.opentelemetry.io/otel/trace v1.46.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
+	go.opentelemetry.io/otel v1.47.0
+	go.opentelemetry.io/otel/metric v1.47.0
+	go.opentelemetry.io/otel/sdk v1.47.0
+	go.opentelemetry.io/otel/sdk/metric v1.47.0
+	go.opentelemetry.io/otel/trace v1.47.0
 	go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b
 	go.yaml.in/yaml/v3 v3.0.5
-	golang.org/x/crypto v0.57.0
-	golang.org/x/sync v0.23.0
-	golang.org/x/sys v0.48.0
-	golang.org/x/term v0.46.0
+	golang.org/x/crypto v0.58.0
+	golang.org/x/sync v0.24.0
+	golang.org/x/sys v0.49.0
+	golang.org/x/term v0.47.0
 	golang.org/x/time v0.16.0
-	golang.org/x/tools v0.50.0
+	golang.org/x/tools v0.52.0
 	gopkg.in/ini.v1 v1.67.3
 )
 
 require (
 	atomicgo.dev/cursor v0.2.0 // indirect
-	atomicgo.dev/keyboard v0.2.9 // indirect
+	atomicgo.dev/keyboard v0.2.10 // indirect
 	atomicgo.dev/schedule v0.1.0 // indirect
 	github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
 	github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
 	github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect
-	github.com/aws/smithy-go v1.28.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
+	github.com/aws/smithy-go v1.28.5 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
@@ -122,7 +122,6 @@ require (
 	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/go-tpm v0.9.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/gookit/color v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -138,6 +137,7 @@ require (
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
+	github.com/jstemmer/go-junit-report/v2 v2.1.0 // indirect
 	github.com/klauspost/reedsolomon v1.14.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
@@ -145,7 +145,7 @@ require (
 	github.com/mattn/go-colorable v0.1.15 // indirect
 	github.com/mattn/go-isatty v0.0.24 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
-	github.com/mattn/go-runewidth v0.0.21 // indirect
+	github.com/mattn/go-runewidth v0.0.24 // indirect
 	github.com/mdlayher/packet v1.1.2 // indirect
 	github.com/mdlayher/socket v0.6.0 // indirect
 	github.com/minio/crc64nvme v1.1.1 // indirect
@@ -190,15 +190,15 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
-	go.opentelemetry.io/otel/log v0.22.0 // indirect
+	go.opentelemetry.io/otel/log v1.47.0 // indirect
 	go.opentelemetry.io/otel/sdk/log v0.22.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
 	golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 // indirect
-	golang.org/x/mod v0.41.0 // indirect
-	golang.org/x/net v0.59.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 // indirect
-	golang.org/x/text v0.42.0 // indirect
+	golang.org/x/mod v0.42.0 // indirect
+	golang.org/x/net v0.61.0 // indirect
+	golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a // indirect
+	golang.org/x/text v0.43.0 // indirect
 	golang.org/x/vuln v1.6.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect
@@ -218,6 +218,7 @@ require (
 replace google.golang.org/genproto => google.golang.org/genproto v0.0.0-20260622175928-b703f567277d
 
 tool (
+	github.com/jstemmer/go-junit-report/v2
 	go.uber.org/nilaway/cmd/nilaway
 	golang.org/x/vuln/cmd/govulncheck
 )

go.sum

@@ -2,19 +2,12 @@ atomicgo.dev/assert v0.0.2 h1:FiKeMiZSgRrZsPo9qn/7vmr7mCsh5SZyXY4YGYiYwrg=
 atomicgo.dev/assert v0.0.2/go.mod h1:ut4NcI3QDdJtlmAxQULOmA13Gz6e2DWbSAS8RUOmNYQ=
 atomicgo.dev/cursor v0.2.0 h1:H6XN5alUJ52FZZUkI7AlJbUc1aW38GWZalpYRPpoPOw=
 atomicgo.dev/cursor v0.2.0/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU=
-atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
-atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
+atomicgo.dev/keyboard v0.2.10 h1:v7mvUKUZLHIggxULEIuWbT+WkkyQSgdbA201EziAhHU=
+atomicgo.dev/keyboard v0.2.10/go.mod h1:ap/z5ilnhLqYq852m6kPeTq5Z6aESGWu5mzRpJlC6aI=
 atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
 atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
-github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
-github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
-github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII=
-github.com/MarvinJWendt/testza v0.2.10/go.mod h1:pd+VWsoGUiFtq+hRKSU1Bktnn+DMCSrDrXDpX2bG66k=
-github.com/MarvinJWendt/testza v0.2.12/go.mod h1:JOIegYyV7rX+7VZ9r77L/eH6CfJHHzXjB69adAhzZkI=
-github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/2oUqKc6bF2c=
-github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE=
 github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4=
 github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY=
 github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 h1:0kQAzHq8vLs7Pptv+7TxjdETLf/nIqJpIB4oC6Ba4vY=
@@ -25,47 +18,46 @@ github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMl
 github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
-github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
-github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
-github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
-github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
+github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
+github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ=
+github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
+github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
-github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
-github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
-github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM=
-github.com/aws/smithy-go v1.28.2 h1:myhcykQcatTul2B/zITjDk203G7t0awUAs1hVry5Bvg=
-github.com/aws/smithy-go v1.28.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM=
+github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
+github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -106,13 +98,12 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ
 github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6dyLYXQ=
 github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw=
-github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc=
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/containerd/api v1.12.0 h1:kuQm82SbDrCuO4n7hf2L8zsBtZLuympyq5X/VotfX2A=
 github.com/containerd/containerd/api v1.12.0/go.mod h1:EBcSzoi9Vl18cdODaXUCskf3D2NT8lsSXeZJnU5jIUc=
-github.com/containerd/containerd/v2 v2.4.1 h1:DUx/ZJN7cEu0WuzHClDB+68H/bqMEH5pWoEjf0ae4hc=
-github.com/containerd/containerd/v2 v2.4.1/go.mod h1:vgLdtvl3prFk1d3ZVqQcsDD5Q9IKM+vAIdU54U85w+I=
+github.com/containerd/containerd/v2 v2.4.2 h1:p6Qg+xbEz8lvwdhHDx0U469xENvp4LgGBx+E7djXkGw=
+github.com/containerd/containerd/v2 v2.4.2/go.mod h1:vgLdtvl3prFk1d3ZVqQcsDD5Q9IKM+vAIdU54U85w+I=
 github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
 github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -163,8 +154,8 @@ github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx5
 github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
 github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
 github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
-github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
-github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
+github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
+github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
 github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -190,6 +181,7 @@ github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9U
 github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
 github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -198,12 +190,6 @@ github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
-github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
-github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
-github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo=
-github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
-github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
@@ -251,20 +237,16 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
-github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
-github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
-github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
-github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
-github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
-github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
+github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc=
+github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ=
+github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
+github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
+github.com/klauspost/cpuid/v2 v2.4.1 h1:I9KxadAkvo+QjtcEE2f2tyfEvmIla/T0m+14PrZNl0U=
+github.com/klauspost/cpuid/v2 v2.4.1/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
 github.com/klauspost/reedsolomon v1.14.2 h1:SafJYwpBBQBI6amHUygcjxZjXeN2HpiENHQDwuPWCCQ=
 github.com/klauspost/reedsolomon v1.14.2/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
@@ -283,9 +265,8 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe
 github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
 github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
 github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
-github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
-github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
+github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
 github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
 github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
 github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU=
@@ -324,8 +305,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
 github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
-github.com/nats-io/nats-server/v2 v2.15.0 h1:M99yf0y05rTr46/qc/Is6ZAowI58Ryp2SjufLCUeVJc=
-github.com/nats-io/nats-server/v2 v2.15.0/go.mod h1:5qLF4CDGzZVFt//3fUrY1ePpwbi05r7QHPNroSUtolk=
+github.com/nats-io/nats-server/v2 v2.15.1 h1:fK3AeYFIji1Smng60jB0hVoAPZezA/UzsaI3BqXFNEI=
+github.com/nats-io/nats-server/v2 v2.15.1/go.mod h1:yeuYRtKVKIFtmMz2/YPk+6hvmNBnk+WwTUzErBdH6IE=
 github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
 github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
 github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
@@ -338,8 +319,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg=
 github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo=
-github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w=
+github.com/oracle/oci-go-sdk/v65 v65.127.0 h1:EfV88zZjJtsIdyALgAYJYL6Pj7Iw11Q2LNtnaGZf+sk=
+github.com/oracle/oci-go-sdk/v65 v65.127.0/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w=
 github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG7acT8gf4=
 github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
 github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
@@ -360,20 +341,12 @@ github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3
 github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
 github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
 github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
-github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
-github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
-github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE=
-github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEejaWgXU=
-github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE=
-github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8=
-github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
-github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA=
-github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk=
+github.com/pterm/pterm v0.12.84 h1:DA9943h1RfwlpcGWCIbD6f6nRTLCe09gK7w6GDXaIzE=
+github.com/pterm/pterm v0.12.84/go.mod h1:LVerZ1tFVFZrTv7lRbLdbrrTLSLAB5oZY8z3IY/z+IQ=
 github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
 github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
 github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A=
 github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
@@ -381,8 +354,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
 github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
-github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
-github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
+github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
 github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
 github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg=
@@ -405,8 +378,6 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/
 github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
 github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
@@ -421,7 +392,6 @@ github.com/tidwall/btree v1.8.2 h1:mTY5iM4UTh2e80TG7RHDQQmnAY3xGNQA/DbU0olkhkA=
 github.com/tidwall/btree v1.8.2/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
 github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA=
 github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264=
-github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
@@ -433,12 +403,12 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=
 go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
 go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0 h1:v4KkRLVvE1cWqTJDfZZkTCG+Z4aolsa6RVos0FX7vqE=
 go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0/go.mod h1:g/xbuPC0XbgwMdKuyF5sKOUUEsorSkN6APydyFP/H9E=
-go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
-go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
+go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
+go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
@@ -447,22 +417,22 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzb
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
-go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=
-go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=
-go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
-go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
-go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU=
-go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4=
-go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
-go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
+go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
+go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
+go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
+go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU=
+go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM=
+go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k=
+go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I=
+go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0=
 go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=
 go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=
 go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg=
 go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s=
-go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
-go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
-go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
-go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
+go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw=
+go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
+go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
+go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
 go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
 go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b h1:FHYZT12VzXnb6P399kd1i9Ai7aLF+CFSUsOFylDt7Lk=
@@ -478,39 +448,35 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
 go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
-golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
+golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
+golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
 golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 h1:GTD/WuaexTazIG/SxLOz4rEKZPDVilmVVC2nz4xhwfE=
 golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
-golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
+golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
+golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
-golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
+golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
+golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
-golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
+golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -518,32 +484,30 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
-golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
-golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 h1:F5BWKvW126NXR74uxkxuc1jQHhm/rwm/J3rSiFyuRs4=
-golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA=
+golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
+golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a h1:wnveMBKHwx8/nIb8WYT5P9vtefdXOvIWxzycCkc5n6Q=
+golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
-golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
+golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
+golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
-golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
+golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
+golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
 golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
 golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
-golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
+golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
+golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
 golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
@@ -562,16 +526,12 @@ google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsok
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
 google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
 gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc=
 pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk=

images/mkosi.profiles/vllm/mkosi.extra/usr/local/sbin/bedrock-serve

@@ -184,7 +184,7 @@ fstype_of() {
 # weights_device_ordinal maps this member's BEDROCK_WEIGHTS_DEVICE (/dev/sdf,
 # /dev/sdg, ...) to its 0-based position among the bundle's members: sdf -> 0,
 # sdg -> 1, and so on, mirroring nthWeightsDevice's own 'f'+n scheme
-# (handlers/bedrock/launch.go) exactly. Named-letter arithmetic, not ASCII
+# (spinifex/domains/ochre/launch.go) exactly. Named-letter arithmetic, not ASCII
 # math: POSIX sh has no portable char-to-int builtin, and a bundle never
 # carries more than a handful of co-served members, so the alphabet is not a
 # real ceiling to enumerate by hand. Prints the ordinal and returns 0, or

internal/awsmodel/smithy.go

@@ -308,8 +308,8 @@ func localName(id string) string {
 	if strings.HasPrefix(id, smithyPrelude) {
 		return id
 	}
-	if separator := strings.LastIndex(id, "#"); separator >= 0 {
-		return id[separator+1:]
+	if _, after, ok := strings.CutLast(id, "#"); ok {
+		return after
 	}
 	return id
 }

internal/testkit/readonlystore/readonlystore.go

@@ -0,0 +1,111 @@
+// Package readonlystore wraps an object store or a JetStream KV bucket so that
+// any write fails the test. It exists to prove a code path is read-only, which
+// an assertion on the final store contents cannot: a write that is undone
+// before the check would pass it.
+package readonlystore
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// ErrWrite is returned by every write after the test has been failed.
+var ErrWrite = errors.New("write to a read-only store")
+
+// ObjectStore fails the test on any write and records every key it reads.
+type ObjectStore struct {
+	objectstore.ObjectStore
+
+	t testing.TB
+
+	mu   sync.Mutex
+	gets []string
+}
+
+var _ objectstore.ObjectStore = (*ObjectStore)(nil)
+
+// NewObjectStore wraps inner for t.
+func NewObjectStore(t testing.TB, inner objectstore.ObjectStore) *ObjectStore {
+	return &ObjectStore{ObjectStore: inner, t: t}
+}
+
+func (s *ObjectStore) GetObject(ctx context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	s.mu.Lock()
+	s.gets = append(s.gets, aws.StringValue(in.Key))
+	s.mu.Unlock()
+	return s.ObjectStore.GetObject(ctx, in)
+}
+
+// Gets returns every key read so far.
+func (s *ObjectStore) Gets() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.gets...)
+}
+
+func (s *ObjectStore) PutObject(_ context.Context, in *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	s.t.Errorf("read-only store: PutObject %s", aws.StringValue(in.Key))
+	return nil, ErrWrite
+}
+
+func (s *ObjectStore) DeleteObject(_ context.Context, in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	s.t.Errorf("read-only store: DeleteObject %s", aws.StringValue(in.Key))
+	return nil, ErrWrite
+}
+
+func (s *ObjectStore) EnsureBucket(_ context.Context, bucket string) error {
+	s.t.Errorf("read-only store: EnsureBucket %s", bucket)
+	return ErrWrite
+}
+
+// KV fails the test on any write to the wrapped bucket.
+type KV struct {
+	jetstream.KeyValue
+
+	t testing.TB
+}
+
+// NewKV wraps inner for t.
+func NewKV(t testing.TB, inner jetstream.KeyValue) *KV {
+	return &KV{KeyValue: inner, t: t}
+}
+
+func (k *KV) write(op, key string) error {
+	k.t.Errorf("read-only KV %s: %s %s", k.Bucket(), op, key)
+	return ErrWrite
+}
+
+func (k *KV) Put(_ context.Context, key string, _ []byte) (uint64, error) {
+	return 0, k.write("Put", key)
+}
+
+func (k *KV) PutString(_ context.Context, key string, _ string) (uint64, error) {
+	return 0, k.write("PutString", key)
+}
+
+func (k *KV) Create(_ context.Context, key string, _ []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
+	return 0, k.write("Create", key)
+}
+
+func (k *KV) Update(_ context.Context, key string, _ []byte, _ uint64) (uint64, error) {
+	return 0, k.write("Update", key)
+}
+
+func (k *KV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
+	return k.write("Delete", key)
+}
+
+func (k *KV) Purge(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
+	return k.write("Purge", key)
+}
+
+func (k *KV) PurgeDeletes(_ context.Context, _ ...jetstream.KVPurgeOpt) error {
+	return k.write("PurgeDeletes", "")
+}

scripts/README.md

@@ -38,14 +38,12 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
 |---|---|
 | `smoke-test.sh` | End-to-end sanity check on a running node: imports an SSH keypair and the Ubuntu AMI, then launches an instance. `--create-vpc` builds the VPC, gateway, subnet and routes first and tears them down after; `--nodes N` launches through a spread placement group so each instance lands on a different physical server. `TEST_GPU=1` adds GPU passthrough checks. |
 | `verify-network.sh` | Asserts a node's datapath matches its declared planes — EIP reachability, which plane carries Geneve, and which interfaces services bind. Catches what `smoke-test.sh` cannot, since none of its traffic leaves the box. |
-| `test-ddil-phase1.sh` | systemd-level smoke test for daemon-local-autonomy tier 1. Destructive on the local node: stops and starts `spinifex-nats` and restarts `spinifex-daemon`, restoring both via a cleanup trap. |
 | `check-coverage.sh` | Fails the build if total Go coverage falls below the minimum threshold, excluding packages exempt from the requirement. |
 | `diff-coverage.sh` | Coverage check scoped to changed lines only, against an auto-detected base ref (`HEAD~1` on main, `origin/main` on dev, otherwise `origin/dev`). |
 | `sync-aws-models.sh` | Replaces the embedded AWS Smithy models under `internal/awsmodel/models/` with the ten service files from `aws/api-models-aws` at the given commit, and records that commit in `internal/awsmodel/model_source.go`. Takes a full 40-character SHA. |
 | `run-bench.sh` | Launches a benchmark against running Ubuntu instances, auto-detecting the AMI for the host architecture. Tracks nbdkit and perf alongside the run. |
 | `disk-performance.sh` | Runs a `fio` random 70/30 read-write benchmark swept across 4k/16k/128k/1M. Writes JSON results to `/tmp/spinifex-disk-bench`; override with `BENCH_DIR`, `OUT_DIR`, `SIZE`, `JOBS`, `BLOCK_SIZES`. |
 | `network-performance.sh` | iperf throughput from several clients to one server, driven over SSH, writing a `summary.txt` of Gbit/s per client. `--server-ip` sets the address clients dial, so pointing it at instance private addresses measures the VPC overlay rather than the external pool. |
-| `workload-performance.sh` | In-guest memory, disk, CPU and network workload test. Installs Go and runs the Badger test suite with jemalloc disabled and reduced parallelism. |
 
 ## Guest images
 
@@ -93,13 +91,6 @@ Installed into the EKS node AMI and run at boot or on a timer. All host communic
 | `mulga-mgmt-net.sh` | Boot oneshot bringing a multi-NIC system VM's interfaces up by MAC from the QEMU fw_cfg netcfg blob, before cloud-init's network stage. cloud-init cannot reliably pick the right NIC out of two. |
 | `mulga-vpc-mtu.sh` | Boot oneshot pinning the VPC data NIC MTU so flannel sizes its VXLAN overlay to fit the OVN geneve underlay. Otherwise large frames are silently dropped while handshakes pass. |
 
-## Launch helpers
-
-| Script | What it does |
-|---|---|
-| `launch-gpu.sh` | Launches a GPU instance from the AMD or NVIDIA GPU AMI, selecting the AMI from the instance type's GPU brand. `--amd` / `--nvidia` override the detection. |
-| `ecs-node-bringup.sh` | Launches an ECS container-instance VM. Renders the cloud-init user-data seeding the agent's control-plane config, IAM credentials and gateway CA — there is no ECS-specific launch API, by design. |
-
 ## `demo/` — GPU demo
 
 | Script | What it does |

scripts/ecs-node-bringup.sh

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-# ecs-node-bringup.sh — launch a Spinifex ECS container-instance VM.
-#
-# ECS container instances are plain EC2 instances booted from the spinifex-ecs-node
-# AMI (Alpine + containerd + ecs-agent), AWS-faithful: there is no ECS-specific
-# launch API. This script renders the cloud-init user-data that seeds the agent's
-# control-plane config + IAM credentials into /etc/spinifex-ecs/agent.env and the
-# gateway TLS CA into /etc/spinifex-ecs/gateway-ca.pem, then runs the instance.
-#
-# The agent then registers, heartbeats, polls assignments, and reports task state
-# entirely through the AWS gateway over TLS+SigV4 — it never touches NATS.
-#
-# Usage:
-#   AWS_PROFILE=spinifex scripts/ecs-node-bringup.sh <cluster-name>
-#
-# Env overrides:
-#   GATEWAY_IP   gateway IP the guest dials        (default: host WAN-bridge IP)
-#   GATEWAY_CA   host path to the gateway CA PEM   (default /etc/spinifex/ca.pem)
-#   AMI_ID       ECS-node AMI                      (default: latest spinifex-ecs-node)
-#   SUBNET_ID / SG_ID / KEY_NAME / INSTANCE_TYPE   (default: first available / t3.small)
-set -euo pipefail
-
-CLUSTER="${1:?usage: ecs-node-bringup.sh <cluster-name>}"
-PROFILE="${AWS_PROFILE:-spinifex}"
-REGION="$(aws configure get region --profile "$PROFILE")"
-
-# VPC guests NAT out through the WAN and reach host-run services on the host's
-# WAN-bridge IP; the mgmt bridge (10.15.8.x) is not routable from a guest netns.
-# Default to the br-wan address, falling back to the mgmt IP if it can't be read.
-default_gateway_ip() {
-  ip -4 addr show br-wan 2>/dev/null | awk '/inet /{print $2; exit}' | cut -d/ -f1
-}
-GATEWAY_IP="${GATEWAY_IP:-$(default_gateway_ip)}"
-GATEWAY_IP="${GATEWAY_IP:-10.15.8.1}"
-GATEWAY_CA="${GATEWAY_CA:-/etc/spinifex/ca.pem}"
-INSTANCE_TYPE="${INSTANCE_TYPE:-t3.small}"
-
-awscli() { aws --profile "$PROFILE" "$@"; }
-
-AMI_ID="${AMI_ID:-$(awscli ec2 describe-images \
-  --filters 'Name=name,Values=spinifex-ecs-node' \
-  --query 'sort_by(Images,&CreationDate)[-1].ImageId' --output text)}"
-SUBNET_ID="${SUBNET_ID:-$(awscli ec2 describe-subnets \
-  --query 'Subnets[0].SubnetId' --output text)}"
-SG_ID="${SG_ID:-$(awscli ec2 describe-security-groups \
-  --query 'SecurityGroups[0].GroupId' --output text)}"
-KEY_NAME="${KEY_NAME:-$(awscli ec2 describe-key-pairs \
-  --query 'KeyPairs[0].KeyName' --output text)}"
-
-ACCESS_KEY="$(aws configure get aws_access_key_id --profile "$PROFILE")"
-SECRET_KEY="$(aws configure get aws_secret_access_key --profile "$PROFILE")"
-
-echo "cluster=$CLUSTER ami=$AMI_ID subnet=$SUBNET_ID sg=$SG_ID key=$KEY_NAME gw=https://$GATEWAY_IP:9999" >&2
-
-# Indent a file's content by six spaces for a cloud-init "content: |" block.
-indent6() { sed 's/^/      /' "$1"; }
-
-USERDATA="$(cat <<EOF
-#cloud-config
-bootcmd:
-  - rm -f /etc/resolv.conf
-  - printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' > /etc/resolv.conf
-write_files:
-  - path: /etc/spinifex-ecs/agent.env
-    permissions: '0600'
-    content: |
-      ECS_GATEWAY_URL=https://$GATEWAY_IP:9999
-      ECS_GATEWAY_CA=/etc/spinifex-ecs/gateway-ca.pem
-      ECS_REGION=$REGION
-      ECS_CLUSTER=$CLUSTER
-      ECS_ACCESS_KEY=$ACCESS_KEY
-      ECS_SECRET_KEY=$SECRET_KEY
-  - path: /etc/spinifex-ecs/gateway-ca.pem
-    permissions: '0644'
-    content: |
-$(indent6 "$GATEWAY_CA")
-EOF
-)"
-
-INSTANCE_ID="$(awscli ec2 run-instances \
-  --image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" \
-  --key-name "$KEY_NAME" --subnet-id "$SUBNET_ID" --security-group-ids "$SG_ID" \
-  --user-data "$USERDATA" \
-  --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=ecs-node-$CLUSTER}]" \
-  --query 'Instances[0].InstanceId' --output text)"
-
-echo "$INSTANCE_ID"

scripts/launch-gpu.sh

@@ -1,248 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-export AWS_PROFILE=spinifex
-
-# Resolve the actual user's home — $HOME is /root when run under sudo.
-_USER_HOME=$(getent passwd "${SUDO_USER:-$(id -un)}" | cut -d: -f6)
-SSH_KEY="${SSH_KEY:-${_USER_HOME}/.ssh/spinifex-key}"
-SSH_USER="${SSH_USER:-ubuntu}"
-SSH_TIMEOUT="${SSH_TIMEOUT:-300}"
-
-usage() {
-    cat <<'EOF'
-launch-gpu.sh — Launch a GPU instance from the Spinifex AMD or NVIDIA GPU AMI.
-
-Detects the GPU brand from the specified instance type and selects the matching
-pre-built AMI automatically. Use --amd or --nvidia to override.
-
-Usage:
-  scripts/launch-gpu.sh --type <instance-type> [--disk GB] [--amd|--nvidia]
-
-Options:
-  --type TYPE    EC2 instance type to launch (required)
-  --disk GB      Root disk size in GB (default: 50)
-  --ami  NAME    Override the AMI name to use (default: ubuntu-amd-gpu or ubuntu-26.04-nvidia-gpu-x86_64)
-  --amd          Force AMD GPU AMI  (ubuntu-amd-gpu)
-  --nvidia       Force NVIDIA GPU AMI (ubuntu-26.04-nvidia-gpu-x86_64)
-
-Env overrides:
-  SSH_KEY      Path to SSH private key (default: ~/.ssh/spinifex-key)
-  SSH_USER     SSH user inside guest  (default: ubuntu)
-  SSH_TIMEOUT  Seconds to wait for initial SSH (default: 300)
-EOF
-}
-
-INSTANCE_TYPE=""
-DISK_GB=50
-GPU_BRAND=""
-AMI_OVERRIDE=""
-
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-        --type)   INSTANCE_TYPE="$2"; shift 2 ;;
-        --disk)   DISK_GB="$2";        shift 2 ;;
-        --ami)    AMI_OVERRIDE="$2";   shift 2 ;;
-        --amd)    GPU_BRAND="amd";     shift ;;
-        --nvidia) GPU_BRAND="nvidia";  shift ;;
-        -h|--help) usage; exit 0 ;;
-        *) echo "Unknown option: $1"; usage; exit 1 ;;
-    esac
-done
-
-list_gpu_types() {
-    aws ec2 describe-instance-types \
-        --query 'InstanceTypes[?GpuInfo].[InstanceType,GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]' \
-        --output table 2>/dev/null || true
-}
-
-if [ -z "$INSTANCE_TYPE" ]; then
-    echo "Error: --type is required."
-    echo ""
-    echo "Available GPU instance types on this node:"
-    list_gpu_types
-    exit 1
-fi
-
-# --- Validate instance type and fetch GPU info in one call ---
-echo "==> Checking instance type: ${INSTANCE_TYPE}"
-INSTANCE_INFO=$(aws ec2 describe-instance-types \
-    --query "InstanceTypes[?InstanceType=='${INSTANCE_TYPE}' && GpuInfo].[GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]" \
-    --output text 2>/dev/null || true)
-
-if [ -z "$INSTANCE_INFO" ] || [ "$INSTANCE_INFO" = "None" ]; then
-    echo "'${INSTANCE_TYPE}' is not available or is not a GPU type on this node."
-    echo ""
-    echo "Available GPU instance types:"
-    list_gpu_types
-    exit 1
-fi
-
-IFS=$'\t' read -r GPU_COUNT GPU_MANUFACTURER GPU_NAME <<< "$INSTANCE_INFO"
-echo "   ${GPU_COUNT}× ${GPU_NAME} (${GPU_MANUFACTURER})"
-
-# --- Resolve GPU brand ---
-if [ -z "$GPU_BRAND" ]; then
-    case "$GPU_MANUFACTURER" in
-        AMD|amd)       GPU_BRAND="amd" ;;
-        NVIDIA|nvidia) GPU_BRAND="nvidia" ;;
-        *)
-            echo "Cannot auto-detect GPU brand from manufacturer '${GPU_MANUFACTURER}'."
-            echo "Use --amd or --nvidia to specify explicitly."
-            exit 1
-            ;;
-    esac
-fi
-
-case "$GPU_BRAND" in
-    amd)    AMI_NAME="ubuntu-amd-gpu" ;;
-    nvidia) AMI_NAME="ubuntu-26.04-nvidia-gpu-x86_64" ;;
-esac
-
-# --- Resolve AMI ---
-AMI_LOOKUP="${AMI_OVERRIDE:-${AMI_NAME}}"
-echo "==> Resolving AMI: ${AMI_LOOKUP}"
-AMI_ID=$(aws ec2 describe-images \
-    --query "Images[?Name=='${AMI_LOOKUP}'].ImageId | [0]" --output text 2>/dev/null || true)
-if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
-    echo "No AMI found with name '${AMI_LOOKUP}'."
-    echo "Build and import it first, or specify the AMI name with --ami <name>"
-    echo "  scripts/build-system-image.sh scripts/images/ubuntu-gpu-${GPU_BRAND}.conf --import"
-    exit 1
-fi
-echo "   $AMI_ID"
-
-# --- Resolve subnet ---
-echo "==> Resolving subnet"
-SUBNET_ID=$(aws ec2 describe-subnets \
-    --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text 2>/dev/null || true)
-if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
-    echo "No public subnet found"
-    exit 1
-fi
-echo "   $SUBNET_ID"
-
-VPC_ID=$(aws ec2 describe-subnets --subnet-ids "$SUBNET_ID" \
-    --query 'Subnets[0].VpcId' --output text 2>/dev/null || true)
-if [ -z "$VPC_ID" ] || [ "$VPC_ID" = "None" ]; then
-    echo "Could not resolve VPC for subnet $SUBNET_ID"
-    exit 1
-fi
-
-# --- Ensure SSH security group ---
-# The default VPC SG only allows intra-SG traffic; we need a group that
-# permits port 22 from anywhere so the SSH wait below can connect.
-SSH_SG_NAME="spinifex-gpu-ssh"
-echo "==> Ensuring SSH security group (${SSH_SG_NAME})"
-SG_ID=$(aws ec2 describe-security-groups \
-    --filters "Name=group-name,Values=${SSH_SG_NAME}" "Name=vpc-id,Values=${VPC_ID}" \
-    --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || true)
-if [ -z "$SG_ID" ] || [ "$SG_ID" = "None" ]; then
-    SG_ID=$(aws ec2 create-security-group \
-        --group-name "$SSH_SG_NAME" \
-        --description "Allow SSH access to GPU instances" \
-        --vpc-id "$VPC_ID" \
-        --query 'GroupId' --output text)
-    aws ec2 authorize-security-group-ingress \
-        --group-id "$SG_ID" \
-        --protocol tcp \
-        --port 22 \
-        --cidr 0.0.0.0/0 >/dev/null
-    echo "   Created $SG_ID"
-else
-    echo "   Reusing $SG_ID"
-fi
-
-# --- Ensure SSH key ---
-if [ ! -f "$SSH_KEY" ]; then
-    echo "==> Generating SSH key"
-    sudo -u "${SUDO_USER:-$(id -un)}" mkdir -p "$(dirname "$SSH_KEY")"
-    sudo -u "${SUDO_USER:-$(id -un)}" ssh-keygen -t ed25519 -f "$SSH_KEY" -N ""
-fi
-# Delete before import — stale key pair material causes import to fail
-# silently, leaving the instance unreachable via SSH.
-echo "==> Importing SSH key"
-aws ec2 delete-key-pair --key-name spinifex-key >/dev/null 2>&1 || true
-aws ec2 import-key-pair --key-name spinifex-key \
-    --public-key-material "fileb://${SSH_KEY}.pub" >/dev/null 2>&1 || true
-
-# --- Launch ---
-echo "==> Launching ${INSTANCE_TYPE} (${DISK_GB} GB)..."
-ID=$(aws ec2 run-instances \
-    --image-id "$AMI_ID" \
-    --instance-type "$INSTANCE_TYPE" \
-    --key-name spinifex-key \
-    --subnet-id "$SUBNET_ID" \
-    --security-group-ids "$SG_ID" \
-    --count 1 \
-    --block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":${DISK_GB},\"VolumeType\":\"gp3\",\"DeleteOnTermination\":true}}]" \
-    --query 'Instances[0].InstanceId' --output text)
-if [ -z "$ID" ] || [ "$ID" = "None" ] || [ "$ID" = "null" ]; then
-    echo "run-instances returned no InstanceId"
-    exit 1
-fi
-echo "   $ID"
-
-# --- Wait for running ---
-echo "==> Waiting for running state..."
-COUNT=0; STATE="unknown"
-while [ $COUNT -lt 120 ]; do
-    STATE=$(aws ec2 describe-instances --instance-ids "$ID" \
-        --query "Reservations[0].Instances[0].State.Name" --output text 2>/dev/null || echo "unknown")
-    [ "$STATE" = "running" ] && break
-    [ "$STATE" = "terminated" ] && { echo "Instance terminated unexpectedly"; exit 1; }
-    sleep 2; COUNT=$((COUNT + 2))
-done
-[ "$STATE" = "running" ] || { echo "Instance did not reach running state (last: $STATE)"; exit 1; }
-
-# --- Wait for public IP ---
-echo "==> Waiting for public IP..."
-IP=""
-for _i in $(seq 1 120); do
-    IP=$(aws ec2 describe-instances --instance-ids "$ID" \
-        --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true)
-    [ -n "$IP" ] && [ "$IP" != "None" ] && [ "$IP" != "null" ] && break
-    sleep 1
-done
-[ -n "$IP" ] && [ "$IP" != "None" ] || { echo "No public IP assigned after 120s"; exit 1; }
-echo "   $IP"
-
-# Clear stale known_hosts entry — instances reuse IPs across runs.
-KNOWN_HOSTS="$HOME/.ssh/known_hosts"
-[ -f "$KNOWN_HOSTS" ] && ssh-keygen -f "$KNOWN_HOSTS" -R "$IP" >/dev/null 2>&1 || true
-
-# --- Wait for SSH ---
-echo "==> Waiting for SSH..."
-_ssh() {
-    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-        -o ConnectTimeout=5 -o BatchMode=yes \
-        -i "$SSH_KEY" "${SSH_USER}@${IP}" "$@"
-}
-
-elapsed=0
-while [ "$elapsed" -lt "$SSH_TIMEOUT" ]; do
-    _ssh 'echo ready' >/dev/null 2>&1 && break
-    sleep 5; elapsed=$((elapsed + 5))
-done
-[ "$elapsed" -lt "$SSH_TIMEOUT" ] || { echo "SSH timeout after ${SSH_TIMEOUT}s"; exit 1; }
-
-# --- GPU visibility check ---
-echo "==> Checking GPU visibility..."
-lspci_out=$(_ssh 'lspci -nn 2>/dev/null' || true)
-if printf '%s\n' "$lspci_out" | grep -qiE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA"; then
-    printf '%s\n' "$lspci_out" \
-        | grep -iE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA" \
-        | sed 's/^/   ✓ /'
-else
-    echo "   WARNING: GPU not visible in lspci — check host-side passthrough assignment"
-    printf '%s\n' "$lspci_out" | head -20 | sed 's/^/   /'
-fi
-
-# --- Done ---
-echo ""
-echo "✅ ${INSTANCE_TYPE} ready — ${ID} (${IP})"
-echo ""
-echo "   ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no ${SSH_USER}@${IP}"
-echo ""
-echo "   Terminate when done:"
-echo "   AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids ${ID}"

scripts/mkosi-builder.Dockerfile

@@ -7,7 +7,7 @@
 # Debian is the build host regardless of what is being built — mkosi bootstraps
 # the target distribution from its own archive, so an Ubuntu image builds fine
 # from here (see ubuntu-keyring below).
-FROM debian:trixie-slim
+FROM debian:trixie-slim@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f
 
 # The runner user is uid 1000; matching it keeps bind-mounted build output
 # owned by the invoking user instead of root.

scripts/test-ddil-phase1.sh

@@ -1,121 +0,0 @@
-#!/bin/bash
-# test-ddil-phase1.sh — systemd-level smoke test for daemon-local-autonomy
-# Tier 1 (1a / 1b / 1c / 1d / 1d-strict). Run after `make deploy` on a single
-# node where spinifex-daemon and spinifex-nats are managed by systemd.
-#
-# Usage: scripts/test-ddil-phase1.sh
-# Requires: sudo, curl, jq
-#
-# Destructive on the local node — stops/starts spinifex-nats and restarts
-# spinifex-daemon. Do NOT run against a host carrying workloads. The cleanup
-# trap restores both services on exit.
-
-set -euo pipefail
-
-DAEMON_URL="${DAEMON_URL:-https://127.0.0.1:4432}"
-DAEMON_SVC=spinifex-daemon
-NATS_SVC=spinifex-nats
-OVERRIDE=/etc/systemd/system/${DAEMON_SVC}.service.d/99-test-ddil.conf
-NATS_BLOCK=/run/spinifex-nats-blocked
-NATS_BLOCK_DROPIN=/run/systemd/system/${NATS_SVC}.service.d/99-test-block.conf
-CURL=(curl -fsS -k --max-time 5)
-
-ok()   { printf '\033[32mPASS\033[0m %s\n' "$1"; }
-fail() { printf '\033[31mFAIL\033[0m %s\n' "$1"; exit 1; }
-info() { printf '\033[36m==>\033[0m %s\n' "$1"; }
-
-cleanup() {
-    info "cleanup: removing nats block, override, restoring services"
-    sudo rm -f "$NATS_BLOCK" "$NATS_BLOCK_DROPIN" "$OVERRIDE"
-    sudo rmdir "$(dirname "$NATS_BLOCK_DROPIN")" 2>/dev/null || true
-    sudo systemctl daemon-reload
-    sudo systemctl start "$NATS_SVC"     2>/dev/null || true
-    sudo systemctl restart "$DAEMON_SVC" 2>/dev/null || true
-}
-trap cleanup EXIT
-
-field() { "${CURL[@]}" "${DAEMON_URL}/local/status" | jq -r ".$1 // empty"; }
-
-wait_field() {
-    local f=$1 want=$2 timeout=${3:-30} elapsed=0
-    while [ "$elapsed" -lt "$timeout" ]; do
-        [ "$(field "$f" 2>/dev/null || true)" = "$want" ] && return 0
-        sleep 1; elapsed=$((elapsed + 1))
-    done
-    return 1
-}
-
-# 0. baseline
-info "baseline: daemon active + cluster mode"
-sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon not running — run \`make deploy\` first"
-sudo systemctl start "$NATS_SVC"
-wait_field mode cluster 30 || fail "never reached mode=cluster (got: $(field mode))"
-[ "$(field nats)" = connected ] || fail "nats not connected (got: $(field nats))"
-ok "mode=cluster nats=connected"
-
-# 1. kill NATS → standalone, daemon survives
-info "1a/1c: kill NATS"
-retry_before=$(field nats_retry_count)
-sudo systemctl stop "$NATS_SVC"
-wait_field mode standalone 30   || fail "never went standalone (got: $(field mode))"
-wait_field nats disconnected 5  || fail "nats not disconnected (got: $(field nats))"
-sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon exited — should stay up"
-ok "standalone, nats=disconnected, daemon alive"
-
-# 2. /local/instances reachable during partition
-info "1b: /local/instances during partition"
-"${CURL[@]}" "${DAEMON_URL}/local/instances" | jq -e . >/dev/null \
-    || fail "/local/instances did not return JSON"
-ok "/local/instances served while NATS down"
-
-# 3. restart NATS → cluster mode + retry count advances + KV push observed
-# nats_retry_count counts disconnect→reconnect *cycles* (bumped from
-# onNATSReconnect, daemon.go:321). It stays flat during disconnect and
-# advances by 1 here when the reconnect callback fires.
-info "1c reconnect: restart NATS"
-before_sync=$(field last_kv_sync_at)
-sudo systemctl start "$NATS_SVC"
-wait_field mode cluster 60   || fail "did not return to cluster mode"
-wait_field nats connected 10 || fail "nats not reconnected"
-sleep 5
-retry_after=$(field nats_retry_count)
-[ "$retry_after" -gt "$retry_before" ] \
-    || fail "nats_retry_count did not advance on reconnect ($retry_before → $retry_after)"
-after_sync=$(field last_kv_sync_at)
-[ -n "$after_sync" ] && [ "$after_sync" != "$before_sync" ] \
-    || fail "last_kv_sync_at did not advance ($before_sync → $after_sync)"
-ok "cluster restored, retry $retry_before → $retry_after, KV resync at $after_sync"
-
-# 4. SPINIFEX_REQUIRE_NATS=1 → daemon exits within ~30s when NATS down.
-# Block spinifex-nats via a runtime drop-in with ConditionPathExists=!. This
-# defeats the daemon unit's Wants=spinifex-nats.service auto-start, which
-# would otherwise pull NATS back up on `systemctl restart spinifex-daemon`
-# and let the daemon connect immediately. mask refuses (real unit lives in
-# /etc) and --runtime mask is shadowed by /etc — the condition drop-in is
-# the only mechanism that works against an /etc-installed unit.
-info "1d-strict: SPINIFEX_REQUIRE_NATS=1 exits when NATS down"
-sudo mkdir -p "$(dirname "$NATS_BLOCK_DROPIN")"
-sudo tee "$NATS_BLOCK_DROPIN" >/dev/null <<EOF
-[Unit]
-ConditionPathExists=!$NATS_BLOCK
-EOF
-sudo touch "$NATS_BLOCK"
-sudo systemctl daemon-reload
-sudo systemctl stop "$NATS_SVC"
-sudo mkdir -p "$(dirname "$OVERRIDE")"
-sudo tee "$OVERRIDE" >/dev/null <<EOF
-[Service]
-Environment=SPINIFEX_REQUIRE_NATS=1
-Restart=no
-EOF
-sudo systemctl daemon-reload
-sudo systemctl restart "$DAEMON_SVC" || true
-start=$(date +%s)
-while sudo systemctl is-active --quiet "$DAEMON_SVC"; do
-    [ $(( $(date +%s) - start )) -gt 45 ] && fail "did not exit within 45s"
-    sleep 1
-done
-ok "exited after $(( $(date +%s) - start ))s"
-
-echo
-echo "DDIL Phase 1 systemd integration: ALL PASS"

scripts/workload-performance.sh

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-# Workload performance testing (memory, disk, cpu, network)
-
-sudo apt update
-sudo apt install -y make gcc git
-
-wget https://go.dev/dl/go1.27.0.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.27.0.linux-amd64.tar.gz
-
-echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc
-source $HOME/.bashrc
-
-git clone https://github.com/dgraph-io/badger.git
-cd badger
-
-# disable jemalloc
-sed -i 's/-tags=jemalloc/-tags=nojemalloc/g' test.sh
-
-# run 4 tests, otherwise OOM on <8GB RAM
-sed -i 's/-parallel 16/-parallel 4/g; s/-parallel=16/-parallel=4/g' test.sh
-
-# run tests
-./test.sh

spinifex/accountteardown/cluster.go

@@ -21,6 +21,7 @@ func NewClusterEngine(
 	ctx context.Context,
 	nc *nats.Conn,
 	expectedNodes int,
+	nodeIDs []string,
 	svc handlers_iam.IAMService,
 	buckets BucketStore,
 ) (*Engine, error) {
@@ -64,12 +65,14 @@ func NewClusterEngine(
 	// live spot request launches a replacement for anything already terminated.
 	reapers = append(reapers, ELBv2Reapers(nc)...)
 	reapers = append(reapers, EC2ExtraReapers(nc)...)
-	reapers = append(reapers, EC2Reapers(nc, expectedNodes)...)
+	reapers = append(reapers, EC2Reapers(nc, expectedNodes, nodeIDs)...)
 
 	reapers = append(reapers, S3Reapers(buckets)...)
 	reapers = append(reapers, ACMReapers(nc)...)
 	reapers = append(reapers, IAMReapers(svc)...)
 	reapers = append(reapers, AccountReapers(svc, names, usage)...)
 
-	return NewEngine(NewIAMAccounts(svc), reapers...), nil
+	engine := NewEngine(NewIAMAccounts(svc), reapers...)
+	engine.ec2Inventory = newEC2PlatformInventoryClient(nc)
+	return engine, nil
 }

spinifex/accountteardown/cluster_test.go

@@ -23,13 +23,13 @@ func TestNewClusterEngineCoversEveryStage(t *testing.T) {
 	svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
 	require.NoError(t, err)
 
-	engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets())
+	engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
 	require.NoError(t, err)
 	require.NotNil(t, engine.Accounts)
 
 	// A teardown with no object store would report every stage drained and
 	// still delete the account record, leaving the tenant's objects behind.
-	_, err = NewClusterEngine(t.Context(), nc, 1, svc, nil)
+	_, err = NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, nil)
 	require.Error(t, err)
 
 	for _, stage := range Stages() {
@@ -51,7 +51,7 @@ func TestClusterEnginePrechecks(t *testing.T) {
 	svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
 	require.NoError(t, err)
 
-	engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets())
+	engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
 	require.NoError(t, err)
 
 	// Account ids are sequential and the first is the protected super admin.

spinifex/accountteardown/ec2_platform_inventory.go

@@ -0,0 +1,92 @@
+package accountteardown
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// ec2PlatformInventoryTimeout bounds one inventory request. The scan reads
+// four stores for one account; a reply later than this is scan-incomplete.
+const ec2PlatformInventoryTimeout = 30 * time.Second
+
+// ec2PlatformInventory is EC2's read-only report of an account's key pairs,
+// placement groups and launch templates. It never deletes, and a failure to
+// obtain the report is returned as scan-incomplete kinds, never as clean.
+type ec2PlatformInventory interface {
+	PlatformInventory(ctx context.Context, accountID, correlationRef string) ec2v1.PlatformInventoryReply
+}
+
+// ec2PlatformInventoryClient reaches the EC2 platform-inventory contract over NATS.
+type ec2PlatformInventoryClient struct {
+	nc      *nats.Conn
+	timeout time.Duration
+}
+
+var _ ec2PlatformInventory = (*ec2PlatformInventoryClient)(nil)
+
+func newEC2PlatformInventoryClient(nc *nats.Conn) *ec2PlatformInventoryClient {
+	return &ec2PlatformInventoryClient{nc: nc, timeout: ec2PlatformInventoryTimeout}
+}
+
+// PlatformInventory asks EC2 for the account's platform inventory. No
+// responder, which is also how a daemon predating the subject answers, a
+// timeout, an error reply or a malformed reply each make every kind incomplete.
+func (c *ec2PlatformInventoryClient) PlatformInventory(ctx context.Context, accountID, correlationRef string) ec2v1.PlatformInventoryReply {
+	reply, err := natsmsg.NATSRequest[ec2v1.PlatformInventoryReply](ctx, c.nc, ec2v1.PlatformInventorySubject,
+		&ec2v1.PlatformInventoryRequest{AccountID: accountID, CorrelationRef: correlationRef}, c.timeout, accountID)
+	if err != nil {
+		return ec2v1.IncompletePlatformInventory(accountID, correlationRef, inventoryFailureReason(err))
+	}
+	return normalisePlatformInventory(*reply, accountID, correlationRef)
+}
+
+func inventoryFailureReason(err error) string {
+	switch {
+	case errors.Is(err, nats.ErrNoResponders):
+		return ec2v1.PlatformReasonNoResponder
+	case errors.Is(err, nats.ErrTimeout), errors.Is(err, context.DeadlineExceeded):
+		return ec2v1.PlatformReasonTimeout
+	}
+	if _, ok := awserrors.ResolveErrorCode(err); ok {
+		return ec2v1.PlatformReasonErrorReply
+	}
+	return ec2v1.PlatformReasonTransportFailed
+}
+
+// normalisePlatformInventory refuses a reply for another account and turns a
+// missing or repeated kind, or an unknown status, into scan-incomplete.
+func normalisePlatformInventory(reply ec2v1.PlatformInventoryReply, accountID, correlationRef string) ec2v1.PlatformInventoryReply {
+	if reply.AccountID != accountID {
+		return ec2v1.IncompletePlatformInventory(accountID, correlationRef, ec2v1.PlatformReasonInvalidReply)
+	}
+	known := []string{ec2v1.PlatformStatusObservedClean, ec2v1.PlatformStatusRecordsFound, ec2v1.PlatformStatusScanIncomplete}
+	byKind := make(map[string]ec2v1.PlatformKindReport, len(reply.Kinds))
+	for _, report := range reply.Kinds {
+		if !slices.Contains(known, report.Status) {
+			report.Status = ec2v1.PlatformStatusScanIncomplete
+			report.IncompleteReasons = append(report.IncompleteReasons, ec2v1.PlatformReasonInvalidReply)
+		}
+		if _, dup := byKind[report.Kind]; dup {
+			report = ec2v1.PlatformKindReport{Kind: report.Kind, Status: ec2v1.PlatformStatusScanIncomplete,
+				IncompleteReasons: []string{ec2v1.PlatformReasonInvalidReply}}
+		}
+		byKind[report.Kind] = report
+	}
+	reply.Kinds = nil
+	for _, kind := range ec2v1.PlatformKinds() {
+		report, ok := byKind[kind]
+		if !ok {
+			report = ec2v1.PlatformKindReport{Kind: kind, Status: ec2v1.PlatformStatusScanIncomplete,
+				IncompleteReasons: []string{ec2v1.PlatformReasonKindNotReported}}
+		}
+		reply.Kinds = append(reply.Kinds, report)
+	}
+	return reply
+}

spinifex/accountteardown/ec2_platform_inventory_test.go

@@ -0,0 +1,192 @@
+package accountteardown
+
+//test:in-package — the consumer interface, its adapter and the engine field
+// that binds it are unexported.
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// serveInventory answers the inventory subject with reply's bytes, or not at all when nil.
+func serveInventory(t *testing.T, nc *nats.Conn, reply func() []byte) {
+	t.Helper()
+	sub, err := nc.QueueSubscribe(ec2v1.PlatformInventorySubject, "spinifex-workers", func(msg *nats.Msg) {
+		if data := reply(); data != nil {
+			_ = msg.Respond(data)
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+}
+
+func assertAllIncomplete(t *testing.T, reply ec2v1.PlatformInventoryReply, reason string) {
+	t.Helper()
+	assert.Equal(t, testAccountID, reply.AccountID)
+	require.Len(t, reply.Kinds, 4)
+	for _, report := range reply.Kinds {
+		assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, report.Status, report.Kind)
+		assert.Equal(t, []string{reason}, report.IncompleteReasons, report.Kind)
+	}
+	assert.Empty(t, reply.Items)
+	assert.Empty(t, reply.Counts)
+}
+
+// The adapter sends the account and correlation reference in the body and
+// returns EC2's report of the owner stores, residue counts included.
+func TestEC2PlatformInventoryClientReturnsEC2sReport(t *testing.T) {
+	o := newPlatformOwners(t)
+	ctx := testCtx(t)
+	inv := accountinventory.New(o.keys, o.groups, o.templates, o.tags)
+	var seen ec2v1.PlatformInventoryRequest
+	sub, err := o.nc.QueueSubscribe(ec2v1.PlatformInventorySubject, "spinifex-workers", func(msg *nats.Msg) {
+		natsmsg.ServeNATSRequestCtx(msg, func(ctx context.Context, req *ec2v1.PlatformInventoryRequest) (*ec2v1.PlatformInventoryReply, error) {
+			seen = *req
+			return inv.PlatformInventory(ctx, req)
+		})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	_, err = o.keys.ImportKeyPair(ctx, &ec2.ImportKeyPairInput{
+		KeyName: aws.String("deploy"), PublicKeyMaterial: []byte("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"),
+	}, testAccountID)
+	require.NoError(t, err)
+	o.store.failDelete = func(key string) bool { return key == "keys/"+testAccountID+"/deploy" }
+	_, err = o.keys.DeleteKeyPair(ctx, &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, testAccountID)
+	require.NoError(t, err)
+
+	reply := newEC2PlatformInventoryClient(o.nc).PlatformInventory(ctx, testAccountID, "req-1")
+
+	assert.Equal(t, ec2v1.PlatformInventoryRequest{AccountID: testAccountID, CorrelationRef: "req-1"}, seen)
+	assert.Equal(t, "req-1", reply.CorrelationRef)
+	assert.Equal(t, []ec2v1.PlatformRecordCount{{
+		Kind: ec2v1.PlatformKindKeyPair, Disposition: ec2v1.PlatformDispositionResidue,
+		Class: ec2v1.PlatformClassPublicKeyWithoutMetadata, Count: 1,
+	}}, reply.Counts)
+	statuses := map[string]string{}
+	for _, report := range reply.Kinds {
+		statuses[report.Kind] = report.Status
+	}
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindKeyPair:        ec2v1.PlatformStatusRecordsFound,
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindLaunchTemplate: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusObservedClean,
+	}, statuses)
+}
+
+// No daemon serving the subject, which is also what a daemon predating it
+// looks like, is scan-incomplete for every kind, never clean.
+func TestEC2PlatformInventoryClientReportsNoResponderAsIncomplete(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	reply := newEC2PlatformInventoryClient(nc).PlatformInventory(testCtx(t), testAccountID, "req-1")
+	assertAllIncomplete(t, reply, ec2v1.PlatformReasonNoResponder)
+	assert.Equal(t, "req-1", reply.CorrelationRef)
+}
+
+func TestEC2PlatformInventoryClientReportsTransportFailuresAsIncomplete(t *testing.T) {
+	t.Run("timeout", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveInventory(t, nc, func() []byte { return nil })
+		client := newEC2PlatformInventoryClient(nc)
+		client.timeout = 50 * time.Millisecond
+		assertAllIncomplete(t, client.PlatformInventory(testCtx(t), testAccountID, ""), ec2v1.PlatformReasonTimeout)
+	})
+	t.Run("error reply", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveInventory(t, nc, func() []byte {
+			return awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal)
+		})
+		assertAllIncomplete(t, newEC2PlatformInventoryClient(nc).PlatformInventory(testCtx(t), testAccountID, ""), ec2v1.PlatformReasonErrorReply)
+	})
+	t.Run("undecodable reply", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveInventory(t, nc, func() []byte { return []byte(`{"kinds":"nope"}`) })
+		assertAllIncomplete(t, newEC2PlatformInventoryClient(nc).PlatformInventory(testCtx(t), testAccountID, ""), ec2v1.PlatformReasonTransportFailed)
+	})
+	t.Run("closed connection", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		nc.Close()
+		assertAllIncomplete(t, newEC2PlatformInventoryClient(nc).PlatformInventory(testCtx(t), testAccountID, ""), ec2v1.PlatformReasonTransportFailed)
+	})
+}
+
+// A reply for another account is discarded; a kind the reply omits or
+// repeats, or a status this consumer does not know, is scan-incomplete.
+func TestEC2PlatformInventoryClientDistrustsAMalformedReply(t *testing.T) {
+	clean := ec2v1.PlatformKindReport{Kind: ec2v1.PlatformKindKeyPair, Status: ec2v1.PlatformStatusObservedClean}
+
+	_, nc, _ := testutil.StartTestJetStream(t)
+	serveInventory(t, nc, func() []byte {
+		data, _ := json.Marshal(ec2v1.PlatformInventoryReply{AccountID: "999999999999", Kinds: []ec2v1.PlatformKindReport{clean}})
+		return data
+	})
+	assertAllIncomplete(t, newEC2PlatformInventoryClient(nc).PlatformInventory(testCtx(t), testAccountID, ""), ec2v1.PlatformReasonInvalidReply)
+
+	reply := normalisePlatformInventory(ec2v1.PlatformInventoryReply{AccountID: testAccountID, Kinds: []ec2v1.PlatformKindReport{
+		clean,
+		clean,
+		{Kind: ec2v1.PlatformKindPlacementGroup, Status: "drained"},
+		{Kind: ec2v1.PlatformKindTagIndex, Status: ec2v1.PlatformStatusRecordsFound},
+	}}, testAccountID, "")
+	assert.Equal(t, []ec2v1.PlatformKindReport{
+		{Kind: ec2v1.PlatformKindKeyPair, Status: ec2v1.PlatformStatusScanIncomplete, IncompleteReasons: []string{ec2v1.PlatformReasonInvalidReply}},
+		{Kind: ec2v1.PlatformKindPlacementGroup, Status: ec2v1.PlatformStatusScanIncomplete, IncompleteReasons: []string{ec2v1.PlatformReasonInvalidReply}},
+		{Kind: ec2v1.PlatformKindLaunchTemplate, Status: ec2v1.PlatformStatusScanIncomplete, IncompleteReasons: []string{ec2v1.PlatformReasonKindNotReported}},
+		{Kind: ec2v1.PlatformKindTagIndex, Status: ec2v1.PlatformStatusRecordsFound},
+	}, reply.Kinds)
+}
+
+// failingInventory fails the test if anything consults it.
+type failingInventory struct {
+	t *testing.T
+}
+
+func (f failingInventory) PlatformInventory(context.Context, string, string) ec2v1.PlatformInventoryReply {
+	f.t.Error("an engine path called the EC2 platform inventory")
+	return ec2v1.PlatformInventoryReply{}
+}
+
+// The cluster wiring binds the NATS adapter, and no engine path consults it:
+// precheck, inventory, dry run and a full teardown all leave it uncalled.
+func TestNoEnginePathCallsTheEC2PlatformInventory(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
+	require.NoError(t, err)
+	cluster, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
+	require.NoError(t, err)
+	assert.IsType(t, &ec2PlatformInventoryClient{}, cluster.ec2Inventory)
+
+	reaper := &fakeReaper{kind: "key-pair", stage: StagePlatform, resources: []Resource{{Kind: "key-pair", ID: "deploy"}}}
+	engine := quietEngine(newFakeAccounts(), reaper)
+	engine.ec2Inventory = failingInventory{t: t}
+
+	require.NoError(t, engine.Precheck(testAccountID, testAccountName))
+	_, err = engine.Inventory(testCtx(t), testAccountID)
+	require.NoError(t, err)
+	dryRun := request()
+	dryRun.DryRun = true
+	_, err = engine.Teardown(testCtx(t), dryRun)
+	require.NoError(t, err)
+	result, err := engine.Teardown(testCtx(t), request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+}

spinifex/accountteardown/engine.go

@@ -28,6 +28,10 @@ type Engine struct {
 	// teardown in the background can publish progress. A teardown takes minutes
 	// and this is the only view into it while it runs.
 	OnStage func(ctx context.Context, stage StageResult)
+
+	// ec2Inventory is EC2's read-only platform inventory. Bound by the cluster
+	// wiring but consulted by no teardown or inventory path yet.
+	ec2Inventory ec2PlatformInventory
 }
 
 // NewEngine builds an engine with the default timeouts.

spinifex/accountteardown/reapers_ec2.go

@@ -31,15 +31,15 @@ import (
 // They drive the same NATS-backed operations the gateway serves a customer
 // request with, which is what keeps OVN teardown, IPAM release and quota
 // accounting correct without teardown having to reimplement any of it.
-func EC2Reapers(nc *nats.Conn, expectedNodes int) []Reaper {
+func EC2Reapers(nc *nats.Conn, expectedNodes int, nodeIDs []string) []Reaper {
 	return []Reaper{
-		&instanceReaper{nc: nc, expectedNodes: expectedNodes},
+		&instanceReaper{nc: nc, nodeIDs: nodeIDs},
 
 		// Snapshots before volumes: a volume with a live snapshot refuses to
 		// delete, because snapshot-backed clones read from its chunk prefix.
 		&snapshotReaper{nc: nc},
 		&imageReaper{nc: nc},
-		&volumeReaper{nc: nc, expectedNodes: expectedNodes},
+		&volumeReaper{nc: nc, nodeIDs: nodeIDs},
 
 		// Reverse of creation order. The VPC cannot go until everything
 		// inside it has, and the gateway enforces that.
@@ -67,17 +67,22 @@ func EC2Reapers(nc *nats.Conn, expectedNodes int) []Reaper {
 var terminalInstanceStates = map[string]bool{"terminated": true, "shutting-down": true}
 
 type instanceReaper struct {
-	nc            *nats.Conn
-	expectedNodes int
+	nc *nats.Conn
+	// nodeIDs is the configured node set; all must answer before an empty
+	// listing counts as drained.
+	nodeIDs []string
+
+	// describeOpts tunes the inventory fan-out; production leaves it empty.
+	describeOpts []ec2instanceapi.DescribeOption
 }
 
 func (r *instanceReaper) Kind() string { return "instance" }
 func (r *instanceReaper) Stage() Stage { return StageCompute }
 
 func (r *instanceReaper) List(ctx context.Context, accountID string) ([]Resource, error) {
-	// Checked: a partial answer here would read as "no instances left" and let
-	// storage deletion start while a guest still holds a volume.
-	out, err := ec2instanceapi.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{}, r.nc, r.expectedNodes, nil, accountID)
+	// Complete or nothing: a partial answer would read as "no instances left"
+	// and let storage deletion start while a guest still holds a volume.
+	out, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, r.nc, r.nodeIDs, accountID, r.describeOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -109,8 +114,10 @@ func (r *instanceReaper) Delete(ctx context.Context, accountID string, resource
 }
 
 type volumeReaper struct {
-	nc            *nats.Conn
-	expectedNodes int
+	nc *nats.Conn
+	// nodeIDs is the configured node set; every one must answer, by identity,
+	// before a volume's holder can be proven gone or the delete guard passes.
+	nodeIDs []string
 }
 
 func (r *volumeReaper) Kind() string { return "volume" }
@@ -144,16 +151,16 @@ func (r *volumeReaper) List(ctx context.Context, accountID string) ([]Resource,
 func (r *volumeReaper) Delete(ctx context.Context, accountID string, resource Resource, force bool) error {
 	_, err := ec2volumeapi.DeleteVolume(ctx, &ec2.DeleteVolumeInput{
 		VolumeId: aws.String(resource.ID),
-	}, r.nc, r.expectedNodes, accountID)
+	}, r.nc, r.nodeIDs, accountID)
 	if isAlreadyGone(err) {
 		return nil
 	}
 	if !force {
 		return err
 	}
-	// Force path: the ordinary delete refuses an attached volume, and an
-	// instance that will not terminate would otherwise strand it forever.
-	return forceDeleteVolume(ctx, r.nc, r.expectedNodes, accountID, resource.ID)
+	// Force path: the ordinary delete refuses an attached volume, and a holder
+	// that is gone but still on the record would otherwise strand it forever.
+	return forceDeleteVolume(ctx, r.nc, r.nodeIDs, accountID, resource.ID, err)
 }
 
 type snapshotReaper struct{ nc *nats.Conn }
@@ -693,22 +700,120 @@ const forceDetachSubject = "ec2.ForceDetachVolume"
 // unwell rather than that the work is large.
 const forceDetachTimeout = 30 * time.Second
 
-// forceDeleteVolume clears the attachment the ordinary delete refuses to
-// override, then deletes.
-//
-// This is the deadlock the force path exists for: a volume attached to an
-// instance that will not terminate leaves both undeletable, and the account
-// can never be emptied.
-func forceDeleteVolume(ctx context.Context, nc *nats.Conn, expectedNodes int, accountID, volumeID string) error {
-	_, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, nc, forceDetachSubject,
+// forceDeleteVolume clears an attachment the ordinary delete refuses to
+// override, then deletes, but only once every instance the volume names is
+// proven absent. Anything short of proof leaves the volume and its record alone.
+func forceDeleteVolume(ctx context.Context, nc *nats.Conn, nodeIDs []string, accountID, volumeID string, refusal error) error {
+	holders, err := volumeHolders(ctx, nc, accountID, volumeID)
+	if err != nil {
+		if isAlreadyGone(err) {
+			return nil
+		}
+		return fmt.Errorf("volume %s retained: cannot read its attachment: %w", volumeID, err)
+	}
+	if len(holders) == 0 {
+		// No holder on record, so the refusal is not an attachment and
+		// clearing the record would change nothing but the evidence.
+		return refusal
+	}
+	for _, instanceID := range holders {
+		if err := proveInstanceAbsent(ctx, nc, nodeIDs, accountID, instanceID); err != nil {
+			return fmt.Errorf("volume %s retained: %w", volumeID, err)
+		}
+	}
+
+	_, err = natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, nc, forceDetachSubject,
 		&ec2.DetachVolumeInput{VolumeId: aws.String(volumeID), Force: aws.Bool(true)},
 		forceDetachTimeout, accountID)
 	if err != nil && !isAlreadyGone(err) {
 		return fmt.Errorf("force detach %s: %w", volumeID, err)
 	}
 
+	// The ordinary delete re-checks that the record is unattached and that no
+	// live instance maps the volume, so a re-attach in between still refuses.
 	_, err = ec2volumeapi.DeleteVolume(ctx, &ec2.DeleteVolumeInput{
 		VolumeId: aws.String(volumeID),
-	}, nc, expectedNodes, accountID)
+	}, nc, nodeIDs, accountID)
 	return ignoreAlreadyGone(err)
 }
+
+// volumeHolders returns the instances the volume's own record says hold it,
+// read fresh rather than from the listing that queued the delete.
+func volumeHolders(ctx context.Context, nc *nats.Conn, accountID, volumeID string) ([]string, error) {
+	out, err := ec2volumeapi.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{
+		VolumeIds: []*string{aws.String(volumeID)},
+	}, nc, accountID)
+	if err != nil {
+		return nil, err
+	}
+	var holders []string
+	for _, volume := range out.Volumes {
+		if volume == nil || aws.StringValue(volume.VolumeId) != volumeID {
+			continue
+		}
+		for _, attachment := range volume.Attachments {
+			if attachment != nil && aws.StringValue(attachment.InstanceId) != "" {
+				holders = append(holders, aws.StringValue(attachment.InstanceId))
+			}
+		}
+	}
+	return holders, nil
+}
+
+// proveInstanceAbsent accepts only a definitive InvalidInstanceID.NotFound or
+// a terminated copy from a describe naming the instance, then confirms both on
+// a listing every configured node answered by identity. Anything else is not proof.
+func proveInstanceAbsent(ctx context.Context, nc *nats.Conn, nodeIDs []string, accountID, instanceID string) error {
+	if len(nodeIDs) == 0 {
+		return fmt.Errorf("cannot prove holding instance %s is gone: the configured node set is unknown", instanceID)
+	}
+	named, err := ec2instanceapi.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}, nc, 0, nodeIDs, accountID)
+	if err != nil {
+		if code, ok := awserrors.ResolveErrorCode(err); !ok || code != awserrors.ErrorInvalidInstanceIDNotFound {
+			return fmt.Errorf("cannot prove holding instance %s is gone: %w", instanceID, err)
+		}
+	} else {
+		found, err := requireTerminated(named.Reservations, instanceID)
+		if err != nil {
+			return err
+		}
+		if !found {
+			return fmt.Errorf("cannot prove holding instance %s is gone: the describe naming it returned neither the instance nor NotFound", instanceID)
+		}
+	}
+
+	// The named describe stops collecting once the id turns up, and a lone
+	// node's 4xx can surface as NotFound, so neither answer is complete alone.
+	listing, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, nc, nodeIDs, accountID)
+	if err != nil {
+		return fmt.Errorf("cannot prove holding instance %s is gone: %w", instanceID, err)
+	}
+	_, err = requireTerminated(listing.Reservations, instanceID)
+	return err
+}
+
+// requireTerminated reports whether instanceID appears in reservations, and
+// fails if any copy of it is in a state other than terminated.
+func requireTerminated(reservations []*ec2.Reservation, instanceID string) (found bool, err error) {
+	for _, reservation := range reservations {
+		if reservation == nil {
+			continue
+		}
+		for _, instance := range reservation.Instances {
+			if instance == nil || aws.StringValue(instance.InstanceId) != instanceID {
+				continue
+			}
+			found = true
+			state := "in an unknown state"
+			if instance.State != nil && aws.StringValue(instance.State.Name) != "" {
+				state = aws.StringValue(instance.State.Name)
+			}
+			if state != ec2.InstanceStateNameTerminated {
+				return true, fmt.Errorf("holding instance %s is still %s", instanceID, state)
+			}
+		}
+	}
+	return found, nil
+}

spinifex/accountteardown/reapers_ec2_inventory_test.go

@@ -0,0 +1,194 @@
+package accountteardown
+
+//test:in-package — the instance reaper is unexported, and whether its listing
+// can be mistaken for an empty account is the safety property under test.
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var inventoryNodes = []string{"node-1", "node-2"}
+
+// testInventoryReaper builds the instance reaper for a two-node cluster, with
+// a short fan-out deadline so a silent node costs the test little.
+func testInventoryReaper(nc *nats.Conn) *instanceReaper {
+	return &instanceReaper{nc: nc, nodeIDs: inventoryNodes, describeOpts: []ec2instanceapi.DescribeOption{
+		ec2instanceapi.WithFanoutTimeout(300 * time.Millisecond),
+	}}
+}
+
+// answerAsNode replies on subject the way a daemon does, carrying the node ID
+// header the fan-out attributes frames by.
+func answerAsNode(t *testing.T, nc *nats.Conn, subject, nodeID string, payload []byte) {
+	t.Helper()
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
+		reply := nats.NewMsg(msg.Reply)
+		reply.Data = payload
+		if nodeID != "" {
+			reply.Header.Set(natsmsg.NodeIDHeader, nodeID)
+		}
+		_ = msg.RespondMsg(reply)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+func describeOutput(t *testing.T, instanceIDs ...string) []byte {
+	t.Helper()
+	out := ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{}}
+	if len(instanceIDs) > 0 {
+		reservation := &ec2.Reservation{ReservationId: aws.String("r-" + instanceIDs[0])}
+		for _, id := range instanceIDs {
+			reservation.Instances = append(reservation.Instances, &ec2.Instance{
+				InstanceId: aws.String(id),
+				State:      &ec2.InstanceState{Name: aws.String("running")},
+			})
+		}
+		out.Reservations = append(out.Reservations, reservation)
+	}
+	payload, err := json.Marshal(out)
+	require.NoError(t, err)
+	return payload
+}
+
+// emptyInstanceBuckets answers the stopped and terminated bucket queries with
+// nothing, so only the running-instance fan-out decides completeness.
+func emptyInstanceBuckets(t *testing.T, nc *nats.Conn) {
+	t.Helper()
+	answerAsNode(t, nc, "ec2.DescribeStoppedInstances", "", describeOutput(t))
+	answerAsNode(t, nc, "ec2.DescribeTerminatedInstances", "", describeOutput(t))
+}
+
+// A node that never answers is the timeout case: its running guests are absent
+// from the aggregate, so an empty answer from the rest is not evidence the
+// account has no instances.
+func TestInstanceReaperFailsClosedWhenANodeDoesNotAnswer(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	emptyInstanceBuckets(t, nc)
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
+	require.NoError(t, nc.Flush())
+
+	found, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
+
+	require.Error(t, err, "a listing missing a node must not read as an empty account, got %v", found)
+	code, _, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, code)
+}
+
+// The partial list is the dangerous shape: the reaper would terminate what it
+// can see and then report the stage drained on the next, equally partial pass.
+func TestInstanceReaperFailsClosedOnAPartialList(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	emptyInstanceBuckets(t, nc)
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t, "i-on-node-1"))
+	require.NoError(t, nc.Flush())
+
+	found, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
+
+	require.Error(t, err, "a partial listing must not be returned as the inventory, got %v", found)
+}
+
+// An error frame from a node is no better than silence: it says nothing about
+// what that node is running.
+func TestInstanceReaperFailsClosedWhenANodeAnswersWithAnError(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	emptyInstanceBuckets(t, nc)
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-2", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	require.NoError(t, nc.Flush())
+
+	_, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
+
+	require.Error(t, err)
+}
+
+// Completeness is judged against the configured node set, not a count of
+// replies: two frames from one node while another is silent is still missing
+// a node.
+func TestInstanceReaperFailsClosedWhenOneNodeAnswersTwice(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	emptyInstanceBuckets(t, nc)
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
+	require.NoError(t, nc.Flush())
+
+	_, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
+
+	require.Error(t, err)
+}
+
+// Stopped instances live in a bucket, not on a node. A failed bucket query
+// hides them as surely as a silent node hides running ones.
+func TestInstanceReaperFailsClosedWhenTheStoppedBucketFails(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	answerAsNode(t, nc, "ec2.DescribeStoppedInstances", "", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	answerAsNode(t, nc, "ec2.DescribeTerminatedInstances", "", describeOutput(t))
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-2", describeOutput(t))
+	require.NoError(t, nc.Flush())
+
+	_, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
+
+	require.Error(t, err)
+}
+
+// The healthy cluster still lists: every configured node answered, so the
+// union is the account's inventory.
+func TestInstanceReaperListsEveryNodeWhenAllAnswer(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	emptyInstanceBuckets(t, nc)
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t, "i-on-node-1"))
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-2", describeOutput(t, "i-on-node-2"))
+	require.NoError(t, nc.Flush())
+
+	started := time.Now()
+	found, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
+	require.NoError(t, err)
+
+	var ids []string
+	for _, resource := range found {
+		ids = append(ids, resource.ID)
+	}
+	assert.ElementsMatch(t, []string{"i-on-node-1", "i-on-node-2"}, ids)
+	assert.Less(t, time.Since(started), 2*time.Second, "a fully answered fan-out must not wait out its deadline")
+}
+
+// End to end through the engine: with a node silent, the compute stage must
+// not read as drained and the account record must survive.
+func TestTeardownDoesNotReportComputeDrainedWhileANodeIsSilent(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	emptyInstanceBuckets(t, nc)
+	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
+	require.NoError(t, nc.Flush())
+
+	accounts := newFakeAccounts()
+	volumes := &fakeReaper{kind: "volume", stage: StageStorage,
+		resources: []Resource{{Kind: "volume", ID: "vol-held-by-node-2"}}}
+	engine := quietEngine(accounts, testInventoryReaper(nc), volumes)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	result, err := engine.Teardown(ctx, request())
+
+	require.Error(t, err, "teardown must not succeed on an incomplete instance inventory")
+	assert.ErrorContains(t, err, "list instance")
+	assert.False(t, accounts.deleted, "the account record must survive an incomplete inventory")
+	for _, stage := range result.Stages {
+		assert.NotEqual(t, StageCompute, stage.Stage, "the compute stage must not be reported finished")
+	}
+	assert.Equal(t, 0, volumes.deletesAttempted, "storage must not be touched before compute is proven drained")
+}

spinifex/accountteardown/reapers_ec2_platform_current_behaviour_test.go

@@ -0,0 +1,219 @@
+package accountteardown
+
+//test:in-package — drives the unexported platform reapers and reuses the
+// engine test doubles.
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
+	ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The TestCurrentBehaviour_AccountTermination_ tests pin what the platform
+// reapers report while owner stores still hold records. They describe current
+// behaviour, not the target design; a failure means it changed.
+
+const platformBucket = "test-bucket"
+
+// deleteFailingStore fails object deletes its predicate selects.
+type deleteFailingStore struct {
+	*objectstore.MemoryObjectStore
+
+	failDelete func(key string) bool
+}
+
+var _ objectstore.ObjectStore = (*deleteFailingStore)(nil)
+
+func (s *deleteFailingStore) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if s.failDelete != nil && s.failDelete(aws.StringValue(in.Key)) {
+		return nil, errors.New("injected store failure")
+	}
+	return s.MemoryObjectStore.DeleteObject(ctx, in)
+}
+
+// platformOwners runs the real key, placement-group, launch-template and tag
+// services behind the subjects the platform reapers call.
+type platformOwners struct {
+	nc        *nats.Conn
+	store     *deleteFailingStore
+	keys      *ec2key.KeyServiceImpl
+	groups    *ec2placementgroup.PlacementGroupServiceImpl
+	templates *ec2launchtemplate.LaunchTemplateServiceImpl
+	tags      *ec2tags.TagsServiceImpl
+	pgKV      jetstream.KeyValue
+	ltKV      jetstream.KeyValue
+	tagsKV    jetstream.KeyValue
+}
+
+// serveOwner answers subject with fn, taking the account from X-Account-ID
+// as the daemon's handlers do.
+func serveOwner[I, O any](t *testing.T, nc *nats.Conn, subject string, fn func(context.Context, *I, string) (*O, error)) {
+	t.Helper()
+	sub, err := nc.QueueSubscribe(subject, "spinifex-workers", func(msg *nats.Msg) {
+		natsmsg.ServeNATSRequestCtx(msg, func(ctx context.Context, in *I) (*O, error) {
+			return fn(ctx, in, natsmsg.AccountIDFromMsg(msg))
+		})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+func newPlatformOwners(t *testing.T) *platformOwners {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	ctx := t.Context()
+	o := &platformOwners{nc: nc, store: &deleteFailingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}}
+
+	var err error
+	o.tagsKV, err = ec2tags.GetOrCreateTagsBucket(ctx, js)
+	require.NoError(t, err)
+	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: platformBucket}}
+	o.tags = ec2tags.NewTagsServiceImplWithStore(cfg, o.store, o.tagsKV)
+	o.keys = ec2key.NewKeyServiceImplWithStore(o.store, platformBucket)
+	o.keys.SetCentralTagStore(o.tags)
+	o.groups, err = ec2placementgroup.NewPlacementGroupServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	o.templates, err = ec2launchtemplate.NewLaunchTemplateServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	o.pgKV, err = js.KeyValue(ctx, ec2placementgroup.KVBucketPlacementGroups)
+	require.NoError(t, err)
+	o.ltKV, err = js.KeyValue(ctx, ec2launchtemplate.KVBucketLaunchTemplates)
+	require.NoError(t, err)
+
+	serveOwner(t, nc, "ec2.DescribeKeyPairs", o.keys.DescribeKeyPairs)
+	serveOwner(t, nc, "ec2.DeleteKeyPair", o.keys.DeleteKeyPair)
+	serveOwner(t, nc, "ec2.DescribePlacementGroups", o.groups.DescribePlacementGroups)
+	serveOwner(t, nc, "ec2.DeletePlacementGroup", o.groups.DeletePlacementGroup)
+	serveOwner(t, nc, "ec2.DescribeLaunchTemplates", o.templates.DescribeLaunchTemplates)
+	serveOwner(t, nc, "ec2.DeleteLaunchTemplate", o.templates.DeleteLaunchTemplate)
+	require.NoError(t, nc.Flush())
+	return o
+}
+
+func (o *platformOwners) objectKeys(t *testing.T) []string {
+	t.Helper()
+	objects, _, err := objectstore.ListAll(t.Context(), o.store, &s3.ListObjectsV2Input{
+		Bucket: aws.String(platformBucket), Prefix: aws.String("keys/" + testAccountID + "/"),
+	})
+	require.NoError(t, err)
+	var keys []string
+	for _, obj := range objects {
+		keys = append(keys, aws.StringValue(obj.Key))
+	}
+	return keys
+}
+
+func accountKVKeys(t *testing.T, kv jetstream.KeyValue) []string {
+	t.Helper()
+	keys, err := kv.Keys(t.Context())
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil
+	}
+	require.NoError(t, err)
+	var mine []string
+	for _, k := range keys {
+		if strings.HasPrefix(k, testAccountID+".") {
+			mine = append(mine, k)
+		}
+	}
+	slices.Sort(mine)
+	return mine
+}
+
+func platformReapers(nc *nats.Conn) []Reaper {
+	return []Reaper{&keyPairReaper{nc: nc}, &placementGroupReaper{nc: nc}, &launchTemplateReaper{nc: nc}}
+}
+
+// Pins current behaviour that differs from AWS: a teardown reports the
+// platform stage drained and deletes the account record while the owner
+// stores still hold key, placement-group, launch-template and tag records.
+func TestCurrentBehaviour_AccountTermination_PlatformReapersReportDrainedOverHiddenResidue(t *testing.T) {
+	o := newPlatformOwners(t)
+	ctx := testCtx(t)
+
+	// Live items: each is listed, and deleted through its owner's subject.
+	_, err := o.keys.ImportKeyPair(ctx, &ec2.ImportKeyPairInput{
+		KeyName:           aws.String("deploy"),
+		PublicKeyMaterial: []byte("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"),
+	}, testAccountID)
+	require.NoError(t, err)
+	pg, err := o.groups.CreatePlacementGroup(ctx, &ec2.CreatePlacementGroupInput{
+		GroupName: aws.String("spread"), Strategy: aws.String(ec2.PlacementStrategySpread),
+	}, testAccountID)
+	require.NoError(t, err)
+	lt, err := o.templates.CreateLaunchTemplate(ctx, &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("web"),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("secret")))},
+	}, testAccountID)
+	require.NoError(t, err)
+	pgID, ltID := aws.StringValue(pg.PlacementGroup.GroupId), aws.StringValue(lt.LaunchTemplate.LaunchTemplateId)
+	_, err = o.tags.CreateTags(ctx, &ec2.CreateTagsInput{
+		Resources: []*string{aws.String(pgID), aws.String(ltID)},
+		Tags:      []*ec2.Tag{{Key: aws.String("team"), Value: aws.String("blue")}},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	// Residue each owner's Describe skips.
+	publicKey := "keys/" + testAccountID + "/deploy"
+	o.store.failDelete = func(key string) bool { return key == publicKey }
+	_, err = o.store.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(platformBucket), Key: aws.String("keys/" + testAccountID + "/key-0badbadbadbadbad0.json"),
+		Body: strings.NewReader("{not json"),
+	})
+	require.NoError(t, err)
+	_, err = o.pgKV.Put(ctx, testAccountID+".broken", []byte("{not json"))
+	require.NoError(t, err)
+	orphanBody := testAccountID + ".lt-0orphan000000000.v1"
+	_, err = o.ltKV.Put(ctx, orphanBody, []byte(`{"version_number":1,"data":{"UserData":"c2VjcmV0"}}`))
+	require.NoError(t, err)
+
+	accounts := newFakeAccounts()
+	result, err := quietEngine(accounts, platformReapers(o.nc)...).Teardown(ctx, request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+	assert.True(t, accounts.deleted)
+	assert.Zero(t, result.StuckCount())
+
+	var deleted []string
+	for _, stage := range result.Stages {
+		for _, r := range stage.Deleted {
+			deleted = append(deleted, r.Kind+"/"+r.ID)
+		}
+	}
+	assert.ElementsMatch(t, []string{"key-pair/deploy", "placement-group/spread", "launch-template/" + ltID}, deleted)
+
+	for _, reaper := range platformReapers(o.nc) {
+		found, err := reaper.List(ctx, testAccountID)
+		require.NoError(t, err)
+		assert.Empty(t, found, "%s lists nothing", reaper.Kind())
+	}
+
+	assert.ElementsMatch(t, []string{publicKey, "keys/" + testAccountID + "/key-0badbadbadbadbad0.json"}, o.objectKeys(t))
+	assert.Equal(t, []string{testAccountID + ".broken"}, accountKVKeys(t, o.pgKV))
+	assert.Equal(t, []string{orphanBody}, accountKVKeys(t, o.ltKV))
+	assert.Equal(t, sortedStrings(testAccountID+"."+pgID, testAccountID+"."+ltID), accountKVKeys(t, o.tagsKV))
+}
+
+func sortedStrings(s ...string) []string {
+	slices.Sort(s)
+	return s
+}

spinifex/accountteardown/reapers_ec2_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,6 +32,7 @@ type fakeCluster struct {
 	requests  map[string][]byte
 	replies   map[string]any
 	errorCode map[string]string
+	handlers  map[string]func(data []byte) (reply any, errorCode string)
 }
 
 func newFakeCluster(t *testing.T) *fakeCluster {
@@ -42,6 +44,7 @@ func newFakeCluster(t *testing.T) *fakeCluster {
 		requests:  map[string][]byte{},
 		replies:   map[string]any{},
 		errorCode: map[string]string{},
+		handlers:  map[string]func([]byte) (any, string){},
 	}
 
 	sub, err := nc.Subscribe("ec2.>", cluster.serve)
@@ -52,26 +55,42 @@ func newFakeCluster(t *testing.T) *fakeCluster {
 	return cluster
 }
 
+// fakeClusterNode is the one node the fake cluster answers as. Replies carry
+// its ID because an inventory fan-out only counts attributable frames.
+const fakeClusterNode = "node-1"
+
 func (c *fakeCluster) serve(msg *nats.Msg) {
 	c.mu.Lock()
 	c.calls = append(c.calls, msg.Subject)
 	c.requests[msg.Subject] = msg.Data
 	code, isError := c.errorCode[msg.Subject]
 	reply, hasReply := c.replies[msg.Subject]
+	handler, hasHandler := c.handlers[msg.Subject]
 	c.mu.Unlock()
 
+	respond := func(data []byte) {
+		out := nats.NewMsg(msg.Reply)
+		out.Data = data
+		out.Header.Set(natsmsg.NodeIDHeader, fakeClusterNode)
+		_ = msg.RespondMsg(out)
+	}
+	if hasHandler && !isError {
+		reply, code = handler(msg.Data)
+		isError, hasReply = code != "", true
+	}
+
 	switch {
 	case isError:
-		_ = msg.Respond(awserrors.GenerateErrorPayload(code))
+		respond(awserrors.GenerateErrorPayload(code))
 	case hasReply:
 		payload, err := json.Marshal(reply)
 		if err != nil {
-			_ = msg.Respond(awserrors.GenerateErrorPayload("InternalError"))
+			respond(awserrors.GenerateErrorPayload("InternalError"))
 			return
 		}
-		_ = msg.Respond(payload)
+		respond(payload)
 	default:
-		_ = msg.Respond([]byte(`{}`))
+		respond([]byte(`{}`))
 	}
 }
 
@@ -87,6 +106,29 @@ func (c *fakeCluster) fail(subject, code string) {
 	c.errorCode[subject] = code
 }
 
+// handle answers subject from state the test owns, for doubles whose answer
+// must change as the reaper acts — an attachment a force detach clears.
+func (c *fakeCluster) handle(subject string, handler func(data []byte) (any, string)) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.handlers[subject] = handler
+}
+
+// echoAsNode adds a second reply to every request on subject, attributed to
+// node, so one node can answer twice while another stays silent.
+func (c *fakeCluster) echoAsNode(t *testing.T, subject, node string) {
+	t.Helper()
+	sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
+		out := nats.NewMsg(msg.Reply)
+		out.Data = []byte(`{}`)
+		out.Header.Set(natsmsg.NodeIDHeader, node)
+		_ = msg.RespondMsg(out)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, c.nc.Flush())
+}
+
 func (c *fakeCluster) called() []string {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -108,10 +150,17 @@ func testCtx(t *testing.T) context.Context {
 	return ctx
 }
 
+// silentClusterCtx outlasts the full fan-out deadline a silent node costs.
+func silentClusterCtx(t *testing.T) context.Context {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	t.Cleanup(cancel)
+	return ctx
+}
+
 // The stage graph is what keeps teardown converging, so it is asserted rather
 // than left to the order the constructor happens to list.
 func TestEC2ReapersAreOrderedByDependency(t *testing.T) {
-	reapers := EC2Reapers(nil, 1)
+	reapers := EC2Reapers(nil, 1, []string{"node-1"})
 
 	var kinds []string
 	for _, reaper := range reapers {
@@ -156,7 +205,7 @@ func TestInstanceReaperSkipsTerminalStates(t *testing.T) {
 		}}},
 	})
 
-	reaper := &instanceReaper{nc: cluster.nc, expectedNodes: 1}
+	reaper := &instanceReaper{nc: cluster.nc, nodeIDs: []string{fakeClusterNode}}
 	found, err := reaper.List(testCtx(t), "000000000042")
 	require.NoError(t, err)
 
@@ -169,12 +218,12 @@ func TestInstanceReaperSkipsTerminalStates(t *testing.T) {
 // makes a stuck host the case --force exists for.
 func TestInstanceReaperTerminates(t *testing.T) {
 	cluster := newFakeCluster(t)
-	reaper := &instanceReaper{nc: cluster.nc, expectedNodes: 1}
+	reaper := &instanceReaper{nc: cluster.nc, nodeIDs: []string{fakeClusterNode}}
 
 	require.NoError(t, reaper.Delete(testCtx(t), "000000000042",
-		Resource{Kind: "instance", ID: "i-abc"}, false))
+		Resource{Kind: "instance", ID: "i-0abc0123456789def"}, false))
 
-	assert.Contains(t, cluster.called(), "ec2.cmd.i-abc")
+	assert.Contains(t, cluster.called(), "ec2.cmd.i-0abc0123456789def")
 }
 
 // The attachment is the reason a volume will not go, so it has to reach the
@@ -190,7 +239,7 @@ func TestVolumeReaperCarriesItsAttachment(t *testing.T) {
 		},
 	})
 
-	reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1}
+	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
 	found, err := reaper.List(testCtx(t), "000000000042")
 	require.NoError(t, err)
 
@@ -205,31 +254,250 @@ func TestVolumeReaperReportsRefusalWithoutForce(t *testing.T) {
 	cluster := newFakeCluster(t)
 	cluster.fail("ec2.DeleteVolume", "VolumeInUse")
 
-	reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1}
+	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
 	err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, false)
 
 	require.Error(t, err)
 	assert.NotContains(t, cluster.called(), forceDetachSubject)
 }
 
-// The deadlock the force path exists for: the ordinary delete refuses, so the
-// attachment is cleared cluster-wide and the delete retried.
-func TestVolumeReaperForceClearsTheAttachment(t *testing.T) {
+// heldVolume is one volume record as its owner keeps it: the ordinary delete
+// refuses while the record names an instance, and a force detach clears it.
+type heldVolume struct {
+	mu       sync.Mutex
+	id       string
+	holder   string
+	deleted  bool
+	detaches int
+}
+
+func newHeldVolume(cluster *fakeCluster, id, holder string) *heldVolume {
+	volume := &heldVolume{id: id, holder: holder}
+	cluster.handle("ec2.DescribeVolumes", func(data []byte) (any, string) {
+		var input ec2.DescribeVolumesInput
+		_ = json.Unmarshal(data, &input)
+		volume.mu.Lock()
+		defer volume.mu.Unlock()
+		if volume.deleted {
+			if len(input.VolumeIds) > 0 {
+				return nil, awserrors.ErrorInvalidVolumeNotFound
+			}
+			return ec2.DescribeVolumesOutput{}, ""
+		}
+		described := &ec2.Volume{VolumeId: aws.String(volume.id), State: aws.String("available")}
+		if volume.holder != "" {
+			described.State = aws.String("in-use")
+			described.Attachments = []*ec2.VolumeAttachment{
+				{VolumeId: aws.String(volume.id), InstanceId: aws.String(volume.holder), State: aws.String("attached")},
+			}
+		}
+		return ec2.DescribeVolumesOutput{Volumes: []*ec2.Volume{described}}, ""
+	})
+	cluster.handle(forceDetachSubject, func([]byte) (any, string) {
+		volume.mu.Lock()
+		defer volume.mu.Unlock()
+		volume.detaches++
+		previous := volume.holder
+		volume.holder = ""
+		return ec2.VolumeAttachment{VolumeId: aws.String(volume.id), InstanceId: aws.String(previous)}, ""
+	})
+	cluster.handle("ec2.DeleteVolume", func([]byte) (any, string) {
+		volume.mu.Lock()
+		defer volume.mu.Unlock()
+		switch {
+		case volume.deleted:
+			return nil, awserrors.ErrorInvalidVolumeNotFound
+		case volume.holder != "":
+			return nil, awserrors.ErrorVolumeInUse
+		}
+		volume.deleted = true
+		return ec2.DeleteVolumeOutput{}, ""
+	})
+	return volume
+}
+
+func (v *heldVolume) state() (deleted bool, holder string, detaches int) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return v.deleted, v.holder, v.detaches
+}
+
+func instancesIn(state string, instanceIDs ...string) ec2.DescribeInstancesOutput {
+	reservation := &ec2.Reservation{ReservationId: aws.String("r-" + state)}
+	for _, id := range instanceIDs {
+		reservation.Instances = append(reservation.Instances, &ec2.Instance{
+			InstanceId: aws.String(id), State: &ec2.InstanceState{Name: aws.String(state)},
+		})
+	}
+	return ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{reservation}}
+}
+
+// Force must not clear an attachment a live guest still holds. The instance
+// carries no block device mapping for the volume, so the attachment record is
+// the only thing standing between the guest and a deleted disk.
+func TestVolumeReaperForceRetainsAVolumeALiveInstanceHolds(t *testing.T) {
+	for _, state := range []string{"running", "stopped", "shutting-down", "pending"} {
+		t.Run(state, func(t *testing.T) {
+			cluster := newFakeCluster(t)
+			volume := newHeldVolume(cluster, "vol-held", "i-live")
+			subject := "ec2.DescribeInstances"
+			if state == "stopped" {
+				subject = "ec2.DescribeStoppedInstances"
+			}
+			cluster.reply(subject, instancesIn(state, "i-live"))
+
+			reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
+			err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true)
+
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "vol-held")
+			assert.Contains(t, err.Error(), "i-live")
+			assert.Contains(t, err.Error(), state)
+			deleted, holder, detaches := volume.state()
+			assert.False(t, deleted, "a volume a live guest holds must survive teardown")
+			assert.Equal(t, "i-live", holder, "the attachment must not be cleared under a live guest")
+			assert.Zero(t, detaches)
+		})
+	}
+}
+
+// "Cannot prove gone" is not "gone": every way the proof can fall short keeps
+// the volume and its attachment exactly as they were.
+func TestVolumeReaperForceRetainsAVolumeWhenTheHolderCannotBeProvenGone(t *testing.T) {
+	twoNodes := []string{fakeClusterNode, "node-2"}
+	cases := []struct {
+		name  string
+		setup func(cluster *fakeCluster)
+		nodes []string
+	}{
+		{"configured node set unknown", func(*fakeCluster) {}, []string{}},
+		{"lone node answers NotFound while another is silent", func(cluster *fakeCluster) {
+			cluster.fail("ec2.DescribeInstances", awserrors.ErrorInvalidInstanceIDNotFound)
+		}, twoNodes},
+		{"terminated copy found while the holder node is silent", func(cluster *fakeCluster) {
+			cluster.reply("ec2.DescribeTerminatedInstances", instancesIn("terminated", "i-held"))
+		}, twoNodes},
+		{"node describe fails", func(cluster *fakeCluster) {
+			cluster.fail("ec2.DescribeInstances", awserrors.ErrorServerInternal)
+		}, nil},
+		{"terminated bucket unreachable", func(cluster *fakeCluster) {
+			cluster.fail("ec2.DescribeTerminatedInstances", awserrors.ErrorServerInternal)
+		}, nil},
+		{"stopped bucket unreachable", func(cluster *fakeCluster) {
+			cluster.fail("ec2.DescribeStoppedInstances", awserrors.ErrorServerInternal)
+		}, nil},
+		{"sources disagree", func(cluster *fakeCluster) {
+			cluster.reply("ec2.DescribeTerminatedInstances", instancesIn("terminated", "i-held"))
+			cluster.reply("ec2.DescribeInstances", instancesIn("running", "i-held"))
+		}, nil},
+		{"volume describe fails", func(cluster *fakeCluster) {
+			cluster.fail("ec2.DescribeVolumes", awserrors.ErrorServerInternal)
+		}, nil},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			cluster := newFakeCluster(t)
+			volume := newHeldVolume(cluster, "vol-held", "i-held")
+			tc.setup(cluster)
+			nodes := tc.nodes
+			if nodes == nil {
+				nodes = []string{fakeClusterNode}
+			}
+
+			reaper := newVolumeReaper(cluster.nc, nodes...)
+			err := reaper.Delete(silentClusterCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true)
+
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "vol-held")
+			deleted, holder, detaches := volume.state()
+			assert.False(t, deleted)
+			assert.Equal(t, "i-held", holder)
+			assert.Zero(t, detaches)
+		})
+	}
+}
+
+// The deadlock force exists for: the instance is gone but its volume record
+// still names it, so the ordinary delete refuses forever. Once the holder is
+// proven absent the attachment is cleared and the volume deleted.
+func TestVolumeReaperForceDeletesOnceTheHolderIsProvenGone(t *testing.T) {
+	cases := map[string]func(cluster *fakeCluster){
+		"terminated": func(cluster *fakeCluster) {
+			cluster.reply("ec2.DescribeTerminatedInstances", instancesIn("terminated", "i-gone"))
+		},
+		"not found": func(*fakeCluster) {},
+	}
+	for name, setup := range cases {
+		t.Run(name, func(t *testing.T) {
+			cluster := newFakeCluster(t)
+			volume := newHeldVolume(cluster, "vol-held", "i-gone")
+			setup(cluster)
+
+			reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
+			require.NoError(t, reaper.Delete(testCtx(t), "000000000042",
+				Resource{Kind: "volume", ID: "vol-held"}, true))
+
+			deleted, holder, detaches := volume.state()
+			assert.True(t, deleted)
+			assert.Empty(t, holder)
+			assert.Equal(t, 1, detaches)
+
+			var input ec2.DetachVolumeInput
+			cluster.request(t, forceDetachSubject, &input)
+			assert.Equal(t, "vol-held", aws.StringValue(input.VolumeId))
+		})
+	}
+}
+
+// With no holder on record, the refusal is about something else — a snapshot,
+// a mapping the delete guard found — and clearing the record cannot help.
+func TestVolumeReaperForceLeavesAnUnattachedRefusalAlone(t *testing.T) {
 	cluster := newFakeCluster(t)
-	cluster.fail("ec2.DeleteVolume", "VolumeInUse")
+	cluster.reply("ec2.DescribeVolumes", ec2.DescribeVolumesOutput{
+		Volumes: []*ec2.Volume{{VolumeId: aws.String("vol-snapped"), State: aws.String("available")}},
+	})
+	cluster.fail("ec2.DeleteVolume", awserrors.ErrorVolumeInUse)
 
-	reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1}
-	err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true)
+	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
+	err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-snapped"}, true)
 
-	// The delete still fails here because the double refuses every attempt;
-	// what matters is that force reached for the detach at all.
 	require.Error(t, err)
-	assert.Contains(t, cluster.called(), forceDetachSubject)
+	assert.Contains(t, err.Error(), awserrors.ErrorVolumeInUse)
+	assert.NotContains(t, cluster.called(), forceDetachSubject)
+}
 
-	var input ec2.DetachVolumeInput
-	cluster.request(t, forceDetachSubject, &input)
-	assert.Equal(t, "vol-held", aws.StringValue(input.VolumeId))
-	assert.True(t, aws.BoolValue(input.Force))
+// End to end through the engine: a forced teardown whose guest is still up
+// leaves the volume, names why, and keeps the account TERMINATING.
+func TestForcedTeardownKeepsTheAccountWhileALiveGuestHoldsAVolume(t *testing.T) {
+	cluster := newFakeCluster(t)
+	volume := newHeldVolume(cluster, "vol-held", "i-live")
+	cluster.reply("ec2.DescribeInstances", instancesIn("running", "i-live"))
+
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, newVolumeReaper(cluster.nc, fakeClusterNode))
+	req := request()
+	req.Force = true
+
+	result, err := engine.Teardown(testCtx(t), req)
+
+	require.ErrorIs(t, err, ErrResourcesStuck)
+	require.Equal(t, 1, result.StuckCount())
+	var stuck Stuck
+	for _, stage := range result.Stages {
+		if len(stage.Stuck) > 0 {
+			stuck = stage.Stuck[0]
+		}
+	}
+	assert.Equal(t, "vol-held", stuck.Resource.ID)
+	assert.Contains(t, stuck.Reason, "vol-held")
+	assert.Contains(t, stuck.Reason, "i-live")
+	assert.Contains(t, stuck.Reason, "running")
+	assert.False(t, result.AccountDeleted)
+	assert.False(t, accounts.deleted)
+	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+	deleted, holder, _ := volume.state()
+	assert.False(t, deleted)
+	assert.Equal(t, "i-live", holder)
 }
 
 // A volume that is already gone is a success. Teardown re-runs after a crash
@@ -238,7 +506,7 @@ func TestVolumeReaperTreatsAMissingVolumeAsDeleted(t *testing.T) {
 	cluster := newFakeCluster(t)
 	cluster.fail("ec2.DeleteVolume", "InvalidVolume.NotFound")
 
-	reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1}
+	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
 
 	assert.NoError(t, reaper.Delete(testCtx(t), "000000000042",
 		Resource{Kind: "volume", ID: "vol-gone"}, false))
@@ -541,7 +809,7 @@ func TestAListingFailureIsReported(t *testing.T) {
 	cluster := newFakeCluster(t)
 	cluster.fail("ec2.DescribeVolumes", "InternalError")
 
-	_, err := (&volumeReaper{nc: cluster.nc, expectedNodes: 1}).List(testCtx(t), "000000000042")
+	_, err := (newVolumeReaper(cluster.nc, fakeClusterNode)).List(testCtx(t), "000000000042")
 
 	assert.Error(t, err)
 }
@@ -678,7 +946,7 @@ func TestNetworkInterfaceReaperTreatsAMissingInterfaceAsDeleted(t *testing.T) {
 // subnet delete is refused while an interface is still in it.
 func TestEC2ReapersRemoveInterfacesBeforeSubnets(t *testing.T) {
 	var eni, subnet = -1, -1
-	for i, reaper := range EC2Reapers(nil, 1) {
+	for i, reaper := range EC2Reapers(nil, 1, []string{"node-1"}) {
 		switch reaper.Kind() {
 		case "network-interface":
 			eni = i
@@ -689,3 +957,60 @@ func TestEC2ReapersRemoveInterfacesBeforeSubnets(t *testing.T) {
 	require.NotEqual(t, -1, eni, "there is no network-interface reaper")
 	require.Less(t, eni, subnet, "interfaces must be reaped before the subnets holding them")
 }
+
+func newVolumeReaper(nc *nats.Conn, nodeIDs ...string) *volumeReaper {
+	return &volumeReaper{nc: nc, nodeIDs: nodeIDs}
+}
+
+// The holder lives on node-2, which never answers, while node-1 answers every
+// fan-out twice. A count of replies reads that as complete and the holder as
+// gone; only judging by responder identity sees that node-2 is missing.
+func TestVolumeReaperForceRetainsAVolumeWhenOneNodeAnswersTwiceAndTheHolderNodeIsSilent(t *testing.T) {
+	cluster := newFakeCluster(t)
+	volume := newHeldVolume(cluster, "vol-held", "i-on-silent-node")
+	cluster.echoAsNode(t, "ec2.DescribeInstances", fakeClusterNode)
+
+	reaper := newVolumeReaper(cluster.nc, fakeClusterNode, "node-2")
+	err := reaper.Delete(silentClusterCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true)
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "vol-held")
+	assert.Contains(t, err.Error(), "i-on-silent-node")
+	deleted, holder, detaches := volume.state()
+	assert.False(t, deleted, "a holder on a silent node is not proven gone")
+	assert.Equal(t, "i-on-silent-node", holder)
+	assert.Zero(t, detaches)
+	assert.NotContains(t, cluster.called(), forceDetachSubject)
+}
+
+func TestForcedTeardownKeepsTheAccountWhenOneNodeAnswersTwiceAndTheHolderNodeIsSilent(t *testing.T) {
+	cluster := newFakeCluster(t)
+	volume := newHeldVolume(cluster, "vol-held", "i-on-silent-node")
+	cluster.echoAsNode(t, "ec2.DescribeInstances", fakeClusterNode)
+
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, newVolumeReaper(cluster.nc, fakeClusterNode, "node-2"))
+	req := request()
+	req.Force = true
+
+	result, err := engine.Teardown(silentClusterCtx(t), req)
+
+	require.ErrorIs(t, err, ErrResourcesStuck)
+	require.Equal(t, 1, result.StuckCount())
+	var stuck Stuck
+	for _, stage := range result.Stages {
+		if len(stage.Stuck) > 0 {
+			stuck = stage.Stuck[0]
+		}
+	}
+	assert.Equal(t, "vol-held", stuck.Resource.ID)
+	assert.Contains(t, stuck.Reason, "vol-held")
+	assert.Contains(t, stuck.Reason, "i-on-silent-node")
+	assert.False(t, result.AccountDeleted)
+	assert.False(t, accounts.deleted)
+	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+	deleted, holder, detaches := volume.state()
+	assert.False(t, deleted)
+	assert.Equal(t, "i-on-silent-node", holder)
+	assert.Zero(t, detaches)
+}

spinifex/accountteardown/reapers_ecs_test.go

@@ -178,7 +178,7 @@ func TestECSClusterNameFromARN(t *testing.T) {
 // stopped first, which is what puts ECS ahead of it in the compute stage.
 func TestECSReapersRunBeforeTheInstanceReaper(t *testing.T) {
 	reapers := ECSReapers(nil)
-	reapers = append(reapers, EC2Reapers(nil, 1)...)
+	reapers = append(reapers, EC2Reapers(nil, 1, []string{"node-1"})...)
 
 	ecsIndex := indexOfKind(reapers, "ecs-cluster")
 	instanceIndex := indexOfKind(reapers, "instance")
@@ -197,7 +197,7 @@ func TestECSReapersRunBeforeTheInstanceReaper(t *testing.T) {
 // the registration order above survives it.
 func TestSortReapersKeepsECSAheadOfInstances(t *testing.T) {
 	reapers := ECSReapers(nil)
-	reapers = append(reapers, EC2Reapers(nil, 1)...)
+	reapers = append(reapers, EC2Reapers(nil, 1, []string{"node-1"})...)
 	SortReapers(reapers)
 
 	assert.Less(t, indexOfKind(reapers, "ecs-cluster"), indexOfKind(reapers, "instance"))

spinifex/accountteardown/reapers_s3_test.go

@@ -277,7 +277,7 @@ func TestS3ReaperRunsInTheStorageStage(t *testing.T) {
 // The engine sorts reapers by stage and the sort is stable, so a bucket reaper
 // registered after the EC2 storage reapers stays after them.
 func TestSortReapersKeepsBucketsAfterVolumes(t *testing.T) {
-	reapers := append(EC2Reapers(nil, 1), S3Reapers(newFakeBuckets())...)
+	reapers := append(EC2Reapers(nil, 1, []string{"node-1"}), S3Reapers(newFakeBuckets())...)
 	SortReapers(reapers)
 
 	storage := make([]string, 0, len(reapers))

spinifex/agents/rds/agent/agent_test.go

@@ -35,6 +35,8 @@ type fakeControlPlane struct {
 
 	bootstrapOut  *handlers_rds.GetDBBootstrapConfigOutput
 	bootstrapErrs []error
+	// Returned once bootstrapErrs is drained, for a failure no retry heals.
+	bootstrapErr  error
 	bootstrapReqs []identity
 	// Scripted responses ahead of bootstrapOut, so a test can answer the first
 	// fetch differently from the ones the agent retries with.
@@ -105,6 +107,9 @@ func (f *fakeControlPlane) GetBootstrapConfig(_ context.Context, id identity) (*
 	if err := nextErr(&f.bootstrapErrs); err != nil {
 		return nil, err
 	}
+	if f.bootstrapErr != nil {
+		return nil, f.bootstrapErr
+	}
 	if len(f.bootstrapQueue) > 0 {
 		out := f.bootstrapQueue[0]
 		f.bootstrapQueue = f.bootstrapQueue[1:]
@@ -639,7 +644,9 @@ func TestMountTableContains(t *testing.T) {
 
 func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
 	cp := newFakeControlPlane()
-	cp.bootstrapErrs = []error{errors.New("403 AccessDenied: instance profile is not authorised")}
+	// Persistent, so every retry republishes the 403 rather than leaving it
+	// visible to a heartbeat for a single retry interval.
+	cp.bootstrapErr = errors.New("403 AccessDenied: instance profile is not authorised")
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)

spinifex/daemon/daemon.go

@@ -950,6 +950,7 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.ModifyLaunchTemplate", handleNATSRequest(d.node, d.launchTemplateService.ModifyLaunchTemplate), "spinifex-workers"},
 		{"ec2.DescribeLaunchTemplates", handleNATSRequest(d.node, d.launchTemplateService.DescribeLaunchTemplates), "spinifex-workers"},
 		{"ec2.DescribeLaunchTemplateVersions", handleNATSRequest(d.node, d.launchTemplateService.DescribeLaunchTemplateVersions), "spinifex-workers"},
+		{ec2v1.PlatformInventorySubject, handleNATSRequest(d.node, d.platformInventory), "spinifex-workers"},
 		{"ec2.PutSpotInstanceRequests", handleNATSRequest(d.node, d.spotInstanceService.PutSpotInstanceRequests), "spinifex-workers"},
 		{"ec2.DescribeSpotInstanceRequests", handleNATSRequest(d.node, d.spotInstanceService.DescribeSpotInstanceRequests), "spinifex-workers"},
 		{"ec2.CancelSpotInstanceRequests", handleNATSRequest(d.node, d.spotInstanceService.CancelSpotInstanceRequests), "spinifex-workers"},

spinifex/daemon/daemon_handlers_platform_inventory.go

@@ -0,0 +1,16 @@
+package daemon
+
+import (
+	"context"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
+)
+
+// platformInventory serves the read-only EC2 platform inventory. The account
+// comes from the request body, not X-Account-ID, so the scanned account is
+// part of the versioned contract; the header is ignored.
+func (d *Daemon) platformInventory(ctx context.Context, req *ec2v1.PlatformInventoryRequest, _ string) (*ec2v1.PlatformInventoryReply, error) {
+	inventory := accountinventory.New(d.keyService, d.placementGroupService, d.launchTemplateService, d.tagsService)
+	return inventory.PlatformInventory(ctx, req)
+}

spinifex/daemon/daemon_handlers_platform_inventory_test.go

@@ -0,0 +1,287 @@
+package daemon
+
+//test:in-package — the subject is served from the daemon's own subscription
+// table, registered by the unexported subscribeAll.
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/readonlystore"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
+	ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	inventoryAccount = "111122223333"
+	inventoryOther   = "444455556666"
+	inventoryEmpty   = "777788889999"
+	inventoryBucket  = "test-bucket"
+	inventoryPubKey  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"
+	inventoryUser    = "#!/bin/sh\necho tenant-user-data-secret\n"
+
+	orphanTemplateID = "lt-0orphan000000000"
+	brokenTemplateID = "lt-0badbadbadbadbad0"
+	brokenKeyPairID  = "key-0badbadbadbadbad0"
+	deletedGroupID   = "pg-0deleted00000000"
+)
+
+// inventoryCluster is a daemon serving its real subscription table over owner
+// stores seeded with live records and every residue class, and a separate
+// client holding no IAM credential.
+type inventoryCluster struct {
+	caller  *nats.Conn
+	js      jetstream.JetStream
+	live    map[string]string
+	heldIDs []string
+	secrets []string
+}
+
+func newInventoryCluster(t *testing.T) *inventoryCluster {
+	t.Helper()
+	ns, nc, js := testutil.StartTestJetStream(t)
+	ctx := t.Context()
+	store := objectstore.NewMemoryObjectStore()
+	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: inventoryBucket}}
+	tagsKV, err := ec2tags.GetOrCreateTagsBucket(ctx, js)
+	require.NoError(t, err)
+	tags := ec2tags.NewTagsServiceImplWithStore(cfg, store, tagsKV)
+	keys := ec2key.NewKeyServiceImplWithStore(store, inventoryBucket)
+	keys.SetCentralTagStore(tags)
+	groups, err := ec2placementgroup.NewPlacementGroupServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	templates, err := ec2launchtemplate.NewLaunchTemplateServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	c := &inventoryCluster{js: js, live: map[string]string{}}
+
+	tagSpec := func(resourceType string) []*ec2.TagSpecification {
+		return []*ec2.TagSpecification{{ResourceType: aws.String(resourceType),
+			Tags: []*ec2.Tag{{Key: aws.String("secret-tag-key"), Value: aws.String("secret-tag-value")}}}}
+	}
+	secretTags := map[string]string{"secret-tag-key": "secret-tag-value"}
+
+	for _, account := range []string{inventoryAccount, inventoryOther} {
+		key, err := keys.ImportKeyPair(ctx, &ec2.ImportKeyPairInput{
+			KeyName: aws.String("live-key-name"), PublicKeyMaterial: []byte(inventoryPubKey), TagSpecifications: tagSpec("key-pair"),
+		}, account)
+		require.NoError(t, err)
+		group, err := groups.CreatePlacementGroup(ctx, &ec2.CreatePlacementGroupInput{
+			GroupName: aws.String("live-group-name"), Strategy: aws.String(ec2.PlacementStrategySpread),
+		}, account)
+		require.NoError(t, err)
+		template, err := templates.CreateLaunchTemplate(ctx, &ec2.CreateLaunchTemplateInput{
+			LaunchTemplateName: aws.String("live-template-name"),
+			LaunchTemplateData: &ec2.RequestLaunchTemplateData{UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(inventoryUser)))},
+			TagSpecifications:  tagSpec("launch-template"),
+		}, account)
+		require.NoError(t, err)
+		if account == inventoryAccount {
+			c.live[aws.StringValue(key.KeyPairId)] = ec2v1.PlatformKindKeyPair
+			c.live[aws.StringValue(group.PlacementGroup.GroupId)] = ec2v1.PlatformKindPlacementGroup
+			c.live[aws.StringValue(template.LaunchTemplate.LaunchTemplateId)] = ec2v1.PlatformKindLaunchTemplate
+		}
+	}
+
+	// A group still holding a launched instance.
+	held, err := groups.CreatePlacementGroup(ctx, &ec2.CreatePlacementGroupInput{
+		GroupName: aws.String("held-group-name"), Strategy: aws.String(ec2.PlacementStrategyCluster),
+	}, inventoryAccount)
+	require.NoError(t, err)
+	reserved, err := groups.ReserveClusterNode(ctx, &ec2placementgroup.ReserveClusterNodeInput{
+		GroupName: "held-group-name", EligibleNodes: []string{"node-1"},
+	}, inventoryAccount)
+	require.NoError(t, err)
+	_, err = groups.FinalizeClusterInstances(ctx, &ec2placementgroup.FinalizeClusterInstancesInput{
+		GroupName: "held-group-name", NodeInstances: map[string][]string{reserved.TargetNode: {"i-0held0000000000"}},
+	}, inventoryAccount)
+	require.NoError(t, err)
+	c.heldIDs = []string{aws.StringValue(held.PlacementGroup.GroupId)}
+
+	// Owner-store residue no Describe subject reports.
+	prefix := "keys/" + inventoryAccount + "/"
+	for objectKey, body := range map[string]string{
+		prefix + "orphan-secret-key-name":                    inventoryPubKey + "\n",
+		prefix + brokenKeyPairID + ".json":                   `{"KeyName":"broken-secret-key-name","KeyPairId":`,
+		"keys/" + inventoryOther + "/orphan-secret-key-name": inventoryPubKey + "\n",
+	} {
+		_, err := store.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(inventoryBucket), Key: aws.String(objectKey), Body: bytes.NewReader([]byte(body))})
+		require.NoError(t, err)
+	}
+	ltKV, err := js.KeyValue(ctx, ec2launchtemplate.KVBucketLaunchTemplates)
+	require.NoError(t, err)
+	orphanBody, err := json.Marshal(ec2launchtemplate.LaunchTemplateVersionRec{VersionNumber: 1,
+		Data: &ec2.ResponseLaunchTemplateData{UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(inventoryUser)))}})
+	require.NoError(t, err)
+	orphanName := inventoryAccount + ".name." + hex.EncodeToString([]byte("orphan-secret-template-name"))
+	for kvKey, body := range map[string][]byte{
+		inventoryAccount + "." + orphanTemplateID + ".v1": orphanBody,
+		inventoryAccount + "." + orphanTemplateID + ".v2": orphanBody,
+		orphanName: []byte(orphanTemplateID),
+		inventoryAccount + "." + brokenTemplateID:         []byte(`{"launch_template_name":"broken-secret-template-name"`),
+		inventoryAccount + "." + brokenTemplateID + ".v1": orphanBody,
+	} {
+		_, err := ltKV.Put(ctx, kvKey, body)
+		require.NoError(t, err)
+	}
+	// Tag-index entries a placement-group and launch-template delete leave.
+	require.NoError(t, tags.PutResourceTags(ctx, inventoryAccount, deletedGroupID, secretTags))
+	require.NoError(t, tags.PutResourceTags(ctx, inventoryAccount, orphanTemplateID, secretTags))
+	require.NoError(t, tags.PutResourceTags(ctx, inventoryAccount, brokenTemplateID, secretTags))
+
+	c.secrets = []string{
+		"i-0held0000000000",
+		"live-key-name", "live-group-name", "live-template-name", "held-group-name",
+		"orphan-secret-key-name", "broken-secret-key-name", "orphan-secret-template-name",
+		hex.EncodeToString([]byte("orphan-secret-template-name")), "broken-secret-template-name",
+		"secret-tag-key", "secret-tag-value", "AAAAC3NzaC1lZDI1NTE5", "tenant-user-data-secret",
+		base64.StdEncoding.EncodeToString([]byte(inventoryUser)), brokenKeyPairID, brokenTemplateID,
+		"keys/", ".json", ".name.", ".v1", ".v2", inventoryAccount + ".", "invalid character", "unexpected end",
+	}
+
+	// The daemon reads the object store and tag index through read-only views.
+	d := newSubscribeTestDaemon(t, nc, "", "")
+	ro := readonlystore.NewObjectStore(t, store)
+	d.keyService = ec2key.NewKeyServiceImplWithStore(ro, inventoryBucket)
+	d.tagsService = ec2tags.NewTagsServiceImplWithStore(cfg, ro, readonlystore.NewKV(t, tagsKV))
+	d.placementGroupService = groups
+	d.launchTemplateService = templates
+	require.NoError(t, d.subscribeAll())
+
+	c.caller, err = nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(c.caller.Close)
+	return c
+}
+
+// request sends body with X-Account-ID set to header and returns the raw reply.
+func (c *inventoryCluster) request(t *testing.T, body any, header string) []byte {
+	t.Helper()
+	data, err := json.Marshal(body)
+	require.NoError(t, err)
+	reply, err := natsRequestAs(c.caller, ec2v1.PlatformInventorySubject, data, header, 5*time.Second)
+	require.NoError(t, err)
+	return reply.Data
+}
+
+func (c *inventoryCluster) lastSeq(t *testing.T, bucket string) uint64 {
+	t.Helper()
+	stream, err := c.js.Stream(t.Context(), "KV_"+bucket)
+	require.NoError(t, err)
+	info, err := stream.Info(t.Context())
+	require.NoError(t, err)
+	return info.State.LastSeq
+}
+
+// The daemon answers for the account in the body, whatever X-Account-ID says,
+// with live and tag-index residue IDs, owner-store residue as counts, and no
+// write to any store.
+func TestPlatformInventorySubjectReportsTheAccountsOwnerStores(t *testing.T) {
+	c := newInventoryCluster(t)
+	pgSeq := c.lastSeq(t, ec2placementgroup.KVBucketPlacementGroups)
+	ltSeq := c.lastSeq(t, ec2launchtemplate.KVBucketLaunchTemplates)
+
+	raw := c.request(t, ec2v1.PlatformInventoryRequest{AccountID: inventoryAccount, CorrelationRef: "req-1"}, inventoryOther)
+	require.Empty(t, replyErrCode(t, raw), string(raw))
+	var reply ec2v1.PlatformInventoryReply
+	require.NoError(t, json.Unmarshal(raw, &reply))
+
+	assert.Equal(t, inventoryAccount, reply.AccountID)
+	assert.Equal(t, "req-1", reply.CorrelationRef)
+	want := []ec2v1.PlatformItem{
+		{ResourceID: c.heldIDs[0], Kind: ec2v1.PlatformKindPlacementGroup, Disposition: ec2v1.PlatformDispositionDependencyHeld},
+		{ResourceID: deletedGroupID, Kind: ec2v1.PlatformKindTagIndex, Disposition: ec2v1.PlatformDispositionResidue},
+		{ResourceID: orphanTemplateID, Kind: ec2v1.PlatformKindTagIndex, Disposition: ec2v1.PlatformDispositionResidue},
+	}
+	for id, kind := range c.live {
+		want = append(want, ec2v1.PlatformItem{ResourceID: id, Kind: kind, Disposition: ec2v1.PlatformDispositionLive})
+	}
+	assert.ElementsMatch(t, want, reply.Items)
+	assert.ElementsMatch(t, []ec2v1.PlatformRecordCount{
+		{Kind: ec2v1.PlatformKindKeyPair, Disposition: ec2v1.PlatformDispositionUnreadable, Class: ec2v1.PlatformClassUnreadableRecord, Count: 1},
+		{Kind: ec2v1.PlatformKindKeyPair, Disposition: ec2v1.PlatformDispositionUnreadable, Class: ec2v1.PlatformClassPublicKeyUnresolved, Count: 1},
+		{Kind: ec2v1.PlatformKindLaunchTemplate, Disposition: ec2v1.PlatformDispositionUnreadable, Class: ec2v1.PlatformClassUnreadableRecord, Count: 1},
+		{Kind: ec2v1.PlatformKindLaunchTemplate, Disposition: ec2v1.PlatformDispositionResidue, Class: ec2v1.PlatformClassVersionBodyWithoutHeader, Count: 2},
+		{Kind: ec2v1.PlatformKindLaunchTemplate, Disposition: ec2v1.PlatformDispositionResidue, Class: ec2v1.PlatformClassNameIndexWithoutHeader, Count: 1},
+	}, reply.Counts)
+	require.Len(t, reply.Kinds, 4)
+	for _, report := range reply.Kinds {
+		assert.Equal(t, ec2v1.PlatformStatusRecordsFound, report.Status, report.Kind)
+		assert.False(t, report.ScanStartedAt.IsZero(), report.Kind)
+		assert.False(t, report.ScanEndedAt.Before(report.ScanStartedAt), report.Kind)
+	}
+
+	for _, secret := range c.secrets {
+		assert.NotContains(t, string(raw), secret, "the reply discloses %q", secret)
+	}
+	assert.Equal(t, pgSeq, c.lastSeq(t, ec2placementgroup.KVBucketPlacementGroups), "placement-group bucket written")
+	assert.Equal(t, ltSeq, c.lastSeq(t, ec2launchtemplate.KVBucketLaunchTemplates), "launch-template bucket written")
+}
+
+// An account with no records is observed clean for every kind, and another
+// account's records never appear.
+func TestPlatformInventorySubjectReportsAnEmptyAccountClean(t *testing.T) {
+	c := newInventoryCluster(t)
+	var reply ec2v1.PlatformInventoryReply
+	require.NoError(t, json.Unmarshal(c.request(t, ec2v1.PlatformInventoryRequest{AccountID: inventoryEmpty}, inventoryAccount), &reply))
+
+	assert.Empty(t, reply.Items)
+	assert.Empty(t, reply.Counts)
+	require.Len(t, reply.Kinds, 4)
+	for _, report := range reply.Kinds {
+		assert.Equal(t, ec2v1.PlatformStatusObservedClean, report.Status, report.Kind)
+	}
+}
+
+// A public-key object with no metadata, where no metadata is unreadable, is
+// residue; the other account's records stay out of the reply.
+func TestPlatformInventorySubjectCountsAnOrphanPublicKeyAsResidue(t *testing.T) {
+	c := newInventoryCluster(t)
+	raw := c.request(t, ec2v1.PlatformInventoryRequest{AccountID: inventoryOther}, inventoryAccount)
+	var reply ec2v1.PlatformInventoryReply
+	require.NoError(t, json.Unmarshal(raw, &reply))
+
+	assert.Equal(t, []ec2v1.PlatformRecordCount{{
+		Kind: ec2v1.PlatformKindKeyPair, Disposition: ec2v1.PlatformDispositionResidue,
+		Class: ec2v1.PlatformClassPublicKeyWithoutMetadata, Count: 1,
+	}}, reply.Counts)
+	require.Len(t, reply.Items, 3)
+	for _, item := range reply.Items {
+		assert.Equal(t, ec2v1.PlatformDispositionLive, item.Disposition)
+		assert.NotContains(t, c.live, item.ResourceID)
+	}
+	for _, id := range append(c.heldIDs, deletedGroupID, orphanTemplateID) {
+		assert.NotContains(t, string(raw), id)
+	}
+}
+
+// A request without a valid account in its body is refused, not scanned.
+func TestPlatformInventorySubjectRefusesARequestWithoutAnAccount(t *testing.T) {
+	c := newInventoryCluster(t)
+	assert.Equal(t, awserrors.ErrorMissingParameter, replyErrCode(t, c.request(t, ec2v1.PlatformInventoryRequest{}, inventoryAccount)))
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, replyErrCode(t, c.request(t, ec2v1.PlatformInventoryRequest{AccountID: "1111"}, "")))
+}
+
+// A daemon that does not serve the subject leaves the request with no responder.
+func TestPlatformInventorySubjectHasNoResponderWithoutTheDaemon(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	_, err := natsRequestAs(nc, ec2v1.PlatformInventorySubject, []byte(`{"account_id":"111122223333"}`), "", time.Second)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+}

spinifex/daemon/daemon_handlers_storage.go

@@ -2,7 +2,9 @@ package daemon
 
 import (
 	"log/slog"
+	"net"
 	"os"
+	"strconv"
 
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 	"github.com/nats-io/nats.go"
@@ -26,9 +28,24 @@ type predastoreRS struct {
 // names the machine and carries no port; the nodes pinned to it supply their
 // own.
 type predastoreHost struct {
-	ID    int              `toml:"id"`
-	Addr  string           `toml:"addr"`
-	Nodes []predastoreNode `toml:"node"`
+	ID        int              `toml:"id"`
+	Addr      string           `toml:"addr"`
+	BindAddr  string           `toml:"bind_addr"`
+	AdminPort int              `toml:"admin_port"`
+	Nodes     []predastoreNode `toml:"node"`
+}
+
+// adminAddr is where another machine dials this host's admin listener, empty
+// when it runs none. A wildcard bind is reached at the host's addr.
+func (h predastoreHost) adminAddr() string {
+	if h.AdminPort == 0 {
+		return ""
+	}
+	host := h.BindAddr
+	if host == "" || host == "0.0.0.0" || host == "::" {
+		host = h.Addr
+	}
+	return net.JoinHostPort(host, strconv.Itoa(h.AdminPort))
 }
 
 // predastoreNode is a role pinned to the host it is written under. Its address
@@ -79,11 +96,16 @@ func (d *Daemon) handleStorageConfig(msg *nats.Msg) string {
 
 	// A node's address is its host's addr plus its own port: colocated nodes
 	// share a machine but never a port.
+	singleHost := len(cfg.Hosts) == 1
 	for _, h := range cfg.Hosts {
 		for _, n := range h.Nodes {
 			switch n.Role {
 			case predastoreRoleMeta:
-				resp.MetaNodes = append(resp.MetaNodes, operatorv1.StorageMetaNode{ID: n.ID, Host: h.Addr, Port: n.Port})
+				meta := operatorv1.StorageMetaNode{ID: n.ID, Host: h.Addr, Port: n.Port}
+				if singleHost {
+					meta.AdminAddr = h.adminAddr()
+				}
+				resp.MetaNodes = append(resp.MetaNodes, meta)
 			case predastoreRoleBlob:
 				resp.BlobNodes = append(resp.BlobNodes, operatorv1.StorageBlobNode{ID: n.ID, Host: h.Addr, Port: n.Port})
 			case predastoreRoleGate:

spinifex/daemon/daemon_handlers_storage_test.go

@@ -2,6 +2,7 @@ package daemon
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -28,6 +29,7 @@ parity = 1
 id = 1
 bind_addr = "0.0.0.0"
 addr = "10.0.0.1"
+admin_port = 8660
 data_dir = "/var/lib/spinifex/predastore/cluster"
 
 [[host.node]]
@@ -94,9 +96,8 @@ func storageConfigResponse(t *testing.T, cfg string, reply string) []byte {
 }
 
 // TestHandleStorageConfig_SplitsNodesByRole pins what the gateway builds its
-// storage report from: every node is reported at its host address and its own
-// port, meta and blob in separate tables, and the gate in neither — it serves
-// the S3 API and holds no cluster state to report.
+// storage report from: every node at its host address and own port, meta and
+// blob apart, no gate, and no admin address since a multi-host cluster has QUIC.
 func TestHandleStorageConfig_SplitsNodesByRole(t *testing.T) {
 	payload := storageConfigResponse(t, clusterTOML, "test.storage.config.ok")
 
@@ -118,6 +119,46 @@ func TestHandleStorageConfig_SplitsNodesByRole(t *testing.T) {
 	assert.Equal(t, "ap-southeast-2", resp.Buckets[0].Region)
 }
 
+// singleHostAdminTOML is the default install: every node on one host, which
+// opens no QUIC socket, so raft status is only on its admin port.
+const singleHostAdminTOML = `
+[rs]
+data = 1
+parity = 0
+
+[[host]]
+id = 1
+addr = "10.0.0.1"
+bind_addr = "%s"
+admin_port = 8660
+
+[[host.node]]
+id = 3
+role = "meta"
+port = 7660
+`
+
+// TestHandleStorageConfig_SingleHostNamesTheAdminListener pins where the
+// gateway reads a single-host node's health: the host's bind address when it
+// names one, its addr when it binds a wildcard.
+func TestHandleStorageConfig_SingleHostNamesTheAdminListener(t *testing.T) {
+	for i, tc := range []struct{ bind, want string }{
+		{"127.0.0.1", "127.0.0.1:8660"},
+		{"0.0.0.0", "10.0.0.1:8660"},
+		{"", "10.0.0.1:8660"},
+	} {
+		t.Run("bind="+tc.bind, func(t *testing.T) {
+			payload := storageConfigResponse(t, fmt.Sprintf(singleHostAdminTOML, tc.bind),
+				fmt.Sprintf("test.storage.config.single.%d", i))
+
+			var resp operatorv1.StorageConfigResponse
+			require.NoError(t, json.Unmarshal(payload, &resp))
+			require.Len(t, resp.MetaNodes, 1)
+			assert.Equal(t, tc.want, resp.MetaNodes[0].AdminAddr)
+		})
+	}
+}
+
 // TestHandleStorageConfig_ReportsAMissingConfig covers the node that has no
 // predastore config: the report is an error rather than an empty topology,
 // which would read as a cluster with no storage at all.

spinifex/daemon/daemon_mgmt_ip_test.go

@@ -173,6 +173,7 @@ func TestMgmtIPAllocator_ReleaseNonexistent(t *testing.T) {
 }
 
 func TestMgmtIPAllocator_Exhaustion(t *testing.T) {
+	t.Parallel()
 	jsm := newTestMgmtJSM(t)
 	a, err := NewMgmtIPAllocator("10.17.8.1")
 	if err != nil {
@@ -372,6 +373,7 @@ func TestMgmtIPAllocator_SharedKV_NoDuplicates(t *testing.T) {
 // unique — this is the exact shape of CI run 29912697556, where independent
 // per-node allocation raced onto the same br-mgmt address.
 func TestMgmtIPAllocator_ConcurrentAcrossNodes(t *testing.T) {
+	t.Parallel()
 	jsm := newTestMgmtJSM(t)
 
 	const nodeCount = 3
@@ -503,6 +505,7 @@ func TestMgmtIPAllocator_Release_FreesAddressClusterWide(t *testing.T) {
 // locally believes the range is full) and a third allocation from either
 // must still fail.
 func TestMgmtIPAllocator_Exhaustion_ClusterWide(t *testing.T) {
+	t.Parallel()
 	jsm := newTestMgmtJSM(t)
 
 	node1, err := NewMgmtIPAllocator("10.25.8.1")

spinifex/daemon/daemon_test.go

@@ -3905,6 +3905,7 @@ func TestCanAllocate(t *testing.T) {
 
 // TestConnectNATS_RetriesOnFailure tests that connectNATS retries when NATS is not immediately available.
 func TestConnectNATS_RetriesOnFailure(t *testing.T) {
+	t.Parallel()
 	// Use a port that nothing is listening on
 	clusterCfg := &config.ClusterConfig{
 		Node:  "node-1",

spinifex/daemon/disconnect_test.go

@@ -55,7 +55,7 @@ func TestDaemonModeFlipsOnConnectAndDisconnect(t *testing.T) {
 	assert.Equal(t, DaemonModeCluster, d.Mode())
 
 	ns.Shutdown()
-	require.Eventually(t, func() bool { return d.Mode() == DaemonModeStandalone }, 3*time.Second, 50*time.Millisecond,
+	require.Eventually(t, func() bool { return d.Mode() == DaemonModeStandalone }, 3*time.Second, 10*time.Millisecond,
 		"mode should flip to standalone after NATS shutdown")
 }
 
@@ -76,10 +76,10 @@ func TestDaemonReconnectBumpsRetryCount(t *testing.T) {
 	require.Equal(t, int64(0), d.NATSRetryCount())
 
 	ns.Shutdown()
-	require.Eventually(t, func() bool { return !d.natsConn.IsConnected() }, 3*time.Second, 50*time.Millisecond)
+	require.Eventually(t, func() bool { return !d.natsConn.IsConnected() }, 3*time.Second, 10*time.Millisecond)
 
 	startTestNATSOnPortForTest(t, port)
-	require.Eventually(t, func() bool { return d.Mode() == DaemonModeCluster }, 5*time.Second, 50*time.Millisecond,
+	require.Eventually(t, func() bool { return d.Mode() == DaemonModeCluster }, 5*time.Second, 10*time.Millisecond,
 		"mode should flip back to cluster on reconnect")
 	assert.GreaterOrEqual(t, d.NATSRetryCount(), int64(1))
 }
@@ -340,7 +340,7 @@ func TestStartCluster_StrictRequireNATS_BoundedAndExits(t *testing.T) {
 
 	d, err := NewDaemon(cfg)
 	require.NoError(t, err)
-	d.requireNATSTimeout = 200 * time.Millisecond
+	d.requireNATSTimeout = 50 * time.Millisecond
 	d.natsRetryOpts = []natsmsg.RetryOption{natsmsg.WithRetryDelay(20 * time.Millisecond)}
 
 	exitCh := make(chan int, 1)
@@ -417,6 +417,7 @@ func TestStartCluster_NoStrictEnv_UsesInfiniteRetry(t *testing.T) {
 // connect loop honours d.ctx. Cancelling the daemon context unblocks an
 // otherwise-infinite retry.
 func TestStartCluster_RetriesUntilContextCancelled(t *testing.T) {
+	t.Parallel()
 	tmpDir := t.TempDir()
 	cfg := &config.ClusterConfig{
 		Node: "node-1",

spinifex/daemon/instance_records_rekey_test.go

@@ -128,6 +128,7 @@ func TestRecordKeys_AreWatchableByASubjectFilter(t *testing.T) {
 // what this change exists to fix. If this ever stops failing to deliver, the
 // re-key was unnecessary and the reasoning above is wrong.
 func TestRecordKeys_SlashSeparatorWasNotWatchable(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js, err := jetstream.New(nc)
 	require.NoError(t, err)

spinifex/daemon/jetstream_test.go

@@ -840,6 +840,15 @@ func TestJetStreamManager_WriteServiceManifest_ClusterKVNotInitialized(t *testin
 // during NATS cluster formation when catchup resets corrupt a stream) and verify
 // that operations recover by re-creating the bucket.
 
+// isolatedJetStreamConn connects to a JetStream server of the test's own. The
+// recovery tests delete a bucket's stream, so a private server lets them run in
+// parallel without pulling the bucket out from under any other test.
+func isolatedJetStreamConn(t *testing.T) *nats.Conn {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	return nc
+}
+
 // deleteInstanceStateBucket deletes the underlying JetStream stream for the instance-state KV bucket.
 func deleteInstanceStateBucket(t *testing.T, nc *nats.Conn) {
 	t.Helper()
@@ -850,9 +859,8 @@ func deleteInstanceStateBucket(t *testing.T, nc *nats.Conn) {
 }
 
 func TestJetStreamManager_WriteState_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -877,9 +885,8 @@ func TestJetStreamManager_WriteState_RecoverAfterStreamLost(t *testing.T) {
 }
 
 func TestJetStreamManager_LoadState_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -896,9 +903,8 @@ func TestJetStreamManager_LoadState_RecoverAfterStreamLost(t *testing.T) {
 }
 
 func TestJetStreamManager_DeleteState_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -913,9 +919,8 @@ func TestJetStreamManager_DeleteState_RecoverAfterStreamLost(t *testing.T) {
 }
 
 func TestJetStreamManager_WriteStoppedInstance_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -939,9 +944,8 @@ func TestJetStreamManager_WriteStoppedInstance_RecoverAfterStreamLost(t *testing
 }
 
 func TestJetStreamManager_LoadStoppedInstance_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -957,9 +961,8 @@ func TestJetStreamManager_LoadStoppedInstance_RecoverAfterStreamLost(t *testing.
 }
 
 func TestJetStreamManager_DeleteStoppedInstance_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -973,9 +976,8 @@ func TestJetStreamManager_DeleteStoppedInstance_RecoverAfterStreamLost(t *testin
 }
 
 func TestJetStreamManager_ListStoppedInstances_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1024,9 +1026,8 @@ func swapToNonJSContext(t *testing.T, jsm *JetStreamManager) {
 }
 
 func TestJetStreamManager_WriteState_RecoveryFailure(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1042,9 +1043,8 @@ func TestJetStreamManager_WriteState_RecoveryFailure(t *testing.T) {
 }
 
 func TestJetStreamManager_LoadState_RecoveryFailure(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1060,9 +1060,8 @@ func TestJetStreamManager_LoadState_RecoveryFailure(t *testing.T) {
 }
 
 func TestJetStreamManager_DeleteState_RecoveryFailure(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1077,9 +1076,8 @@ func TestJetStreamManager_DeleteState_RecoveryFailure(t *testing.T) {
 }
 
 func TestJetStreamManager_WriteStoppedInstance_RecoveryFailure(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1095,9 +1093,8 @@ func TestJetStreamManager_WriteStoppedInstance_RecoveryFailure(t *testing.T) {
 }
 
 func TestJetStreamManager_LoadStoppedInstance_RecoveryFailure(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1113,9 +1110,8 @@ func TestJetStreamManager_LoadStoppedInstance_RecoveryFailure(t *testing.T) {
 }
 
 func TestJetStreamManager_DeleteStoppedInstance_RecoveryFailure(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1130,9 +1126,8 @@ func TestJetStreamManager_DeleteStoppedInstance_RecoveryFailure(t *testing.T) {
 }
 
 func TestJetStreamManager_ListStoppedInstances_RecoveryFailure(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1425,9 +1420,8 @@ func deleteTerminatedInstanceBucket(t *testing.T, nc *nats.Conn) {
 }
 
 func TestJetStreamManager_WriteTerminatedInstance_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1448,9 +1442,8 @@ func TestJetStreamManager_WriteTerminatedInstance_RecoverAfterStreamLost(t *test
 }
 
 func TestJetStreamManager_LoadTerminatedInstance_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1464,9 +1457,8 @@ func TestJetStreamManager_LoadTerminatedInstance_RecoverAfterStreamLost(t *testi
 }
 
 func TestJetStreamManager_DeleteTerminatedInstance_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1479,9 +1471,8 @@ func TestJetStreamManager_DeleteTerminatedInstance_RecoverAfterStreamLost(t *tes
 }
 
 func TestJetStreamManager_ListTerminatedInstances_RecoverAfterStreamLost(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1525,9 +1516,8 @@ func TestJetStreamManager_InitBuckets_WritesVersion(t *testing.T) {
 }
 
 func TestJetStreamManager_ListStoppedInstances_SkipsVersionKey(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)
@@ -1540,9 +1530,8 @@ func TestJetStreamManager_ListStoppedInstances_SkipsVersionKey(t *testing.T) {
 }
 
 func TestJetStreamManager_ListTerminatedInstances_SkipsVersionKey(t *testing.T) {
-	nc, err := nats.Connect(sharedJSNATSURL)
-	require.NoError(t, err)
-	defer nc.Close()
+	t.Parallel()
+	nc := isolatedJetStreamConn(t)
 
 	jsm, err := NewJetStreamManager(nc)
 	require.NoError(t, err)

spinifex/daemon/main_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/nats-io/nats-server/v2/server"
 )
 
@@ -75,6 +76,13 @@ func TestMain(m *testing.M) {
 	sharedJSServer = jsNS
 	sharedJSNATSURL = jsNS.ClientURL()
 
+	// The shared JetStream server is a one-node cluster. Declaring it here keeps
+	// tests that use it independent of whichever earlier test happened to declare.
+	if err := clustersize.Declare(1); err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to declare cluster size: %v\n", err)
+		os.Exit(1)
+	}
+
 	code := m.Run()
 
 	ns.Shutdown()

spinifex/daemon/platform_disclosure_current_behaviour_test.go

@@ -0,0 +1,252 @@
+package daemon
+
+//test:in-package — the evidence is the daemon's own subscription table,
+// registered by the unexported subscribeAll.
+
+import (
+	"bytes"
+	"encoding/json"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
+	ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The TestCurrentBehaviour_AccountTermination_ tests pin what any NATS client
+// can read through the daemon's existing per-kind EC2 subjects today. They
+// describe current behaviour, not the target design; a failure means it changed.
+
+const (
+	disclosureAccount = "111122223333"
+	disclosureOther   = "444455556666"
+	disclosureBucket  = "test-bucket"
+)
+
+// disclosureCluster is a daemon serving its real subscription table, the
+// stores behind it, and a separate NATS client that holds no IAM credential.
+type disclosureCluster struct {
+	caller *nats.Conn
+	store  *objectstore.MemoryObjectStore
+	js     jetstream.JetStream
+}
+
+func newDisclosureCluster(t *testing.T) *disclosureCluster {
+	t.Helper()
+	ns, nc, js := testutil.StartTestJetStream(t)
+	d := newSubscribeTestDaemon(t, nc, "", "")
+
+	store := objectstore.NewMemoryObjectStore()
+	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: disclosureBucket}}
+	tagsKV, err := ec2tags.GetOrCreateTagsBucket(t.Context(), js)
+	require.NoError(t, err)
+	d.keyService = ec2key.NewKeyServiceImplWithStore(store, disclosureBucket)
+	d.tagsService = ec2tags.NewTagsServiceImplWithStore(cfg, store, tagsKV)
+	d.keyService.SetCentralTagStore(d.tagsService)
+	d.placementGroupService, err = ec2placementgroup.NewPlacementGroupServiceImplWithNATS(t.Context(), nil, nc)
+	require.NoError(t, err)
+	d.launchTemplateService, err = ec2launchtemplate.NewLaunchTemplateServiceImplWithNATS(t.Context(), nil, nc)
+	require.NoError(t, err)
+	require.NoError(t, d.subscribeAll())
+
+	caller, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(caller.Close)
+	return &disclosureCluster{caller: caller, store: store, js: js}
+}
+
+// call sends one request with X-Account-ID set to account, the only account
+// selector the subjects read, and decodes a successful reply into out.
+func (c *disclosureCluster) call(t *testing.T, subject, account string, in, out any) {
+	t.Helper()
+	data, err := json.Marshal(in)
+	require.NoError(t, err)
+	reply, err := natsRequestAs(c.caller, subject, data, account, 5*time.Second)
+	require.NoError(t, err)
+	require.Empty(t, replyErrCode(t, reply.Data), "%s answered %s", subject, reply.Data)
+	if out != nil {
+		require.NoError(t, json.Unmarshal(reply.Data, out))
+	}
+}
+
+// platformIDs is the account-scoped identity set each Describe subject returns.
+type platformIDs struct {
+	keyPairs, placementGroups, launchTemplates, taggedResources []string
+}
+
+// seededAccount records the identities created for one account.
+type seededAccount struct {
+	keyName, keyID, groupName, groupID, templateName, templateID string
+}
+
+func (c *disclosureCluster) seed(t *testing.T, account, suffix string) seededAccount {
+	t.Helper()
+	tag := []*ec2.Tag{{Key: aws.String("owner"), Value: aws.String(suffix)}}
+
+	var key ec2.CreateKeyPairOutput
+	c.call(t, "ec2.CreateKeyPair", account, &ec2.CreateKeyPairInput{
+		KeyName:           aws.String("key-" + suffix),
+		TagSpecifications: []*ec2.TagSpecification{{ResourceType: aws.String("key-pair"), Tags: tag}},
+	}, &key)
+	var pg ec2.CreatePlacementGroupOutput
+	c.call(t, "ec2.CreatePlacementGroup", account, &ec2.CreatePlacementGroupInput{
+		GroupName: aws.String("pg-" + suffix), Strategy: aws.String(ec2.PlacementStrategySpread),
+	}, &pg)
+	var lt ec2.CreateLaunchTemplateOutput
+	c.call(t, "ec2.CreateLaunchTemplate", account, &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("lt-" + suffix),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{ImageId: aws.String("ami-123")},
+	}, &lt)
+	s := seededAccount{
+		keyName: aws.StringValue(key.KeyName), keyID: aws.StringValue(key.KeyPairId),
+		groupName: aws.StringValue(pg.PlacementGroup.GroupName), groupID: aws.StringValue(pg.PlacementGroup.GroupId),
+		templateName: aws.StringValue(lt.LaunchTemplate.LaunchTemplateName), templateID: aws.StringValue(lt.LaunchTemplate.LaunchTemplateId),
+	}
+	c.call(t, "ec2.CreateTags", account, &ec2.CreateTagsInput{
+		Resources: []*string{aws.String(s.groupID), aws.String(s.templateID)}, Tags: tag,
+	}, nil)
+	return s
+}
+
+// describe reads every platform identity the four Describe subjects return
+// for account, as "<id>/<name>" pairs (tag rows as "<resource-id>").
+func (c *disclosureCluster) describe(t *testing.T, account string) platformIDs {
+	t.Helper()
+	var ids platformIDs
+
+	var keys ec2.DescribeKeyPairsOutput
+	c.call(t, "ec2.DescribeKeyPairs", account, &ec2.DescribeKeyPairsInput{}, &keys)
+	for _, kp := range keys.KeyPairs {
+		ids.keyPairs = append(ids.keyPairs, aws.StringValue(kp.KeyPairId)+"/"+aws.StringValue(kp.KeyName))
+	}
+	var groups ec2.DescribePlacementGroupsOutput
+	c.call(t, "ec2.DescribePlacementGroups", account, &ec2.DescribePlacementGroupsInput{}, &groups)
+	for _, g := range groups.PlacementGroups {
+		ids.placementGroups = append(ids.placementGroups, aws.StringValue(g.GroupId)+"/"+aws.StringValue(g.GroupName))
+	}
+	var templates ec2.DescribeLaunchTemplatesOutput
+	c.call(t, "ec2.DescribeLaunchTemplates", account, &ec2.DescribeLaunchTemplatesInput{}, &templates)
+	for _, lt := range templates.LaunchTemplates {
+		ids.launchTemplates = append(ids.launchTemplates, aws.StringValue(lt.LaunchTemplateId)+"/"+aws.StringValue(lt.LaunchTemplateName))
+	}
+	var tags ec2.DescribeTagsOutput
+	c.call(t, "ec2.DescribeTags", account, &ec2.DescribeTagsInput{}, &tags)
+	for _, row := range tags.Tags {
+		if id := aws.StringValue(row.ResourceId); !slices.Contains(ids.taggedResources, id) {
+			ids.taggedResources = append(ids.taggedResources, id)
+		}
+	}
+	for _, s := range [][]string{ids.keyPairs, ids.placementGroups, ids.launchTemplates, ids.taggedResources} {
+		slices.Sort(s)
+	}
+	return ids
+}
+
+func sortedIDs(ids ...string) []string {
+	slices.Sort(ids)
+	return ids
+}
+
+// A NATS client with no IAM credential chooses the account in X-Account-ID
+// and receives that account's key pair, placement group and launch template
+// IDs and names, and every tagged key-, pg- and lt- ID, scoped to it alone.
+func TestCurrentBehaviour_AccountTermination_DirectNATSCallerReadsAnyAccountsPlatformIdentities(t *testing.T) {
+	c := newDisclosureCluster(t)
+	a := c.seed(t, disclosureAccount, "a")
+	b := c.seed(t, disclosureOther, "b")
+
+	for _, tc := range []struct {
+		account string
+		s       seededAccount
+	}{{disclosureAccount, a}, {disclosureOther, b}} {
+		got := c.describe(t, tc.account)
+		assert.Equal(t, []string{tc.s.keyID + "/" + tc.s.keyName}, got.keyPairs, "key pairs of %s", tc.account)
+		assert.Equal(t, []string{tc.s.groupID + "/" + tc.s.groupName}, got.placementGroups, "placement groups of %s", tc.account)
+		assert.Equal(t, []string{tc.s.templateID + "/" + tc.s.templateName}, got.launchTemplates, "launch templates of %s", tc.account)
+		assert.Equal(t, sortedIDs(tc.s.keyID, tc.s.groupID, tc.s.templateID), got.taggedResources, "tag index of %s", tc.account)
+	}
+
+	// Without the header the subjects scope to the empty account and return nothing.
+	assert.Equal(t, platformIDs{}, c.describe(t, ""))
+}
+
+// Records the owner stores hold but Describe skips are not reachable through
+// the per-kind subjects: orphan and undecodable key-pair objects, undecodable
+// placement-group records, and launch-template bodies or headers with no live template.
+func TestCurrentBehaviour_AccountTermination_DirectNATSCallerCannotSeeHiddenResidue(t *testing.T) {
+	c := newDisclosureCluster(t)
+	c.seed(t, disclosureAccount, "a")
+	before := c.describe(t, disclosureAccount)
+
+	prefix := "keys/" + disclosureAccount + "/"
+	for key, body := range map[string]string{
+		prefix + "orphan-public-key":          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n",
+		prefix + "key-0badbadbadbadbad0.json": "{not json",
+	} {
+		_, err := c.store.PutObject(t.Context(), &s3.PutObjectInput{
+			Bucket: aws.String(disclosureBucket), Key: aws.String(key), Body: bytes.NewReader([]byte(body)),
+		})
+		require.NoError(t, err)
+	}
+	pgKV, err := c.js.KeyValue(t.Context(), ec2placementgroup.KVBucketPlacementGroups)
+	require.NoError(t, err)
+	_, err = pgKV.Put(t.Context(), disclosureAccount+".broken", []byte("{not json"))
+	require.NoError(t, err)
+	ltKV, err := c.js.KeyValue(t.Context(), ec2launchtemplate.KVBucketLaunchTemplates)
+	require.NoError(t, err)
+	for key, body := range map[string]string{
+		disclosureAccount + ".lt-0orphan000000000.v1":  `{"version_number":1,"data":{"UserData":"c2VjcmV0"}}`,
+		disclosureAccount + ".name.6f727068616e":       "lt-0orphan000000000",
+		disclosureAccount + ".lt-0badbadbadbadbad0":    "{not json",
+		disclosureAccount + ".lt-0badbadbadbadbad0.v1": `{"version_number":1}`,
+	} {
+		_, err := ltKV.Put(t.Context(), key, []byte(body))
+		require.NoError(t, err)
+	}
+
+	after := c.describe(t, disclosureAccount)
+	assert.Equal(t, before.keyPairs, after.keyPairs)
+	assert.Equal(t, before.placementGroups, after.placementGroups)
+	assert.Equal(t, before.launchTemplates, after.launchTemplates)
+	assert.Equal(t, before.taggedResources, after.taggedResources)
+}
+
+// Pins current behaviour that differs from AWS: deleting a placement group or
+// launch template leaves its tag-index entry, which DescribeTags keeps
+// returning with its values. A key-pair delete clears its entry.
+func TestCurrentBehaviour_AccountTermination_PlatformDeletesLeaveTagIndexEntries(t *testing.T) {
+	c := newDisclosureCluster(t)
+	s := c.seed(t, disclosureAccount, "a")
+
+	c.call(t, "ec2.DeleteKeyPair", disclosureAccount, &ec2.DeleteKeyPairInput{KeyName: aws.String(s.keyName)}, nil)
+	c.call(t, "ec2.DeletePlacementGroup", disclosureAccount, &ec2.DeletePlacementGroupInput{GroupName: aws.String(s.groupName)}, nil)
+	c.call(t, "ec2.DeleteLaunchTemplate", disclosureAccount, &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(s.templateID)}, nil)
+
+	got := c.describe(t, disclosureAccount)
+	assert.Empty(t, got.keyPairs)
+	assert.Empty(t, got.placementGroups)
+	assert.Empty(t, got.launchTemplates)
+	assert.Equal(t, sortedIDs(s.groupID, s.templateID), got.taggedResources)
+
+	var tags ec2.DescribeTagsOutput
+	c.call(t, "ec2.DescribeTags", disclosureAccount, &ec2.DescribeTagsInput{}, &tags)
+	for _, row := range tags.Tags {
+		assert.Equal(t, "owner", aws.StringValue(row.Key))
+		assert.Equal(t, "a", aws.StringValue(row.Value))
+		assert.True(t, strings.HasPrefix(aws.StringValue(row.ResourceId), "pg-") || strings.HasPrefix(aws.StringValue(row.ResourceId), "lt-"))
+	}
+}

spinifex/daemon/shutdown_test.go

@@ -315,7 +315,9 @@ func TestHandleShutdownGate(t *testing.T) {
 // local drain off every other node: the phase subjects are fan-out, so a
 // request naming another node must be dropped before any service is stopped.
 func TestShutdownRequestTarget(t *testing.T) {
+	t.Parallel()
 	t.Run("request for another node is ignored with no ack", func(t *testing.T) {
+		t.Parallel()
 		daemon := createTestDaemon(t, sharedNATSURL)
 		daemon.config.Services = []string{"awsgw", "ui", "vpcd"}
 		pidDir := configurePidDir(t, daemon)
@@ -341,6 +343,7 @@ func TestShutdownRequestTarget(t *testing.T) {
 	})
 
 	t.Run("request for this node is honoured", func(t *testing.T) {
+		t.Parallel()
 		daemon := createTestDaemon(t, sharedNATSURL)
 		daemon.config.Services = []string{}
 		configurePidDir(t, daemon)
@@ -368,6 +371,7 @@ func TestShutdownRequestTarget(t *testing.T) {
 	// STORAGE and PERSIST stop another node's storage, and INFRA exits its
 	// daemon outright.
 	t.Run("every phase handler ignores another node's request", func(t *testing.T) {
+		t.Parallel()
 		daemon := createTestDaemon(t, sharedNATSURL)
 		daemon.config.Services = []string{}
 		configurePidDir(t, daemon)
@@ -379,6 +383,7 @@ func TestShutdownRequestTarget(t *testing.T) {
 			"infra":   asMsgHandler(daemon.handleShutdownInfra),
 		} {
 			t.Run(phase, func(t *testing.T) {
+				t.Parallel()
 				subject := "spinifex.cluster.shutdown." + phase + ".elsewhere"
 				sub, err := daemon.natsConn.Subscribe(subject, handler)
 				require.NoError(t, err)
@@ -395,6 +400,7 @@ func TestShutdownRequestTarget(t *testing.T) {
 	})
 
 	t.Run("untargeted request still reaches every node", func(t *testing.T) {
+		t.Parallel()
 		daemon := createTestDaemon(t, sharedNATSURL)
 		daemon.config.Services = []string{}
 		configurePidDir(t, daemon)
@@ -643,9 +649,11 @@ func TestHandleShutdownDrain(t *testing.T) {
 		require.NoError(t, json.Unmarshal(reply.Data, &ack))
 		assert.Empty(t, ack.Error)
 
+		// Progress is published on the daemon's connection before the ACK, so by
+		// the time the reply lands every progress message is already queued.
 		var progresses []ShutdownProgress
 		for {
-			msg, err := progressSub.NextMsg(500 * time.Millisecond)
+			msg, err := progressSub.NextMsg(noReplyWait)
 			if err != nil {
 				break
 			}

spinifex/daemon/tagging_current_behaviour_test.go

@@ -0,0 +1,47 @@
+package daemon
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The TestCurrentBehaviour_Tagging_ tests pin what the daemon's EC2 tag
+// handlers accept today. They describe current behaviour, not the target
+// design, and a failure means the behaviour changed.
+
+// A non-instance ID is never resolved to a resource before the central write.
+// Another account's ID, a well-formed ID naming nothing, and an ID with no
+// recognised prefix all succeed, and each is stored under the caller's account.
+func TestCurrentBehaviour_Tagging_EC2CreateTagsStoresUnresolvedIDsInCallerNamespace(t *testing.T) {
+	const caller = "999999999999"
+	d := tagTestDaemon(t, "i-tag-cb-unrelated", nil)
+
+	ids := []string{"sg-0a1b2c3d4e5f60718", "vol-00000000000000000", "not-a-real-id"}
+	resources := make([]*string, 0, len(ids))
+	for _, id := range ids {
+		resources = append(resources, aws.String(id))
+	}
+
+	_, err := d.createTags(context.Background(), &ec2.CreateTagsInput{
+		Resources: resources,
+		Tags:      []*ec2.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
+	}, caller)
+	require.NoError(t, err)
+
+	for _, id := range ids {
+		assert.Equal(t, map[string]string{"env": "prod"}, centralTags(t, d, caller, id), id)
+		assert.Empty(t, centralTags(t, d, testAccountID, id), id)
+	}
+
+	out, err := d.tagsService.DescribeTags(t.Context(), &ec2.DescribeTagsInput{
+		Filters: []*ec2.Filter{{Name: aws.String("resource-id"), Values: []*string{aws.String("not-a-real-id")}}},
+	}, caller)
+	require.NoError(t, err)
+	require.Len(t, out.Tags, 1)
+	assert.Equal(t, "unknown", aws.StringValue(out.Tags[0].ResourceType))
+}

spinifex/domains/acm/awsapi/registration.go

@@ -89,8 +89,8 @@ func resolveAction(r *http.Request) string {
 // actionFromTarget extracts the action suffix from an X-Amz-Target header.
 // Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
 func actionFromTarget(target string) string {
-	if i := strings.LastIndex(target, "."); i >= 0 {
-		return target[i+1:]
+	if _, after, ok := strings.CutLast(target, "."); ok {
+		return after
 	}
 	return target
 }

spinifex/domains/acm/certfields_test.go

@@ -31,10 +31,11 @@ const (
 
 // genAuditShapedCert builds a certificate shaped like the one the audit
 // imported into real ACM (openssl req -x509 -newkey rsa:2048 -subj /CN=...):
-// RSA-2048, SHA-256 with RSA, CN only, no SAN, key-usage or EKU extension.
+// SHA-256 with RSA, CN only, no SAN, key-usage or EKU extension. The key is
+// 1024 bits rather than the audit's 2048 to keep generation fast.
 func genAuditShapedCert(t *testing.T) (certPEM, keyPEM []byte) {
 	t.Helper()
-	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
 	require.NoError(t, err)
 	serial, ok := new(big.Int).SetString(awsAuditSerialHex, 16)
 	require.True(t, ok)
@@ -82,6 +83,7 @@ func wireJSON(t *testing.T, v any) map[string]any {
 // Expected values are copied from real ACM's DescribeCertificate response for
 // the audit's certificate. Export is absent: aws-sdk-go v1 cannot carry it.
 func TestDescribeCertificate_ImportedMatchesAWSAuditResponse(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, keyPEM := genAuditShapedCert(t)
 	d := describeCert(t, svc, importCert(t, svc, certPEM, keyPEM))
@@ -105,6 +107,7 @@ func TestDescribeCertificate_ImportedMatchesAWSAuditResponse(t *testing.T) {
 }
 
 func TestListCertificates_ImportedSummaryMatchesAWSAuditResponse(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, keyPEM := genAuditShapedCert(t)
 	importCert(t, svc, certPEM, keyPEM)
@@ -125,6 +128,7 @@ func TestListCertificates_ImportedSummaryMatchesAWSAuditResponse(t *testing.T) {
 }
 
 func TestImportCertificate_ReimportKeepsCreatedAtAndMovesImportedAt(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genAuditShapedCert(t)
 	certArn := importCert(t, svc, c1, k1)
@@ -144,6 +148,7 @@ func TestImportCertificate_ReimportKeepsCreatedAtAndMovesImportedAt(t *testing.T
 // A record stored before these fields existed holds the old Serial and Issuer
 // spellings, no SANs and no CreatedAt; it must still describe as AWS does.
 func TestDescribeCertificate_LegacyImportedRecordDerivedFromPEM(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, keyPEM := genAuditShapedCert(t)
 	certArn := svc.mintCertificateArn(testAccountID)
@@ -175,6 +180,7 @@ func TestDescribeCertificate_LegacyImportedRecordDerivedFromPEM(t *testing.T) {
 // a self-signed P-384 import carrying an O, SANs that omit the CN, and both
 // usage extensions.
 func TestDescribeCertificate_ImportedWithExtensionsMatchesAWS(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	key, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
 	require.NoError(t, err)

spinifex/domains/acm/getcertificate_test.go

@@ -13,6 +13,7 @@ import (
 )
 
 func TestGetCertificate_ReturnsBodyAndChain(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	leafPEM, keyPEM := genCert(t, "get.example.com")
 	chainPEM, _ := genCert(t, "issuer.example.com")
@@ -36,6 +37,7 @@ func TestGetCertificate_ReturnsBodyAndChain(t *testing.T) {
 // itself as the chain. The substitution is response-only: the stored record
 // keeps no chain, which is what ELBv2 builds its leaf+chain bundle from.
 func TestGetCertificate_AbsentChainReturnsCertificate(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	leafPEM, keyPEM := genCert(t, "nochain.example.com")
 
@@ -62,6 +64,7 @@ func TestGetCertificate_AbsentChainReturnsCertificate(t *testing.T) {
 // A re-import without a chain drops the chain the first import carried, so
 // GetCertificate falls back to the new certificate, not the stale chain.
 func TestGetCertificate_ReimportWithoutChainReturnsNewCertificate(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	leafPEM, keyPEM := genCert(t, "reimport.example.com")
 	chainPEM, _ := genCert(t, "issuer.example.com")
@@ -90,6 +93,7 @@ func TestGetCertificate_ReimportWithoutChainReturnsNewCertificate(t *testing.T)
 }
 
 func TestGetCertificate_UnknownArnNotFound(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 
 	_, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{
@@ -103,6 +107,7 @@ func TestGetCertificate_UnknownArnNotFound(t *testing.T) {
 // body is the material itself, so a cross-account read must not confirm the
 // certificate even exists.
 func TestGetCertificate_CrossAccountNotFound(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	leafPEM, keyPEM := genCert(t, "other.example.com")
 
@@ -123,6 +128,7 @@ func TestGetCertificate_CrossAccountNotFound(t *testing.T) {
 // and distinct from not-found, so a client polling for issuance can tell them
 // apart.
 func TestGetCertificate_PendingIssuanceIsRequestInProgress(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/pending-1"
 	require.NoError(t, svc.store.PutCert(t.Context(), &CertRecord{
@@ -140,6 +146,7 @@ func TestGetCertificate_PendingIssuanceIsRequestInProgress(t *testing.T) {
 }
 
 func TestGetCertificate_EmptyArnRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 
 	_, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{}, testAccountID)
@@ -170,6 +177,7 @@ func putUndecryptableCert(t *testing.T, store *Store, arn, domain string) {
 // than instrumentation: an undecryptable key is fatal to a decrypting read, so a
 // list that still returns the certificate cannot have decrypted it.
 func TestListPath_NeverTouchesKeyMaterial(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/nokeyread-1"
 	putUndecryptableCert(t, svc.store, arn, "nokeyread.example.com")
@@ -202,6 +210,7 @@ func TestListPath_NeverTouchesKeyMaterial(t *testing.T) {
 // caller cannot mistake ciphertext for usable key material in a field that is
 // plaintext PEM everywhere else.
 func TestMetadataReads_ClearPrivateKey(t *testing.T) {
+	t.Parallel()
 	store := setupACMStore(t)
 	arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/cleared-1"
 	_, keyPEM := genCert(t, "cleared.example.com")
@@ -230,6 +239,7 @@ func TestMetadataReads_ClearPrivateKey(t *testing.T) {
 }
 
 func TestGetCertMetadata_AbsentReturnsNil(t *testing.T) {
+	t.Parallel()
 	store := setupACMStore(t)
 
 	rec, err := store.GetCertMetadata(t.Context(), "arn:aws:acm:ap-southeast-2:000000000001:certificate/missing")

spinifex/domains/acm/keyalgorithm_test.go

@@ -20,9 +20,9 @@ import (
 
 // genRSACert returns a self-signed RSA leaf with only a CN, the shape of the
 // certificate the AWS conformance audit imported.
-func genRSACert(t *testing.T, cn string, bits int) (certPEM, keyPEM []byte) {
+func genRSACert(t *testing.T, cn string) (certPEM, keyPEM []byte) {
 	t.Helper()
-	key, err := rsa.GenerateKey(rand.Reader, bits)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
 	require.NoError(t, err)
 	tmpl := &x509.Certificate{
 		SerialNumber: big.NewInt(time.Now().UnixNano()),
@@ -57,11 +57,12 @@ func describeAndListKeyAlgorithm(t *testing.T, svc *ACMServiceImpl, certArn stri
 	return "", ""
 }
 
-// AWS returns RSA-2048 with a hyphen from both Describe and List, although the
-// API enum (and the ListCertificates keyTypes filter) spells it RSA_2048.
+// AWS returns RSA-1024 with a hyphen from both Describe and List, although the
+// API enum (and the ListCertificates keyTypes filter) spells it RSA_1024.
 func TestImportCertificate_RSAKeyAlgorithmIsHyphenated(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
-	certPEM, keyPEM := genRSACert(t, "rsa.example.com", 2048)
+	certPEM, keyPEM := genRSACert(t, "rsa.example.com")
 
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
 		Certificate: certPEM,
@@ -70,18 +71,19 @@ func TestImportCertificate_RSAKeyAlgorithmIsHyphenated(t *testing.T) {
 	require.NoError(t, err)
 
 	described, listed := describeAndListKeyAlgorithm(t, svc, aws.StringValue(out.CertificateArn))
-	assert.Equal(t, "RSA-2048", described)
-	assert.Equal(t, "RSA-2048", listed)
+	assert.Equal(t, "RSA-1024", described)
+	assert.Equal(t, "RSA-1024", listed)
 
 	// The read path normalises, so check the stored spelling separately.
 	raw, _, err := svc.store.store.Get(t.Context(), certKey(aws.StringValue(out.CertificateArn)))
 	require.NoError(t, err)
-	assert.Equal(t, "RSA-2048", raw.KeyAlgorithm)
+	assert.Equal(t, "RSA-1024", raw.KeyAlgorithm)
 }
 
 // Records written before the spelling fix hold RSA_<bits> or EC_<Go curve>;
 // they must read back in the AWS spelling without a migration.
 func TestKeyAlgorithm_StoredUnderscoreSpellingReadsBackHyphenated(t *testing.T) {
+	t.Parallel()
 	cases := []struct {
 		stored, want string
 	}{
@@ -93,6 +95,7 @@ func TestKeyAlgorithm_StoredUnderscoreSpellingReadsBackHyphenated(t *testing.T)
 	svc := setupACMService(t)
 	for i, tc := range cases {
 		t.Run(tc.stored, func(t *testing.T) {
+			t.Parallel()
 			certArn := "arn:aws:acm:ap-southeast-2:" + testAccountID + ":certificate/legacy-" + string(rune('a'+i))
 			require.NoError(t, svc.store.PutCert(t.Context(), &CertRecord{
 				CertificateArn: certArn,
@@ -111,8 +114,9 @@ func TestKeyAlgorithm_StoredUnderscoreSpellingReadsBackHyphenated(t *testing.T)
 // AWS refuses a re-import whose key type differs from the current one, naming
 // both in the API enum spelling.
 func TestImportCertificate_ReimportWithDifferentKeyTypeRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
-	c1, k1 := genRSACert(t, "k.example.com", 2048)
+	c1, k1 := genRSACert(t, "k.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID)
 	require.NoError(t, err)
 
@@ -123,12 +127,13 @@ func TestImportCertificate_ReimportWithDifferentKeyTypeRejected(t *testing.T) {
 	code, msg, ok := awserrors.ResolveErrorDetail(err)
 	require.True(t, ok)
 	assert.Equal(t, awserrors.ErrorValidationException, code)
-	assert.Equal(t, "New certificate has a key of EC_prime256v1 which is different from RSA_2048 in the current certificate.", msg)
+	assert.Equal(t, "New certificate has a key of EC_prime256v1 which is different from RSA_1024 in the current certificate.", msg)
 }
 
 func TestImportCertificate_SerialZeroRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
-	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
 	require.NoError(t, err)
 	tmpl := &x509.Certificate{
 		SerialNumber: big.NewInt(0),

spinifex/domains/acm/privateca_test.go

@@ -17,13 +17,11 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestMain lowers the RSA key sizes for the whole privateca test file. 2048
-// (CA) and 1024 (leaf) bits are still valid RSA keys and generate several
-// times faster than the production defaults, and the tests here care about
-// certificate structure, not cryptographic strength — mirrors
-// admin.certKeyBits's test seam.
+// TestMain lowers the RSA key sizes for the whole package to 1024 bits, the
+// smallest crypto/rsa accepts; the tests care about certificate structure,
+// not cryptographic strength — mirrors admin.certKeyBits's test seam.
 func TestMain(m *testing.M) {
-	caKeyBits = 2048
+	caKeyBits = 1024
 	leafKeyBits = 1024
 	os.Exit(m.Run())
 }
@@ -35,6 +33,7 @@ func tenantCAPaths(t *testing.T) (certPath, keyPath string) {
 }
 
 func TestLoadOrCreateTenantCA_CreatesConstrainedRoot(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
@@ -79,6 +78,7 @@ func TestLoadOrCreateTenantCA_CreatesConstrainedRoot(t *testing.T) {
 }
 
 func TestLoadOrCreateTenantCA_RefusesEmptyPermittedDomains(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	_, err := LoadOrCreateTenantCA(certPath, keyPath, nil)
@@ -88,6 +88,7 @@ func TestLoadOrCreateTenantCA_RefusesEmptyPermittedDomains(t *testing.T) {
 }
 
 func TestLoadOrCreateTenantCA_Idempotent(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	first, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
@@ -111,6 +112,7 @@ func TestLoadOrCreateTenantCA_Idempotent(t *testing.T) {
 }
 
 func TestLoadOrCreateTenantCA_DriftIsRejectedAndLeavesRootUntouched(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	_, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
@@ -135,6 +137,7 @@ func TestLoadOrCreateTenantCA_DriftIsRejectedAndLeavesRootUntouched(t *testing.T
 }
 
 func TestLoadOrCreateTenantCA_SubdomainRequestCoveredByExistingApex(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	_, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"example.com"})
@@ -147,6 +150,7 @@ func TestLoadOrCreateTenantCA_SubdomainRequestCoveredByExistingApex(t *testing.T
 }
 
 func TestLoadOrCreateTenantCA_IncompletePairIsRejected(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	_, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
@@ -161,6 +165,7 @@ func TestLoadOrCreateTenantCA_IncompletePairIsRejected(t *testing.T) {
 }
 
 func TestIssueLeaf_ChainsToRootAndCoversWildcard(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
 	require.NoError(t, err)
@@ -193,6 +198,7 @@ func TestIssueLeaf_ChainsToRootAndCoversWildcard(t *testing.T) {
 }
 
 func TestIssueLeaf_RejectsDomainOutsidePermittedDomains(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
 	require.NoError(t, err)
@@ -211,6 +217,7 @@ func TestIssueLeaf_RejectsDomainOutsidePermittedDomains(t *testing.T) {
 // calls out. If this leaf were to verify, the constraints baked into the
 // root would be decorative, not enforced.
 func TestLeafOutsidePermittedDomains_FailsVerify(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
 	require.NoError(t, err)
@@ -240,6 +247,7 @@ func TestLeafOutsidePermittedDomains_FailsVerify(t *testing.T) {
 // the CA key so the test does not depend on IssueLeaf ever choosing to
 // offer an IP SAN option.
 func TestLeafWithIPSAN_FailsVerify(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
 	require.NoError(t, err)
@@ -270,6 +278,7 @@ func TestLeafWithIPSAN_FailsVerify(t *testing.T) {
 // one), uses that intermediate to sign a leaf, and asserts the resulting
 // chain fails verification even though every individual signature is valid.
 func TestTenantRootCannotSignSubordinateCA(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
 	require.NoError(t, err)
@@ -320,6 +329,7 @@ func TestTenantRootCannotSignSubordinateCA(t *testing.T) {
 }
 
 func TestAuthorized(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
 	require.NoError(t, err)
@@ -342,12 +352,14 @@ func TestAuthorized(t *testing.T) {
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
 			assert.Equal(t, tc.want, ca.Authorized(tc.domain))
 		})
 	}
 }
 
 func TestLoadTenantCA_MissingReturnsActionableError(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	_, err := LoadTenantCA(certPath, keyPath)
@@ -356,6 +368,7 @@ func TestLoadTenantCA_MissingReturnsActionableError(t *testing.T) {
 }
 
 func TestLoadTenantCA_IncompletePairReturnsActionableError(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	_, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
@@ -369,6 +382,7 @@ func TestLoadTenantCA_IncompletePairReturnsActionableError(t *testing.T) {
 }
 
 func TestLoadTenantCA_RoundTripsCreatedCA(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 
 	created, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com", "lab.example.org"})
@@ -391,6 +405,7 @@ func TestLoadTenantCA_RoundTripsCreatedCA(t *testing.T) {
 }
 
 func TestPermittedDomains_ReturnsDefensiveCopy(t *testing.T) {
+	t.Parallel()
 	certPath, keyPath := tenantCAPaths(t)
 	ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"})
 	require.NoError(t, err)

spinifex/domains/acm/renewal_test.go

@@ -21,6 +21,7 @@ import (
 // exactly on renewalDueAt rather than depending on it landing on a whole
 // unit of time.
 func TestRenewable_ProportionalWindow(t *testing.T) {
+	t.Parallel()
 	const arn = "arn:aws:acm:ap-southeast-2:1:certificate/window-test"
 	jitter := renewalJitter(arn)
 	now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -38,6 +39,7 @@ func TestRenewable_ProportionalWindow(t *testing.T) {
 	}
 
 	t.Run("not yet due", func(t *testing.T) {
+		t.Parallel()
 		lifetime := 90 * 24 * time.Hour
 		window := renewalWindow(lifetime)
 		rec := newRec(lifetime, window+jitter-time.Minute)
@@ -45,6 +47,7 @@ func TestRenewable_ProportionalWindow(t *testing.T) {
 	})
 
 	t.Run("exactly at threshold", func(t *testing.T) {
+		t.Parallel()
 		lifetime := 90 * 24 * time.Hour
 		window := renewalWindow(lifetime)
 		rec := newRec(lifetime, window+jitter)
@@ -52,6 +55,7 @@ func TestRenewable_ProportionalWindow(t *testing.T) {
 	})
 
 	t.Run("overdue", func(t *testing.T) {
+		t.Parallel()
 		lifetime := 90 * 24 * time.Hour
 		window := renewalWindow(lifetime)
 		rec := newRec(lifetime, window+jitter+24*time.Hour)
@@ -59,6 +63,7 @@ func TestRenewable_ProportionalWindow(t *testing.T) {
 	})
 
 	t.Run("very short lifetime hits the floor", func(t *testing.T) {
+		t.Parallel()
 		lifetime := 10 * time.Minute
 		window := renewalWindow(lifetime)
 		require.Equal(t, minRenewalWindow, window,
@@ -78,6 +83,7 @@ func TestRenewable_ProportionalWindow(t *testing.T) {
 // only a PRIVATE, ISSUED, ELIGIBLE certificate is ever renewable, regardless
 // of how overdue it is.
 func TestRenewable_GatesOnTypeStatusEligibility(t *testing.T) {
+	t.Parallel()
 	now := time.Now()
 	overdue := func() *CertRecord {
 		return &CertRecord{
@@ -102,6 +108,7 @@ func TestRenewable_GatesOnTypeStatusEligibility(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
 			rec := overdue()
 			tt.mutate(rec)
 			assert.False(t, renewable(rec, now))
@@ -112,6 +119,7 @@ func TestRenewable_GatesOnTypeStatusEligibility(t *testing.T) {
 // --- Deterministic per-ARN jitter -------------------------------------------
 
 func TestRenewalJitter_DeterministicPerARN(t *testing.T) {
+	t.Parallel()
 	const arn = "arn:aws:acm:ap-southeast-2:1:certificate/jitter-test"
 	first := renewalJitter(arn)
 	second := renewalJitter(arn)
@@ -121,6 +129,7 @@ func TestRenewalJitter_DeterministicPerARN(t *testing.T) {
 }
 
 func TestRenewalJitter_SpreadsAcrossDistinctARNs(t *testing.T) {
+	t.Parallel()
 	offsets := make(map[time.Duration]bool)
 	for i := range 8 {
 		arn := fmt.Sprintf("arn:aws:acm:ap-southeast-2:1:certificate/cert-%d", i)
@@ -145,6 +154,7 @@ func backdateToOverdue(t *testing.T, svc *ACMServiceImpl, certArn string) {
 // TestWorker_RenewOne_PreservesIdentityChangesKeyAndSerial is the core
 // renewal contract: same ARN, same domain and SANs, fresh key and serial.
 func TestWorker_RenewOne_PreservesIdentityChangesKeyAndSerial(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{
 		"renew.example.com":     true,
@@ -185,6 +195,7 @@ func TestWorker_RenewOne_PreservesIdentityChangesKeyAndSerial(t *testing.T) {
 // mandatory push described in the ACM design doc's ELBv2 propagation
 // section.
 func TestWorker_CertMaterialUpdated_InvokedExactlyOnceOnSuccess(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"fanout.example.com": true}}
 	var calls int
@@ -215,6 +226,7 @@ func TestWorker_CertMaterialUpdated_InvokedExactlyOnceOnSuccess(t *testing.T) {
 // scan over a mix of certificate shapes and asserts only the eligible,
 // overdue PRIVATE_CA certificate is renewed.
 func TestWorker_ScanOnce_SkipsIneligibleAndImported_RenewsEligible(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{
 		"eligible.example.com": true,
@@ -267,6 +279,7 @@ func TestWorker_ScanOnce_SkipsIneligibleAndImported_RenewsEligible(t *testing.T)
 // RenewalSummary (the sole DescribeCertificate diagnostic surface) and must
 // not re-attempt on the very next scan.
 func TestWorker_ScanOnce_DomainOutsideConstraints_RecordsFailureAndBacksOff(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	ca := &fakeCertAuthority{permitted: map[string]bool{"narrowed.example.com": true}}
 	svc.TenantCA = ca
@@ -305,6 +318,7 @@ func TestWorker_ScanOnce_DomainOutsideConstraints_RecordsFailureAndBacksOff(t *t
 // lease actually excludes a second worker: node-a's live lease must stop
 // node-b's worker from reissuing the same certificate underneath it.
 func TestWorker_LeaseContention_OnlyOneNodeRenews(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"contended.example.com": true}}
 
@@ -338,6 +352,7 @@ func TestWorker_LeaseContention_OnlyOneNodeRenews(t *testing.T) {
 // path: a freshly issued certificate, nowhere near its renewal window, must
 // still reissue on demand.
 func TestForceRenewCertificate_BypassesRenewalWindow(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"force.example.com": true}}
 
@@ -365,6 +380,7 @@ func TestForceRenewCertificate_BypassesRenewalWindow(t *testing.T) {
 // an imported certificate rather than silently no-op'ing or crashing — this
 // worker only ever drives PRIVATE_CA reissuance.
 func TestForceRenewCertificate_RejectsNonPrivateType(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c, k := genCert(t, "force-import.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c, PrivateKey: k}, testAccountID)
@@ -379,6 +395,7 @@ func TestForceRenewCertificate_RejectsNonPrivateType(t *testing.T) {
 // ownership check: an ARN that does not exist, or belongs to another
 // account, must be rejected rather than disclosing anything about it.
 func TestForceRenewCertificate_UnknownArnRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	worker := NewWorker(svc, "node-1")
 	_, err := worker.ForceRenewCertificate(context.Background(), &ForceRenewCertificateInput{

spinifex/domains/acm/service_impl.go

@@ -24,6 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 )
 
@@ -724,15 +725,21 @@ func (s *ACMServiceImpl) AddTagsToCertificate(ctx context.Context, input *acm.Ad
 	if input == nil || aws.StringValue(input.CertificateArn) == "" {
 		return nil, errors.New(awserrors.ErrorACMInvalidArn)
 	}
-	rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID)
-	if err != nil {
+	if _, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID); err != nil {
 		return nil, err
 	}
-	if rec.Tags == nil {
-		rec.Tags = map[string]string{}
+	newTags := tagsToMap(input.Tags)
+	err := s.store.UpdateTags(ctx, aws.StringValue(input.CertificateArn), func(tags map[string]string) map[string]string {
+		if tags == nil {
+			tags = map[string]string{}
+		}
+		maps.Copy(tags, newTags)
+		return tags
+	})
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil, errors.New(awserrors.ErrorResourceNotFound)
 	}
-	maps.Copy(rec.Tags, tagsToMap(input.Tags))
-	if err := s.store.PutCert(ctx, rec); err != nil {
+	if err != nil {
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
 	return &acm.AddTagsToCertificateOutput{}, nil
@@ -745,21 +752,26 @@ func (s *ACMServiceImpl) RemoveTagsFromCertificate(ctx context.Context, input *a
 	if input == nil || aws.StringValue(input.CertificateArn) == "" {
 		return nil, errors.New(awserrors.ErrorACMInvalidArn)
 	}
-	rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID)
-	if err != nil {
+	if _, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID); err != nil {
 		return nil, err
 	}
-	for _, tag := range input.Tags {
-		if tag == nil {
-			continue
+	err := s.store.UpdateTags(ctx, aws.StringValue(input.CertificateArn), func(tags map[string]string) map[string]string {
+		for _, tag := range input.Tags {
+			if tag == nil {
+				continue
+			}
+			key := aws.StringValue(tag.Key)
+			if tag.Value != nil && tags[key] != aws.StringValue(tag.Value) {
+				continue
+			}
+			delete(tags, key)
 		}
-		key := aws.StringValue(tag.Key)
-		if tag.Value != nil && rec.Tags[key] != aws.StringValue(tag.Value) {
-			continue
-		}
-		delete(rec.Tags, key)
+		return tags
+	})
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil, errors.New(awserrors.ErrorResourceNotFound)
 	}
-	if err := s.store.PutCert(ctx, rec); err != nil {
+	if err != nil {
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
 	return &acm.RemoveTagsFromCertificateOutput{}, nil

spinifex/domains/acm/service_impl_test.go

@@ -67,6 +67,7 @@ func genCert(t *testing.T, cn string, dnsNames ...string) (certPEM, keyPEM []byt
 }
 
 func TestImportCertificate_MintsArnAndParsesLeaf(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, keyPEM := genCert(t, "example.com", "example.com", "www.example.com")
 
@@ -93,6 +94,7 @@ func TestImportCertificate_MintsArnAndParsesLeaf(t *testing.T) {
 }
 
 func TestImportCertificate_MismatchedKeyRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, _ := genCert(t, "a.example.com")
 	_, otherKey := genCert(t, "b.example.com")
@@ -106,6 +108,7 @@ func TestImportCertificate_MismatchedKeyRejected(t *testing.T) {
 }
 
 func TestImportCertificate_GarbagePEMRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	_, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
 		Certificate: []byte("not a pem"),
@@ -116,6 +119,7 @@ func TestImportCertificate_GarbagePEMRejected(t *testing.T) {
 }
 
 func TestListCertificates_ScopedToAccount(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "one.example.com")
 	c2, k2 := genCert(t, "two.example.com")
@@ -132,6 +136,7 @@ func TestListCertificates_ScopedToAccount(t *testing.T) {
 }
 
 func TestDescribeCertificate_CrossAccountHidden(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "secret.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID)
@@ -145,6 +150,7 @@ func TestDescribeCertificate_CrossAccountHidden(t *testing.T) {
 }
 
 func TestDeleteCertificate(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "del.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID)
@@ -165,6 +171,7 @@ func TestDeleteCertificate(t *testing.T) {
 }
 
 func TestImportCertificate_ReimportPreservesInUseBy(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "inuse.example.com")
 
@@ -190,6 +197,7 @@ func TestImportCertificate_ReimportPreservesInUseBy(t *testing.T) {
 }
 
 func TestImportCertificate_ReimportInvokesFanoutHook(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "hook.example.com")
 
@@ -215,6 +223,7 @@ func TestImportCertificate_ReimportInvokesFanoutHook(t *testing.T) {
 }
 
 func TestImportCertificate_ReimportZeroInUseBy_HookNotCalled(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "unused.example.com")
 
@@ -240,6 +249,7 @@ func TestImportCertificate_ReimportZeroInUseBy_HookNotCalled(t *testing.T) {
 }
 
 func TestImportCertificate_FreshImportDoesNotInvokeHook(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	called := false
 	svc.CertMaterialUpdated = func(ctx context.Context, arn string) error {
@@ -254,6 +264,7 @@ func TestImportCertificate_FreshImportDoesNotInvokeHook(t *testing.T) {
 }
 
 func TestImportCertificate_ReimportUnknownArnRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "re.example.com")
 	_, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
@@ -270,6 +281,7 @@ func TestImportCertificate_ReimportUnknownArnRejected(t *testing.T) {
 // contain the plaintext PEM key, and reading it back through the Store must
 // still recover the original plaintext.
 func TestImportCertificate_PrivateKeyEncryptedAtRest(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, keyPEM := genCert(t, "atrest.example.com")
 
@@ -297,6 +309,7 @@ func TestImportCertificate_PrivateKeyEncryptedAtRest(t *testing.T) {
 // record is stored. Precedent: TestSensitiveDataNotLogged_MasterKey in
 // handlers/iam/service_impl_test.go.
 func TestDescribeCertificate_DoesNotLeakPrivateKey(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, keyPEM := genCert(t, "noleak.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
@@ -370,6 +383,7 @@ func genLeafPEM(cn string, dnsNames []string) (certPEM, keyPEM string, err error
 }
 
 func TestRequestCertificate_ProviderAPI_PendingValidationNoResourceRecord(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.dnsProviderConfigured = true
 
@@ -395,6 +409,7 @@ func TestRequestCertificate_ProviderAPI_PendingValidationNoResourceRecord(t *tes
 }
 
 func TestRequestCertificate_ManualTXT_EmitsTXTResourceRecordAndIneligible(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.NorthstarHostsZone = func(domain string) bool { return true }
 
@@ -423,6 +438,7 @@ func TestRequestCertificate_ManualTXT_EmitsTXTResourceRecordAndIneligible(t *tes
 // PRIVATE_CA (automatic, offline, renewable), not MANUAL_TXT (which can never
 // complete against an internal zone a public CA cannot resolve).
 func TestRequestCertificate_PrivateCA_WinsOverNorthstarHostedZone(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"home.example.com": true}}
 	svc.NorthstarHostsZone = func(domain string) bool { return domain == "home.example.com" }
@@ -442,6 +458,7 @@ func TestRequestCertificate_PrivateCA_WinsOverNorthstarHostedZone(t *testing.T)
 }
 
 func TestRequestCertificate_PrivateCA_IssuesSynchronously(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"private.example.com": true}}
 
@@ -470,6 +487,7 @@ func TestRequestCertificate_PrivateCA_IssuesSynchronously(t *testing.T) {
 // the daemon-wiring slice adds no coverage for the seam between
 // RequestCertificate and the real implementation without a test like this one.
 func TestRequestCertificate_PrivateCA_RealTenantCA_IssuesAndChains(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	dir := t.TempDir()
 	ca, err := LoadOrCreateTenantCA(
@@ -509,6 +527,7 @@ func TestRequestCertificate_PrivateCA_RealTenantCA_IssuesAndChains(t *testing.T)
 }
 
 func TestRequestCertificate_PrivateCA_DomainOutsidePermittedRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"allowed.example.com": true}}
 
@@ -520,6 +539,7 @@ func TestRequestCertificate_PrivateCA_DomainOutsidePermittedRejected(t *testing.
 }
 
 func TestRequestCertificate_PrivateCA_SANOutsidePermittedRejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"allowed.example.com": true}}
 
@@ -532,6 +552,7 @@ func TestRequestCertificate_PrivateCA_SANOutsidePermittedRejected(t *testing.T)
 }
 
 func TestRequestCertificate_PrivateCA_NoTenantCAWired_FailsLoudly(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	// TenantCA deliberately left nil: PRIVATE_CA is derived (no dns provider, no
 	// northstar zone) but nothing can drive it.
@@ -542,6 +563,7 @@ func TestRequestCertificate_PrivateCA_NoTenantCAWired_FailsLoudly(t *testing.T)
 }
 
 func TestRequestCertificate_PrivateCA_IssueLeafErrorNotStored(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{
 		permitted: map[string]bool{"broken.example.com": true},
@@ -559,6 +581,7 @@ func TestRequestCertificate_PrivateCA_IssueLeafErrorNotStored(t *testing.T) {
 }
 
 func TestRequestCertificate_NoDomainName_Rejected(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	_, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{}, testAccountID)
 	require.Error(t, err)
@@ -566,6 +589,7 @@ func TestRequestCertificate_NoDomainName_Rejected(t *testing.T) {
 }
 
 func TestRequestCertificate_PrivateCA_DescribeDoesNotLeakPrivateKey(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"noleak-priv.example.com": true}}
 
@@ -583,6 +607,7 @@ func TestRequestCertificate_PrivateCA_DescribeDoesNotLeakPrivateKey(t *testing.T
 }
 
 func TestImportCertificate_RenewalEligibilityIneligible(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "imported-ineligible.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID)
@@ -595,6 +620,7 @@ func TestImportCertificate_RenewalEligibilityIneligible(t *testing.T) {
 }
 
 func TestDeleteCertificate_RefusesWhenInUse(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "inuse-delete.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID)
@@ -612,6 +638,7 @@ func TestDeleteCertificate_RefusesWhenInUse(t *testing.T) {
 }
 
 func TestDeleteCertificate_SucceedsWhenNotInUse(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "notinuse-delete.example.com")
 	out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID)
@@ -625,6 +652,8 @@ func TestDeleteCertificate_SucceedsWhenNotInUse(t *testing.T) {
 	assert.Contains(t, err.Error(), awserrors.ErrorResourceNotFound)
 }
 
+// Not parallel: it swaps the process-wide slog default, which parallel tests
+// would write to concurrently.
 func TestSensitiveDataNotLogged_ACMPrivateKey(t *testing.T) {
 	var buf strings.Builder
 	logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
@@ -654,6 +683,7 @@ func TestSensitiveDataNotLogged_ACMPrivateKey(t *testing.T) {
 // on one either: every spelling below names a real certificate under certKey's
 // old last-slash rule while resolving account-wide at the gate.
 func TestCertificateArnSpellingsTheGateCannotRead(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	certPEM, keyPEM := genCert(t, "fenced.example.com")
 
@@ -681,6 +711,7 @@ func TestCertificateArnSpellingsTheGateCannotRead(t *testing.T) {
 
 	for _, tt := range spellings {
 		t.Run(tt.certArn, func(t *testing.T) {
+			t.Parallel()
 			_, err := svc.DescribeCertificate(context.Background(),
 				&acm.DescribeCertificateInput{CertificateArn: aws.String(tt.certArn)}, testAccountID)
 			require.Error(t, err)
@@ -710,6 +741,7 @@ func TestCertificateArnSpellingsTheGateCannotRead(t *testing.T) {
 // The handler mints the ARN the gate builds, or every resource-scoped statement
 // names an object that does not exist.
 func TestMintedArnRoundTripsThroughTheGatesFormatter(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	minted := svc.mintCertificateArn(testAccountID)
 
@@ -722,6 +754,7 @@ func TestMintedArnRoundTripsThroughTheGatesFormatter(t *testing.T) {
 // empty: the SDK marshaller drops a nil slice, so a caller that ranges over the
 // result without a nil check breaks on every account's first state.
 func TestListCertificates_EmptyAccountReturnsEmptyList(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 
 	list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
@@ -737,6 +770,7 @@ func TestListCertificates_EmptyAccountReturnsEmptyList(t *testing.T) {
 // The populated case must keep working: an empty slice that never fills is just
 // a different way of losing the list.
 func TestListCertificates_PopulatedAccountReturnsSummaries(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	c1, k1 := genCert(t, "filled-one.example.com")
 	c2, k2 := genCert(t, "filled-two.example.com")
@@ -760,6 +794,7 @@ func TestListCertificates_PopulatedAccountReturnsSummaries(t *testing.T) {
 // fault. InternalErrorException is a retryable 500, so Terraform sat on
 // "Still creating..." instead of failing; this asserts the 4xx and the reason.
 func TestRequestCertificate_NoTenantCAIsAClientErrorNamingTheCause(t *testing.T) {
+	t.Parallel()
 	svc := setupACMService(t)
 	require.Nil(t, svc.TenantCA, "this test only means anything with no tenant CA wired")