fix: write MariaDB's startup spelling for time_zone into the option file

mariadbd has no time_zone startup option — the name only exists for SET — so
the resolved parameter group aborted the server while it parsed defaults, before
it opened the datadir. No pidfile was ever written and the reconciler failed
every instance at the 20-minute engine-health deadline.

ParameterSpec gains an option-file spelling, used only when rendering the
include. time_zone is the one setting that needs it, as default_time_zone, which
resolves @@global.time_zone to the same value. The customer-facing name is
unchanged: the API still reports time_zone and a live apply still issues
SET GLOBAL time_zone. Names read back out of a generated file return to the
catalog's namespace first, so a startup spelling cannot read as an unknown name
and report a pending restart nothing could clear.

setup.sh now feeds the catalog's resolved defaults to the image's own mariadbd
at build time, so a name the server refuses fails the build rather than every
instance built from it.

.githooks/pre-commit

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-branch=$(git rev-parse --abbrev-ref HEAD)
-
-# Only enforce preflight on dev and main
-if [ "$branch" != "dev" ] && [ "$branch" != "main" ]; then
-  exit 0
-fi
-
-# Only run preflight if Go files were staged
-if git diff --cached --name-only | grep -q '\.go$'; then
-  echo "Go files staged on $branch — running make preflight..."
-  make preflight
-else
-  echo "No Go files staged, skipping preflight."
-fi

.github/CODEOWNERS

@@ -6,16 +6,10 @@
 # Set the matching branch protection rule on `dev` and `main` to
 # "Require review from Code Owners" or this file is advisory only.
 
-# Fall-through: any change needs a team review.
-*                           @mulgadc/mulgadc-engineers
-
 # Network layer is governed by ADR-0006 invariant tests.
-# Changes here must respect those clauses — see
-# spinifex/network/invariants/ and
+# Changes here must respect those clauses
+# see spinifex/network/invariants/
 # docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md
 /spinifex/network/          @joshsiv-mulga
 /spinifex/vpcd/             @joshsiv-mulga
-
-# Invariants suite is the executable ADR-0006 fence — any widening or
-# narrowing of the scan roots needs an explicit owner review.
 /spinifex/network/invariants/ @joshsiv-mulga

.github/actions/e2e-analyze/README.md

@@ -70,7 +70,7 @@ When the report format changes intentionally, regenerate the snapshots:
 go test ./.github/actions/e2e-analyze/ -update
 ```
 
-## Per-failure bundles (Stage 2)
+## Per-failure bundles
 
 After parsing, the action materialises a bundle directory per failure
 under `<log-dir>/analysis/NN-<TestName>/`. The report links to it from

.github/actions/e2e-analyze/analyze.go

@@ -1,7 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1 of docs/development/improvements/e2e-go-failure-analysis.md:
-// cluster failures from go-junit-report's junit-*.xml by error signature,
+// Cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -74,8 +73,14 @@ type Failure struct {
 	// `test-<suite>.start` file written by the workflow before the suite
 	// runs) can recompute StartAt without re-parsing the XML.
 	OffsetFromSuiteStart time.Duration
+	// SuiteSpan is the suite's own wall-clock span, set only when the suite
+	// ran its tests in parallel. Overlapping tests make the cumulative
+	// offset above pure fiction — it lands minutes past the real failure —
+	// so consumers widen the journal window to the suite instead of
+	// slicing a plausible-looking window around the wrong minute.
+	SuiteSpan time.Duration
 	// BundlePath is the relative path (from log-dir) to the per-failure
-	// bundle written by Stage 2's WriteBundles. Empty when bundles were
+	// bundle written by WriteBundles. Empty when bundles were
 	// not generated (e.g. unit tests that call Render directly).
 	BundlePath string
 }
@@ -89,6 +94,12 @@ type SuiteReport struct {
 	Root      *Failure    // nil if no failures
 	Cascades  []Failure   // everything else that failed in this suite
 	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
+	// the XML was produced — i.e. when the suite finished, not when it began.
+	EndedAt time.Time
+	// TopLevelTotal is the summed duration of the parentless tests. Longer than
+	// the suite's wall clock means they overlapped.
+	TopLevelTotal time.Duration
 }
 
 // Report is the full analysis across every junit file.
@@ -120,6 +131,28 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
+// teardownPatterns match the content a test logs from t.Cleanup or from the
+// harness's on-failure diagnostics — i.e. after it has already failed. They are
+// the LAST file:line lines in the failure body, so extractErrorLine's
+// keep-the-last rule picks them over the assertion unless they are skipped.
+// Anchored deliberately: a fragment loose enough to match an assertion would
+// blank out the one line the report exists to show.
+var teardownPatterns = []*regexp.Regexp{
+	regexp.MustCompile(`^DB instance \S+ is gone$`),
+	regexp.MustCompile(`^db diagnostics \S+:`),
+}
+
+// isTeardownLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for.
+func isTeardownLine(content string) bool {
+	for _, re := range teardownPatterns {
+		if re.MatchString(content) {
+			return true
+		}
+	}
+	return false
+}
+
 // noiseReplacers normalise dynamic identifiers so that "ten failures, same
 // bug" bucket together. Pattern ordering matters: longer/more-specific
 // patterns must precede the catch-alls.
@@ -207,11 +240,14 @@ func extractErrorLine(body string) string {
 		if m := goFileLineRe.FindStringSubmatch(l); m != nil {
 			// Keep the LAST occurrence: t.Log / harness traces emit
 			// progress lines from the same file:line pattern before the
-			// failing assertion finally fires.
-			goFileLine = strings.TrimSpace(m[1])
+			// failing assertion finally fires. Cleanup runs after it, though,
+			// so its lines are last of all and have to be skipped.
+			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+				goFileLine = content
+			}
 			continue
 		}
-		if fileHintRe.MatchString(l) {
+		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -255,11 +291,19 @@ func extractFileHint(body string) string {
 	if m := errorTraceRe.FindStringSubmatch(body); m != nil {
 		return m[1]
 	}
-	all := fileHintRe.FindAllString(body, -1)
-	if len(all) == 0 {
-		return ""
+	// Scanned line by line rather than across the body so a cleanup line's own
+	// file:line does not become the site the report points a reader at.
+	hint := ""
+	for _, raw := range strings.Split(body, "\n") {
+		l := strings.TrimSpace(raw)
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+			continue
+		}
+		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
+			hint = all[len(all)-1]
+		}
 	}
-	return all[len(all)-1]
+	return hint
 }
 
 // signature normalises an error line so different runs of the same root
@@ -314,6 +358,47 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 	return leaf
 }
 
+// computeRolledUpSet returns the testcase names whose failure is already
+// reported by a failing descendant. go-junit-report gives such a parent a
+// testcase of its own, and its body is the parent's t.Log output rather than an
+// assertion — a cleanup line like "DB instance … is gone", which passing tests
+// emit too, reads to extractErrorLine exactly like a failure site.
+func computeRolledUpSet(cases []junitTC) map[string]bool {
+	rolledUp := make(map[string]bool, len(cases))
+	for _, tc := range cases {
+		if tc.Failure == nil && tc.Error == nil {
+			continue
+		}
+		// Mark every ancestor: the failure belongs to the deepest case that
+		// carries it, not to the tests it is nested under.
+		name := tc.Name
+		for {
+			i := strings.LastIndex(name, "/")
+			if i <= 0 {
+				break
+			}
+			name = name[:i]
+			rolledUp[name] = true
+		}
+	}
+	return rolledUp
+}
+
+// topLevelTotal sums the durations of the tests with no parent, which are the
+// units t.Parallel overlaps. Compared against the suite's wall clock it says
+// whether the suite ran its tests concurrently — see ApplySuiteStartFiles.
+// suite.Time cannot answer that: go-junit-report sets it to the sum of every
+// testcase, parents included, so it is never less than any subset of itself.
+func topLevelTotal(cases []junitTC) time.Duration {
+	total := 0.0
+	for _, tc := range cases {
+		if !strings.Contains(tc.Name, "/") {
+			total += tc.Time
+		}
+	}
+	return time.Duration(total * float64(time.Second))
+}
+
 // parseStartTime turns a junit timestamp attr (RFC3339) into a time.Time;
 // zero value if unparseable.
 func parseStartTime(s string) time.Time {
@@ -360,6 +445,9 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 		// summing every testcase double-counts. Pre-compute the leaf set so
 		// only leaves contribute to the suite's running offset.
 		isLeaf := computeLeafSet(suite.Cases)
+		rolledUp := computeRolledUpSet(suite.Cases)
+		rep.EndedAt = start
+		rep.TopLevelTotal += topLevelTotal(suite.Cases)
 		cumul := 0.0
 		for _, tc := range suite.Cases {
 			rep.Total++
@@ -377,11 +465,12 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 				continue
 			}
 
-			// Skip the synthetic parent-test "Failed" stub that go-junit-report
-			// emits for any Go test whose subtests failed. It has an empty body
-			// and would otherwise outrank real failures by XML order.
+			// Skip the parent-test entry go-junit-report emits for any Go test
+			// whose subtests failed: it is a rollup of failures already counted
+			// below it, and it would outrank them by XML order. An empty body is
+			// the same stub in the case where the parent logged nothing.
 			trimmed := strings.TrimSpace(stripANSI(body))
-			if trimmed == "" {
+			if trimmed == "" || rolledUp[tc.Name] {
 				if leaf {
 					cumul += tc.Time
 				}
@@ -463,15 +552,24 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }
 
-// Render writes the markdown report. Format is documented in
-// docs/development/improvements/e2e-go-failure-analysis.md (Stage 1).
+// Render writes the markdown report.
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
 
-	totalFail := 0
+	totalFail, totalTests := 0, 0
 	for _, s := range r.Suites {
 		totalFail += s.FailCount
+		totalTests += s.Total
+	}
+
+	// A cell that dies before the suites run emits junit with no testcases at
+	// all. Reporting that as clean turns "nothing ran" into "nothing wrong", so
+	// name it and point above the suites.
+	if totalTests == 0 {
+		b.WriteString("⚠️ No tests ran — no testcases in any junit file. " +
+			"The failure is upstream of the suites (provisioning, bootstrap, or reimage); check the job log.\n")
+		return b.String()
 	}
 	if totalFail == 0 {
 		b.WriteString("✅ No failures across any suite.\n")
@@ -486,12 +584,22 @@ func Render(r Report) string {
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
 
 		if s.Root != nil {
-			b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			// XML order is completion order, so "earliest" only means earliest
+			// in time for a suite whose tests ran one after another.
+			if s.Root.SuiteSpan > 0 {
+				b.WriteString("**First non-cascade failure** (the suite ran in parallel, so this is report order, not time order)\n\n")
+			} else {
+				b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			}
 			fmt.Fprintf(&b, "- Test: `%s`\n", s.Root.Name)
-			if !s.Root.StartAt.IsZero() {
+			switch {
+			case s.Root.SuiteSpan > 0:
+				fmt.Fprintf(&b, "- Start: unknown, the suite ran in parallel (duration %.1fs)\n",
+					s.Root.Duration)
+			case !s.Root.StartAt.IsZero():
 				fmt.Fprintf(&b, "- Start: %s (duration %.1fs)\n",
 					s.Root.StartAt.Format("15:04:05"), s.Root.Duration)
-			} else {
+			default:
 				fmt.Fprintf(&b, "- Duration: %.1fs\n", s.Root.Duration)
 			}
 			if s.Root.Error != "" {

.github/actions/e2e-analyze/analyze_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
 // updateGolden lets `go test ./.github/actions/e2e-analyze/ -update` rewrite
@@ -124,6 +125,22 @@ func TestExtractErrorLine_FallsBackToLastTestLine(t *testing.T) {
 	}
 }
 
+// A test's cleanup logs after it has failed, so its lines are the last in the
+// body — and the cleanup line here is one passing tests emit too.
+func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
+	body := `    failure_test.go:42: DB instance rds-e2e-failure-1 reached status available
+    failure_test.go:60: EventuallyErr: condition not met within 4m0s: rds-e2e-failure-1 status=available want=failed
+    rds.go:364: db diagnostics rds-e2e-failure-1: no VM to capture a console from (<nil>)
+    main_test.go:325: DB instance rds-e2e-failure-1 is gone`
+	got := extractErrorLine(body)
+	if !strings.HasPrefix(got, "EventuallyErr:") {
+		t.Errorf("extractErrorLine = %q, want the assertion at failure_test.go:60", got)
+	}
+	if hint := extractFileHint(body); hint != "failure_test.go:60" {
+		t.Errorf("extractFileHint = %q, want failure_test.go:60", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -157,6 +174,116 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
+	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
+	// "DB instance … is gone" — which passing tests emit too — reads exactly
+	// like an assertion site. Its subtest holds the real failure.
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="2">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    main_test.go:325: DB instance rds-e2e-backup-1 is gone
+]]></failure></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (the parent rollup should be skipped)", sr.FailCount)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the subtest as root, got %+v", sr.Root)
+	}
+}
+
+// A parent that fails on its own after its subtests passed is a real failure,
+// and dropping it would lose the only record of it.
+func TestParseFile_KeepsParentWithNoFailingSubtest(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    foo_test.go:42:
+        Messages: the parent asserted after its subtests
+]]></failure></testcase>
+  <testcase name="TestX/Passing" time="0.5"></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX" {
+		t.Fatalf("expected the parent as root, got %+v", sr.Root)
+	}
+}
+
+// suiteFixture writes a junit file plus its .start sidecar into a temp log dir
+// and returns the parsed, start-file-corrected report for one suite.
+func suiteFixture(t *testing.T, start string, xml []byte) SuiteReport {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "test-x.start"), []byte(start), 0o644); err != nil {
+		t.Fatalf("write start file: %v", err)
+	}
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	rep := Report{Suites: []SuiteReport{sr}}
+	ApplySuiteStartFiles(&rep, dir)
+	return rep.Suites[0]
+}
+
+// Parallel tests overlap, so summing the ones before a failure puts its start
+// minutes past where it really was — and the journal slice cut from that window
+// misses the failure entirely. The tell is the top-level tests taking longer in
+// total than the suite's own wall clock.
+func TestApplySuiteStartFiles_DropsTheOffsetWhenTheSuiteRanInParallel(t *testing.T) {
+	// timestamp is when go-junit-report wrote the XML, i.e. the suite's end.
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestSlow" time="100.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 2*time.Minute {
+		t.Errorf("SuiteSpan = %s, want 2m0s to mark the suite parallel", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 0, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start %s rather than a summed offset", s.Root.StartAt, want)
+	}
+}
+
+// The sequential case must keep its offsets: they are the only thing that makes
+// a per-failure journal slice narrow enough to read.
+func TestApplySuiteStartFiles_KeepsTheOffsetWhenTheSuiteRanSequentially(t *testing.T) {
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestFirst" time="30.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 0 {
+		t.Errorf("SuiteSpan = %s, want 0 for a sequential suite", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 30, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start plus 30s of preceding tests %s", s.Root.StartAt, want)
+	}
+}
+
 func TestParseFile_CascadeFallbackWhenAllCascades(t *testing.T) {
 	// If every failure is a cascade marker, the earliest one becomes root
 	// so the report isn't blank.
@@ -189,3 +316,25 @@ func TestRender_NoFailuresIsClean(t *testing.T) {
 		t.Errorf("expected zero-failure banner, got:\n%s", out)
 	}
 }
+
+// A cell that fails in provisioning writes junit with no testcases. Rendering
+// the clean banner there reports "nothing ran" as "nothing wrong".
+func TestRender_NoTestsRanIsNotClean(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		suites []SuiteReport
+	}{
+		{"empty junit file", []SuiteReport{{File: "junit-baremetal.xml", Label: "baremetal", Total: 0, FailCount: 0}}},
+		{"no junit files at all", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			out := Render(Report{Title: "E2E failure analysis", Suites: tc.suites})
+			if strings.Contains(out, "No failures across any suite") {
+				t.Errorf("clean banner rendered when no test ran, got:\n%s", out)
+			}
+			if !strings.Contains(out, "No tests ran") {
+				t.Errorf("expected a no-tests-ran warning, got:\n%s", out)
+			}
+		})
+	}
+}

.github/actions/e2e-analyze/bundle.go

@@ -1,7 +1,6 @@
 package main
 
-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// for each failure surfaced by Stage 1, materialise a per-failure bundle
+// Stage 2: for each failure surfaced by Stage 1, materialise a per-failure bundle
 // directory containing the failure summary, a time-window slice of the
 // daemon journal, a testcase-window slice of the suite's stdout log, and
 // any per-test artifacts the harness already dumped at run-time.
@@ -18,9 +17,9 @@ import (
 )
 
 // bundlePad is the per-failure window padding either side of the
-// [StartAt, StartAt+Duration] testcase span. Five seconds matches the
-// plan doc: long enough to catch the daemon's pre-test setup and
-// post-test cleanup chatter, short enough to keep the slice focused.
+// [StartAt, StartAt+Duration] testcase span. Five seconds catches the
+// daemon's pre-test setup and post-test cleanup chatter while keeping
+// the slice focused.
 const bundlePad = 5 * time.Second
 
 // nameSanitiseRe replaces characters unsafe in directory names with `_`.
@@ -59,21 +58,57 @@ func ApplySuiteStartFiles(rep *Report, logDir string) {
 			fmt.Fprintf(os.Stderr, "::warning::e2e-analyze: parse %s: %v\n", startPath, err)
 			continue
 		}
-		if s.Root != nil {
-			s.Root.StartAt = ts.Add(s.Root.OffsetFromSuiteStart)
-		}
-		for ci := range s.Cascades {
-			s.Cascades[ci].StartAt = ts.Add(s.Cascades[ci].OffsetFromSuiteStart)
-		}
-		for bi := range s.Buckets {
-			for fi := range s.Buckets[bi] {
-				f := &s.Buckets[bi][fi]
-				f.StartAt = ts.Add(f.OffsetFromSuiteStart)
+		// The start file is the first wall clock the analyzer has, so it is also
+		// the first chance to tell a parallel suite from a sequential one — and
+		// a parallel one's per-test offsets are fiction, since overlapping tests
+		// never started at the sum of the ones before them.
+		span := suiteSpanIfParallel(s, ts)
+		for _, f := range collectFailures(s) {
+			f.SuiteSpan = span
+			if span > 0 {
+				f.StartAt = ts
+				continue
 			}
+			f.StartAt = ts.Add(f.OffsetFromSuiteStart)
 		}
 	}
 }
 
+// suiteSpanIfParallel returns the suite's wall-clock span when its top-level
+// tests overlapped, and zero when they ran one after another. Consumers read a
+// non-zero span as "no per-test offset is derivable from this suite".
+func suiteSpanIfParallel(s *SuiteReport, start time.Time) time.Duration {
+	if s.EndedAt.IsZero() || s.TopLevelTotal == 0 {
+		return 0
+	}
+	wall := s.EndedAt.Sub(start)
+	// The 5% margin keeps per-testcase rounding from calling a sequential
+	// suite parallel; real overlap runs to several times the wall clock.
+	if wall <= 0 || s.TopLevelTotal <= time.Duration(float64(wall)*1.05) {
+		return 0
+	}
+	return wall
+}
+
+// collectFailures returns every failure in the report by pointer. Root, cascades
+// and buckets hold independent copies of the same failures, so a field that has
+// to reach the bundle writer must be set on all three.
+func collectFailures(s *SuiteReport) []*Failure {
+	var all []*Failure
+	if s.Root != nil {
+		all = append(all, s.Root)
+	}
+	for ci := range s.Cascades {
+		all = append(all, &s.Cascades[ci])
+	}
+	for bi := range s.Buckets {
+		for fi := range s.Buckets[bi] {
+			all = append(all, &s.Buckets[bi][fi])
+		}
+	}
+	return all
+}
+
 // WriteBundles materialises per-failure bundles under <logDir>/analysis/.
 // Mutates rep in-place so each Failure.BundlePath holds the relative
 // path the renderer will link from the summary.
@@ -144,7 +179,11 @@ func writeOneBundle(logDir, base, suiteLabel string, f *Failure) {
 
 	writeFailureSummary(dir, suiteLabel, f)
 
-	if !f.StartAt.IsZero() {
+	// A parallel suite offers no derivable per-test window, and the narrowing is
+	// the whole point of the slice: cutting one from the wrong minute leaves out
+	// the lines that explain the failure, and widening it to the suite just
+	// copies the artifact's own journal into every bundle.
+	if !f.StartAt.IsZero() && f.SuiteSpan == 0 {
 		start := f.StartAt.Add(-bundlePad)
 		end := f.StartAt.Add(time.Duration(f.Duration * float64(time.Second))).Add(bundlePad)
 		journalSrc := filepath.Join(logDir, "spinifex-journal.log")
@@ -189,7 +228,11 @@ func writeFailureSummary(dir, suiteLabel string, f *Failure) {
 	var b strings.Builder
 	fmt.Fprintf(&b, "Suite:    %s\n", suiteLabel)
 	fmt.Fprintf(&b, "Test:     %s\n", f.Name)
-	if !f.StartAt.IsZero() {
+	switch {
+	case f.SuiteSpan > 0:
+		b.WriteString("Start:    unknown, the suite ran in parallel; " +
+			"no journal window is derivable — read ../../spinifex-journal.log\n")
+	case !f.StartAt.IsZero():
 		fmt.Fprintf(&b, "Start:    %s\n", f.StartAt.Format(time.RFC3339))
 	}
 	fmt.Fprintf(&b, "Duration: %.3fs\n", f.Duration)

.github/actions/e2e-analyze/main.go

@@ -45,7 +45,7 @@ func main() {
 		report.Suites = append(report.Suites, sr)
 	}
 
-	// Stage 2: prefer authoritative per-suite start times written by the
+	// Prefer authoritative per-suite start times written by the
 	// workflow over junit-report's conversion-time `timestamp` attribute,
 	// then materialise per-failure bundles so Render can link them.
 	if logDir != "" {

.github/actions/e2e-analyze/slice.go

@@ -1,7 +1,6 @@
 package main
 
-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// per-failure log slicers used by bundle.go to materialise time-window /
+// Per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.
 
 import (

.github/actions/otel-job-id/action.yml

@@ -0,0 +1,52 @@
+name: Resolve OTEL job id
+description: >-
+  Append ci.job_id to SPINIFEX_OTEL_ATTRS so telemetry emitted by this job can
+  be pivoted straight back to its GitHub job log. Must run before the step that
+  bakes SPINIFEX_OTEL_ATTRS into a VM's telemetry.env, and requires
+  `actions: read` on the calling job.
+
+inputs:
+  job-name:
+    description: >-
+      Rendered name of the calling job, exactly as it appears in the Actions UI.
+      github.job is only the YAML key and no context exposes the numeric id, so
+      the id has to be looked up by name.
+    required: true
+  token:
+    description: Token used for the Actions jobs API lookup.
+    required: false
+    default: ${{ github.token }}
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve CI job id (best-effort)
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+        JOB_NAME: ${{ inputs.job-name }}
+        REPO: ${{ github.repository }}
+        RUN_ID: ${{ github.run_id }}
+      run: |
+        # Never fail the caller: on any lookup error or name mismatch ci.job_id
+        # is simply omitted and the run proceeds with the labels it already has.
+        set +e
+
+        if [ -z "${SPINIFEX_OTEL_ATTRS:-}" ]; then
+          echo "SPINIFEX_OTEL_ATTRS is unset or empty — telemetry is disabled, nothing to label"
+          exit 0
+        fi
+
+        # env.JOB_NAME rather than shell interpolation into the jq filter: job
+        # names carry quotes, slashes and non-ASCII separators that would
+        # otherwise break the expression.
+        JOB_ID=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}/jobs" --paginate \
+          --jq '.jobs[] | select(.name == env.JOB_NAME) | .id' 2>/dev/null | head -n1)
+
+        if [ -n "$JOB_ID" ]; then
+          echo "SPINIFEX_OTEL_ATTRS=${SPINIFEX_OTEL_ATTRS},ci.job_id=${JOB_ID}" >> "$GITHUB_ENV"
+          echo "resolved ci.job_id=${JOB_ID} for job '${JOB_NAME}'"
+        else
+          echo "::warning::could not resolve numeric job id for '${JOB_NAME}' — continuing without ci.job_id"
+        fi
+        exit 0

.github/assets/architecture.svg

@@ -1,225 +1,114 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="900" height="620" viewBox="0 0 900 620">
-  <defs>
-    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
-      <stop offset="0%" style="stop-color:#0a0a0a"/>
-      <stop offset="100%" style="stop-color:#0d1117"/>
-    </linearGradient>
-    <linearGradient id="bar-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#161b22"/>
-      <stop offset="45%" style="stop-color:#1f2028"/>
-      <stop offset="100%" style="stop-color:#161b22"/>
-    </linearGradient>
-    <linearGradient id="client-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#3fb95022"/>
-      <stop offset="100%" style="stop-color:#3fb95008"/>
-    </linearGradient>
-    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#DC262633"/>
-      <stop offset="100%" style="stop-color:#7F1D1D14"/>
-    </linearGradient>
-    <linearGradient id="nats-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#F87171"/>
-      <stop offset="50%" style="stop-color:#DC2626"/>
-      <stop offset="100%" style="stop-color:#7F1D1D"/>
-    </linearGradient>
-  </defs>
-
-  <rect width="900" height="620" fill="url(#bg)" rx="12"/>
-
-  <!-- Title bar -->
-  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ARCHITECTURE</text>
-  <text x="24" y="52" font-family="'Segoe UI', system-ui, sans-serif" font-size="16" fill="#f5f5f5" font-weight="500">AWS SDK → Gateway → NATS → Service daemons</text>
-  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">stateless · message-driven · horizontally scalable</text>
-
-  <!-- ═══ CLIENT LAYER ════════════════════════════════════ -->
-  <rect x="16" y="72" width="868" height="78" rx="8" fill="url(#client-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.5"/>
-  <text x="32" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" font-weight="700" letter-spacing="1">CLIENTS</text>
-  <text x="32" y="108" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">Standard AWS tooling — unmodified, no shims, no custom SDK</text>
-
-  <g transform="translate(32, 120)" font-family="'SF Mono', ui-monospace, monospace" font-size="10">
-    <rect x="0" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="43" y="15" fill="#7ee787" text-anchor="middle">aws-cli</text>
-
-    <rect x="94" y="0" width="96" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="142" y="15" fill="#7ee787" text-anchor="middle">Terraform</text>
-
-    <rect x="198" y="0" width="76" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="236" y="15" fill="#7ee787" text-anchor="middle">boto3</text>
-
-    <rect x="282" y="0" width="98" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="331" y="15" fill="#7ee787" text-anchor="middle">aws-sdk-go</text>
-
-    <rect x="388" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="431" y="15" fill="#7ee787" text-anchor="middle">kubectl</text>
-
-    <rect x="482" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="525" y="15" fill="#7ee787" text-anchor="middle">Packer</text>
-
-    <rect x="576" y="0" width="120" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="636" y="15" fill="#7ee787" text-anchor="middle">your application</text>
-  </g>
-
-  <!-- ═══ HTTPS + SigV4 connector ═══════════════════════════ -->
-  <line x1="450" y1="150" x2="450" y2="178" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
-  <rect x="390" y="154" width="120" height="18" rx="9" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
-  <text x="450" y="166" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#F87171" text-anchor="middle">HTTPS · SigV4 · :9999</text>
-
-  <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
-  <rect x="16" y="178" width="868" height="68" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
-  <text x="32" y="200" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY</text>
-  <text x="32" y="215" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
-
-  <g transform="translate(32, 224)" font-family="'SF Mono', ui-monospace, monospace" font-size="9">
-    <rect x="0" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="39" y="11" fill="#FCA5A5" text-anchor="middle">TLS 1.3</text>
-
-    <rect x="86" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="125" y="11" fill="#FCA5A5" text-anchor="middle">SigV4 auth</text>
-
-    <rect x="172" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="221" y="11" fill="#FCA5A5" text-anchor="middle">policy eval</text>
-
-    <rect x="278" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="327" y="11" fill="#FCA5A5" text-anchor="middle">req routing</text>
-
-    <rect x="384" y="0" width="88" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="428" y="11" fill="#FCA5A5" text-anchor="middle">rate-limit</text>
-
-    <rect x="480" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="529" y="11" fill="#FCA5A5" text-anchor="middle">audit log</text>
-  </g>
-
-  <!-- ═══ NATS BUS ═══════════════════════════════════════ -->
-  <line x1="450" y1="246" x2="450" y2="272" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
-  <rect x="16" y="272" width="868" height="34" rx="6" fill="url(#nats-grad)" opacity="0.92"/>
-  <text x="32" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#f5f5f5" font-weight="700" letter-spacing="1.5">NATS BUS</text>
-  <text x="140" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#fee2e2" opacity="0.95">ec2.*   ebs.*   s3.*   vpc.*   iam.*</text>
-  <text x="876" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#fee2e2" text-anchor="end" opacity="0.9">queue groups · fan-out · request/reply</text>
-
-  <!-- ═══ DAEMON FANOUT CONNECTORS ═══════════════════════ -->
-  <g stroke="#DC2626" stroke-width="0.9" stroke-dasharray="3,3" opacity="0.55" fill="none">
-    <line x1="100" y1="306" x2="100" y2="328"/>
-    <line x1="275" y1="306" x2="275" y2="328"/>
-    <line x1="450" y1="306" x2="450" y2="328"/>
-    <line x1="625" y1="306" x2="625" y2="328"/>
-    <line x1="800" y1="306" x2="800" y2="328"/>
-  </g>
-
-  <!-- ═══ SERVICE DAEMONS ROW ═══════════════════════════ -->
-  <!-- EC2 -->
-  <g transform="translate(28, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ec2.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">RunInstances</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">TerminateInstances</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">QMP · cloud-init</text>
-  </g>
-
-  <!-- EBS -->
-  <g transform="translate(204, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ebs.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">CreateVolume</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">AttachVolume</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">snapshots · WAL</text>
-  </g>
-
-  <!-- S3 -->
-  <g transform="translate(380, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">s3.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">PutObject · GetObject</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Multipart · presigned</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">erasure-coded</text>
-  </g>
-
-  <!-- VPC -->
-  <g transform="translate(556, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">vpc.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Subnets · SGs · RTs</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">ALB · NAT · IGW</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">OVN · OVS</text>
-  </g>
-
-  <!-- IAM -->
-  <g transform="translate(732, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">iam.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Users · Roles · Keys</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Policy evaluation</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">audit trail</text>
-  </g>
-
-  <!-- ═══ DAEMON → BACKEND CONNECTORS ═══════════════════ -->
-  <g stroke="#71717a" stroke-width="0.8" stroke-dasharray="2,3" opacity="0.55" fill="none">
-    <line x1="100" y1="410" x2="100" y2="440"/>
-    <line x1="276" y1="410" x2="276" y2="440"/>
-    <line x1="452" y1="410" x2="452" y2="440"/>
-    <line x1="628" y1="410" x2="628" y2="440"/>
-    <line x1="804" y1="410" x2="804" y2="440"/>
-  </g>
-
-  <!-- ═══ BACKEND RESOURCES BAR ═════════════════════════ -->
-  <rect x="16" y="440" width="868" height="26" rx="5" fill="url(#bar-grad)"/>
-  <text x="450" y="458" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle" letter-spacing="1.5">BACKEND RESOURCES · local hardware · no cloud dependency</text>
-
-  <!-- ═══ BACKEND ROW ═══════════════════════════════════ -->
-  <!-- QEMU/KVM -->
-  <g transform="translate(28, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">hypervisor</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">host kernel · virtio</text>
-  </g>
-
-  <!-- Viperblock -->
-  <g transform="translate(204, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Viperblock</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">block storage</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">NVMe WAL · NBD</text>
-  </g>
-
-  <!-- Predastore -->
-  <g transform="translate(380, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Predastore</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">object storage</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">Reed–Solomon</text>
-  </g>
-
-  <!-- OVN/OVS -->
-  <g transform="translate(556, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">OVN / OVS</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">SDN fabric</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">overlays · flow rules</text>
-  </g>
-
-  <!-- IAM store -->
-  <g transform="translate(732, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Identity store</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">users · keys · rules</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">local · audit-logged</text>
-  </g>
-
-  <!-- ═══ FOOTER NOTES ══════════════════════════════════ -->
-  <g transform="translate(16, 558)">
-    <rect width="868" height="48" rx="6" fill="#0a0a0a" stroke="#27272a" stroke-width="0.8"/>
-    <text x="24" y="20" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
-      <tspan fill="#F87171" font-weight="600">Queue groups</tspan> load-balance ops across daemon replicas.
-      <tspan fill="#F87171" font-weight="600"> Fan-out</tspan> (no queue group) dispatches describe calls to every node.
-    </text>
-    <text x="24" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
-      Daemons are <tspan fill="#F87171" font-weight="600">stateless</tspan> — scale horizontally by starting more.
-      No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.
-    </text>
-  </g>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="770" viewBox="0 0 1020 770" role="img" aria-label="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus; stateless ec2/ebs/s3/vpc/iam daemons answer and drive local backends (QEMU/KVM, Viperblock, Predastore, OVN/OVS, identity store).">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0.0" y="0.0" width="1020.0" height="770.0" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28.0" y="42.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ARCHITECTURE</text>
+<text x="992.0" y="42.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">stateless · message-driven · horizontally scalable</text>
+<text x="28.0" y="72.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">AWS SDK → Gateway → NATS → service daemons</text>
+<rect x="28.0" y="92.0" width="964.0" height="92.0" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48.0" y="119.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">CLIENTS</text>
+<text x="48.0" y="140.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#5B636C" font-weight="400" text-anchor="start">Standard AWS tooling, unmodified. No shims, no custom SDK.</text>
+<rect x="46.0" y="153.0" width="68.2" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="80.1" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli</text>
+<rect x="123.2" y="153.0" width="81.4" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="163.9" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Terraform</text>
+<rect x="213.6" y="153.0" width="55.0" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="241.1" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">boto3</text>
+<rect x="277.6" y="153.0" width="88.0" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="321.6" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-sdk-go</text>
+<rect x="374.6" y="153.0" width="68.2" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="408.7" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">kubectl</text>
+<rect x="451.8" y="153.0" width="61.6" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="482.6" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Packer</text>
+<rect x="522.4" y="153.0" width="74.8" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="559.8" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">your app</text>
+<line x1="510.0" y1="186.0" x2="510.0" y2="192.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="429.9" y="191.0" width="160.3" height="22.0" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.4"/>
+<text x="510.0" y="205.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="400" text-anchor="middle" fill-opacity="0.9">HTTPS · SigV4 · :9999</text>
+<line x1="510.0" y1="212.0" x2="510.0" y2="220.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28.0" y="224.0" width="964.0" height="94.0" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="48.0" y="251.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="700" text-anchor="start" letter-spacing="1.5">AWS GATEWAY</text>
+<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
+<rect x="46.0" y="287.0" width="68.2" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="80.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">TLS 1.3</text>
+<rect x="123.2" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="167.2" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">SigV4 auth</text>
+<rect x="220.2" y="287.0" width="94.6" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="267.5" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">policy eval</text>
+<rect x="323.8" y="287.0" width="94.6" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="371.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">req routing</text>
+<rect x="427.4" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.4" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">rate-limit</text>
+<rect x="524.4" y="287.0" width="81.4" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="565.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">audit log</text>
+<line x1="510.0" y1="320.0" x2="510.0" y2="332.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28.0" y="334.0" width="964.0" height="52.0" rx="12" fill="url(#nats)" fill-opacity="1.0" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<rect x="28.0" y="334.0" width="964.0" height="52.0" rx="12" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55"/>
+<text x="50.0" y="365.6" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">NATS BUS</text>
+<text x="178.0" y="364.6" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#F5F3EE" font-weight="400" text-anchor="start" fill-opacity="0.92">ec2.*  ebs.*  s3.*  vpc.*  iam.*</text>
+<text x="972.0" y="363.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F5F3EE" font-weight="400" text-anchor="end" fill-opacity="0.9">queue groups · fan-out · request/reply</text>
+<line x1="119.6" y1="388.0" x2="119.6" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="314.8" y1="388.0" x2="314.8" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="510.0" y1="388.0" x2="510.0" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="705.2" y1="388.0" x2="705.2" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="900.4" y1="388.0" x2="900.4" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="28.0" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="44.0" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">ec2.* daemon</text>
+<line x1="44.0" y1="448.0" x2="195.2" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="44.0" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">RunInstances</text>
+<text x="44.0" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">TerminateInstances</text>
+<text x="44.0" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">QMP · cloud-init</text>
+<rect x="223.2" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="239.2" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">ebs.* daemon</text>
+<line x1="239.2" y1="448.0" x2="390.4" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="239.2" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">CreateVolume</text>
+<text x="239.2" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">AttachVolume</text>
+<text x="239.2" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">snapshots · WAL</text>
+<rect x="418.4" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="434.4" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">s3.* daemon</text>
+<line x1="434.4" y1="448.0" x2="585.6" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="434.4" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">PutObject · GetObject</text>
+<text x="434.4" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Multipart · presigned</text>
+<text x="434.4" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">erasure-coded</text>
+<rect x="613.6" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="629.6" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">vpc.* daemon</text>
+<line x1="629.6" y1="448.0" x2="780.8" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="629.6" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Subnets · SGs · RTs</text>
+<text x="629.6" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">ALB · NAT · IGW</text>
+<text x="629.6" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">OVN · OVS</text>
+<rect x="808.8" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="824.8" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">iam.* daemon</text>
+<line x1="824.8" y1="448.0" x2="976.0" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="824.8" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Users · Roles · Keys</text>
+<text x="824.8" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Policy evaluation</text>
+<text x="824.8" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">audit trail</text>
+<line x1="119.6" y1="516.0" x2="119.6" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="314.8" y1="516.0" x2="314.8" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="510.0" y1="516.0" x2="510.0" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="705.2" y1="516.0" x2="705.2" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="900.4" y1="516.0" x2="900.4" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<rect x="28.0" y="530.0" width="964.0" height="30.0" rx="9" fill="#1E242B" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="510.0" y="548.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle" letter-spacing="1">BACKEND RESOURCES  ·  local hardware  ·  no cloud dependency</text>
+<rect x="28.0" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="119.6" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">QEMU / KVM</text>
+<text x="119.6" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">hypervisor</text>
+<text x="119.6" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">host kernel · virtio</text>
+<rect x="223.2" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="314.8" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Viperblock</text>
+<text x="314.8" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">block storage</text>
+<text x="314.8" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">NVMe WAL · NBD</text>
+<rect x="418.4" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="510.0" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Predastore</text>
+<text x="510.0" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">object storage</text>
+<text x="510.0" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Reed-Solomon</text>
+<rect x="613.6" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="705.2" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">OVN / OVS</text>
+<text x="705.2" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">SDN fabric</text>
+<text x="705.2" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">overlays · flow rules</text>
+<rect x="808.8" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="900.4" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Identity store</text>
+<text x="900.4" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">users · keys · rules</text>
+<text x="900.4" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">local · audit-logged</text>
+<rect x="28.0" y="670.0" width="964.0" height="72.0" rx="11" fill="#181D23" fill-opacity="0.6" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48.0" y="697.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400"><tspan fill="#EE4B12" font-weight="700">Queue groups</tspan><tspan fill="#5B636C"> load-balance ops across daemon replicas. </tspan><tspan fill="#EE4B12" font-weight="700">Fan-out</tspan><tspan fill="#5B636C"> (no queue group) dispatches describe calls to every node.</tspan></text>
+<text x="48.0" y="720.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400"><tspan fill="#5B636C">Daemons are </tspan><tspan fill="#EE4B12" font-weight="700">stateless</tspan><tspan fill="#5B636C">: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.</tspan></text>
+</svg>

.github/assets/diagrams/ecs-topology.svg

@@ -0,0 +1,102 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">ECS · EC2 LAUNCH TYPE ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">container instances from spinifex-ecs-node AMI · awsvpc + ALB</text>
+
+  <!-- Internet -->
+  <rect x="120" y="52" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="220" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+
+  <!-- connector to ALB -->
+  <line x1="220" y1="92" x2="220" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="216,108 224,108 220,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- ALB -->
+  <g transform="translate(90, 116)">
+    <rect width="260" height="60" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">Application Load Balancer</text>
+    <line x1="16" y1="32" x2="244" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787">target group · target_type = ip</text>
+  </g>
+
+  <!-- Spinifex Gateway (control plane) -->
+  <g transform="translate(560, 110)">
+    <rect width="316" height="72" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">Spinifex Gateway :9999</text>
+    <text x="296" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">control plane</text>
+    <line x1="16" y1="32" x2="300" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">ECS agent registers · TLS + SigV4 (IMDS creds)</text>
+    <text x="20" y="63" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">task role creds → container at 169.254.170.2</text>
+  </g>
+
+  <!-- ALB fan-out to subnets (traffic) -->
+  <line x1="220" y1="176" x2="220" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="650" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="220" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="650" y1="200" x2="650" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="435" y="195" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">balances to each task's ENI</text>
+
+  <!-- agent path: subnets up to gateway -->
+  <line x1="718" y1="228" x2="718" y2="182" stroke="#F87171" stroke-width="1.1" stroke-dasharray="3,3" opacity="0.6"/>
+  <polygon points="714,186 722,186 718,178" fill="#F87171" opacity="0.8"/>
+
+  <!-- subnet_a -->
+  <g transform="translate(60, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- container instance -->
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="225" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(520, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="103" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">image pull</text>
+    <text x="225" y="117" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">registry / ECR</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="410" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">No ECS launch API — you add capacity by booting EC2 instances from the spinifex-ecs-node AMI (the console's "Provision capacity" wraps RunInstances)</text>
+  <text x="450" y="426" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">The service keeps the desired task count running and registers each task's ENI IP with the ALB target group</text>
+</svg>

.github/assets/diagrams/eks-topology.svg

@@ -0,0 +1,96 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="cp-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">EKS · MANAGED KUBERNETES ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">control plane VM + NLB · workers from eks-node AMI</text>
+
+  <!-- Internet / kubectl -->
+  <rect x="330" y="52" width="240" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="450" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet · kubectl + aws eks</text>
+
+  <!-- connector to control plane -->
+  <line x1="450" y1="92" x2="450" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="446,108 454,108 450,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- Control plane (Spinifex-managed) -->
+  <g transform="translate(290, 116)">
+    <rect width="320" height="76" rx="8" fill="url(#cp-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">EKS Control Plane</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">Spinifex-managed</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="87" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">API NLB (public)</text>
+    <rect x="165" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="232" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">apiserver VM</text>
+  </g>
+
+  <!-- authn note -->
+  <rect x="624" y="138" width="172" height="32" rx="6" fill="#0a0a0a" stroke="#71717a" stroke-width="0.6" stroke-opacity="0.5"/>
+  <text x="710" y="151" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">authn: API · access entries</text>
+  <text x="710" y="163" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">aws eks get-token (IAM → RBAC)</text>
+
+  <!-- fan-out to node-group subnets -->
+  <line x1="450" y1="192" x2="450" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="670" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="230" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="670" y1="210" x2="670" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="450" y="206" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">managed node group · workers join the control plane</text>
+
+  <!-- subnet_a -->
+  <g transform="translate(108, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- worker node -->
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(472, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">ECR</text>
+    <text x="248" y="107" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">pull</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="404" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">You provide the VPC, subnets, and IAM roles · Spinifex provisions the control plane, node-group VMs, and security groups</text>
+  <text x="450" y="420" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Node groups always boot the registered eks-node image · workers need egress (IGW / NAT) to pull images</text>
+</svg>

.github/assets/platform.svg

@@ -1,144 +1,84 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="900" height="560" viewBox="0 0 900 560">
-  <defs>
-    <linearGradient id="pf-bg" x1="0%" y1="0%" x2="100%" y2="100%">
-      <stop offset="0%" style="stop-color:#0a0a0a"/>
-      <stop offset="100%" style="stop-color:#0d1117"/>
-    </linearGradient>
-    <linearGradient id="hw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#52525b;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#52525b;stop-opacity:0.06"/>
-    </linearGradient>
-    <linearGradient id="os-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#a855f7;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#a855f7;stop-opacity:0.05"/>
-    </linearGradient>
-    <linearGradient id="spx-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0.28"/>
-      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0.1"/>
-    </linearGradient>
-    <linearGradient id="app-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#3fb950;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#3fb950;stop-opacity:0.05"/>
-    </linearGradient>
-  </defs>
-
-  <rect width="900" height="560" fill="url(#pf-bg)" rx="12"/>
-
-  <!-- Subtle mesh dots background -->
-  <g opacity="0.12" fill="#7F1D1D">
-    <circle cx="40" cy="40" r="1"/><circle cx="120" cy="30" r="1"/><circle cx="220" cy="50" r="1"/>
-    <circle cx="320" cy="25" r="1"/><circle cx="440" cy="45" r="1"/><circle cx="560" cy="30" r="1"/>
-    <circle cx="680" cy="50" r="1"/><circle cx="780" cy="30" r="1"/><circle cx="860" cy="45" r="1"/>
-    <circle cx="60" cy="530" r="1"/><circle cx="160" cy="520" r="1"/><circle cx="260" cy="540" r="1"/>
-    <circle cx="380" cy="525" r="1"/><circle cx="500" cy="540" r="1"/><circle cx="620" cy="520" r="1"/>
-    <circle cx="740" cy="540" r="1"/><circle cx="840" cy="525" r="1"/>
-  </g>
-
-  <!-- Title -->
-  <text x="450" y="32" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" text-anchor="middle" letter-spacing="2.8">THE MULGA PLATFORM</text>
-  <text x="450" y="58" font-family="'Segoe UI', system-ui, sans-serif" font-size="20" font-weight="600" fill="#f5f5f5" text-anchor="middle">AWS-style workflows, on your own stack</text>
-
-  <!-- Layer 4: Applications (top) -->
-  <g transform="translate(60, 95)">
-    <rect width="780" height="88" rx="10" fill="url(#app-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
-    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" letter-spacing="1.5" font-weight="600">LAYER 4 · APPLICATIONS &amp; TOOLING</text>
-    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">Your workloads — unmodified AWS tooling</text>
-
-    <g transform="translate(20, 60)" font-family="'Segoe UI', sans-serif" font-size="10.5">
-      <rect x="0" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="35" y="14" fill="#7ee787" text-anchor="middle">aws-cli</text>
-
-      <rect x="78" y="0" width="78" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="117" y="14" fill="#7ee787" text-anchor="middle">Terraform</text>
-
-      <rect x="164" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="208" y="14" fill="#7ee787" text-anchor="middle">AWS SDKs</text>
-
-      <rect x="260" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="304" y="14" fill="#7ee787" text-anchor="middle">Kubernetes</text>
-
-      <rect x="356" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="391" y="14" fill="#7ee787" text-anchor="middle">Packer</text>
-
-      <rect x="434" y="0" width="98" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="483" y="14" fill="#7ee787" text-anchor="middle">Your Services</text>
-    </g>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 188)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 3: Spinifex (highlighted) -->
-  <g transform="translate(60, 215)">
-    <rect width="780" height="140" rx="10" fill="url(#spx-grad)" stroke="#DC2626" stroke-width="1.5" stroke-opacity="0.75"/>
-    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" letter-spacing="1.5" font-weight="600">LAYER 3 · SPINIFEX · AWS ORCHESTRATION</text>
-    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">AWS Gateway → NATS → Service Daemons · SigV4, IAM, VPC-aware</text>
-
-    <g transform="translate(20, 66)" font-family="'Segoe UI', sans-serif">
-      <!-- EC2 -->
-      <rect x="0" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="59" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EC2</text>
-      <text x="59" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">QEMU/KVM VMs</text>
-      <text x="59" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">cloud-init · AMIs</text>
-
-      <!-- EBS -->
-      <rect x="126" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="185" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EBS</text>
-      <text x="185" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Viperblock</text>
-      <text x="185" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">WAL · NBD · snapshots</text>
-
-      <!-- S3 -->
-      <rect x="252" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="311" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">S3</text>
-      <text x="311" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Predastore</text>
-      <text x="311" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Reed–Solomon EC</text>
-
-      <!-- VPC -->
-      <rect x="378" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
-      <text x="437" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">VPC</text>
-      <text x="437" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">OVN / OVS</text>
-      <text x="437" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">subnets · SG · ALB</text>
-
-      <!-- IAM -->
-      <rect x="504" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
-      <text x="563" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">IAM</text>
-      <text x="563" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">SigV4 · policies</text>
-      <text x="563" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">users · roles · keys</text>
-
-      <!-- Control -->
-      <rect x="630" y="0" width="110" height="58" rx="6" fill="#27272a" fill-opacity="0.8" stroke="#71717a" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="685" y="18" font-size="13" fill="#e4e4e7" text-anchor="middle" font-weight="600">Control</text>
-      <text x="685" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">NATS bus</text>
-      <text x="685" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">spx · daemons</text>
-    </g>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 360)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 2: OS -->
-  <g transform="translate(60, 387)">
-    <rect width="780" height="62" rx="10" fill="url(#os-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.5"/>
-    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" letter-spacing="1.5" font-weight="600">LAYER 2 · OPERATING SYSTEM</text>
-    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · headless autoinstall</text>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 454)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 1: Hardware -->
-  <g transform="translate(60, 481)">
-    <rect width="780" height="62" rx="10" fill="url(#hw-grad)" stroke="#71717a" stroke-width="1.2" stroke-opacity="0.6"/>
-    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" letter-spacing="1.5" font-weight="600">LAYER 1 · HARDWARE</text>
-    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Single node · multi-node cluster · edge appliance · air-gapped rack · commodity x86 / ARM</text>
-  </g>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="664" viewBox="0 0 1000 664" role="img" aria-label="The Spinifex stack: applications and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU/GPU hardware, deployable to Neocloud, on-prem or edge.">
+<rect x="0.0" y="0.0" width="1000.0" height="664.0" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<rect x="28.0" y="204.0" width="950.0" height="438.0" rx="16" fill="none" fill-opacity="1.0" stroke="#9AA1A8" stroke-width="1" stroke-opacity="0.12" stroke-dasharray="2 4"/>
+<rect x="25.0" y="201.0" width="950.0" height="438.0" rx="16" fill="none" fill-opacity="1.0" stroke="#9AA1A8" stroke-width="1" stroke-opacity="0.2" stroke-dasharray="2 4"/>
+<rect x="22.0" y="198.0" width="950.0" height="438.0" rx="16" fill="#181D23" fill-opacity="0.25" stroke="#9AA1A8" stroke-width="1.3" stroke-opacity="0.45" stroke-dasharray="2 4"/>
+<text x="36.0" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Apps</text>
+<rect x="136.0" y="34.0" width="836.0" height="54.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="66.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">Your infrastructure &amp; apps</text>
+<rect x="516.7" y="51.0" width="89.3" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.16" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<text x="561.4" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="middle">AI training</text>
+<rect x="613.0" y="51.0" width="76.7" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.16" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<text x="651.4" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="middle">Inference</text>
+<rect x="696.7" y="51.0" width="79.0" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="736.2" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Web &amp; APIs</text>
+<rect x="782.7" y="51.0" width="72.7" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="819.1" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Databases</text>
+<rect x="862.4" y="51.0" width="91.6" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="908.2" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Batch &amp; data</text>
+<text x="36.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Tooling</text>
+<rect x="136.0" y="102.0" width="836.0" height="54.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="134.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">AWS tooling</text>
+<rect x="600.9" y="119.0" width="64.1" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="633.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">aws-cli</text>
+<rect x="672.0" y="119.0" width="76.7" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="710.4" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">Terraform</text>
+<rect x="755.7" y="119.0" width="64.1" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="787.8" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">kubectl</text>
+<rect x="826.8" y="119.0" width="66.4" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="860.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">AWS SDKs</text>
+<rect x="900.2" y="119.0" width="53.8" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="927.1" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Packer</text>
+<text x="484.0" y="184.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">↓  AWS API · SigV4</text>
+<text x="624.0" y="184.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">RESPONSE  ↑</text>
+<text x="136.0" y="231.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">One portable stack.</text>
+<text x="136.0" y="249.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="start">Runs the same on a single node, cluster, edge, or air-gapped rack.</text>
+<rect x="752.3" y="222.0" width="72.4" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="788.5" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">NEOCLOUD</text>
+<rect x="832.7" y="222.0" width="66.1" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="865.7" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">ON-PREM</text>
+<rect x="906.8" y="222.0" width="47.2" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="930.4" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">EDGE</text>
+<text x="152.0" y="295.6" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">AWS orchestration</text>
+<text x="956.0" y="293.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.9">Gateway → NATS → daemons</text>
+<rect x="150.0" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="164.0" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EC2</text>
+<text x="164.0" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Compute</text>
+<text x="164.0" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">QEMU / KVM</text>
+<rect x="422.7" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="436.7" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EBS</text>
+<text x="436.7" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Block storage</text>
+<text x="436.7" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">Viperblock</text>
+<rect x="695.3" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="709.3" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">S3</text>
+<text x="709.3" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Object storage</text>
+<text x="709.3" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">Predastore</text>
+<rect x="150.0" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="164.0" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">VPC</text>
+<text x="164.0" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Networking</text>
+<text x="164.0" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">OVN / OVS</text>
+<rect x="422.7" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="436.7" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">IAM</text>
+<text x="436.7" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Identity</text>
+<text x="436.7" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">SigV4 · policies</text>
+<rect x="695.3" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="709.3" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EKS</text>
+<text x="709.3" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Kubernetes</text>
+<text x="709.3" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">K8s-conformant</text>
+<rect x="150.0" y="452.0" width="808.0" height="32.0" rx="8" fill="#14181D" fill-opacity="0.6" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0" stroke-dasharray="3 3"/>
+<text x="164.0" y="472.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">+ ALB · ECS · ECR · RDS and more of the core AWS services</text>
+<text x="944.0" y="471.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end">See all →</text>
+<text x="36.0" y="386.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="start">Spinifex</text>
+<rect x="136.0" y="268.0" width="836.0" height="230.0" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="36.0" y="536.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">OS</text>
+<rect x="136.0" y="510.0" width="836.0" height="46.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="537.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">Operating system</text>
+<text x="952.0" y="536.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="end">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · Open vSwitch</text>
+<text x="36.0" y="596.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Hardware</text>
+<rect x="136.0" y="568.0" width="413.0" height="50.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="597.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">CPU</text>
+<text x="533.0" y="596.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="end">Intel · AMD · ARM</text>
+<rect x="559.0" y="568.0" width="413.0" height="50.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="579.0" y="597.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">GPU</text>
+<text x="956.0" y="596.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end">NVIDIA · AMD</text>
+</svg>

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -19,7 +19,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -38,7 +38,7 @@ updates:
     directory: /spinifex/services/spinifexui/frontend
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3

.github/pull_request_template.md

@@ -0,0 +1,7 @@
+## Summary
+
+<!-- What does this PR do, and why? -->
+
+## Changes
+
+<!-- The notable changes, as bullets. -->

.github/resolve-sibling-refs.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Resolve which branch of each sibling repo (viperblock, predastore) the CI
+# workspace should build against. A cross-repo change is developed on a branch
+# of the SAME name in each repo; build against that branch when it exists so the
+# set can be tested together, and fall back to dev otherwise. This mirrors the
+# resolution the e2e path already does in
+# mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
+# agree on how sibling branches are selected.
+#
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
+# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
+set -euo pipefail
+
+: "${BRANCH:?BRANCH must be set to the triggering branch name}"
+: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
+
+for repo in viperblock predastore; do
+  if git ls-remote --exit-code --heads \
+      "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
+    ref="$BRANCH"
+  else
+    ref="dev"
+  fi
+  echo "${repo}: building against ${ref}"
+  echo "${repo}=${ref}" >>"$GITHUB_OUTPUT"
+done

.github/scripts/suite-sets.sh

@@ -0,0 +1,62 @@
+# shellcheck shell=bash
+# shellcheck disable=SC2034  # every variable here is consumed by the sourcing workflow
+# Source from a workflow `run:` block:
+#   . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+#
+# Which e2e suites each topology can run. Sourced by every workflow that drives
+# suites (e2e.yml, e2e-suites.yml, e2e-nightly.yml) so the lists cannot drift
+# apart — they previously disagreed three ways.
+#
+# Eligibility is a property of the suite, not of the workflow: a suite belongs
+# to a topology when all of its assertions are reachable there. lb is
+# multi-only for exactly that reason — on a single node every internet-facing
+# subtest skips for want of a peer, so a single-node lb run reports green
+# having exercised a fraction of the suite.
+
+# Suites runnable against a one-node environment.
+#
+# partialblock gates merges because it earned it, not because it is new: on one
+# node, same binary and workload, it fails against the pre-fix engine (1758 of
+# 8192 halves reverted a generation) and passes against the fixed one (0 of
+# 8192). A suite that has never been shown to go red for the reason it claims
+# does not belong here.
+# rds is single-only for the opposite reason to lb: v1 has no multi-AZ
+# behaviour, so a second node adds no assertion. Its DB VMs are capped at four
+# concurrent by the suite's own semaphore, which is what keeps it inside the
+# job's 50-minute budget alongside the suites above.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+
+# Suites runnable against a multi-node environment.
+E2E_SUITES_MULTI="multinode lb cert"
+
+# grep -xE alternation form, for narrowing a requested set to the eligible one.
+E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
+E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
+
+# e2e-nightly runs a narrower set than the two sets above, deliberately: its
+# ~19 cells each have a 35-minute budget and exist to prove that every
+# install / network / host-OS permutation boots and serves. Suite breadth is
+# the PR gate's job, and it already runs on every push. These are subsets, and
+# e2e_suites_assert_subset below enforces that.
+E2E_SUITES_NIGHTLY_SINGLE="single cert"
+E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
+
+# Fail loudly when a narrowed list names a suite its topology cannot run. This
+# is the guard that stops the lists drifting again: a suite added to a nightly
+# subset without also being declared eligible above stops the job here rather
+# than silently skipping half its assertions in CI.
+#   e2e_suites_assert_subset <label> <subset> <eligible-set>
+e2e_suites_assert_subset() {
+  local label="$1" subset="$2" eligible="$3" suite found eligible_suite
+  for suite in ${subset}; do
+    found=0
+    for eligible_suite in ${eligible}; do
+      if [ "${suite}" = "${eligible_suite}" ]; then found=1; break; fi
+    done
+    if [ "${found}" -ne 1 ]; then
+      echo "::error::${label} names suite '${suite}', which is not eligible for that topology (eligible: ${eligible}). Fix the list in .github/scripts/suite-sets.sh." >&2
+      return 1
+    fi
+  done
+  return 0
+}

.github/workflows/apt-packages.yml

@@ -0,0 +1,48 @@
+name: Apt Packages
+
+# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
+# list against each supported suite's real indices, so a renamed, dropped, or
+# ambiguously virtual package fails here rather than on a user's machine.
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  apt_packages:
+    name: Resolve apt list (${{ matrix.image }})
+    runs-on: ubuntu-26.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
+        # accepted by detect_os but has never been exercised and does not resolve.
+        image:
+          - ubuntu:24.04
+          - ubuntu:26.04
+          - debian:13
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Resolve runtime package list
+        run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}

.github/workflows/e2e-baremetal.yml

@@ -2,6 +2,18 @@ name: E2E Bare-Metal
 
 on:
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: "spinifex ref to checkout/test (default: triggering ref)"
+        required: false
+        type: string
+        default: ""
+      otel-attrs:
+        description: "OTEL_RESOURCE_ATTRIBUTES value for this run, e.g. ci.run_id=123,ci.cell=20 (caller-specific, so it's an input rather than a workflow-level default)"
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -9,10 +21,14 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
+  # no default here before — this cell's telemetry never reached the sink at
+  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
-    name: Bare-Metal Multi-Node E2E
+    name: Bare-Metal Single-Node E2E
     runs-on: [ self-hosted, ci-baremetal ]
     timeout-minutes: 120
     concurrency:
@@ -20,11 +36,20 @@ jobs:
       cancel-in-progress: false
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
+      # Caller-supplied (workflow_call input); empty for a manual
+      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
     steps:
       - name: Clean Workspace
-        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
+          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - uses: actions/create-github-app-token@v3
         id: app-token
@@ -33,7 +58,7 @@ jobs:
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -91,6 +116,8 @@ jobs:
             echo "Ubuntu 26.04 cloud image cached ($(du -h "$UBUNTU_IMG" | cut -f1))"
           fi
 
+      - name: Install go-junit-report on runner
+        run: go install github.com/jstemmer/go-junit-report/v2@latest
 
       - name: Run Bare-Metal E2E
         env:
@@ -100,8 +127,34 @@ jobs:
           BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
           SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
         run: |
+          mkdir -p "${ARTIFACT_DIR}"
+          set -o pipefail
           sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
             --squashfs  "${BM_ISO_CACHE}/spinifex.squashfs" \
-            --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+            --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
+            2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
+
+      - name: Generate JUnit report
+        if: always()
+        run: |
+          LOG="${ARTIFACT_DIR}/test-baremetal.log"
+          [ -f "$LOG" ] || exit 0
+          go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: ./.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E failure analysis (cell 20)"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-20-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/e2e-ddil.yml

@@ -1,8 +1,7 @@
 name: E2E DDIL
 
-# Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge
-# land in mulga-siv-5.8 once ≥4 scenarios are asserting. See
-# docs/development/improvements/ddil-e2e-test-harness.md §4 / §Dependencies.
+# This workflow is manually triggered while the DDIL scenarios are being
+# validated.
 on:
   workflow_dispatch:
     inputs:
@@ -13,21 +12,28 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the otel-job-id action look up this job's numeric id via
+  # the Actions API — github.job is only the YAML key, no context exposes it.
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   ddil:
     name: DDIL Real Multi-Node
     runs-on: [ self-hosted, ci-multi ]
-    # Budget: ~45m go test + tofu provision/bootstrap/teardown on ci-multi.
-    # In Phase 1 the scenarios all t.Skip, so the real-world runtime is
-    # dominated by provisioning; the 60m ceiling leaves headroom for
-    # Phase 2/3 when assertions come online.
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
+      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
+      # this workflow's telemetry so a DDIL run's partition-induced errors are
+      # never mistaken for a healthy multi-node run's in the sink.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace
         run: sudo rm -rf "$GITHUB_WORKSPACE"/*
@@ -39,16 +45,12 @@ jobs:
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      # Full spinifex checkout: the Go harness must run on the runner
-      # (outside the cluster) so its SSH path survives scenario partitions
-      # that drop peer-to-peer cluster traffic. Running inside the cluster
-      # would self-sever when Scenario C isolates a node.
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         name: Checkout Spinifex
         with:
           path: spinifex
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         name: Checkout Mulga (tofu-cluster)
         with:
           repository: mulgadc/mulga
@@ -76,6 +78,13 @@ jobs:
           echo "${BIN_DIR}" >> "${GITHUB_PATH}"
           "${BIN_DIR}/gotestsum" --version
 
+      # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
+      # the VMs, so a telemetry doc points back at this job's log directly.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: DDIL Real Multi-Node
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -88,7 +97,6 @@ jobs:
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
@@ -140,10 +148,6 @@ jobs:
             exit 1
           fi
 
-          # Guest SSH key: bootstrap-install.sh imports spinifex-key into EC2;
-          # the matching private material was generated on the primary node at
-          # ~/.ssh/spinifex-key. Pull it to the runner so the harness can SSH
-          # the witness via its MapPublicIpOnLaunch-assigned EIP.
           GUEST_KEY="${RUNNER_TEMP}/spinifex-key"
           scp -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
             "${SSH_USER}@${PRIMARY_IP}:~/.ssh/spinifex-key" "${GUEST_KEY}"
@@ -151,11 +155,6 @@ jobs:
 
           mkdir -p "${RUNNER_TEMP}/ddil-results"
           cd "${GITHUB_WORKSPACE}/spinifex"
-          # gotestsum: human-readable PASS/FAIL/SKIP lines in Actions stdout,
-          # plus machine-readable jsonfile + JUnit XML artifacts. The JUnit
-          # file feeds the junit-report composite action below so failures
-          # become summary table rows + workflow annotations.
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           banner "DDIL scenarios"
           DDIL_NODES="${NODE_IPS}" \
@@ -185,10 +184,6 @@ jobs:
         if: failure()
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          # Pull spinifex-* journals + QEMU process listings from every
-          # node into the artifact bundle so failures can be triaged
-          # without re-provisioning the cluster. Best-effort — a missing
-          # node should not mask the original test failure.
           set +e
           INVENTORY=$(tofu output -json inventory)
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -197,7 +192,6 @@ jobs:
           LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs"
           mkdir -p "${LOG_DIR}"
 
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip')
           for ip in "${NODE_IPS[@]}"; do
@@ -221,9 +215,18 @@ jobs:
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
           quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-ddil-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 30

.github/workflows/e2e-nightly.yml

@@ -11,12 +11,20 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the "Resolve CI job id" step below look up this job's
+  # numeric id via the Actions API — github.job is only the YAML key, no
+  # context exposes the id itself.
+  actions: read
 
 # Scheduled runs are loaded from the default branch (main), so the schedule
 # trigger must stay in sync on main for cron to fire.
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
-  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   matrix:
@@ -28,18 +36,25 @@ jobs:
       matrix:
         include:
           - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
+          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
           - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
+          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
           - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
           - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
+          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
 
     runs-on: [ self-hosted, "${{ matrix.runner }}" ]
-    timeout-minutes: 35
+    timeout-minutes: ${{ matrix.extra == 'tofu-examples' && 45 || 35 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
+      # Job-level env wins over the workflow-level default above, so telemetry
+      # from this cell carries ci.cell/ci.matrix. Nine cells run under one
+      # run id and every cell names its hosts node1/node2/node3, so without
+      # this every cell's telemetry collapses into one unattributable bucket.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
       # blank or scheduled run = full matrix.
@@ -60,7 +75,10 @@ jobs:
 
       - name: Clean Workspace
         if: steps.gate.outputs.selected == 'true'
-        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
         if: steps.gate.outputs.selected == 'true'
@@ -70,7 +88,7 @@ jobs:
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         if: steps.gate.outputs.selected == 'true'
         with:
           repository: mulgadc/mulga
@@ -78,6 +96,21 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
+      - uses: actions/checkout@v7
+        if: steps.gate.outputs.selected == 'true'
+        with:
+          path: spinifex
+
+      # Nine cells share one run id and every cell names its hosts
+      # node1/node2/node3, so ci.cell above separates them and ci.job_id here
+      # links each cell's telemetry to the job log that produced it. Runs
+      # before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
+      - name: Resolve CI job id
+        if: steps.gate.outputs.selected == 'true'
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
+
       - name: Build Install Artifacts
         if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
@@ -95,10 +128,13 @@ jobs:
         run: |
           export TF_VAR_git_branch="${GIT_BRANCH}"
 
+          # Match the family, not a pinned version: an exact match silently stops
+          # overriding the image when the matrix bumps release, dropping the cell
+          # back onto the debian gold.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
 
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
@@ -112,7 +148,12 @@ jobs:
         env:
           MATRIX_NET: ${{ matrix.net }}
         run: |
-          if [ "$MATRIX_NET" = "dhcp" ]; then
+          if [ "$MATRIX_NET" = "nat" ]; then
+            # Routed-NAT mode with a static public pool: exercises both the
+            # Tier-1 transit datapath and the Tier-2 EIP ingress lane.
+            export SPX_EXTERNAL_MODE=nat
+            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
+          elif [ "$MATRIX_NET" = "dhcp" ]; then
             unset SPX_EXTERNAL_POOL
             export SPX_EXTERNAL_SOURCE=dhcp
           else
@@ -128,48 +169,129 @@ jobs:
         if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set -euo pipefail
+          # Nightly runs a documented subset of the single-node set; the assert
+          # fails the cell if that subset ever names a suite this topology
+          # cannot fully exercise.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
+
           INVENTORY=$(tofu output -json inventory)
           WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
-          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
+             for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
+               echo \"::group::suite \$SUITE\"; \
+               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
+                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
+               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
+             done; exit \$FAIL"
+
+      - name: Run NAT Uplink E2E
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
+        working-directory: mulga/scripts/tofu-cluster
+        # Skips cleanly (exit 0) when the tested branch predates the natuplink
+        # suite so scheduled runs against dev stay green until the merge lands.
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
+             echo '::group::suite natuplink'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./natuplink.test -test.v -test.timeout=30m \
+               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
 
       - name: Run Multi Node E2E
         if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set -euo pipefail
+          # See the single-node step: same subset guard, multi-node set.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly multi" "${E2E_SUITES_NIGHTLY_MULTI}" "${E2E_SUITES_MULTI}"
+
           INVENTORY=$(tofu output -json inventory)
           NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
 
-          NODE_IPS=""
+          NODES_CSV=""
           NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
           for i in $(seq 0 $((NODE_COUNT - 1))); do
-            NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
           done
 
-          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
+             for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
+               echo \"::group::suite \$SUITE\"; \
+               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+                 SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+                 SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
+                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
+               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
+             done; exit \$FAIL"
 
-      - name: Run Tofu Workbooks
+      - name: Import RDS PostgreSQL AMI
         if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set -euo pipefail
           INVENTORY=$(tofu output -json inventory)
           WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh"
+            "spx admin images import --name spinifex-rds-postgres"
+
+      - name: Run Tofu Workbooks
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        working-directory: mulga/scripts/tofu-cluster
+        # The driver emits go test -v markers per workbook; teeing to
+        # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
+        # chain the Go suites use, so cell 17 produces an RCA bundle too.
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
+            2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
+          rc=${PIPESTATUS[0]}
+          if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
+          exit $rc
 
       - name: Run Reboot Resilience E2E
         if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
         working-directory: mulga/scripts/tofu-cluster
-        # Driver runs from the runner (NOT via ssh-and-run) so it survives the
+        # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
         # systemctl-reboot-induced ssh disconnect mid-run.
         run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
           INVENTORY=$(tofu output -json inventory)
           WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -178,41 +300,129 @@ jobs:
           REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
           mkdir -p "$REBOOT_DIR"
           tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
-          chmod +x "$REBOOT_DIR/run-reboot-e2e.sh"
+          chmod +x "$REBOOT_DIR/_bin/reboot.test"
 
-          WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
+          CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
+          AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"
 
-      - name: Dump Cluster Journals on Failure
-        if: failure() && steps.gate.outputs.selected == 'true'
+          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
+          GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+            SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
+            SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
+            SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
+            ARTIFACT_DIR="${ARTIFACT_DIR}" \
+            "$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
+            2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
+          rc=${PIPESTATUS[0]}
+          if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
+          exit $rc
+
+      - name: Collect e2e artifacts
+        if: always() && steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
           INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
-          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
-          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
           NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
+
+          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+
           for i in $(seq 0 $((NODE_COUNT - 1))); do
-            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
-            echo "===== node$((i+1)) ($IP) ====="
-            for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd dhcpcd; do
-              echo "--- $svc ---"
-              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
-                "sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)"
-            done
-            echo "--- DHCP client state ---"
-            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
-              "systemctl is-enabled dhcpcd.service 2>/dev/null; systemctl is-active dhcpcd.service 2>/dev/null; sudo ss -lupn 'sport = 68' 2>/dev/null; pgrep -laf dhcpcd 2>/dev/null" 2>&1 || echo "(node unreachable)"
+            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
+            mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+              | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
           done
 
+          cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
+            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Install go-junit-report
+        if: always() && steps.gate.outputs.selected == 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Convert test logs to JUnit XML
+        if: always() && steps.gate.outputs.selected == 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.selected == 'true'
+        uses: ./spinifex/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E failure analysis (cell ${{ matrix.cell }})"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.selected == 'true'
+        with:
+          name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
       - name: Teardown
         if: always()
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
         run: |
+          # Must mirror the Provision step's override, or destroy runs against a
+          # different image path than apply did.
           TOFU_VAR_ARGS=""
-          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
-            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
-          fi
+          case "$MATRIX_HOST" in
+            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
+          esac
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
+          timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
+  # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
+  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
+  # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
+  baremetal:
+    name: cell-20 (baremetal/pxe/real-hw)
+    if: >-
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',20,')
+    permissions:
+      contents: read
+      statuses: write
+    uses: ./.github/workflows/e2e-baremetal.yml
+    with:
+      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      # A reusable-workflow call gets none of this workflow's context (env:
+      # above, matrix.cell) for free — cell-20 needs its own attrs string.
+      # There's no matrix leg here, just the one fixed baremetal cell.
+      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
+    secrets: inherit

.github/workflows/e2e-suites.yml

@@ -0,0 +1,908 @@
+# E2E per-suite blocks. Each suite appears as its own node in the Actions
+# graph:
+#
+#   single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
+#                     └─ single_teardown (if: always)
+#   multi_provision  ─┬─ multi_suite  (matrix: the multi-node-eligible suites)
+#                     └─ multi_teardown (if: always)
+#
+# Both matrices come from .github/scripts/suite-sets.sh, the one definition of
+# which suites a topology can run, shared with e2e.yml and e2e-nightly.yml.
+#
+# Constraint: tofu state is runner-filesystem-local, so *_provision and
+# *_teardown must execute on the same self-hosted runner (one runner per label
+# today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does
+# NOT re-checkout mulga (that would wipe local state). Suite-matrix legs reach
+# the already-provisioned VM over its WAN IP using the key path the provision
+# job emits; they share the runner home dir, so the key file persists.
+#
+# This file does NOT post the gating `e2e` commit status — it runs alongside
+# e2e.yml for validation. Once green across a few manual runs, fold the
+# triggers from e2e.yml here and retire the loop.
+
+name: E2E (suite blocks)
+
+on:
+  workflow_dispatch:
+    inputs:
+      single_suites:
+        description: "Single-node suites to run as matrix legs (empty = every single-node-eligible suite, per .github/scripts/suite-sets.sh)"
+        required: false
+        default: ""
+      multi_suites:
+        description: "Multi-node suites to run as matrix legs (empty = every multi-node-eligible suite, per .github/scripts/suite-sets.sh)"
+        required: false
+        default: ""
+      run_multinode:
+        description: "Also run the multinode env block"
+        type: boolean
+        required: false
+        default: true
+      test_run:
+        description: "Optional -test.run regex (empty = all tests in suite)"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+  # actions: read lets the otel-job-id action look up a provision job's numeric
+  # id via the Actions API — github.job is only the YAML key, no context
+  # exposes it.
+  actions: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Single-node env
+  # ---------------------------------------------------------------------------
+  single_provision:
+    name: Single env · provision
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 40
+    outputs:
+      wan_ip: ${{ steps.vm.outputs.wan_ip }}
+      ssh_key: ${{ steps.vm.outputs.ssh_key }}
+      suites: ${{ steps.suites.outputs.list }}
+      matrix: ${{ steps.suites.outputs.matrix }}
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      # This job (not single_suite) is where "Bootstrap via Binary Install"
+      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
+      # suite matrix even exists — single_suite's legs share this one
+      # already-booted VM, so daemon telemetry can't be scoped any finer
+      # than "single env" per suite leg. ci.cell=single distinguishes it
+      # from multi_provision's VMs within the same run id.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM; sleep 3; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      # Default and allow-list both come from the shared definition, so this
+      # workflow cannot drift from e2e.yml about what a single node can run.
+      - name: Compute suite matrix
+        id: suites
+        env:
+          REQUESTED: ${{ inputs.single_suites }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
+          e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
+          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
+
+      - name: Reclaim docker build cache
+        run: |
+          docker container prune -f || true
+          docker image prune -f || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
+      # ci.job_id names the provision job, not the suite job that later drives
+      # the env: the suite legs share this env and its baked telemetry.env, so
+      # the provisioner is the only job the daemon telemetry can belong to.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Single env · provision
+
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Provision VM
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "VM provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve VM SSH details
+        id: vm
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: contains(steps.suites.outputs.list, 'eks')
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          banner "build eks-node AMI (libguestfs — no host devices)"
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          group_open "make build-eks-node-image"
+          make build-eks-node-image
+          group_close
+          ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
+
+      - name: Import eks-node AMI on VM
+        if: contains(steps.suites.outputs.list, 'eks')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/eks-node/manifest.conf)"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "import eks-node AMI to VM image store"
+          scp "${SSH_OPTS[@]}" \
+            /tmp/eks-node-image-build/eks-node-alpine.raw \
+            "$VM":/tmp/eks-node-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/eks-node-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
+          ok "eks-node AMI imported"
+
+      - name: Install kubectl on VM
+        if: contains(steps.suites.outputs.list, 'eks')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "install kubectl on VM (eks data-plane subtest)"
+          ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
+          set -euo pipefail
+          KVER=v1.32.5
+          case "$(uname -m)" in
+            x86_64)  ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+            *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
+          esac
+          cd /tmp
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
+          echo "$(cat kubectl.sha256)  kubectl" | sha256sum -c -
+          sudo install -m 0755 kubectl /usr/local/bin/kubectl
+          kubectl version --client
+          EOF
+          ok "kubectl installed on VM"
+
+      - name: Build E2E test binaries on runner
+        working-directory: spinifex/tests/e2e
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # teardown always builds: the run-scoped sweep binary runs in the
+          # teardown job, it is never a selectable suite leg.
+          SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
+          banner "build ${SUITES_BUILD// /, }"
+          group_open "go test -c"
+          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
+          group_close
+          for f in _bin/*.test; do
+            ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
+          done
+
+      - name: Ship test binaries to VM
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
+          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
+          ok "shipped binaries to $VM"
+
+      - name: Start packet capture on VM br-wan
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+            "$VM":/tmp/start-net-capture.sh
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  single_suite:
+    name: Single env · suite
+    needs: single_provision
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
+    env:
+      WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
+      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Install go-junit-report
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Run ${{ matrix.suite }}
+        id: e2e
+        run: |
+          set +e
+          SUITE="${{ matrix.suite }}"
+          VM_HOST="tf-user@${WAN_IP}"
+          TEST_RUN_FLAG=""
+          [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          cleanup_remote() {
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
+              "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
+              2>/dev/null || true
+          }
+          trap cleanup_remote EXIT INT TERM
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
+             R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
+             printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"${SUITE}\" \"\$R\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
+                    | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
+             exit \${PIPESTATUS[0]}"
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect suite artifacts from VM
+        if: always()
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          true
+
+      - name: Convert test log to JUnit XML
+        if: always()
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always()
+        uses: ./spinifex/.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "E2E ${{ matrix.suite }} results"
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: ./spinifex/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E ${{ matrix.suite }} failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Fail job if suite failed
+        if: always()
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
+            exit 1
+          fi
+
+  single_teardown:
+    name: Single env · teardown
+    needs: [single_provision, single_suite]
+    if: always()
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 20
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-single-diag-${{ github.run_id }}
+      WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
+      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
+    steps:
+      # Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
+      # Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
+      # again would wipe the local tofu state and orphan the VM.
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Capture network/IPsec diagnostics on VM
+        if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+            "$VM":/tmp/capture-net-diag.sh 2>/dev/null
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" 2>/dev/null
+          ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "90 min ago"' \
+            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
+          mkdir -p "${ARTIFACT_DIR}/lb-console"
+          ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+            | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
+          true
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-single-diag-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+      # Reclaim any resource leaked by a crashed/interrupted suite before the VM
+      # is destroyed. Scoped to this run's tag, best-effort, never fails the job.
+      - name: Sweep leaked run resources on VM
+        if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          banner "sweep run-tagged resources (run ${{ github.run_id }})"
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
+             SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+             SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+             ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
+            2>&1 || true
+          true
+
+      - name: Teardown VM
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          if [ -n "${WAN_IP}" ] && [ -n "${SSH_KEY}" ]; then
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
+              tf-user@${WAN_IP} \
+              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
+              2>/dev/null || true
+          fi
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 2; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  # ---------------------------------------------------------------------------
+  # Multinode env
+  # ---------------------------------------------------------------------------
+  multi_provision:
+    name: Multi env · provision
+    if: ${{ inputs.run_multinode }}
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 45
+    outputs:
+      node1_ip: ${{ steps.cluster.outputs.node1_ip }}
+      ssh_key: ${{ steps.cluster.outputs.ssh_key }}
+      nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
+      suites: ${{ steps.suites.outputs.list }}
+      matrix: ${{ steps.suites.outputs.matrix }}
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
+      # See single_provision above — daemon telemetry is baked in here, at
+      # provision time, before the multi_suite matrix exists.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) echo "reap $sig pid=$pid"; kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 3; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      - name: Reclaim docker build cache
+        run: |
+          docker container prune -f || true
+          docker image prune -f || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
+      # See the matching note on single_provision.
+      - name: Resolve CI job id
+        uses: ./spinifex/.github/actions/otel-job-id
+        with:
+          job-name: Multi env · provision
+
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      # Mirrors single_provision's step: the multi leg previously ran multinode
+      # alone, silently dropping the lb and cert coverage that only a
+      # multi-node env can complete.
+      - name: Compute suite matrix
+        id: suites
+        env:
+          REQUESTED: ${{ inputs.multi_suites }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
+          e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
+          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
+
+      - name: Build multinode binaries on runner
+        working-directory: spinifex/tests/e2e
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # teardown always builds: the run-scoped sweep binary runs in the
+          # teardown job, it is never a selectable suite leg.
+          SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
+          banner "build ${SUITES_BUILD// /, }"
+          group_open "go test -c"
+          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
+          group_close
+          ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)"
+
+      - name: Ship test binaries to node1
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
+          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
+          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
+          ok "shipped binaries to $VM"
+
+      - name: Start packet capture on all nodes
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            scp "${SSH_OPTS[@]}" \
+              "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+              tf-user@"$ip":/tmp/start-net-capture.sh
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
+          done
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  multi_suite:
+    name: Multi env · suite
+    needs: multi_provision
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
+    env:
+      NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
+      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
+      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Install go-junit-report
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Run ${{ matrix.suite }}
+        id: e2e
+        run: |
+          set +e
+          SUITE="${{ matrix.suite }}"
+          VM_HOST="tf-user@${NODE1_IP}"
+          TEST_RUN_FLAG=""
+          [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          cleanup_remote() {
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
+              "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
+              2>/dev/null || true
+          }
+          trap cleanup_remote EXIT INT TERM
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
+             printf '\n━━ suite %s ━━\n' \"${SUITE}\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+               SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
+                    | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
+             exit \${PIPESTATUS[0]}"
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect suite artifacts from node1
+        if: always()
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
+          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          true
+
+      - name: Convert test log to JUnit XML
+        if: always()
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always()
+        uses: ./spinifex/.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "Multi-Node E2E ${{ matrix.suite }} results"
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: ./spinifex/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Fail job if suite failed
+        if: always()
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
+            exit 1
+          fi
+
+  multi_teardown:
+    name: Multi env · teardown
+    needs: [multi_provision, multi_suite]
+    if: always() && inputs.run_multinode
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 20
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log
+      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
+      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      # Reclaim leaked run-tagged resources before the cluster is destroyed.
+      - name: Sweep leaked run resources on node1
+        if: ${{ needs.multi_provision.result == 'success' && needs.multi_provision.outputs.node1_ip != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${{ needs.multi_provision.outputs.node1_ip }}
+          banner "sweep run-tagged resources (run ${{ github.run_id }})"
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
+             SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+             SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+             ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
+            2>&1 || true
+          true
+
+      - name: Teardown cluster
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          if [ -n "${NODES_CSV}" ] && [ -n "${SSH_KEY}" ]; then
+            IFS=',' read -ra NODE_LIST <<< "${NODES_CSV}"
+            for ip in "${NODE_LIST[@]}"; do
+              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
+                tf-user@"$ip" \
+                'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
+                2>/dev/null || true
+            done
+          fi
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 2; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  e2e_suites_status:
+    name: E2E suite blocks · summary
+    if: always()
+    needs: [single_suite, multi_suite]
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    steps:
+      - name: Summarize
+        run: |
+          echo "## E2E suite-block results" >> "$GITHUB_STEP_SUMMARY"
+          echo "single_suite: ${{ needs.single_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "multi_suite:  ${{ needs.multi_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
+          case "${{ needs.single_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
+          case "${{ needs.multi_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac

.github/workflows/gpu-e2e.yml

@@ -0,0 +1,345 @@
+# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
+# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
+# exposure).
+#
+# ---------------------------------------------------------------------------
+# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
+#
+#   1. Register ONE self-hosted GitHub Actions runner on a banksia host with
+#      the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
+#      NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
+#      other runner the `ci-gpu` label — a second labelled runner would
+#      let two GPU jobs race the single RTX A1000. With exactly one runner
+#      carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
+#      serialization; the `concurrency:` block below queues same-group runs
+#      when that runner is already busy with a live job.
+#   2. Drop the wattle SSH private key on that runner's filesystem and wire
+#      it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
+#      contents). Optionally override `vars.WATTLE_SSH_HOST` /
+#      `vars.WATTLE_SSH_USER` if they differ from the defaults below.
+#   3. Confirm the runner user can drive wattle's control plane over SSH. The
+#      job calls `spx` (read-only checks) and the `aws` CLI with the already-
+#      configured `spinifex` profile directly — no wrapper scripts, no install
+#      on wattle (tf-user is already in the `spinifex` group there):
+#        ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
+#   4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
+#      block CI by touching a sentinel file in the RUNNER's home directory
+#      (NOT on wattle):
+#        touch ~/.ci-gpu-paused     # blocks CI
+#        rm ~/.ci-gpu-paused        # resumes CI
+#      or by setting the repo/environment variable `CI_GPU_PAUSED=true`
+#      (no runner filesystem access needed). Either one causes this workflow
+#      to skip cleanly (job succeeds, all work steps skipped) instead of
+#      queuing behind manual work indefinitely.
+#
+# ---------------------------------------------------------------------------
+
+name: GPU E2E
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_run:
+        description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+# Single dedicated runner (see setup note 1 above) + this group together
+# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
+# else to land (unique label) and, if it somehow overlaps in scheduling,
+# queues here rather than cancelling/stomping the in-flight run.
+concurrency:
+  group: ci-gpu
+  cancel-in-progress: false
+
+env:
+  # OTLP export to the central Elastic sink (wattle is the sink itself — see
+  # Part A.2). Repo variable overrides; carried for correlation even though
+  # the GPU suite's own daemon-side spans are emitted by wattle's already-
+  # configured obs-agent, not by this job.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
+  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
+  # note above), so the label only takes effect once wattle's obs-agent is
+  # given them; there is deliberately no ci.job_id lookup for the same reason.
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
+  WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
+  WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+
+jobs:
+  gpu_e2e:
+    name: GPU E2E (wattle)
+    runs-on: [self-hosted, ci-gpu]
+    timeout-minutes: 75
+    env:
+      ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
+    steps:
+      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
+      # SSH to wattle so a paused run touches the box zero times.
+      - name: Manual-work mutex gate
+        id: gate
+        env:
+          REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
+        run: |
+          set -euo pipefail
+          PAUSED=false
+          if [ -f "$HOME/.ci-gpu-paused" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
+          fi
+          if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
+          fi
+          echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
+
+      - name: Paused — nothing to do
+        if: steps.gate.outputs.paused == 'true'
+        run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
+
+      - name: Clean Workspace
+        if: steps.gate.outputs.paused != 'true'
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v7
+        if: steps.gate.outputs.paused != 'true'
+
+      - uses: actions/setup-go@v7
+        if: steps.gate.outputs.paused != 'true'
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install wattle SSH key
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
+          echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
+          echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
+
+      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
+      # actually up before spending time building/shipping the test binary.
+      - name: Healthcheck — describe-regions
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
+            'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
+
+      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
+      # test is the always-on baseline, so CI self-provisions it rather than
+      # depending on a manual import. `--name` selects the catalog entry (pulls
+      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
+      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
+      # falls back to the generic ami-<distro>-<version>-<arch>, which the
+      # tests/e2e/gpu filter would never match. Idempotent — skipped once
+      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
+      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
+      # `spx admin images import` writes local spinifex storage, so it needs
+      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
+      # HTTPS and don't.
+      - name: Ensure base GPU AMI present (import-if-absent)
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$BASE" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            ID=$(present)
+            if [ -n "$ID" ] && [ "$ID" != "None" ]; then
+              echo "base GPU AMI already present: $BASE = $ID"; exit 0
+            fi
+            echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
+            sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
+            for i in 1 2 3 4 5 6; do
+              ID=$(present)
+              [ -n "$ID" ] && [ "$ID" != "None" ] && break
+              sleep 5
+            done
+            [ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
+            echo "imported base GPU AMI: $BASE = $ID"
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
+      # ensured it, so a miss here means the import silently failed. Hard-fail
+      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
+      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
+      # self-skip when absent, so only warn.
+      - name: Preflight — verify GPU AMIs
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$1" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            ID=$(present "$BASE")
+            if [ -z "$ID" ] || [ "$ID" = "None" ]; then
+              echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
+              exit 1
+            fi
+            echo "required: $BASE = $ID"
+            for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
+              OID=$(present "$OPT")
+              if [ -z "$OID" ] || [ "$OID" = "None" ]; then
+                echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
+              else
+                echo "opt-in: $OPT = $OID"
+              fi
+            done
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Same build command CI's other e2e workflows use to cross-compile a
+      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
+      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
+      # toolchain / module cache off wattle entirely.
+      - name: Build GPU e2e test binary on runner
+        if: steps.gate.outputs.paused != 'true'
+        working-directory: tests/e2e
+        run: |
+          set -euo pipefail
+          make e2e-build SUITES_BUILD=gpu
+          ls -la _bin/gpu.test
+
+      - name: Install go-junit-report on runner
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Ship GPU e2e test binary to wattle
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
+          scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
+
+      # Same invocation shape e2e.yml uses to run a pre-built suite binary
+      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
+      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
+      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
+      # the advertised gateway, so the override is required here. The harness
+      # should prefer spinifex.toml's advertised bridge.
+      - name: Run GPU e2e suite on wattle
+        id: e2e
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
+          ssh -tt ${{ env.ssh_opts }} "$VM" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
+             exit \${PIPESTATUS[0]}"
+          E2E_RC=$?
+          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect artifacts from wattle
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Convert test log to JUnit XML
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log)
+            SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "GPU E2E results"
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "GPU E2E failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.paused != 'true'
+        with:
+          name: gpu-e2e-artifacts-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
+      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
+      # terminate any GPU-typed instance still pending/running (the Go
+      # suite's own t.Cleanup already does this on normal pass/fail exit;
+      # this is the backstop for a hard `-test.timeout` kill or a workflow
+      # cancellation, which skip deferred Go cleanups). Termination triggers
+      # gpuManager.Release() on wattle, returning the A1000 to the pool for
+      # the next queued run or manual test.
+      - name: Cleanup — terminate leftover GPU instances, release GPU
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          REMOTE='
+            set +e
+            GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
+              --query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
+            if [ -z "$GPU_TYPES" ]; then
+              echo "no GPU instance types advertised; nothing to reconcile"
+              exit 0
+            fi
+            IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
+              --filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
+              --query "Reservations[].Instances[].InstanceId" --output text)
+            if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
+              echo "no leftover GPU instances — GPU already released"
+              exit 0
+            fi
+            echo "terminating leftover GPU instance(s): $IDS"
+            AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
+          '
+          ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
+
+      - name: Fail job if GPU suite failed
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
+            exit 1
+          fi

.github/workflows/image-scripts.yml

@@ -0,0 +1,39 @@
+name: Image Scripts
+
+# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
+# and mkosi assets baked into system images, not Go, so they get their own
+# path-filtered gate instead of running on every Go contributor's push (see
+# ui.yml for the same pattern applied to the frontend).
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  image_scripts:
+    name: Image Scripts
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Run image script tests + shellcheck
+        run: make test-images

.github/workflows/publish-system-ami.yml

@@ -0,0 +1,160 @@
+name: Publish System AMIs
+
+# Builds the system AMIs (libguestfs, on the runner) and publishes each
+# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
+# iso.mulgadc.com/system-ami/. Six images ship:
+#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node       Alpine ECS container instance
+#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
+#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
+# Operators then install them on any cluster with:
+#   spx admin images import --name spinifex-eks-node
+#   spx admin images import --name spinifex-ecs-node
+#   spx admin images import --name spinifex-eks-node-gpu
+#   spx admin images import --name spinifex-ecs-node-gpu
+#   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
+#
+# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
+# action, not something every push should trigger. The `variant` input scopes
+# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
+# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
+
+on:
+  workflow_dispatch:
+    inputs:
+      variant:
+        description: "Which images to build"
+        type: choice
+        default: all
+        options:
+          - all
+          - cpu
+          - gpu
+          - rds
+      dry_run:
+        description: "Build + convert only; skip the R2 upload"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: publish-eks-ami
+  cancel-in-progress: false
+
+jobs:
+  matrix:
+    name: Resolve image matrix
+    runs-on: [ self-hosted, ci-single ]
+    outputs:
+      images: ${{ steps.gen.outputs.images }}
+    steps:
+      - name: Select images for variant
+        id: gen
+        env:
+          VARIANT: ${{ inputs.variant }}
+        run: |
+          set -euo pipefail
+          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
+          case "${VARIANT}" in
+            cpu) items="${cpu}" ;;
+            gpu) items="${gpu}" ;;
+            rds) items="${rds}" ;;
+            *)   items="${cpu},${gpu},${rds}" ;;
+          esac
+          echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    name: Build + publish ${{ matrix.name }}
+    needs: matrix
+    runs-on: [ self-hosted, ci-single ]
+    strategy:
+      fail-fast: false
+      # ci-single is one runner and each GPU build needs ~16G of scratch disk;
+      # serialise the legs so they never contend for /_temp or the mirror.
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.matrix.outputs.images) }}
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: dev
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: dev
+          path: predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: "spinifex/go.mod"
+          # Self-hosted runner: the Go module + build cache already persists on
+          # local disk between runs. setup-go's cache-save ("Post Setup Go")
+          # otherwise re-uploads hundreds of MB to the Actions cache backend,
+          # taking longer than the build itself.
+          cache: false
+
+      - name: Setup Go Workspace
+        # Idempotent: the ci-single runner reuses one workspace across matrix
+        # legs and runs, and checkout never scrubs the root go.work. Remove any
+        # stale file so each leg re-inits cleanly instead of failing on
+        # "go.work already exists".
+        run: |
+          rm -f go.work go.work.sum
+          go work init ./spinifex ./viperblock ./predastore
+
+      - name: Install build + upload tooling
+        run: |
+          set -euo pipefail
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          if ! command -v aws &>/dev/null; then
+            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+            unzip -q /tmp/awscliv2.zip -d /tmp/aws-install
+            sudo /tmp/aws-install/aws/install
+            rm -rf /tmp/awscliv2.zip /tmp/aws-install
+          fi
+
+      - name: Build + publish ${{ matrix.name }} to R2
+        working-directory: spinifex
+        env:
+          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
+          R2_BUCKET: spinifex-iso
+          DRY_RUN: ${{ inputs.dry_run }}
+          MANIFEST: ${{ matrix.manifest }}
+          # Ubuntu GPU images are 16G — a RAM-backed /tmp cannot hold the qcow2 +
+          # expanded raw, so point the build scratch at the runner's on-disk
+          # _temp. Harmless for the small Alpine legs too.
+          SYSTEM_IMAGE_BUILD_DIR: ${{ runner.temp }}/system-image-build
+        run: |
+          set -euo pipefail
+          # libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
+          # direct backend = no libvirt on the runner.
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          mkdir -p "${SYSTEM_IMAGE_BUILD_DIR}"
+          FLAGS=(--build)
+          if [[ "${DRY_RUN}" == "true" ]]; then
+            FLAGS+=(--dry-run)
+          fi
+          ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"

.github/workflows/release.yml

@@ -9,21 +9,21 @@ permissions:
 
 jobs:
   distro-amd64:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           path: spinifex
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
           path: viperblock
@@ -32,9 +32,9 @@ jobs:
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
-          go-version: '1.26.4'
+          go-version: "1.26.5"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -54,21 +54,21 @@ jobs:
             spinifex/scripts/setup.sh
 
   distro-arm64:
-    runs-on: ubuntu-24.04-arm
+    runs-on: ubuntu-26.04-arm
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           path: spinifex
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
           path: viperblock
@@ -91,7 +91,7 @@ jobs:
 
   release:
     needs: [ distro-amd64, distro-arm64 ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     steps:
       - name: Download amd64 Artifacts
         uses: actions/download-artifact@v8
@@ -131,7 +131,10 @@ jobs:
       REPO: ${{ github.repository }}
     steps:
       - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
         id: app-token
@@ -140,7 +143,7 @@ jobs:
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -169,8 +172,30 @@ jobs:
           TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
           ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" "$SETUP_SH" "$TARBALL"
 
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
+
+  baremetal:
+    needs: release
+    permissions:
+      contents: read
+      statuses: write
+    uses: ./.github/workflows/e2e-baremetal.yml
+    secrets: inherit
+
+  promote:
+    needs: [ verify, baremetal ]
+    runs-on: ubuntu-26.04
+    env:
+      TAG: ${{ github.ref_name }}
+      REPO: ${{ github.repository }}
+    steps:
       - name: Promote Release
-        if: success()
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -182,41 +207,41 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
+  # Annotate the (still-draft) release if either gate failed.
+  mark-failed:
+    needs: [ verify, baremetal ]
+    if: failure()
+    runs-on: ubuntu-26.04
+    env:
+      TAG: ${{ github.ref_name }}
+      REPO: ${{ github.repository }}
+    steps:
       - name: Mark Release Failed
-        if: failure()
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
           NOTES_FILE=$(mktemp)
-          printf 'Verification failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
+          printf 'Verification failed (tofu install or bare-metal boot) — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
           gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
           rm -f "$NOTES_FILE"
 
-      - name: Teardown
-        if: always()
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
-          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
-          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-
   iso:
     needs: release
     runs-on: [ self-hosted, ci-single ]
-    timeout-minutes: 90
+    # ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
+    # upload gate then waits for baremetal to finish, so the job must outlast it.
+    timeout-minutes: 180
     steps:
       - name: Clean Workspace
         run: |
           ROOTFS="${GITHUB_WORKSPACE}/mulga/scripts/iso-builder/.cache/rootfs"
-          # Mirror make clean's umount sequence unconditionally — lazy unmount
-          # detaches immediately even if the mount is busy, so rm -rf is safe.
           sudo umount -l -R "${ROOTFS}" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/dev/pts" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/sys" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
-          sudo rm -rf "$GITHUB_WORKSPACE"/*
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
         id: app-token
@@ -226,25 +251,25 @@ jobs:
           repositories: mulga
 
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           path: spinifex
           ref: ${{ github.ref }}
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
           path: viperblock
 
       - name: Checkout Mulga iso-builder
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -255,7 +280,7 @@ jobs:
         run: sudo make -C mulga/scripts/iso-builder clean
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           go-version-file: spinifex/go.mod
           cache: false
@@ -289,24 +314,25 @@ jobs:
             SPINIFEX_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spx" \
             INSTALLER_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spinifex-installer"
 
-      - name: Wait for verify
+      - name: Wait for verify + baremetal
         id: wait-verify
         if: steps.build-iso.outcome == 'success'
         continue-on-error: true
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          echo "Waiting for verify job to complete..."
+          echo "Waiting for verify + baremetal jobs to complete..."
           while true; do
-            CONCLUSION=$(gh api \
-              "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
-              --jq '.jobs[] | select(.name == "verify") | .conclusion // empty')
-            if [ -n "$CONCLUSION" ]; then
-              echo "Verify conclusion: $CONCLUSION"
-              [ "$CONCLUSION" = "success" ] && exit 0
+            JOBS=$(gh api \
+              "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" --paginate)
+            VERIFY=$(echo "$JOBS" | jq -r '.jobs[] | select(.name == "verify") | .conclusion // empty')
+            BAREMETAL=$(echo "$JOBS" | jq -r '.jobs[] | select(.name | startswith("baremetal")) | .conclusion // empty')
+            if [ -n "$VERIFY" ] && [ -n "$BAREMETAL" ]; then
+              echo "verify=$VERIFY baremetal=$BAREMETAL"
+              { [ "$VERIFY" = "success" ] && [ "$BAREMETAL" = "success" ]; } && exit 0
               exit 1
             fi
-            echo "Verify still running, retrying in 30s..."
+            echo "Still waiting (verify=${VERIFY:-pending} baremetal=${BAREMETAL:-pending}), retry 30s..."
             sleep 30
           done
 
@@ -327,14 +353,13 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
           CF_R2_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT_URL }}
-          VERSION: ${{ github.ref_name }}
         run: |
           aws --endpoint-url="${CF_R2_ENDPOINT_URL}" \
-            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso" \
-                  "s3://spinifex-iso/spinifex-${VERSION}.iso"
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex.iso" \
+                  "s3://spinifex-iso/spinifex.iso"
           aws --endpoint-url="${CF_R2_ENDPOINT_URL}" \
-            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso.sha256" \
-                  "s3://spinifex-iso/spinifex-${VERSION}.iso.sha256"
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex.iso.sha256" \
+                  "s3://spinifex-iso/spinifex.iso.sha256"
 
       - name: Mark ISO failed
         if: always() && (steps.build-iso.outcome == 'failure' || steps.upload.outcome == 'failure')

.github/workflows/segscan-oracle.yml

@@ -0,0 +1,81 @@
+# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only,
+# deliberately NOT wired into spinifex.yml's pull_request path.
+#
+# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
+# scripts/segscan checked out alongside spinifex (it is a separate Go module,
+# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
+# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
+# every PR; adding a second-repo checkout there would make every spinifex PR
+# depend on mulga main being healthy and pay that checkout cost on every run,
+# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger
+# This never gates a PR before merge, matching how e2e.yml's own
+# "front gate" job works.
+name: Segscan Storage Oracle
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  segscan_oracle:
+    name: Segscan Oracle
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Mulga (scripts/segscan only)
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/segscan
+          path: mulga
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Segscan Oracle Test
+        env:
+          SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan
+        run: make -C spinifex test-segscan-oracle

.github/workflows/spinifex.yml

@@ -23,31 +23,38 @@ permissions:
 jobs:
   build_and_test:
     name: Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           path: spinifex
           fetch-depth: 2
 
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
@@ -58,6 +65,9 @@ jobs:
       - name: Build Spinifex
         run: make -C spinifex build
 
+      - name: Manifest check + drift lint
+        run: make -C spinifex manifest-check manifest-lint
+
       - name: Run Tests with Coverage
         run: make -C spinifex test-cover
 
@@ -67,32 +77,88 @@ jobs:
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
-  race_detection:
-    name: Race Detection
-    runs-on: ubuntu-24.04
+  # In-process integration tier (gateway.SetupRoutes over embedded NATS
+  # JetStream). Its own job so it runs in parallel with the unit tiers rather
+  # than adding to their wall time, and it still blocks a PR — unlike e2e.yml's
+  # "Integration (front gate)" job, which is push-triggered against main/dev on
+  # a self-hosted runner and so never gates a PR before merge.
+  integration_tests:
+    name: Integration Tests
+    runs-on: ubuntu-26.04
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Integration Tests
+        run: make -C spinifex test-integration
+
+  race_detection:
+    name: Race Detection
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
         with:
           cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
@@ -105,30 +171,37 @@ jobs:
 
   lint_and_security:
     name: Lint and Security
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           path: spinifex
 
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7
         with:
           cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"

.github/workflows/ui.yml

@@ -23,14 +23,14 @@ permissions:
 jobs:
   ui_build_and_test:
     name: UI Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     defaults:
       run:
         working-directory: spinifex/services/spinifexui/frontend
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           sparse-checkout: spinifex/services/spinifexui/frontend
 
@@ -40,7 +40,7 @@ jobs:
           package_json_file: spinifex/services/spinifexui/frontend/package.json
 
       - name: Setup Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           cache: "pnpm"
           cache-dependency-path: spinifex/services/spinifexui/frontend/pnpm-lock.yaml
@@ -57,3 +57,6 @@ jobs:
 
       - name: Build Frontend
         run: pnpm build
+
+      - name: Check for generated route tree drift
+        run: git diff --exit-code -- src/routeTree.gen.ts

.gitignore

@@ -38,12 +38,5 @@ Thumbs.db
 # Test reports
 *-report.txt
 
-# Internal docs (moved to private mulga repo)
-docs/INTERNAL.md
-docs/PRIVATE.md
-docs/DEVNOTES.md
-docs/PLAN.md
-docs/TODO.md
-docs/development/
-docs/internal/
-scripts/iac/aws/spinifex-test.pem
+# Per-worktree golangci-lint cache
+/.cache/

.golangci.yml

@@ -1,5 +1,8 @@
 version: "2"
 
+issues:
+  max-same-issues: 10
+
 formatters:
   enable:
     - gofmt
@@ -14,24 +17,29 @@ linters:
     - bidichk # checks for dangerous unicode bidi character sequences
     - bodyclose # checks whether HTTP response body is closed successfully
     - canonicalheader # checks whether net/http.Header uses canonical header keys
-    - decorder # checks declaration order of types, consts, vars, and funcs
+    - depguard # blocks imports of packages with a preferred replacement
     - durationcheck # checks for two durations multiplied together
     - embeddedstructfieldcheck # checks embedded sync.Mutex/RWMutex in structs
     - errcheck # checks for unchecked errors in type assertions and I/O
     - errchkjson # checks types passed to json encoding functions, flags omittable error checks
     - errname # checks that sentinel errors are prefixed with Err, error types suffixed with Error
+    - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forcetypeassert # flags unchecked type assertions that can panic
+    - forbidigo # blocks the legacy JetStream/KV symbols
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
+    - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
     - gosec # inspects source code for security problems
     - govet # reports suspicious constructs (Printf alignment, shadow, etc.)
+    - iface # detects interface pollution — duplicate interface declarations
     - ineffassign # detects assignments to existing variables that are not used
+    - iotamixing # flags const blocks that mix iota with non-iota declarations
     - makezero # finds slice declarations with non-zero initial length followed by append
     - mirror # catches wrong mirror patterns in bytes/strings usage
-    - musttag # checks structs passed to marshal/unmarshal have proper tags
+    - misspell # finds commonly misspelled English words
     - modernize # modernizes code to use newer Go idioms (go fix analyzers)
+    - musttag # checks structs passed to marshal/unmarshal have proper tags
     - nakedret # finds naked returns in functions
     - nilerr # catches code that returns nil even after checking err != nil
     - nilnesserr # catches returning a different nil error variable after err != nil check
@@ -43,6 +51,8 @@ linters:
     - recvcheck # checks for receiver type consistency (pointer vs value)
     - sloglint # enforces consistent log/slog usage (key-value pairs, no globals)
     - staticcheck # extensive static analysis checks
+    - testifylint # checks common testify mistakes (assert.Len over assert.Equal on len, etc.)
+    - tparallel # detects inappropriate usage of t.Parallel()
     - unconvert # removes unnecessary type conversions
     - usestdlibvars # detects hardcoded values that should use stdlib constants
     - usetesting # catches os.TempDir() etc. in tests — should use t.TempDir()
@@ -50,14 +60,40 @@ linters:
     - whitespace # detects leading and trailing whitespace in functions
 
   settings:
+    depguard:
+      rules:
+        main:
+          deny:
+            - pkg: log$
+              desc: Use log/slog instead, see https://go.dev/blog/slog
+            - pkg: math/rand$
+              desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
+
     embeddedstructfieldcheck:
       forbid-mutex: true
 
     errcheck:
-      check-type-assertions: true
+      check-type-assertions: true # replaces forcetypeassert
       exclude-functions:
         - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
-        - (*github.com/kdomanski/iso9660.ImageWriter).Cleanup # deferred cleanup, errors not actionable
+
+    forbidigo:
+      analyze-types: true
+      forbid:
+        - pattern: '\.JetStream$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
+        - pattern: '^nats\.(JetStreamContext|KeyValue.*|KeyWatcher|Err(KeyNotFound|KeyExists|BucketNotFound|NoKeysFound|StreamNotFound))$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nats KV types/sentinels are superseded; use github.com/nats-io/nats.go/jetstream.
+
+    gosec:
+      excludes:
+        - G104 # unchecked errors — errcheck is stricter
+        - G116 # bidi chars — bidichk is more thorough
+        - G117 # use of net/http serve functions (we handle TLS separately)
+        - G703 # path traversal taint analysis — false positives on config file loading
+        - G705 # XSS taint analysis — false positives on internal error responses
 
     govet:
       enable-all: true
@@ -66,6 +102,28 @@ linters:
         - shadow # too noisy to retrofit — all 50 findings are err re-declarations
         - unusedwrite # false positives on struct field assignments before marshal/send
 
+    modernize:
+      disable:
+        - newexpr # don't replace aws.String() with new() — AWS SDK idiom
+        - stringsbuilder # don't replace small string concatenations with strings.Builder
+
+    nakedret:
+      max-func-lines: 0
+
+    nolintlint:
+      require-explanation: true
+      require-specific: true
+
+    perfsprint:
+      concat-loop: false
+      err-error: false # don't flag err.Error()
+      errorf: false # don't convert fmt.Errorf to errors.New
+      strconcat: false
+
+    reassign:
+      patterns:
+        - ".*"
+
     staticcheck:
       checks:
         - all
@@ -77,40 +135,14 @@ linters:
         - -ST1020 # godoc on exported types — not enforcing yet
         - -ST1021 # godoc on exported type groups — not enforcing yet
         - -ST1022 # godoc on exported vars/consts — not enforcing yet
-        - -SA1019 # deprecated usage (handle separately)
         - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
-        - -SA9005 # ineffective break (staticcheck false positives)
+        - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
 
-    gosec:
-      excludes:
-        - G104 # unchecked errors — errcheck is stricter
-        - G116 # bidi chars — bidichk is more thorough
-        - G117 # use of net/http serve functions (we handle TLS separately)
-        - G703 # path traversal taint analysis — false positives on config file loading
-        - G705 # XSS taint analysis — false positives on internal error responses
-        - G706 # log injection taint analysis — false positives on structured slog usage
-
-    modernize:
+    testifylint:
       disable:
-        - newexpr # don't replace aws.String() with new() — AWS SDK idiom
-        - stringsbuilder # don't replace small string concatenations with strings.Builder
-
-    perfsprint:
-      concat-loop: false
-      err-error: false # don't flag err.Error()
-      errorf: false # don't convert fmt.Errorf to errors.New
-      strconcat: false
-
-    nakedret:
-      max-func-lines: 0
-
-    nolintlint:
-      require-explanation: true
-      require-specific: true
-
-    reassign:
-      patterns:
-        - ".*"
+        - encoded-compare # misidentifies non-JSON strings when identifiers contain "JSON"
+        - float-compare # false positives
+        - require-error # assert-on-error is deliberate where a test continues to collect further failures
 
   exclusions:
     presets:
@@ -134,3 +166,8 @@ linters:
         linters: [gosec] # subprocess execution — legitimate in VM orchestration
       - text: "G304"
         linters: [gosec] # file path from variable — necessary for config loading
+
+      # aws-sdk-go v1 is deprecated wholesale (EOL 2025-07-31); the v1->v2
+      # migration is tracked separately, so suppress its SA1019 noise for now.
+      - text: "aws-sdk-go is deprecated"
+        linters: [staticcheck]

INSTALL.md

@@ -1,3 +0,0 @@
-# Spinifex Installation
-
-For documentation on using and installing Spinifex see [https://docs.mulgadc.com](https://docs.mulgadc.com/)

Makefile

@@ -1,22 +1,22 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
+# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
+# path, not worktree path, so parallel worktrees of this module collide and
+# surface each other's issues. Scope it per worktree.
+export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
   GO_ARCH := amd64
   AWS_ARCH := x86_64
-  # On x86, we can run ARM VMs via emulation
-  QEMU_PACKAGES := qemu-system-x86 qemu-system-arm
 else ifeq ($(ARCH),aarch64)
   GO_ARCH := arm64
   AWS_ARCH := aarch64
-  # On ARM, we can run x86 VMs via emulation
-  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
 else ifeq ($(ARCH),arm64)
   GO_ARCH := arm64
   AWS_ARCH := aarch64
-  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
 else
   $(error Unsupported architecture: $(ARCH). Only x86_64 and aarch64/arm64 are supported.)
 endif
@@ -49,7 +49,7 @@ else
   _SECQ  = 2>&1 | tee
 endif
 
-build: go_build build-installer build-lb-agent
+build: go_build build-installer build-lb-agent generate-aws-model-coverage
 
 # Build spinifex-ui frontend (requires pnpm)
 build-ui:
@@ -73,24 +73,49 @@ build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
 	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
-build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-server)
+build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
+	@echo -e "\n....Building ecs-agent (static)"
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
+build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
 	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
 endif
 	@if [ -f scripts/images/$(IMAGE).conf ]; then \
-		./scripts/build-system-image.sh scripts/images/$(IMAGE).conf; \
+		./scripts/build-system-image.sh scripts/images/$(IMAGE).conf $(if $(IMPORT),--import); \
 	elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \
-		./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf; \
+		./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf $(if $(IMPORT),--import); \
 	else \
 		echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \
 		exit 1; \
 	fi
 
-build-eks-server-image: ## Build the eks-server AMI (K3s control-plane variant)
-	$(MAKE) build-system-image IMAGE=eks-server
+build-eks-node-image: ## Build the unified eks-node AMI (K3s server+agent; role at first boot; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=eks-node
 
-build-eks-agent-image: ## Build the eks-agent AMI (K3s worker variant)
-	$(MAKE) build-system-image IMAGE=eks-agent
+import-eks-node-image: ## Build + register the eks-node AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=eks-node IMPORT=1
+
+publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
+	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
+
+build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=ecs-agent
+
+import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
+
+build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + PostgreSQL 18 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-postgres
+
+import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
+
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
 
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
@@ -112,14 +137,10 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
 
-go_run:
-	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
-	$(GOPATH)/bin/$(GO_PROJECT_NAME)
-
-# Preflight — runs the same checks as GitHub Actions (lint + vuln + tests).
-# Use this before committing to catch CI failures locally.
+# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
+# integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check lint govulncheck test-cover diff-coverage test-race test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -129,29 +150,67 @@ test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
 	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
 
+# In-process integration tier: the real gateway router against embedded NATS
+# JetStream, with only the daemon-side NATS subjects stubbed.
+AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
+AWS_MODEL_CONFORMANCE_MODE ?= fail
+AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+test-integration:
+	@echo -e "\n....Running in-process integration tests...."
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
+	@$(MAKE) --no-print-directory generate-aws-model-coverage
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+
+generate-aws-model-coverage:
+	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
+	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+aws-model-coverage: generate-aws-model-coverage
+	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
+# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
+# is not the default local or CI layout, so this is a separate target from
+# test-integration rather than folded into it. Skips itself when segscan's
+# source isn't found.
+test-segscan-oracle:
+	@echo -e "\n....Running segscan storage oracle test...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
+
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
-# of services/suites/fixtures + on-disk path existence. Subject content
-# vs source is enforced separately in Bead 5 drift lint.
+# of services/suites/fixtures + on-disk path existence.
 manifest-check:
 	@echo -e "\n....Checking service-interfaces.yaml...."
 	@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
 
+# Drift guards: direct-create fixture lint + NATS subject lint,
+# ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
+# drift beyond the baseline.
+manifest-lint:
+	@echo -e "\n....Linting manifest drift (fixtures + subjects)...."
+	@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root .
+
+# Accept current drift into the baseline. Run after an intentional change.
+manifest-lint-update:
+	@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . -update
+
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
+	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... $(_RACEQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -159,19 +218,47 @@ test-actions:
 	@echo -e "\n....Running action tests...."
 	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
+# Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
+# helpers baked into system images. Kept out of `preflight` (a dedicated CI
+# job gates it on scripts/images/**+images/mkosi.profiles/** changes instead)
+# so image-asset churn doesn't run on every Go contributor's commit.
+#
+# images/mkosi.profiles/ ships scripts with no .sh suffix (mkosi's own
+# mkosi.*.chroot lifecycle hooks, and wrapper binaries like vllm-serve under
+# mkosi.extra/) so a plain `-name '*.sh'` glob silently skips exactly the
+# files most worth linting. Sources there are found by shebang instead of
+# name; this also means a profile that ships no scripts at all yields an
+# empty match rather than an error.
+test-images:
+	@echo -e "\n....Running scripts/images/**/*_test.sh...."
+	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running images/mkosi.profiles/**/*_test.sh...."
+	@for t in $$(find images/mkosi.profiles -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
+	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo -e "\n....Running shellcheck over images/mkosi.profiles/** shell sources...."
+	@srcs="$$(grep -rlIE '^#!/bin/(bash|sh)' images/mkosi.profiles 2>/dev/null | sort)"; \
+	if [ -n "$$srcs" ]; then \
+		shellcheck -S warning $$srcs; \
+	else \
+		echo "  (no shell sources found under images/mkosi.profiles)"; \
+	fi
+	@echo "  test-images ok"
+
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
-	$(MAKE) easyjson
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
 
-run:
-	$(MAKE) go_build
-	$(MAKE) go_run
-
 # Fast iteration: build + install binary + restart all services.
 # Microvm artifacts are reinstalled when they already exist on disk — the rule's
 # input timestamps drive a rebuild only if anything actually changed. On a fresh
@@ -179,6 +266,20 @@ run:
 # `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
+		if ! sudo /usr/local/bin/spx admin preflight; then \
+			echo ""; \
+			echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
+			echo "  grants don't match the binary just installed. Remediate with:"; \
+			echo "    scripts/update-nodes.sh   (from the mulga root)"; \
+			echo "  or:"; \
+			echo "    make reinstall"; \
+			echo "  Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \
+			exit 1; \
+		fi; \
+	else \
+		echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
+	fi
 	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
 		$(MAKE) install-microvm; \
 	else \
@@ -198,20 +299,19 @@ clean:
 
 install-system:
 	@echo -e "\n....Installing system dependencies for $(ARCH)...."
-	@echo "QEMU packages: $(QEMU_PACKAGES)"
 	sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		-o Dpkg::Options::="--force-confdef" \
 		-o Dpkg::Options::="--force-confold" \
-		nbdkit nbdkit-plugin-dev pkg-config $(QEMU_PACKAGES) qemu-utils qemu-kvm \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
 		ovmf qemu-efi-aarch64 \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.4 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -231,24 +331,26 @@ install-aws:
 quickinstall: install-system install-go install-aws
 	@echo -e "\n✅ Quickinstall complete for $(ARCH)."
 	@echo "   Please ensure /usr/local/go/bin is in your PATH."
-	@echo "   Installed: Go ($(GO_ARCH)), AWS CLI ($(AWS_ARCH)), QEMU ($(QEMU_PACKAGES))"
 
-# Lint all Go code via golangci-lint (replaces check-format, vet, gosec, staticcheck)
 lint:
 	@echo "Running golangci-lint..."
-	$(_Q)golangci-lint run ./...
+	$(_Q)scripts/run-gate.sh golangci-lint golangci-lint run ./...
 	@echo "  golangci-lint ok"
 
-# Auto-fix all linter issues that have fixers
 fix:
 	golangci-lint run --fix ./...
 
-# Govulncheck — dependency vulnerability scanning (not covered by golangci-lint)
 govulncheck:
 	@echo "Running govulncheck..."
-	$(_Q)go tool govulncheck ./...
+	$(_Q)scripts/run-gate.sh govulncheck go tool govulncheck ./...
 	@echo "  govulncheck ok"
 
+# NilAway — advisory nil-panic analysis. Not in preflight due to false positives
+nilaway:
+	@echo "Running nilaway..."
+	$(_Q)scripts/run-gate.sh nilaway go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
+	@echo "  nilaway ok"
+
 # Build release tarballs — use distro-ARCH for single arch, distro for both
 distro: distro-amd64 distro-arm64
 	@echo ""
@@ -292,55 +394,8 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-# Ansible dev lifecycle (experimental, parallel to dev-*.sh / reset-dev-env.sh).
-# See scripts/ansible/README.md and docs/development/improvements/ansible-dev-lifecycle.md.
-#
-# Variable overrides: pass EXTRA_VARS="key=val key2=val2" to forward as
-# `--extra-vars` to ansible-playbook. Plain `-e` on the make command line is
-# make's own flag and does NOT reach ansible.
-#   make ansible-dev-install EXTRA_VARS="spinifex_external_pool_start=192.168.1.90 spinifex_external_pool_end=192.168.1.99"
-EXTRA_VARS ?=
-_ANSIBLE_EXTRA = $(if $(strip $(EXTRA_VARS)),--extra-vars "$(EXTRA_VARS)",)
-
-ansible-dev-preflight:
-	cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-teardown:
-	cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-install:
-	cd scripts/ansible && ansible-playbook playbooks/dev-install.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-reset:
-	cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-deploy:
-	cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-status:
-	cd scripts/ansible && ansible-playbook playbooks/dev-status.yml
-
-ansible-dev-logs:
-	cd scripts/ansible && ansible-playbook playbooks/dev-logs.yml
-
-# Snapshot / restore require an explicit -e snapshot_name=<name>. The
-# wrapper passes ANSIBLE_EXTRA through so devs can run
-# `make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge'`.
-ansible-dev-snapshot:
-	cd scripts/ansible && ansible-playbook playbooks/dev-snapshot.yml $(ANSIBLE_EXTRA)
-
-ansible-dev-restore:
-	cd scripts/ansible && ansible-playbook playbooks/dev-restore.yml $(ANSIBLE_EXTRA)
-
-ansible-dev-version:
-	cd scripts/ansible && ansible-playbook playbooks/dev-version.yml
-
-ansible-dev-vpc:
-	cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA)
-
-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-server-image build-eks-agent-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
-	lint fix govulncheck \
-	distro distro-amd64 distro-arm64 distro-clean \
-	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy ansible-dev-status ansible-dev-logs ansible-dev-snapshot ansible-dev-restore ansible-dev-version ansible-dev-vpc
+	lint fix govulncheck nilaway \
+	distro distro-amd64 distro-arm64 distro-clean

README.md

@@ -1,16 +1,18 @@
 <p align="center">
-  <img src=".github/assets/banner.svg" alt="Spinifex — AWS-Compatible Infrastructure Stack" width="900">
+  <img src=".github/assets/banner.svg" alt="Spinifex: the open, AWS-compatible cloud you run yourself. EC2, EBS, S3, VPC, and IAM on your infrastructure, at the edge, or on a Neocloud." width="900">
 </p>
 
 <p align="center">
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-docs.mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
 
 <p align="center">
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
+  <a href="#three-ways-to-deploy">Deploy</a> ·
+  <a href="#aws-compatibility">AWS services</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
   <a href="#installation">Installation</a> ·
@@ -19,31 +21,69 @@
 
 ---
 
-# Spinifex: An Open Source AWS-Compatible Stack for Bare-Metal, Edge, and On-Prem Deployments
+# Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex developed by [Mulga Defense Corporation](https://mulgadc.com) is an open source infrastructure platform that brings the core services of AWS—like EC2, VPC, EBS, and S3—to environments where running in the cloud isn't an option. Whether you're deploying to edge sites, private data-centers, or need to operate in low-connectivity or highly contested environments, Spinifex gives you AWS-style workflows on your own hardware.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
 
 ## What is Spinifex?
 
-Spinifex replicates essential AWS primitives—virtual machines, block volumes, and object storage—using lightweight, self-contained components that are easy to deploy and integrate.
+Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-It's designed for developers and operators who need:
+Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
 
-- Cloud-like infrastructure without the cloud
-- Minimal dependencies, full control
-- Drop-in compatibility with tools like the AWS CLI, SDKs, and Terraform
-- A secure cloud environment you control and own the entire hardware, network and software stack
+It's built for teams that need:
 
-You can run Spinifex on a few servers in a rack, a field site, or anywhere centralized cloud services aren't feasible.
+- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+- Full control of the stack: your hardware, your network, your data, your keys
+- A cloud that keeps running offline, through disconnection, with no external control plane
+- An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
+
+You change *where* your software runs, not *what* it is.
 
 ## The Platform
 
 <p align="center">
-  <img src=".github/assets/platform.svg" alt="The Mulga Platform — Hardware to Applications" width="900">
+  <img src=".github/assets/platform.svg" alt="The Spinifex stack: your apps and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU and GPU hardware, deployable to Neocloud, on-premise, or the edge." width="900">
 </p>
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
+## Three ways to deploy
+
+One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
+
+### Neocloud
+
+Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
+
+### On-premise
+
+Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
+
+### Edge
+
+Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
+
+## AWS compatibility
+
+Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
+
+| Service | What it is | Status |
+| --- | --- | --- |
+| **EC2** | Compute | Available |
+| **EBS** | Block storage | Available |
+| **S3** | Object storage | Available |
+| **VPC** | Networking | Available |
+| **IAM** | Identity & auth | Available |
+| **ALB / NLB** | Load balancers | Available |
+| **EKS** | Kubernetes | Available |
+| **ECR** | Container registry | Available |
+| **ECS** | Container service | Available |
+| **RDS** | Databases | Available |
+| **Bedrock** | AI Deployment | Q3 2026 |
+
+Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+
 ## Core Components
 
 ### Spinifex (Compute Service – EC2 Alternative)
@@ -75,19 +115,19 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 ## Architecture at a Glance
 
 <p align="center">
-  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture — AWS Gateway, NATS bus, service daemons" width="900">
+  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
 </p>
 
-Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless — scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
 ## Key Features
 
-AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, or SDKs you already use.
-
-- Designed for Control – Run on your own terms, whether that's in a datacenter, remote site, or sensitive environment.
-- Minimal Dependencies – Each service is standalone and avoids complex orchestration layers.
-- Works Offline – No reliance on centralized cloud services or external networks.
-- Open Source – Licensed under AGPL 3.0. Fork it, extend it, or deploy it as-is.
+- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
+- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
+- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
+- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
+- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
+- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
 
 ## Installation
 
@@ -192,6 +232,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 
 This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
 
+## Trademarks
+
+"AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
+
 ## License
 
 Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

build/logrotate/spinifex

@@ -1,4 +1,8 @@
 /var/log/spinifex/*.log {
+    # The directory is 0775 root:spinifex so each service user can write its own
+    # log. logrotate refuses to rotate inside a group-writable directory unless
+    # it is told which user to rotate as.
+    su root spinifex
     daily
     rotate 14
     compress

build/microvm/init.sh

@@ -6,20 +6,58 @@ mount -t devtmpfs dev /dev
 # virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
 # already enumerated by the time PID 1 runs — no modprobe needed.
 # qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
-modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true
-modprobe virtio_net 2>/dev/null || true
+#
+# modprobe needs a usable modules.dep; when the image is built on a host whose
+# depmod can't produce one, busybox modprobe silently no-ops and the module
+# never loads — fw_cfg then never appears and init panics. insmod by explicit
+# path needs no modules.dep, so fall back to it. Handles .ko and .ko.gz.
+load_mod() {
+    modprobe "$1" 2>/dev/null && return 0
+    for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
+        insmod "$ko" 2>/dev/null && return 0
+        case "$ko" in
+            *.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
+                  && insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
+        esac
+    done
+    return 1
+}
+load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
+# virtio_net depends on net_failover, which depends on failover. The load_mod
+# insmod fallback loads a single .ko with no dependency resolution, and
+# modules.dep is absent when the image is built on a kmod-less host (depmod
+# skipped), so modprobe can't resolve the chain either. Load it explicitly
+# bottom-up so the NIC comes up regardless of how the image was built —
+# without this the agent has no network and the LB hangs in provisioning.
+load_mod failover || true
+load_mod net_failover || true
+load_mod virtio_net || true
 
-for i in 1 2 3 4 5; do
-    [ -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ] && break
-    sleep 0.02
+NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+i=0
+while [ "$i" -lt 25 ]; do
+    [ -f "$NETCFG" ] && break
+    sleep 0.04
+    i=$((i + 1))
 done
 
-if [ ! -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ]; then
+if [ ! -f "$NETCFG" ]; then
+    # Dump host/guest fw_cfg state to the serial console before init exits so a
+    # "netcfg not available" panic is diagnosable from the captured console log
+    # (module-not-loaded vs qemu-not-exposing-by_name vs wrong machine type).
     echo "[init] FATAL: fw_cfg netcfg not available" >&2
+    echo "[init] dmi: vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null) product=$(cat /sys/class/dmi/id/product_name 2>/dev/null) bios=$(cat /sys/class/dmi/id/bios_version 2>/dev/null)" >&2
+    echo "[init] qemu_fw_cfg dir present: $([ -d /sys/firmware/qemu_fw_cfg ] && echo yes || echo NO)" >&2
+    echo "[init] modules: $(lsmod 2>/dev/null | awk 'NR>1{print $1}' | tr '\n' ',')" >&2
+    echo "[init] by_name listing:" >&2
+    ls -la /sys/firmware/qemu_fw_cfg/by_name/ 2>&1 | sed 's/^/[init]   /' >&2
+    ls -la /sys/firmware/qemu_fw_cfg/by_name/opt/ 2>&1 | sed 's/^/[init]   /' >&2
+    echo "[init] dmesg fw_cfg:" >&2
+    dmesg 2>/dev/null | grep -i fw_cfg | sed 's/^/[init]   /' >&2
     exit 1
 fi
 
-. /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+. "$NETCFG"
 
 for n in 0 1 2 3 4 5; do
     eval mac=\$NIC${n}_MAC

build/scripts/nats-js-watchdog.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+# JetStream liveness watchdog for spinifex-nats.
+#
+# NATS can stop accepting JetStream writes at runtime while the process stays
+# alive and keeps serving reads. That is not a process failure, so
+# Restart=on-failure never fires and KV-backed services (awsgw IAM bucket init,
+# etc.) stall until NATS is restarted by hand.
+#
+# Two distinct latches produce that symptom, and they need different probes:
+#
+#   1. Server-wide. handleOutOfSpace calls ShutdownJetStream and sets an
+#      internal disabled flag that only a restart clears. healthz reports it.
+#   2. Per message block. filestore records a write error on the block and
+#      returns it on every later write; nothing clears it short of a restart.
+#      healthz does NOT report it, because the server-wide state is untouched —
+#      only an actual write observes it.
+#
+# So a health check alone misses case 2 entirely. Both probes run here, and a
+# sustained failure of either restarts the service. The multi-sample window
+# avoids restarting during transient startup/quorum 503s.
+
+set -eu
+
+HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
+SERVICE="spinifex-nats.service"
+SPX="${SPX_BIN:-/usr/local/bin/spx}"
+SAMPLES=3        # consecutive failing samples required before acting
+SAMPLE_GAP=2     # seconds between samples
+PROBE_TIMEOUT=5  # seconds for one canary round-trip
+
+# Only act on a running service: a stopped/activating unit is systemd's to manage,
+# and try-restart on it would either no-op or fight an in-flight (re)start.
+if ! systemctl is-active --quiet "$SERVICE"; then
+    exit 0
+fi
+
+# curl is the probe transport; if it is unavailable, do nothing rather than
+# false-restart a healthy server.
+if ! command -v curl >/dev/null 2>&1; then
+    echo "nats-js-watchdog: curl not found, skipping JS health probe" >&2
+    exit 0
+fi
+
+js_healthy() {
+    curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
+}
+
+# Round-trips a canary key through the cluster-state bucket. Absent spx this
+# degrades to the health check alone rather than restarting on a missing tool.
+js_writable() {
+    if [ ! -x "$SPX" ]; then
+        return 0
+    fi
+    "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+}
+
+failure=""
+i=1
+while [ "$i" -le "$SAMPLES" ]; do
+    if js_healthy; then
+        if js_writable; then
+            exit 0
+        fi
+        failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
+    else
+        failure="JetStream unhealthy at $HEALTH_URL"
+    fi
+    if [ "$i" -lt "$SAMPLES" ]; then
+        sleep "$SAMPLE_GAP"
+    fi
+    i=$((i + 1))
+done
+
+echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
+systemctl try-restart "$SERVICE"

build/scripts/predastore-start.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-# Helper script for spinifex-predastore.service
-# Detects host, port, backend, and node-id from spinifex.toml
-# then execs spx. Falls back to single-node defaults if no config found.
-
-CONF=/etc/spinifex/spinifex.toml
-BIND="0.0.0.0:8443"
-# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
-# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
-# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
-# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
-NODE_ID="-1"
-
-if [ -f "$CONF" ]; then
-    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
-    if [ -n "$H" ]; then
-        BIND="$H"
-    fi
-
-    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ]; then
-        NODE_ID="$N"
-    fi
-fi
-
-export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
-export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
-export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
-
-exec /usr/local/bin/spx service predastore start

build/systemd/regenerate-ssh-host-keys.service

@@ -7,6 +7,9 @@ Before=ssh.service
 Type=oneshot
 ExecStart=/usr/bin/ssh-keygen -A
 RemainAfterExit=yes
+# No ExecStop: the keygen process has already exited by the time this unit
+# is stopped, so there is nothing to signal.
+TimeoutStopSec=5
 
 [Install]
 WantedBy=multi-user.target

build/systemd/spinifex-awsgw.service

@@ -5,15 +5,24 @@ After=spinifex-nats.service spinifex-daemon.service
 Wants=spinifex-nats.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-gw
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
+# No SIGTERM handler in this process: it exits immediately on signal, so this
+# only bounds an unexpectedly wedged process before SIGKILL.
+TimeoutStopSec=10
 
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/
 Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-daemon.service

@@ -2,66 +2,36 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
-# Wants=openvswitch-ipsec.service so systemd brings the IPsec userspace
-# daemon up before spinifex flips other_config:ipsec_encapsulation=true.
-# Without that daemon, OVS holds the cert pointers unused and strongSwan
-# never gets IKE config — tunnel traffic is encrypted with no SAs and
-# silently dropped. Wants= (not Requires=) keeps daemon bootable on
-# single-node / lab hosts where the package is intentionally absent.
+# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
 Wants=openvswitch-ipsec.service
-# No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during
-# `systemctl restart spinifex-daemon`, which would mask the standalone-startup
-# path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers
-# cold boot.
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see
+EnvironmentFile=-/etc/spinifex/host.env
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-daemon
 Group=spinifex
-# DDIL Tier 1 (siv-17): no ExecStartPre wait-for-nats. The daemon starts in
-# standalone mode and reconnects to NATS in the background — process-exiting
-# on a missing NATS would kill the local VM management plane.
-#
-# Opt-in strict-startup (§1d-strict, siv-44): set SPINIFEX_REQUIRE_NATS=1 to
-# restore pre-DDIL fail-fast UX. First NATS connect uses a ~30s bounded retry
-# and the daemon aborts on timeout. Intended for dev/test/single-node deploys
-# where silent NATS misconfig is more confusing than helpful. Do NOT set this
-# in the shipped unit — it re-introduces the SPOF that Tier 1 removed.
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5
-# DDIL Tier 1: KillMode=process so systemd only signals the
-# daemon's main PID, not the cgroup. QEMU children stay alive across
-# `systemctl restart spinifex-daemon` and the new daemon reattaches to them
-# via the persisted local state file + pidfile + QMP socket. With the
-# default control-group mode, restart would terminate every guest VM —
-# defeating Scenario B's "daemon restart preserves running instances".
+# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)
 KillMode=process
-# 15s upper bound on the SIGTERM grace window. The new SIGTERM handler
-# skips VM teardown (it now just persists state and exits), so 15s is
-# plenty of slack for the cluster-manager + NATS unsubscribe path.
 TimeoutStopSec=15
+# RG-4: infra tier — reaped only after every guest
 OOMScoreAdjust=-500
+# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead
+MemoryHigh=75%
+MemoryMax=85%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
-# DDIL Tier 1: QEMU pidfiles, QMP sockets, and the
-# shared qemu.log all resolve via utils.pidPath() — which falls back
-# to os.TempDir() (=/tmp) when $HOME/spinifex doesn't exist. With
-# PrivateTmp=yes, each daemon start gets a fresh /tmp namespace, so
-# pidfiles written by QEMU (running in the OLD daemon's namespace)
-# are invisible to the new daemon. The /local/instances handler then
-# reports PID=0 and Scenario B fails its "live QEMU PID" assertion.
-# Pin XDG_RUNTIME_DIR to the host-wide tmpfs at /run/spinifex (created
-# by tmpfiles.d, already in ReadWritePaths) so pidPath() returns a
-# stable, namespace-independent path that both daemon incarnations and
-# QEMU children resolve identically.
+# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)
 Environment=XDG_RUNTIME_DIR=/run/spinifex
 
-# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd)
-# CAP_SYS_ADMIN: required for GPU passthrough sysfs operations (vfio group management).
-# CAP_DAC_OVERRIDE: sysfs driver bind/unbind paths (/sys/bus/pci/drivers/vfio-pci/bind,
-# unbind) are root:root 0200; non-root processes need CAP_DAC_OVERRIDE to open them.
+# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,
+# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.
 AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no
@@ -71,9 +41,11 @@ DeviceAllow=/dev/net/tun rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
+# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
+# ProtectKernelTunables=yes would otherwise keep read-only.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf
 
-# Hardening
+# RG-9: hardening baseline
 ProtectHome=yes
 PrivateTmp=yes
 RestrictSUIDSGID=yes
@@ -88,8 +60,6 @@ ProtectControlGroups=yes
 ProtectProc=invisible
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
-# DHCP lease acquisition moved to spinifex-vpcd (see dhcp-allocate-error.md);
-# the daemon no longer needs AF_PACKET or the softened kernel protections.
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
 ProtectHostname=yes
 KeyringMode=private
@@ -97,8 +67,7 @@ RemoveIPC=yes
 LimitCORE=0
 LimitNOFILE=65536
 LimitNPROC=4096
-# VFIO GPU passthrough requires locking all guest RAM into physical memory.
-# Without this, QEMU fails with RLIMIT_MEMLOCK exceeded when pinning guest pages.
+# LimitMEMLOCK=infinity: VFIO pins all guest RAM
 LimitMEMLOCK=infinity
 
 [Install]

build/systemd/spinifex-guests.slice

@@ -0,0 +1,8 @@
+[Unit]
+Description=Spinifex guest VM tier
+Before=slices.target
+
+[Slice]
+# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of
+# spinifex-system. Parent for the per-instance spinifex-instance-<id>.slice
+# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.

build/systemd/spinifex-nats-watchdog.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog
+PartOf=spinifex.target
+After=spinifex-nats.service
+
+[Service]
+Type=oneshot
+# Runs as root: the probes are a local curl to the NATS monitoring port and a
+# canary KV round-trip via spx, and the remedy is `systemctl try-restart
+# spinifex-nats` when JetStream stops accepting writes while the process stays
+# alive (which Restart=on-failure cannot catch).
+ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)
+# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than
+# cutting it off. Only the failing path runs that long; a healthy node exits on
+# the first sample.
+TimeoutStopSec=45
+
+# Hardening (kept compatible with talking to PID1 over /run/systemd).
+NoNewPrivileges=yes
+ProtectHome=yes
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+LockPersonality=yes
+RestrictSUIDSGID=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -0,0 +1,13 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog timer
+PartOf=spinifex.target
+
+[Timer]
+# Delay the first probe so NATS startup + JetStream cluster quorum settle (a
+# forming cluster reports JS not-current transiently, which is not a fault).
+OnBootSec=60s
+OnUnitActiveSec=30s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target

build/systemd/spinifex-nats.service

@@ -4,13 +4,23 @@ PartOf=spinifex.target
 Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-nats
 Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
 Restart=on-failure
 RestartSec=5
+# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft
+# stepdown, client drain) with no internal deadline; every dependent service
+# stops before nats does, so size this for nats's own graceful-stop needs.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 
 # Filesystem access

build/systemd/spinifex-northstar.service

@@ -0,0 +1,61 @@
+[Unit]
+Description=Spinifex Northstar (Authoritative DNS)
+PartOf=spinifex.target
+After=spinifex-nats.service spinifex-predastore.service
+Wants=spinifex-predastore.service
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+User=spinifex-northstar
+Group=spinifex
+ExecStart=/usr/local/bin/spx service northstar start
+Restart=on-failure
+RestartSec=5
+# server.Shutdown() is called with an internal 5s context; margin covers the
+# NATS unsubscribe/close and process exit that follow it.
+TimeoutStopSec=10
+OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/
+
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/northstar /etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/northstar /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+# RG-9: exactly CAP_NET_BIND_SERVICE so the unprivileged user binds :53 (no root).
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
+[Install]
+WantedBy=spinifex.target

build/systemd/spinifex-predastore.service

@@ -5,14 +5,29 @@ After=spinifex-nats.service
 Wants=spinifex-nats.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+# Guest block I/O arrives with no caller trace; sample those roots at 5% so
+# they don't drown request traces. Request-linked spans are always kept.
+Environment=MULGA_ROOT_TRACE_RATIO=0.05
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-storage
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
-ExecStart=/var/lib/spinifex/predastore-start.sh
+ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
+# Matches viperblock's drain budget: viperblock's WAL flush writes into
+# predastore, and After= already stops viperblock first, but this is the
+# backstop if that ordering is ever bypassed (direct stop, future edit).
+TimeoutStopSec=30
 OOMScoreAdjust=-500
+# Legitimately the most memory-hungry infra service (S3 buffering/caching);
+# High/Max stay well above idle so normal load isn't throttled, while still
+# bounding a leak short of the slice-wide cap.
+MemoryHigh=40%
+MemoryMax=55%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem

build/systemd/spinifex-qmp-collector.service

@@ -0,0 +1,63 @@
+[Unit]
+Description=Spinifex QMP Telemetry Collector (guest metrics)
+PartOf=spinifex.target
+After=spinifex-nats.service spinifex-daemon.service
+Wants=spinifex-nats.service
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+# spinifex-daemon user: the collector reads the daemon's telemetry QMP
+# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.
+User=spinifex-daemon
+Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/usr/local/bin/spx service qmp-collector start
+Restart=on-failure
+RestartSec=5
+# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown
+# is near-instant, so this only bounds an unexpectedly wedged process.
+TimeoutStopSec=10
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
+# Same runtime dir as the daemon: telemetry sockets + metadata live there
+Environment=XDG_RUNTIME_DIR=/run/spinifex
+
+# RG-9: read-mostly telemetry tier — no capabilities, no device access
+ProtectSystem=full
+PrivateDevices=yes
+NoNewPrivileges=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex
+
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads
+# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=512
+
+[Install]
+WantedBy=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Spinifex graceful guest drain
+PartOf=spinifex.target
+# After= all services so it stops FIRST on target/host shutdown: the drain runs
+# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).
+After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/true
+# ExecStop fires on ANY target stop, including the stop half of `systemctl
+# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
+# of whether the host is actually going down. --only-if-host-stopping drains
+# only when systemd itself is unwinding into shutdown.target; a plain restart
+# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
+# (KillMode=process) and the daemon reconnects to them on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
+TimeoutStopSec=180
+
+[Install]
+WantedBy=spinifex.target

build/systemd/spinifex-system.slice

@@ -0,0 +1,9 @@
+[Unit]
+Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
+Before=slices.target
+
+[Slice]
+# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the
+# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU
+# moves to the sibling spinifex-guests.slice (per-instance placement lands in
+# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.

build/systemd/spinifex-ui.service

@@ -4,22 +4,32 @@ PartOf=spinifex.target
 After=spinifex-awsgw.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-ui
 Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
+# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers
+# long streaming proxy responses, e.g. console passthrough); add margin above it.
+TimeoutStopSec=40
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
 
 # Filesystem access
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
 ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
-ReadWritePaths=/var/log/spinifex
+ReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex
 
 # Hardening
 ProtectHome=yes

build/systemd/spinifex-viperblock.service

@@ -5,7 +5,9 @@ After=spinifex-nats.service spinifex-predastore.service
 Wants=spinifex-nats.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-viperblock
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
@@ -13,6 +15,16 @@ ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+# Per-service ceiling bounds a leak in this service alone rather than
+# tripping the slice-wide cap, which would pick an arbitrary victim.
+MemoryHigh=15%
+MemoryMax=25%
+# KillMode=process: nbdkit still serving a guest survives a restart here, the
+# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
+# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
+KillMode=process
+# Bounds this process's own WAL drain, which the plugin caps at 20s.
+TimeoutStopSec=30
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/

build/systemd/spinifex-vpcd.service

@@ -5,27 +5,44 @@ After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ope
 Wants=spinifex-nats.service ovn-controller.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-vpcd
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5
+# Waits for an in-flight OVN drift-reconcile pass to return before exiting;
+# no sub-timeout bounds that call today, so this covers a normal pass.
+TimeoutStopSec=20
 OOMScoreAdjust=-500
+# Per-service ceiling: bounds a leak here instead of the slice-wide cap.
+MemoryHigh=10%
+MemoryMax=20%
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
 
-# Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl)
+# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
+# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
+# sandbox is restored.
+#
+# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and
+# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR
+# update that retires the last escalation path.
+AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+NoNewPrivileges=no
 ProtectSystem=full
-PrivateDevices=no
-ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
 
-# Hardening
+# RG-9: hardening baseline
 ProtectHome=yes
 PrivateTmp=yes
-RestrictSUIDSGID=no
+RestrictSUIDSGID=yes
 LockPersonality=yes
 RestrictRealtime=yes
 RestrictNamespaces=yes
@@ -36,18 +53,9 @@ ProtectKernelLogs=yes
 ProtectControlGroups=yes
 ProtectProc=invisible
 SystemCallArchitectures=native
-MemoryDenyWriteExecute=no
-# AF_PACKET: required by the in-process DHCP client (github.com/insomniacslk/dhcp
-# / nclient4) for raw L2 BOOTP on br-wan before an IP is configured on the
-# interface. See docs/development/bugs/dhcp-allocate-error.md.
+MemoryDenyWriteExecute=yes
+# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
-# CAP_NET_RAW: open AF_PACKET sockets.
-# CAP_NET_BIND_SERVICE: covers unicast RENEW to UDP :68.
-# CAP_NET_ADMIN: already used for ovs-vsctl / ip link wiring on the bridge.
-# Note, disabled for now, sudo issue with ovs
-AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
-CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
-NoNewPrivileges=no
 ProtectHostname=yes
 KeyringMode=private
 RemoveIPC=yes

build/systemd/spinifex.slice

@@ -0,0 +1,13 @@
+[Unit]
+Description=Spinifex resource-bounded slice (host ceiling)
+Before=slices.target
+
+[Slice]
+# RG-1: no CPUWeight — slice roots are siblings; capping below the default would
+# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on
+# system.slice (system.slice.d/spinifex-reserve.conf).
+# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).
+MemoryHigh=80%
+MemoryMax=90%
+# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.
+MemorySwapMax=8G

build/systemd/spinifex.target

@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
 
 [Install]
 WantedBy=multi-user.target

cmd/aws-model-coverage/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+)
+
+func main() {
+	dispatch := gateway.AWSOperationInventory()
+	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
+	for _, service := range awsmodel.Services() {
+		inventory, ok := dispatch[string(service)]
+		var modelInventory awsmodel.DispatchInventory
+		switch {
+		case ok:
+			modelInventory = awsmodel.DispatchInventory{
+				Registered:  inventory.Registered,
+				Stubbed:     inventory.Stubbed,
+				Unsupported: inventory.Unsupported,
+			}
+		case service == awsmodel.S3:
+			modelInventory = awsmodel.DispatchInventory{
+				Opaque: true,
+				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
+			}
+		default:
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
+			os.Exit(1)
+		}
+
+		coverage, err := awsmodel.CompareOperations(service, modelInventory)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+			os.Exit(1)
+		}
+		coverages = append(coverages, coverage)
+	}
+
+	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+}

cmd/dhcptest/main.go

@@ -6,7 +6,7 @@
 // Usage (must run as root — needs AF_PACKET):
 //
 //	sudo dhcptest --iface=br-wan
-//	sudo dhcptest --iface=br-wan --mac=02:00:00:7c:b7:b9 --promisc
+//	sudo dhcptest --iface=br-wan --mac=02:54:df:71:22:5c --promisc
 //	sudo dhcptest --iface=br-wan --spinifex-id=gateway-wan1
 package main
 
@@ -14,7 +14,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
+	"log/slog"
 	"net"
 	"os"
 	"os/exec"
@@ -23,6 +23,7 @@ import (
 
 	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
+	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
 )
 
 func main() {
@@ -36,6 +37,7 @@ func main() {
 	timeout := flag.Duration("timeout", 10*time.Second, "Per-attempt DHCP timeout")
 	retries := flag.Int("retries", 3, "Number of DORA attempts before giving up")
 	verbose := flag.Bool("verbose", false, "Print full DHCP packet contents")
+	broadcast := flag.Bool("broadcast", false, "Set the BOOTP broadcast flag, as vpcd's Acquire path does, so replies come back to ff:ff:ff:ff:ff:ff rather than unicast to the bridge MAC")
 	flag.Parse()
 
 	// Resolve the hardware address to use.
@@ -44,20 +46,25 @@ func main() {
 
 	switch {
 	case *spinifexID != "":
-		hwAddr, err = net.ParseMAC(generateMAC(*spinifexID))
+		// vpcd's own derivation, so the probe leases under the MAC the daemon
+		// would have used for this id.
+		hwAddr, err = dhcp.DeriveMAC(*spinifexID)
 		if err != nil {
-			log.Fatalf("generateMAC(%q): %v", *spinifexID, err)
+			slog.Error("Could not derive MAC", "spinifex-id", *spinifexID, "error", err)
+			os.Exit(1)
 		}
 		fmt.Printf("spinifex virtual MAC for %q: %s\n", *spinifexID, hwAddr)
 	case *macStr != "":
 		hwAddr, err = net.ParseMAC(*macStr)
 		if err != nil {
-			log.Fatalf("parse --mac %q: %v", *macStr, err)
+			slog.Error("Could not parse --mac", "mac", *macStr, "error", err)
+			os.Exit(1)
 		}
 	default:
 		hwAddr, err = ifaceMAC(*iface)
 		if err != nil {
-			log.Fatalf("get MAC for %s: %v", *iface, err)
+			slog.Error("Could not get MAC for interface", "iface", *iface, "error", err)
+			os.Exit(1)
 		}
 		fmt.Printf("using real interface MAC: %s\n", hwAddr)
 	}
@@ -83,7 +90,7 @@ func main() {
 
 	if *promisc {
 		if err := setPromisc(*iface, true); err != nil {
-			log.Printf("WARNING: failed to set promisc on %s: %v", *iface, err)
+			slog.Warn("Failed to set promisc on interface", "iface", *iface, "error", err)
 		} else {
 			fmt.Printf("set %s promisc ON\n\n", *iface)
 		}
@@ -96,25 +103,21 @@ func main() {
 		nclient4.WithRetry(*retries),
 	}
 	if *verbose {
-		opts = append(opts, nclient4.WithLogger(nclient4.ShortSummaryLogger{Printfer: log.New(os.Stderr, "[nclient4] ", 0)}))
+		opts = append(opts, nclient4.WithLogger(nclient4.ShortSummaryLogger{Printfer: slogPrintfer{}}))
 	}
 
 	client, err := nclient4.New(*iface, opts...)
 	if err != nil {
-		log.Fatalf("nclient4.New(%s): %v", *iface, err)
+		slog.Error("Could not create nclient4 client", "iface", *iface, "error", err)
+		os.Exit(1)
 	}
 	defer client.Close()
 
-	// Identity modifiers — client-id / hostname / vendor-class for upstream lease tagging.
-	var mods []dhcpv4.Modifier
-	if cid != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(cid))))
-	}
-	if *hostname != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptHostName(*hostname)))
-	}
-	if *vendorClass != "" {
-		mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClassIdentifier(*vendorClass)))
+	// Identity modifiers — client-id / hostname / vendor-class for upstream lease
+	// tagging. Built by the vpcd code path so the probe cannot drift from it.
+	mods := dhcp.IdentityModifiers(cid, *hostname, *vendorClass, hwAddr)
+	if *broadcast {
+		mods = append(mods, dhcpv4.WithBroadcast(true))
 	}
 
 	fmt.Println("Sending DHCP Discover...")
@@ -141,14 +144,12 @@ func main() {
 	}
 }
 
-// generateMAC is the same deterministic hash used by vpcd's topology.go.
-// It produces a locally-administered MAC (02:00:00:xx:xx:xx) from a resource ID.
-func generateMAC(resourceID string) string {
-	h := uint32(0)
-	for _, c := range resourceID {
-		h = h*31 + uint32(c)
-	}
-	return fmt.Sprintf("02:00:00:%02x:%02x:%02x", (h>>16)&0xff, (h>>8)&0xff, h&0xff)
+// slogPrintfer adapts nclient4's Printfer to slog. nclient4 only emits through
+// it under --verbose, so the packet summaries log at info level to stay visible.
+type slogPrintfer struct{}
+
+func (slogPrintfer) Printf(format string, v ...any) {
+	slog.Info(fmt.Sprintf(format, v...), "component", "nclient4")
 }
 
 func ifaceMAC(name string) (net.HardwareAddr, error) {

cmd/ecr-credential-provider/main.go

@@ -0,0 +1,175 @@
+// ecr-credential-provider is a kubelet exec credential provider that lets a
+// Spinifex EKS worker node pull images from the internal Spinifex ECR registry.
+//
+// kubelet invokes it per-pull, feeding a CredentialProviderRequest on STDIN and
+// reading a CredentialProviderResponse on STDOUT. The provider fetches the
+// node IAM-role credentials from IMDS, SigV4-signs GetAuthorizationToken against
+// the AWS gateway's ECR endpoint, and returns the minted AWS:<jwt> as Basic auth
+// for the requested registry host.
+//
+// Static node config (gateway URL, CA, region) is read from the cloud-init env
+// file /etc/spinifex-eks/agent.env (KEY=value); real env vars override it.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/guestenv"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+const (
+	credProviderAPIVersion = "credentialprovider.kubelet.k8s.io/v1"
+	defaultCacheDuration   = "10h"
+	// expiryMargin trims the token's real TTL so kubelet re-invokes the provider
+	// before the JWT actually expires mid-run.
+	expiryMargin = 30 * time.Minute
+
+	defaultEnvFile   = "/etc/spinifex-eks/agent.env"
+	defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
+	defaultIMDSBase  = "http://169.254.169.254/latest"
+)
+
+// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
+type CredentialProviderRequest struct {
+	APIVersion string `json:"apiVersion"`
+	Kind       string `json:"kind"`
+	Image      string `json:"image"`
+}
+
+// AuthConfig is a single registry credential entry.
+type AuthConfig struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// CredentialProviderResponse is the kubelet exec-provider response on STDOUT.
+type CredentialProviderResponse struct {
+	APIVersion    string                `json:"apiVersion"`
+	Kind          string                `json:"kind"`
+	CacheKeyType  string                `json:"cacheKeyType"`
+	CacheDuration string                `json:"cacheDuration"`
+	Auth          map[string]AuthConfig `json:"auth"`
+}
+
+// config holds the static node settings the provider needs.
+type config struct {
+	GatewayURL string
+	GatewayCA  string
+	Region     string
+	IMDSBase   string
+}
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
+	if err != nil {
+		slog.Error("ecr-credential-provider: build gateway client", "err", err)
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+
+	if err := run(os.Stdin, os.Stdout, cfg, httpClient); err != nil {
+		slog.Error("ecr-credential-provider: run failed", "err", err)
+		// Still emit a valid empty-auth response so kubelet has a parseable reply.
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+}
+
+// run is the testable core: it reads a CredentialProviderRequest from in,
+// resolves credentials via IMDS + the gateway, and writes a
+// CredentialProviderResponse to out. The httpClient must trust the gateway CA.
+func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error {
+	var req CredentialProviderRequest
+	if err := json.NewDecoder(in).Decode(&req); err != nil {
+		return fmt.Errorf("decode request: %w", err)
+	}
+	if req.Image == "" {
+		return fmt.Errorf("request image is empty")
+	}
+	imageHost := ecrauth.HostFromImage(req.Image)
+
+	creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
+	if err != nil {
+		return fmt.Errorf("fetch IMDS credentials: %w", err)
+	}
+
+	tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
+		creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
+	if err != nil {
+		return fmt.Errorf("get authorization token: %w", err)
+	}
+
+	auth := AuthConfig{Username: tok.Username, Password: tok.Password}
+	authMap := map[string]AuthConfig{imageHost: auth}
+	if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
+		authMap[tok.ProxyHost] = auth
+	}
+
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: cacheDurationFrom(tok.ExpiresAt),
+		Auth:          authMap,
+	}
+	return json.NewEncoder(out).Encode(&resp)
+}
+
+// cacheDurationFrom derives a Go-duration string from the token expiry minus a
+// safety margin, falling back to the default when expiry is unknown or too near.
+func cacheDurationFrom(expiresAt time.Time) string {
+	if expiresAt.IsZero() {
+		return defaultCacheDuration
+	}
+	d := time.Until(expiresAt) - expiryMargin
+	if d <= 0 {
+		return defaultCacheDuration
+	}
+	return d.Round(time.Second).String()
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override. The
+// IMDS base is fixed (not in the env file) but overridable for tests.
+func loadConfig(envFile string) config {
+	get := guestenv.Load(envFile).Get
+
+	cfg := config{
+		GatewayURL: get("EKS_GATEWAY_URL"),
+		GatewayCA:  get("EKS_GATEWAY_CA"),
+		Region:     get("EKS_REGION"),
+		IMDSBase:   get("EKS_IMDS_BASE"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	return cfg
+}
+
+// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
+func emitEmpty(out io.Writer) {
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: defaultCacheDuration,
+		Auth:          map[string]AuthConfig{},
+	}
+	if err := json.NewEncoder(out).Encode(&resp); err != nil {
+		slog.Error("ecr-credential-provider: encode empty response", "err", err)
+	}
+}

cmd/ecr-credential-provider/main_test.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// fakeBackend serves both the IMDS role/credentials endpoints and the gateway
+// ECR GetAuthorizationToken on one TLS server, routed by request path.
+func fakeBackend(t *testing.T) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+
+	// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
+	// header; the SDK client parses it from there, not the body.
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("imds-v2-token"))
+	})
+	// IMDS role name.
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			if r.Header.Get("X-Aws-Ec2-Metadata-Token") != "imds-v2-token" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		// Role credentials.
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"Code":            "Success",
+			"Type":            "AWS-HMAC",
+			"AccessKeyId":     "AKIATEST",
+			"SecretAccessKey": "secrettest",
+			"Token":           "sessiontoken",
+			"AccountId":       "123456789012",
+		})
+	})
+	// Gateway ECR GetAuthorizationToken (X-Amz-Target dispatch).
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		authToken := base64.StdEncoding.EncodeToString([]byte("AWS:testjwt"))
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": authToken,
+				"proxyEndpoint":      "https://123456789012.dkr.ecr.us-east-1.spinifex.internal",
+			}},
+		})
+	})
+
+	srv := httptest.NewTLSServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func clientTrusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestRun_ReturnsBasicAuth(t *testing.T) {
+	srv := fakeBackend(t)
+	client := clientTrusting(srv)
+
+	cfg := config{
+		GatewayURL: srv.URL,
+		Region:     "us-east-1",
+		IMDSBase:   srv.URL + "/latest",
+	}
+
+	const image = "123456789012.dkr.ecr.us-east-1.spinifex.internal/spinifex-demo:latest"
+	reqBody, err := json.Marshal(CredentialProviderRequest{
+		APIVersion: credProviderAPIVersion,
+		Kind:       "CredentialProviderRequest",
+		Image:      image,
+	})
+	if err != nil {
+		t.Fatalf("marshal request: %v", err)
+	}
+
+	var out bytes.Buffer
+	if err := run(bytes.NewReader(reqBody), &out, cfg, client); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+
+	var resp CredentialProviderResponse
+	if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
+		t.Fatalf("unmarshal response: %v\n%s", err, out.String())
+	}
+
+	if resp.Kind != "CredentialProviderResponse" {
+		t.Errorf("Kind = %q, want CredentialProviderResponse", resp.Kind)
+	}
+	if resp.CacheKeyType != "Registry" {
+		t.Errorf("CacheKeyType = %q, want Registry", resp.CacheKeyType)
+	}
+
+	host := "123456789012.dkr.ecr.us-east-1.spinifex.internal"
+	auth, ok := resp.Auth[host]
+	if !ok {
+		t.Fatalf("auth map missing host %q; got %v", host, resp.Auth)
+	}
+	if auth.Username != "AWS" {
+		t.Errorf("username = %q, want AWS", auth.Username)
+	}
+	if auth.Password != "testjwt" {
+		t.Errorf("password = %q, want testjwt", auth.Password)
+	}
+}

cmd/ecs-agent/agent.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	goruntime "runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// version is the agent build version, reported in the register message.
+// Overridable via -ldflags "-X main.version=...".
+var version = "dev"
+
+// Agent wires the ecs-agent's runtime seams: a gateway control-plane client, a
+// container runtime, and an ECR resolver. It registers the host as a container
+// instance at boot, heartbeats (re-registers) while alive, polls the gateway for
+// task assignments, and runs them through containerd, reporting state back over
+// the gateway. It never connects to NATS — the bus stays host-internal.
+type Agent struct {
+	cfg      config
+	id       identity
+	cp       controlPlane
+	puller   ctrruntime.ImagePuller
+	runner   ctrruntime.Runner
+	resolver ctrruntime.Resolver
+
+	reg *registrar
+	hb  *heartbeater
+
+	// netns builds awsvpc task network namespaces (nil-safe; bridge/host skip it).
+	netns *taskNetns
+
+	// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
+	cred *credEndpoint
+
+	// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
+	// Always non-nil (newAgent seeds an empty one); New() reseeds it from
+	// nvidia-smi discovery.
+	gpu *gpuLedger
+
+	closers []func() error
+}
+
+// newAgent assembles an Agent from already-built seams. Tests use this directly
+// with fakes; New builds the production seams and delegates here. runner may be
+// nil when containerd is unavailable; the assign path then reports the task
+// STOPPED rather than crashing.
+func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImagePuller, runner ctrruntime.Runner, resolver ctrruntime.Resolver) *Agent {
+	return &Agent{
+		cfg:      cfg,
+		id:       id,
+		cp:       cp,
+		puller:   puller,
+		runner:   runner,
+		resolver: resolver,
+		reg:      newRegistrar(cp, id),
+		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
+		netns:    newTaskNetns(execNetRunner{}),
+		gpu:      newGPULedger(id.Capacity.GPUIDs),
+	}
+}
+
+// New builds an Agent from config: it resolves the host identity from IMDS,
+// builds the gateway control-plane client, the ECR resolver and (best-effort)
+// the containerd runtime. A containerd connect failure is logged, not fatal —
+// registration and heartbeat still run so the instance is visible while the
+// runtime recovers. The ECR gateway client is built lazily on first image pull
+// (not here), so a missing or malformed gateway CA does not stop registration.
+func New(cfg config) (*Agent, error) {
+	imdsClient := &http.Client{Timeout: 5 * time.Second}
+
+	meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
+	if err != nil {
+		return nil, fmt.Errorf("instance metadata: %w", err)
+	}
+	host, _ := os.Hostname()
+	id := identity{
+		AccountID:    meta.AccountID,
+		ClusterName:  cfg.ClusterName,
+		InstanceID:   meta.InstanceID,
+		AZ:           meta.AZ,
+		Hostname:     host,
+		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
+		AgentVersion: version,
+	}
+
+	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
+	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
+
+	var puller ctrruntime.ImagePuller
+	var runner ctrruntime.Runner
+	if rt, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
+		slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
+	} else {
+		puller = rt
+		runner = rt
+	}
+
+	cp, err := newGatewayControlPlane(cfg, creds)
+	if err != nil {
+		return nil, fmt.Errorf("build gateway control-plane: %w", err)
+	}
+
+	a := newAgent(cfg, id, cp, puller, runner, resolver)
+	a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
+		cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
+	if puller != nil {
+		a.closers = append(a.closers, puller.Close)
+	}
+	return a, nil
+}
+
+// Run registers the instance, starts the heartbeat and assignment-poll loops,
+// and blocks until ctx is cancelled, then tears down.
+func (a *Agent) Run(ctx context.Context) error {
+	if err := a.reg.Register(); err != nil {
+		slog.Error("ecs-agent: initial registration failed", "err", err)
+	} else {
+		slog.Info("ecs-agent: registered", "cluster", a.id.ClusterName, "instance", a.id.InstanceID)
+	}
+
+	if a.cred != nil {
+		if err := a.cred.Start(); err != nil {
+			slog.Error("ecs-agent: credential endpoint start failed", "err", err)
+		}
+	}
+
+	go a.hb.Run(ctx)
+
+	// Re-adopt containers still running from before this restart, then poll with
+	// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
+	go a.pollAssignments(ctx, a.reconcile(ctx))
+
+	<-ctx.Done()
+	return a.Stop()
+}
+
+// Stop closes the runtime. Safe to call once.
+func (a *Agent) Stop() error {
+	var firstErr error
+	if a.cred != nil {
+		_ = a.cred.Stop()
+	}
+	for _, c := range a.closers {
+		if err := c(); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
+// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
+// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
+// non-GPU host).
+func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
+	return bus.InstanceCapacity{
+		CPU:       goruntime.NumCPU() * 1024,
+		MemoryMiB: memTotalMiB(),
+		GPU:       len(gpuUUIDs),
+		GPUIDs:    gpuUUIDs,
+	}
+}
+
+// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
+func memTotalMiB() int {
+	f, err := os.Open("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			return 0
+		}
+		kb, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return 0
+		}
+		return kb / 1024
+	}
+	return 0
+}

cmd/ecs-agent/agent_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
+	puller := &ctrruntime.FakePuller{}
+	a := newAgent(cfg, testIdentity(), cp, puller, puller, nil)
+	a.closers = append(a.closers, puller.Close)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	// Give it time to register + re-register (heartbeat) a few times.
+	time.Sleep(30 * time.Millisecond)
+	cancel()
+
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("Run returned error: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Run did not return after cancel")
+	}
+
+	// One boot register + at least one heartbeat re-register.
+	if cp.registerCount() < 2 {
+		t.Errorf("registers = %d, want >= 2 (boot + heartbeat)", cp.registerCount())
+	}
+	if !puller.Closed {
+		t.Error("Stop did not close the puller")
+	}
+}
+
+func TestDetectCapacity_Positive(t *testing.T) {
+	c := detectCapacity(nil)
+	if c.CPU <= 0 {
+		t.Errorf("CPU = %d, want > 0", c.CPU)
+	}
+	// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
+	if c.MemoryMiB < 0 {
+		t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
+	}
+	if c.GPU != 0 || c.GPUIDs != nil {
+		t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
+	}
+}
+
+func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
+	c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
+	if c.GPU != 2 {
+		t.Errorf("GPU = %d, want 2", c.GPU)
+	}
+	if len(c.GPUIDs) != 2 {
+		t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
+	}
+}

cmd/ecs-agent/config.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/guestenv"
+)
+
+const (
+	defaultEnvFile          = "/etc/spinifex-ecs/agent.env"
+	defaultGatewayCA        = "/etc/spinifex-ecs/gateway-ca.pem"
+	defaultIMDSBase         = "http://169.254.169.254/latest"
+	defaultContainerdSocket = "/run/containerd/containerd.sock"
+	defaultHeartbeat        = 30 * time.Second
+	defaultPollInterval     = 5 * time.Second
+)
+
+// config holds the static settings the agent reads at boot. Cluster identity
+// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
+// only the cluster *name* the instance was launched into is static. AccessKey /
+// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
+// gateway calls with instance-role credentials from IMDS; the agent never holds
+// a NATS token.
+type config struct {
+	GatewayURL       string
+	GatewayCA        string
+	Region           string
+	ClusterName      string
+	AccessKey        string
+	SecretKey        string
+	IMDSBase         string
+	ContainerdSocket string
+	Heartbeat        time.Duration
+	PollInterval     time.Duration
+	CredEndpointIP   string
+	CredEndpointPort int
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override.
+func loadConfig(envFile string) config {
+	get := guestenv.Load(envFile).Get
+
+	cfg := config{
+		GatewayURL:       get("ECS_GATEWAY_URL"),
+		GatewayCA:        get("ECS_GATEWAY_CA"),
+		Region:           get("ECS_REGION"),
+		ClusterName:      get("ECS_CLUSTER"),
+		AccessKey:        get("ECS_ACCESS_KEY"),
+		SecretKey:        get("ECS_SECRET_KEY"),
+		IMDSBase:         get("ECS_IMDS_BASE"),
+		ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	if cfg.ContainerdSocket == "" {
+		cfg.ContainerdSocket = defaultContainerdSocket
+	}
+	if cfg.ClusterName == "" {
+		cfg.ClusterName = "default"
+	}
+	cfg.Heartbeat = defaultHeartbeat
+	if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.Heartbeat = d
+		}
+	}
+	cfg.PollInterval = defaultPollInterval
+	if v := get("ECS_POLL_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.PollInterval = d
+		}
+	}
+	cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
+	if cfg.CredEndpointIP == "" {
+		cfg.CredEndpointIP = defaultCredEndpointIP
+	}
+	cfg.CredEndpointPort = defaultCredEndpointPort
+	if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
+		if p, err := strconv.Atoi(v); err == nil && p > 0 {
+			cfg.CredEndpointPort = p
+		}
+	}
+	return cfg
+}

cmd/ecs-agent/config_test.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestLoadConfig_Defaults(t *testing.T) {
+	// Ensure env vars don't leak in from the host.
+	for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
+		"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
+		"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
+		t.Setenv(k, "")
+	}
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	if cfg.GatewayCA != defaultGatewayCA {
+		t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
+	}
+	if cfg.IMDSBase != defaultIMDSBase {
+		t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
+	}
+	if cfg.ContainerdSocket != defaultContainerdSocket {
+		t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
+	}
+	if cfg.ClusterName != "default" {
+		t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
+	}
+	if cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
+	}
+	if cfg.PollInterval != defaultPollInterval {
+		t.Errorf("PollInterval = %v, want %v", cfg.PollInterval, defaultPollInterval)
+	}
+}
+
+func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
+	dir := t.TempDir()
+	envFile := filepath.Join(dir, "agent.env")
+	body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
+	if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
+	t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
+
+	cfg := loadConfig(envFile)
+	if cfg.GatewayURL != "https://gw.env" {
+		t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
+	}
+	if cfg.Region != "us-west-2" {
+		t.Errorf("Region = %q, want from file", cfg.Region)
+	}
+	if cfg.ClusterName != "prod" {
+		t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
+	}
+	if cfg.AccessKey != "AKIAENV" {
+		t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
+	}
+}
+
+func TestLoadConfig_HeartbeatOverride(t *testing.T) {
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
+		t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
+	}
+	// Garbage value falls back to the default, not zero.
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
+	}
+}

cmd/ecs-agent/controlplane.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ecs"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecsgw"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// controlPlane is the agent's channel to the AWS gateway. It replaces the direct
+// NATS publisher: every method is a SigV4-signed gateway request, so the NATS bus
+// stays host-internal. Tests inject a fake; production uses gatewayControlPlane.
+type controlPlane interface {
+	// Register (re-)registers the container instance. Called at boot and on every
+	// heartbeat tick — the gateway's RegisterContainerInstance is idempotent and
+	// refreshes LastSeen, folding liveness into registration (no bus.Heartbeat).
+	Register(id identity) error
+	// SubmitTaskState reports a task's RUNNING/STOPPED transition.
+	SubmitTaskState(st bus.TaskState) error
+	// PollAssignments drains this instance's assignment + stop inboxes, acking the
+	// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
+	// still-pending assignments and stop directives.
+	PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
+	// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
+	// containers, so the control plane can populate DescribeTasks gpuIds.
+	ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
+}
+
+// gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
+type gatewayControlPlane struct {
+	client *ecsgw.Client
+}
+
+var _ controlPlane = (*gatewayControlPlane)(nil)
+
+// pollTimeout bounds a single PollAssignments request; larger than the register
+// timeout to leave room for a future server-side long-poll.
+const pollTimeout = 30 * time.Second
+
+// newGatewayControlPlane builds the SigV4 client signing with the instance-role
+// credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
+func newGatewayControlPlane(cfg config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
+	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return &gatewayControlPlane{client: client}, nil
+}
+
+// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
+// per-call credentials hook.
+func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
+	return func(ctx context.Context) (string, string, string, error) {
+		c, err := p.Retrieve(ctx)
+		if err != nil {
+			return "", "", "", err
+		}
+		return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
+	}
+}
+
+func (g *gatewayControlPlane) Register(id identity) error {
+	in := &ecs.RegisterContainerInstanceInput{
+		Cluster:                  aws.String(id.ClusterName),
+		InstanceIdentityDocument: aws.String(id.InstanceID),
+		TotalResources: []*ecs.Resource{
+			{Name: aws.String("CPU"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.CPU))},
+			{Name: aws.String("MEMORY"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.MemoryMiB))},
+		},
+		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
+	}
+	if len(id.Capacity.GPUIDs) > 0 {
+		in.TotalResources = append(in.TotalResources, &ecs.Resource{
+			Name: aws.String("GPU"), Type: aws.String("STRINGSET"), StringSetValue: aws.StringSlice(id.Capacity.GPUIDs),
+		})
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal register: %w", err)
+	}
+	if _, err := g.client.Call("RegisterContainerInstance", body); err != nil {
+		return fmt.Errorf("register: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
+	in := &ecs.SubmitTaskStateChangeInput{
+		Cluster: aws.String(st.ClusterName),
+		Task:    aws.String(st.TaskID),
+		Status:  aws.String(st.LastStatus),
+		Reason:  aws.String(st.Reason),
+	}
+	for _, c := range st.Containers {
+		csc := &ecs.ContainerStateChange{
+			ContainerName: aws.String(c.Name),
+			Status:        aws.String(c.Status),
+			RuntimeId:     aws.String(c.ContainerID),
+		}
+		if c.ExitCode != nil {
+			csc.ExitCode = aws.Int64(int64(*c.ExitCode))
+		}
+		in.Containers = append(in.Containers, csc)
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal task-state: %w", err)
+	}
+	if _, err := g.client.Call("SubmitTaskStateChange", body); err != nil {
+		return fmt.Errorf("submit task-state: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	in := &handlers_ecs.PollAssignmentsInput{
+		Cluster:           cluster,
+		ContainerInstance: instance,
+		AckTaskIDs:        ackAssigns,
+		AckStopIDs:        ackStops,
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal poll: %w", err)
+	}
+	resp, err := g.client.Call("PollAssignments", body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("poll: %w", err)
+	}
+	var out handlers_ecs.PollAssignmentsOutput
+	if err := json.Unmarshal(resp, &out); err != nil {
+		return nil, nil, fmt.Errorf("decode poll: %w", err)
+	}
+	return out.Assignments, out.Stops, nil
+}
+
+func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	in := &handlers_ecs.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal report-task-gpu: %w", err)
+	}
+	if _, err := g.client.Call("ReportTaskGPU", body); err != nil {
+		return fmt.Errorf("report task gpu: %w", err)
+	}
+	return nil
+}

cmd/ecs-agent/controlplane_test.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+)
+
+// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// that only need a valid SigV4 signature, not real IMDS rotation.
+type fixedCredsProvider struct{}
+
+func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
+	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+}
+
+func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
+	t.Helper()
+	cp, err := newGatewayControlPlane(config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
+	if err != nil {
+		t.Fatalf("newGatewayControlPlane: %v", err)
+	}
+	return cp
+}
+
+// Register carries the discovered GPU UUIDs as a STRINGSET resource, matching
+// the control plane's RegisterContainerInstance GPU parsing (Epic C3).
+func TestGatewayControlPlane_Register_SendsGPUStringSet(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{
+		ClusterName: "default", InstanceID: "i-1",
+		Capacity: detectCapacity([]string{"GPU-aaa", "GPU-bbb"}),
+	}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	var gpu *ecs.Resource
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			gpu = r
+		}
+	}
+	if gpu == nil {
+		t.Fatal("no GPU resource in TotalResources")
+	}
+	if *gpu.Type != "STRINGSET" {
+		t.Errorf("GPU resource type = %q, want STRINGSET", *gpu.Type)
+	}
+	got := make([]string, len(gpu.StringSetValue))
+	for i, v := range gpu.StringSetValue {
+		got[i] = *v
+	}
+	if len(got) != 2 || got[0] != "GPU-aaa" || got[1] != "GPU-bbb" {
+		t.Errorf("GPU UUIDs = %v, want [GPU-aaa GPU-bbb]", got)
+	}
+}
+
+// A non-GPU host (no discovered UUIDs) registers with no GPU resource at all.
+func TestGatewayControlPlane_Register_NoGPUOmitsResource(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{ClusterName: "default", InstanceID: "i-1", Capacity: detectCapacity(nil)}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			t.Fatalf("unexpected GPU resource on a non-GPU host: %+v", r)
+		}
+	}
+}
+
+// ReportTaskGPU posts the per-container device UUIDs under the ReportTaskGPU
+// action, decodable as the handlers_ecs internal shape.
+func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
+	var gotTarget string
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotTarget = r.Header.Get("X-Amz-Target")
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	err := cp.ReportTaskGPU("web", "t-1", []handlers_ecs.ContainerGPUReport{
+		{Name: "trainer", GPUIDs: []string{"GPU-aaa"}},
+	})
+	if err != nil {
+		t.Fatalf("ReportTaskGPU: %v", err)
+	}
+	if gotTarget == "" {
+		t.Fatal("no X-Amz-Target header sent")
+	}
+
+	var in handlers_ecs.ReportTaskGPUInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	if in.Cluster != "web" || in.Task != "t-1" {
+		t.Errorf("cluster/task = %q/%q, want web/t-1", in.Cluster, in.Task)
+	}
+	if len(in.Containers) != 1 || in.Containers[0].Name != "trainer" || len(in.Containers[0].GPUIDs) != 1 {
+		t.Errorf("containers = %+v, want one trainer entry with one UUID", in.Containers)
+	}
+}

cmd/ecs-agent/credendpoint.go

@@ -0,0 +1,401 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+const (
+	// defaultCredEndpointIP is the AWS-reserved link-local address tasks reach for
+	// container credentials; defaultCredEndpointPort is the standard HTTP port.
+	defaultCredEndpointIP   = "169.254.170.2"
+	defaultCredEndpointPort = 80
+	// defaultCredProxyPort is the high port the listener actually binds; a nat
+	// DNAT rule rewrites 169.254.170.2:80 to it so nothing holds a socket on :80
+	// and host/bridge tasks sharing the netns can bind :80 themselves.
+	defaultCredProxyPort = 51679
+	// credRefreshMargin refreshes a cached credential set this long before expiry
+	// so a container never starts a request with a token about to lapse.
+	credRefreshMargin = 5 * time.Minute
+	// credDummyIface is the dummy interface the endpoint IP is added to in the host
+	// netns so the agent can bind it.
+	credDummyIface = "ecs-cred"
+)
+
+// credResponse is the ECS task-credentials JSON the container's AWS SDK expects
+// at AWS_CONTAINER_CREDENTIALS_RELATIVE_URI.
+type credResponse struct {
+	RoleArn         string `json:"RoleArn"`
+	AccessKeyID     string `json:"AccessKeyId"`
+	SecretAccessKey string `json:"SecretAccessKey"`
+	Token           string `json:"Token"`
+	Expiration      string `json:"Expiration"`
+}
+
+// credEntry caches one task's assumed-role credentials alongside the role it was
+// assumed from.
+type credEntry struct {
+	roleARN string
+	creds   stsauth.Credentials
+}
+
+// credEndpoint is the host-netns HTTP server that hands a task its IAM role
+// credentials. It maps credID -> taskRoleARN (registered as tasks start) and
+// mints/caches credentials on demand by assuming the role over the gateway with
+// the agent's instance credentials.
+type credEndpoint struct {
+	provider   credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+	ip         string
+	port       int
+	proxyPort  int
+	run        netCmdRunner
+
+	// assume mints fresh credentials for a role; defaults to AssumeRole over the
+	// gateway and is overridable in tests.
+	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
+
+	mu         sync.Mutex
+	roles      map[string]string    // credID -> taskRoleARN
+	cache      map[string]credEntry // credID -> cached creds
+	httpClient *http.Client
+
+	srv *http.Server
+	ln  net.Listener
+}
+
+// newCredEndpoint builds the endpoint. An empty ip/port falls back to the
+// AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
+// adds the link-local address to the dummy interface (real or fake).
+func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
+	if ip == "" {
+		ip = defaultCredEndpointIP
+	}
+	if port == 0 {
+		port = defaultCredEndpointPort
+	}
+	c := &credEndpoint{
+		provider:   provider,
+		region:     region,
+		gatewayURL: gatewayURL,
+		caPath:     caPath,
+		ip:         ip,
+		port:       port,
+		proxyPort:  defaultCredProxyPort,
+		run:        run,
+		roles:      map[string]string{},
+		cache:      map[string]credEntry{},
+	}
+	c.assume = c.assumeOverGateway
+	return c
+}
+
+// Register associates a credID with the task role it serves; Deregister removes
+// it and drops any cached credentials. Both are no-ops for an empty credID.
+func (c *credEndpoint) Register(credID, roleARN string) {
+	if credID == "" || roleARN == "" {
+		return
+	}
+	c.mu.Lock()
+	c.roles[credID] = roleARN
+	c.mu.Unlock()
+}
+
+func (c *credEndpoint) Deregister(credID string) {
+	if credID == "" {
+		return
+	}
+	c.mu.Lock()
+	delete(c.roles, credID)
+	delete(c.cache, credID)
+	c.mu.Unlock()
+}
+
+// relativeURI is the value injected into a container's
+// AWS_CONTAINER_CREDENTIALS_RELATIVE_URI for the given credID.
+func credRelativeURI(credID string) string { return "/v2/credentials/" + credID }
+
+// ServeHTTP serves GET /v2/credentials/{credID}. Unknown paths/credIDs return
+// 404; non-GET returns 405; an AssumeRole failure returns 502.
+func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	credID, ok := strings.CutPrefix(r.URL.Path, "/v2/credentials/")
+	if !ok || credID == "" || strings.Contains(credID, "/") {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	creds, roleARN, err := c.fetch(r.Context(), credID)
+	if err != nil {
+		if errors.Is(err, errUnknownCredID) {
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		http.Error(w, "credentials unavailable", http.StatusBadGateway)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(credResponse{
+		RoleArn:         roleARN,
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		Token:           creds.SessionToken,
+		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
+	}); err != nil {
+		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+	}
+}
+
+// errUnknownCredID signals a credID with no registered task role.
+var errUnknownCredID = fmt.Errorf("unknown credId")
+
+// fetch returns valid credentials for credID, reusing the cache until within the
+// refresh margin and otherwise assuming the task role over the gateway.
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+	c.mu.Lock()
+	roleARN, known := c.roles[credID]
+	if !known {
+		c.mu.Unlock()
+		return stsauth.Credentials{}, "", errUnknownCredID
+	}
+	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+		c.mu.Unlock()
+		return e.creds, roleARN, nil
+	}
+	c.mu.Unlock()
+
+	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	if err != nil {
+		return stsauth.Credentials{}, "", err
+	}
+
+	c.mu.Lock()
+	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.mu.Unlock()
+	return creds, roleARN, nil
+}
+
+// AssumeProvider returns a CredentialsProvider that mints credentials by assuming
+// roleARN over the gateway — the execution-role path for ECR image pulls. It
+// caches the assumed set and re-assumes within the refresh margin. session scopes
+// the STS session name.
+func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
+}
+
+// assumeProvider adapts a credEndpoint's AssumeRole path to the agent's
+// CredentialsProvider interface, caching the assumed credentials.
+type assumeProvider struct {
+	ep      *credEndpoint
+	roleARN string
+	session string
+
+	mu     sync.Mutex
+	cached stsauth.Credentials
+}
+
+var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+
+func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !credValid(p.cached, credRefreshMargin) {
+		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
+		if err != nil {
+			return credentials.Credentials{}, err
+		}
+		p.cached = creds
+	}
+	return credentials.Credentials{
+		AccessKeyID:     p.cached.AccessKeyID,
+		SecretAccessKey: p.cached.SecretAccessKey,
+		SessionToken:    p.cached.SessionToken,
+		Expiration:      p.cached.Expiration,
+	}, nil
+}
+
+// assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
+// against the gateway with the agent's instance credentials.
+func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+	instCreds, err := c.provider.Retrieve(ctx)
+	if err != nil {
+		return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
+	}
+	client, err := c.client()
+	if err != nil {
+		return stsauth.Credentials{}, err
+	}
+	return stsauth.AssumeRole(c.region, c.gatewayURL, client,
+		instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
+		roleARN, sessionName)
+}
+
+// client builds the pinned-CA gateway HTTP client on first use and caches it.
+func (c *credEndpoint) client() (*http.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.httpClient != nil {
+		return c.httpClient, nil
+	}
+	hc, err := ecrauth.GatewayHTTPClient(c.caPath)
+	if err != nil {
+		return nil, err
+	}
+	c.httpClient = hc
+	return hc, nil
+}
+
+// Start adds the endpoint IP to a dummy interface in the host netns, serves HTTP
+// on the high proxy port, and DNATs the advertised :80 to it. A bind, address,
+// or nat failure is returned so the caller can log it without aborting
+// register/heartbeat.
+func (c *credEndpoint) Start() error {
+	if err := c.addEndpointAddr(); err != nil {
+		return err
+	}
+	ln, err := net.Listen("tcp", net.JoinHostPort(c.ip, strconv.Itoa(c.listenPort())))
+	if err != nil {
+		return fmt.Errorf("listen %s:%d: %w", c.ip, c.listenPort(), err)
+	}
+	if err := c.addRedirect(); err != nil {
+		_ = ln.Close()
+		return err
+	}
+	c.ln = ln
+	c.srv = &http.Server{Handler: c, ReadHeaderTimeout: 5 * time.Second}
+	go func() {
+		if err := c.srv.Serve(ln); err != nil && err != http.ErrServerClosed {
+			slog.Error("ecs-agent: credential endpoint serve", "err", err)
+		}
+	}()
+	slog.Info("ecs-agent: credential endpoint listening", "addr", ln.Addr().String())
+	return nil
+}
+
+// Stop shuts the server down and removes the dummy interface. Safe on a nil/
+// never-started endpoint.
+func (c *credEndpoint) Stop() error {
+	if c == nil {
+		return nil
+	}
+	if c.srv != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		_ = c.srv.Shutdown(ctx)
+	}
+	c.delRedirect()
+	c.delEndpointAddr()
+	return nil
+}
+
+// listenPort is the port the socket binds. Loopback test mode binds the
+// caller-supplied (ephemeral) port directly; the real endpoint binds the high
+// proxy port and reaches :80 via the DNAT rules.
+func (c *credEndpoint) listenPort() int {
+	if c.ip == "127.0.0.1" {
+		return c.port
+	}
+	return c.proxyPort
+}
+
+// addRedirect installs nat DNAT rules rewriting the advertised :80 to the proxy
+// port, for both locally generated traffic (host/bridge task containers, OUTPUT)
+// and traffic arriving over the awsvpc credential veth (PREROUTING). The dst IP
+// is left on the endpoint IP so no route_localnet handling is needed. Skipped in
+// loopback test mode, which binds the port directly.
+func (c *credEndpoint) addRedirect() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		if _, err := c.run.Run("iptables", c.natArgs("-C", chain)...); err == nil {
+			continue // rule already present
+		}
+		if _, err := c.run.Run("iptables", c.natArgs("-A", chain)...); err != nil {
+			return fmt.Errorf("add nat redirect %s: %w", chain, err)
+		}
+	}
+	return nil
+}
+
+func (c *credEndpoint) delRedirect() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		_, _ = c.run.Run("iptables", c.natArgs("-D", chain)...)
+	}
+}
+
+// natArgs builds the iptables nat rule for op (-A/-C/-D) on chain: DNAT the
+// advertised endpoint :80 to the proxy port on the same IP.
+func (c *credEndpoint) natArgs(op, chain string) []string {
+	return []string{
+		"-t", "nat", op, chain,
+		"-d", c.ip, "-p", "tcp", "--dport", strconv.Itoa(c.port),
+		"-j", "DNAT", "--to-destination", net.JoinHostPort(c.ip, strconv.Itoa(c.proxyPort)),
+	}
+}
+
+// addEndpointAddr brings up the dummy interface holding the endpoint IP. Skipped
+// for loopback (tests bind 127.0.0.1 directly, no interface plumbing needed).
+func (c *credEndpoint) addEndpointAddr() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	_, _ = c.run.Run("ip", "link", "add", credDummyIface, "type", "dummy")
+	if _, err := c.run.Run("ip", "addr", "add", c.ip+"/32", "dev", credDummyIface); err != nil &&
+		!strings.Contains(err.Error(), "exists") {
+		return fmt.Errorf("add endpoint addr %s: %w", c.ip, err)
+	}
+	if _, err := c.run.Run("ip", "link", "set", credDummyIface, "up"); err != nil {
+		return fmt.Errorf("bring up %s: %w", credDummyIface, err)
+	}
+	return nil
+}
+
+func (c *credEndpoint) delEndpointAddr() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	_, _ = c.run.Run("ip", "link", "del", credDummyIface)
+}
+
+// credValid reports whether creds are usable now with margin to spare.
+func credValid(c stsauth.Credentials, margin time.Duration) bool {
+	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
+		return false
+	}
+	if c.Expiration.IsZero() {
+		return true
+	}
+	return time.Now().Add(margin).Before(c.Expiration)
+}
+
+// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// STS [\w+=,.@-]{2,64} constraint.
+func sessionName(credID string) string {
+	name := "ecs-" + credID
+	if len(name) > 64 {
+		name = name[:64]
+	}
+	return name
+}

cmd/ecs-agent/credendpoint_test.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+// newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
+// counting calls so cache behaviour is observable. run is nil (no iface plumbing).
+func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+	var calls int32
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		atomic.AddInt32(&calls, 1)
+		return creds, nil
+	}
+	return c, &calls
+}
+
+func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
+	exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
+	})
+	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String())
+	}
+	var got credResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || got.SecretAccessKey != "secret" || got.Token != "tok" {
+		t.Errorf("bad creds in response: %+v", got)
+	}
+	if got.RoleArn != "arn:aws:iam::111122223333:role/task" {
+		t.Errorf("RoleArn = %q", got.RoleArn)
+	}
+	if got.Expiration != exp.Format(time.RFC3339) {
+		t.Errorf("Expiration = %q, want %q", got.Expiration, exp.Format(time.RFC3339))
+	}
+}
+
+func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404", rr.Code)
+	}
+	if *calls != 0 {
+		t.Errorf("assume called %d times for unknown credID, want 0", *calls)
+	}
+}
+
+func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusMethodNotAllowed {
+		t.Fatalf("status = %d, want 405", rr.Code)
+	}
+}
+
+func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	for hit := range 3 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+		if rr.Code != http.StatusOK {
+			t.Fatalf("hit %d: status %d", hit, rr.Code)
+		}
+	}
+	if *calls != 1 {
+		t.Errorf("assume called %d times, want 1 (cached)", *calls)
+	}
+}
+
+func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
+	// Expiry inside the refresh margin -> every hit re-assumes.
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
+	})
+	c.Register("cred-1", "arn:role")
+	for range 2 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	}
+	if *calls != 2 {
+		t.Errorf("assume called %d times, want 2 (expiry within margin)", *calls)
+	}
+}
+
+func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	c.Deregister("cred-1")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404 after deregister", rr.Code)
+	}
+}
+
+func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		return stsauth.Credentials{}, context.DeadlineExceeded
+	}
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusBadGateway {
+		t.Fatalf("status = %d, want 502", rr.Code)
+	}
+}
+
+func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	c.port = 0 // bind an ephemeral port instead of privileged 80
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	defer c.Stop()
+
+	url := "http://" + c.ln.Addr().String() + credRelativeURI("cred-1")
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+}
+
+func TestCredEndpoint_RedirectInstallsAndRemovesDNAT(t *testing.T) {
+	// failOn the -C existence check so addRedirect falls through to -A.
+	f := &fakeNetRunner{failOn: "-t nat -C"}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "169.254.170.2", 80, f)
+	if c.listenPort() != defaultCredProxyPort {
+		t.Fatalf("listenPort = %d, want %d (proxy)", c.listenPort(), defaultCredProxyPort)
+	}
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	target := "-d 169.254.170.2 -p tcp --dport 80 -j DNAT --to-destination 169.254.170.2:51679"
+	for _, chain := range []string{"-A OUTPUT", "-A PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing add rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+	c.delRedirect()
+	for _, chain := range []string{"-D OUTPUT", "-D PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing delete rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+}
+
+func TestCredEndpoint_RedirectSkippedOnLoopback(t *testing.T) {
+	f := &fakeNetRunner{}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, f)
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	c.delRedirect()
+	if f.sawAny("iptables") {
+		t.Errorf("loopback mode must not touch iptables; calls=%v", f.joined())
+	}
+	if c.listenPort() != c.port {
+		t.Errorf("loopback listenPort = %d, want caller port %d", c.listenPort(), c.port)
+	}
+}
+
+func TestCredRelativeURI(t *testing.T) {
+	if got := credRelativeURI("abc"); got != "/v2/credentials/abc" {
+		t.Errorf("credRelativeURI = %q", got)
+	}
+}

cmd/ecs-agent/credentials/imds.go

@@ -0,0 +1,41 @@
+package credentials
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+)
+
+// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
+// (which refetches once the real expiry passes; ec2rolecreds also caps
+// Expires to at most an hour out). Safe for concurrent use.
+type IMDSProvider struct {
+	cache *aws.CredentialsCache
+}
+
+var _ CredentialsProvider = (*IMDSProvider)(nil)
+
+// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
+// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
+// default HTTP client.
+func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
+	provider := imdscreds.NewProvider(client, base)
+	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
+}
+
+// Retrieve returns cached credentials when still valid, else fetches fresh ones.
+func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
+	creds, err := p.cache.Retrieve(ctx)
+	if err != nil {
+		return Credentials{}, err
+	}
+	return Credentials{
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		SessionToken:    creds.SessionToken,
+		Expiration:      creds.Expires,
+	}, nil
+}

cmd/ecs-agent/credentials/imds_test.go

@@ -0,0 +1,107 @@
+package credentials
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
+// real /latest prefix the SDK IMDS client always requests.
+func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits != nil {
+			atomic.AddInt32(hits, 1)
+		}
+		_ = json.NewEncoder(w).Encode(creds)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestRetrieve_FetchesAndCaches(t *testing.T) {
+	var hits int32
+	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
+	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
+	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Token":           "session",
+		"Expiration":      exp.Format(time.RFC3339),
+	})
+
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	got, err := p.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("Retrieve: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
+		t.Errorf("creds mismatch: %+v", got)
+	}
+
+	// Second call well within validity must not re-hit IMDS.
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatalf("Retrieve 2: %v", err)
+	}
+	if hits != 1 {
+		t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
+	}
+}
+
+func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
+	var hits int32
+	// Already expired: aws.CredentialsCache refetches once the stored
+	// credentials' real expiry has passed, regardless of ExpiryWindow.
+	srv := imdsStub(t, &hits, map[string]string{
+		"Code":            "Success",
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+	})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if hits != 2 {
+		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
+	}
+}
+
+func TestRetrieve_CancelledContext(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := p.Retrieve(ctx); err == nil {
+		t.Fatal("expected error from cancelled context")
+	}
+}
+
+func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	if _, err := p.Retrieve(context.Background()); err == nil {
+		t.Fatal("expected error when Code != Success")
+	}
+}

cmd/ecs-agent/credentials/provider.go

@@ -0,0 +1,24 @@
+// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
+// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
+// with the host's instance-role credentials, fetched from IMDS and cached until
+// near expiry.
+package credentials
+
+import (
+	"context"
+	"time"
+)
+
+// Credentials is a SigV4 credential set with an expiry.
+type Credentials struct {
+	AccessKeyID     string
+	SecretAccessKey string
+	SessionToken    string
+	Expiration      time.Time
+}
+
+// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
+// must be safe for concurrent use.
+type CredentialsProvider interface {
+	Retrieve(ctx context.Context) (Credentials, error)
+}

cmd/ecs-agent/credinject_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+func TestTaskCredID(t *testing.T) {
+	cases := []struct {
+		name string
+		as   bus.Assign
+		want string
+	}{
+		{"no role", bus.Assign{TaskID: "t1"}, ""},
+		{"role, no credID -> taskID", bus.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role + credID", bus.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
+	}
+	for _, tc := range cases {
+		if got := taskCredID(&tc.as); got != tc.want {
+			t.Errorf("%s: taskCredID = %q, want %q", tc.name, got, tc.want)
+		}
+	}
+}
+
+func TestWithCredEnv(t *testing.T) {
+	base := map[string]string{"FOO": "bar"}
+
+	// No credID -> env returned unchanged (same map).
+	if got := withCredEnv(base, ""); got["FOO"] != "bar" || got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "" {
+		t.Errorf("blank credID mutated env: %v", got)
+	}
+
+	got := withCredEnv(base, "cred-1")
+	if got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "/v2/credentials/cred-1" {
+		t.Errorf("missing relative-uri: %v", got)
+	}
+	if got["FOO"] != "bar" {
+		t.Errorf("dropped existing env: %v", got)
+	}
+	// Original map must be untouched (copy semantics).
+	if _, ok := base["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]; ok {
+		t.Errorf("source env mutated: %v", base)
+	}
+}
+
+func TestSessionName(t *testing.T) {
+	if got := sessionName("abc"); got != "ecs-abc" {
+		t.Errorf("sessionName = %q", got)
+	}
+	long := sessionName(string(make([]byte, 100)))
+	if len(long) != 64 {
+		t.Errorf("sessionName not truncated to 64: len=%d", len(long))
+	}
+}

cmd/ecs-agent/ecrresolver.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+)
+
+// ecrResolver implements runtime.Resolver by minting an ECR authorization token
+// per pull: it fetches the host's IMDS credentials, SigV4-signs
+// GetAuthorizationToken against the gateway, and returns the AWS:<jwt> as a
+// user/password pair for containerd.
+//
+// The gateway HTTP client is built lazily on first Authorize, not at agent boot:
+// a missing or malformed gateway CA only matters when an image is actually
+// pulled, so the agent can still register + heartbeat on a host with absent or
+// wrong ECR config.
+type ecrResolver struct {
+	creds      credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+
+	mu         sync.Mutex
+	httpClient *http.Client
+}
+
+var _ runtime.Resolver = (*ecrResolver)(nil)
+
+// newECRResolver builds a resolver with an injected HTTP client (tests). The
+// client is used as-is and never rebuilt from caPath.
+func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
+}
+
+// newLazyECRResolver builds a resolver that constructs its gateway HTTP client
+// from caPath on first use (production path).
+func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
+}
+
+// client returns the gateway HTTP client, building it from caPath on first call
+// and caching the result. An injected client (newECRResolver) is returned as-is.
+func (e *ecrResolver) client() (*http.Client, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.httpClient != nil {
+		return e.httpClient, nil
+	}
+	c, err := ecrauth.GatewayHTTPClient(e.caPath)
+	if err != nil {
+		return nil, err
+	}
+	e.httpClient = c
+	return c, nil
+}
+
+// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
+// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
+// public registries (docker.io and friends) pull anonymously, so refs without
+// an ECR host must not drag in IMDS host credentials.
+func isECRRef(ref string) bool {
+	host, _, _ := strings.Cut(ref, "/")
+	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
+		return false // no registry host -> docker.io default
+	}
+	return strings.Contains(host, ".dkr.ecr.")
+}
+
+// Authorize returns the ECR token's user/password and proxy endpoint for ref.
+// Non-ECR refs resolve to anonymous pull (empty credentials).
+func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
+	if !isECRRef(ref) {
+		return "", "", "", nil
+	}
+	c, err := e.creds.Retrieve(ctx)
+	if err != nil {
+		return "", "", "", err
+	}
+	httpClient, err := e.client()
+	if err != nil {
+		return "", "", "", err
+	}
+	tok, err := ecrauth.GetAuthorizationToken(e.region, e.gatewayURL, httpClient,
+		c.AccessKeyID, c.SecretAccessKey, c.SessionToken)
+	if err != nil {
+		return "", "", "", err
+	}
+	return tok.Username, tok.Password, tok.ProxyHost, nil
+}

cmd/ecs-agent/ecrresolver_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+)
+
+// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
+type stubCreds struct {
+	c   credentials.Credentials
+	err error
+}
+
+func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
+	return s.c, s.err
+}
+
+func gatewayStub(t *testing.T, token, proxy string, status int) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if status != http.StatusOK {
+			w.WriteHeader(status)
+			return
+		}
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": base64.StdEncoding.EncodeToString([]byte(token)),
+				"proxyEndpoint":      proxy,
+			}},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func trusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestECRResolver_Authorize(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	user, pass, endpoint, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+	if endpoint != "123456789012.dkr.ecr.us-east-1.spinifex.internal" {
+		t.Errorf("endpoint = %q", endpoint)
+	}
+}
+
+func TestECRResolver_CredsError(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("no imds")}, "us-east-1", "https://x", http.DefaultClient)
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected creds error to propagate")
+	}
+}
+
+func TestECRResolver_GatewayError(t *testing.T) {
+	srv := gatewayStub(t, "", "", http.StatusInternalServerError)
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected gateway error")
+	}
+}
+
+func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	// Construction with a bogus CA path must not fail — only Authorize does.
+	r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected lazy gateway-client build to fail on missing CA")
+	}
+}
+
+// TestECRResolver_NonECRAnonymous asserts that a public-registry ref pulls
+// anonymously: Authorize returns empty creds without ever touching IMDS, even
+// when the credentials provider would error.
+func TestECRResolver_NonECRAnonymous(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("imds must not be called")}, "us-east-1", "https://x", http.DefaultClient)
+	for _, ref := range []string{"nginx:alpine", "docker.io/library/nginx:alpine", "ghcr.io/org/app:1"} {
+		user, pass, endpoint, err := r.Authorize(context.Background(), ref)
+		if err != nil || user != "" || pass != "" || endpoint != "" {
+			t.Errorf("ref %q: got (%q,%q,%q,%v), want anonymous", ref, user, pass, endpoint, err)
+		}
+	}
+}
+
+func TestIsECRRef(t *testing.T) {
+	cases := map[string]bool{
+		"nginx:alpine":            false,
+		"docker.io/library/nginx": false,
+		"ghcr.io/org/app:1":       false,
+		"registry:5000/app:1":     false,
+		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
+		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	}
+	for ref, want := range cases {
+		if got := isECRRef(ref); got != want {
+			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+		}
+	}
+}

cmd/ecs-agent/gpu_discovery.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os/exec"
+	"strings"
+)
+
+// commandRunner abstracts exec.Command so discovery is unit-testable without
+// shelling out. Production uses execCommandRunner; tests inject a fake.
+type commandRunner func(name string, args ...string) ([]byte, error)
+
+// execCommandRunner runs a real host command and returns its stdout.
+func execCommandRunner(name string, args ...string) ([]byte, error) {
+	return exec.Command(name, args...).Output()
+}
+
+// discoverNvidiaGPUs enumerates local NVIDIA GPU device UUIDs via nvidia-smi.
+// A missing binary or any run failure (no GPU host) yields an empty list, not
+// an error — the agent still registers with zero GPU capacity.
+func discoverNvidiaGPUs(run commandRunner) []string {
+	out, err := run("nvidia-smi", "--query-gpu=uuid", "--format=csv,noheader")
+	if err != nil {
+		return nil
+	}
+	return parseNvidiaSMIUUIDs(string(out))
+}
+
+// parseNvidiaSMIUUIDs splits nvidia-smi's newline-delimited UUID output,
+// dropping blank lines. Exported as a pure function so fixture strings can be
+// tested without exec.
+func parseNvidiaSMIUUIDs(out string) []string {
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	uuids := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		uuids = append(uuids, line)
+	}
+	return uuids
+}

cmd/ecs-agent/gpu_discovery_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseNvidiaSMIUUIDs(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want []string
+	}{
+		{"empty", "", nil},
+		{"whitespace only", "   \n\n  ", nil},
+		{"single", "GPU-11111111-1111-1111-1111-111111111111\n", []string{"GPU-11111111-1111-1111-1111-111111111111"}},
+		{"multi", "GPU-aaa\nGPU-bbb\nGPU-ccc\n", []string{"GPU-aaa", "GPU-bbb", "GPU-ccc"}},
+		{"blank lines between", "GPU-aaa\n\nGPU-bbb\n", []string{"GPU-aaa", "GPU-bbb"}},
+		{"trailing whitespace per line", "GPU-aaa \r\nGPU-bbb\r\n", []string{"GPU-aaa", "GPU-bbb"}},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := parseNvidiaSMIUUIDs(c.in)
+			if !reflect.DeepEqual(got, c.want) && (len(got) != 0 || len(c.want) != 0) {
+				t.Errorf("parseNvidiaSMIUUIDs(%q) = %v, want %v", c.in, got, c.want)
+			}
+		})
+	}
+}
+
+func TestDiscoverNvidiaGPUs_AbsentBinaryReturnsEmptyNoError(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return nil, errors.New(`exec: "nvidia-smi": executable file not found in $PATH`)
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list on absent nvidia-smi, got %v", got)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_ParsesFixtureOutput(t *testing.T) {
+	var gotName string
+	var gotArgs []string
+	run := func(name string, args ...string) ([]byte, error) {
+		gotName = name
+		gotArgs = args
+		return []byte("GPU-aaa\nGPU-bbb\n"), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	want := []string{"GPU-aaa", "GPU-bbb"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if gotName != "nvidia-smi" {
+		t.Errorf("command = %q, want nvidia-smi", gotName)
+	}
+	if len(gotArgs) != 2 || gotArgs[0] != "--query-gpu=uuid" || gotArgs[1] != "--format=csv,noheader" {
+		t.Errorf("args = %v, want the uuid/csv,noheader query", gotArgs)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_NoDevicesReturnsEmpty(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return []byte(""), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list, got %v", got)
+	}
+}

cmd/ecs-agent/gpu_ledger.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// gpuLedger is the agent's local source of truth for which of the host's
+// discovered GPU UUIDs are free vs pinned to a running task's container. C4
+// reads a task's pinned UUIDs to build the CDI device list for containerd.
+type gpuLedger struct {
+	mu     sync.Mutex
+	free   []string
+	pinned map[string][]string // key ("taskID/containerName") -> uuids
+}
+
+// newGPULedger seeds the ledger with the host's discovered device UUIDs. A nil
+// or empty slice is a valid non-GPU host: every Pin then fails, which callers
+// treat as best-effort (no device, no CDI injection later).
+func newGPULedger(uuids []string) *gpuLedger {
+	free := append([]string(nil), uuids...)
+	return &gpuLedger{free: free, pinned: make(map[string][]string)}
+}
+
+// gpuKey builds the ledger key for one container's pin/release.
+func gpuKey(taskID, containerName string) string {
+	return taskID + "/" + containerName
+}
+
+// Pin reserves n free UUIDs under key, returning them. An error means fewer
+// than n devices were free; the key is left unpinned so a retry (or a
+// best-effort caller) can proceed without a partial reservation.
+func (l *gpuLedger) Pin(key string, n int) ([]string, error) {
+	if n <= 0 {
+		return nil, nil
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if len(l.free) < n {
+		return nil, fmt.Errorf("gpu ledger: requested %d GPUs, %d free", n, len(l.free))
+	}
+	uuids := append([]string(nil), l.free[:n]...)
+	l.free = l.free[n:]
+	l.pinned[key] = uuids
+	return uuids, nil
+}
+
+// ReleaseTask frees every UUID pinned under any container key belonging to
+// taskID, called once the task's terminal STOPPED state is reported.
+func (l *gpuLedger) ReleaseTask(taskID string) {
+	prefix := taskID + "/"
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for key, uuids := range l.pinned {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		l.free = append(l.free, uuids...)
+		delete(l.pinned, key)
+	}
+}

cmd/ecs-agent/gpu_ledger_test.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestGPULedger_PinReturnsDistinctUUIDs(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c"})
+	got, err := l.Pin("t-1/web", 2)
+	if err != nil {
+		t.Fatalf("Pin: %v", err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("want 2 UUIDs, got %v", got)
+	}
+	if got[0] == got[1] {
+		t.Errorf("pinned UUIDs must be distinct, got %v", got)
+	}
+	if len(l.free) != 1 {
+		t.Errorf("free = %d, want 1 remaining", len(l.free))
+	}
+}
+
+func TestGPULedger_PinInsufficientFreeErrors(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	_, err := l.Pin("t-1/web", 2)
+	if err == nil {
+		t.Fatal("want error when requesting more GPUs than free")
+	}
+	if len(l.free) != 1 {
+		t.Errorf("failed pin must not consume any device, free = %d", len(l.free))
+	}
+}
+
+func TestGPULedger_PinZeroIsNoop(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	got, err := l.Pin("t-1/web", 0)
+	if err != nil || got != nil {
+		t.Errorf("Pin(0) = %v, %v; want nil, nil", got, err)
+	}
+}
+
+func TestGPULedger_ReleaseTaskFreesOnlyThatTasksDevices(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c", "GPU-d"})
+	if _, err := l.Pin(gpuKey("t-1", "web"), 2); err != nil {
+		t.Fatalf("pin t-1: %v", err)
+	}
+	if _, err := l.Pin(gpuKey("t-2", "worker"), 1); err != nil {
+		t.Fatalf("pin t-2: %v", err)
+	}
+	if len(l.free) != 1 {
+		t.Fatalf("free = %d, want 1 before release", len(l.free))
+	}
+
+	l.ReleaseTask("t-1")
+	if len(l.free) != 3 {
+		t.Errorf("free = %d, want 3 after releasing t-1's two devices", len(l.free))
+	}
+	if _, ok := l.pinned[gpuKey("t-2", "worker")]; !ok {
+		t.Error("t-2's pin should survive releasing t-1")
+	}
+	if _, ok := l.pinned[gpuKey("t-1", "web")]; ok {
+		t.Error("t-1's pin should be gone after release")
+	}
+}
+
+// Multi-device pinning across concurrent task assigns must hand out disjoint
+// UUID sets — no two tasks may ever share a device.
+func TestGPULedger_ConcurrentPinsHandOutDistinctUUIDs(t *testing.T) {
+	const nDevices = 20
+	uuids := make([]string, nDevices)
+	for i := range uuids {
+		uuids[i] = "GPU-" + string(rune('a'+i))
+	}
+	l := newGPULedger(uuids)
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	seen := make(map[string]bool)
+	errs := 0
+	for i := range nDevices {
+		wg.Add(1)
+		go func(n int) {
+			defer wg.Done()
+			key := gpuKey("t-concurrent", string(rune('A'+n)))
+			got, err := l.Pin(key, 1)
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				errs++
+				return
+			}
+			for _, u := range got {
+				if seen[u] {
+					t.Errorf("UUID %s handed out to more than one concurrent pin", u)
+				}
+				seen[u] = true
+			}
+		}(i)
+	}
+	wg.Wait()
+	if errs != 0 {
+		t.Errorf("want all %d pins to succeed (exactly enough devices), got %d errors", nDevices, errs)
+	}
+	if len(seen) != nDevices {
+		t.Errorf("distinct UUIDs handed out = %d, want %d", len(seen), nDevices)
+	}
+	if len(l.free) != 0 {
+		t.Errorf("free = %d, want 0 after all devices pinned", len(l.free))
+	}
+}
+
+// Releasing a task's devices concurrently with new pins never double-hands a
+// UUID: a device is only ever free or pinned to exactly one key at a time.
+func TestGPULedger_ConcurrentPinAndReleaseNoDoubleAssignment(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b"})
+	var wg sync.WaitGroup
+	for round := range 50 {
+		wg.Add(2)
+		taskID := "t-" + string(rune('a'+round%26))
+		go func(id string) {
+			defer wg.Done()
+			_, _ = l.Pin(gpuKey(id, "web"), 1)
+		}(taskID)
+		go func(id string) {
+			defer wg.Done()
+			l.ReleaseTask(id)
+		}(taskID)
+	}
+	wg.Wait()
+	// Invariant: every currently-pinned UUID appears in exactly one pinned entry
+	// and never simultaneously in free.
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	freeSet := make(map[string]bool, len(l.free))
+	for _, u := range l.free {
+		if freeSet[u] {
+			t.Errorf("duplicate UUID %s in free list", u)
+		}
+		freeSet[u] = true
+	}
+	pinnedSet := make(map[string]bool)
+	for _, uuids := range l.pinned {
+		for _, u := range uuids {
+			if pinnedSet[u] {
+				t.Errorf("UUID %s pinned under more than one key", u)
+			}
+			if freeSet[u] {
+				t.Errorf("UUID %s is both free and pinned", u)
+			}
+			pinnedSet[u] = true
+		}
+	}
+}

cmd/ecs-agent/heartbeat.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// heartbeater keeps the container instance alive by periodically re-registering
+// it through the gateway. Real ECS folds liveness into ACS; v1 folds it into an
+// idempotent RegisterContainerInstance, which refreshes the instance's LastSeen
+// so the scheduler can tell live instances from dead ones (no bus.Heartbeat).
+type heartbeater struct {
+	cp       controlPlane
+	id       identity
+	interval time.Duration
+}
+
+func newHeartbeater(cp controlPlane, id identity, interval time.Duration) *heartbeater {
+	if interval <= 0 {
+		interval = defaultHeartbeat
+	}
+	return &heartbeater{cp: cp, id: id, interval: interval}
+}
+
+// beat re-registers the instance once, refreshing its LastSeen.
+func (h *heartbeater) beat() error {
+	return h.cp.Register(h.id)
+}
+
+// Run beats every interval until ctx is cancelled. A failed beat is logged and
+// retried on the next tick rather than killing the loop.
+func (h *heartbeater) Run(ctx context.Context) {
+	ticker := time.NewTicker(h.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := h.beat(); err != nil {
+				slog.Warn("ecs-agent: heartbeat (re-register) failed", "err", err)
+			}
+		}
+	}
+}

cmd/ecs-agent/main.go

@@ -0,0 +1,39 @@
+// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
+// the ECS-AMI, with containerd baked in). It registers the host with the ECS
+// scheduler through the AWS gateway over TLS+SigV4 (never NATS), heartbeats by
+// re-registering, polls the gateway for task assignments, runs them through
+// containerd, and reports state back over the gateway.
+//
+// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
+// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
+// (KEY=value); real env vars override it. Host identity (account, instance, AZ)
+// comes from IMDS at boot.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	agent, err := New(cfg)
+	if err != nil {
+		slog.Error("ecs-agent: startup failed", "err", err)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if err := agent.Run(ctx); err != nil {
+		slog.Error("ecs-agent: run failed", "err", err)
+		os.Exit(1)
+	}
+}

cmd/ecs-agent/metadata.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+)
+
+// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
+type instanceMetadata struct {
+	AccountID  string
+	InstanceID string
+	AZ         string
+}
+
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
+// via the SDK IMDS client. base is the IMDS root, e.g.
+// "http://169.254.169.254/latest".
+func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
+	imdsClient := imdscreds.NewClient(client, base)
+	ctx := context.Background()
+
+	instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
+	}
+	az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
+	}
+
+	doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
+	}
+
+	return instanceMetadata{
+		AccountID:  doc.AccountID,
+		InstanceID: strings.TrimSpace(instanceID),
+		AZ:         strings.TrimSpace(az),
+	}, nil
+}
+
+// getMetadata reads a single meta-data path and returns its body as a string.
+func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
+	out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
+	if err != nil {
+		return "", err
+	}
+	defer out.Content.Close()
+	body, err := io.ReadAll(out.Content)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}

cmd/ecs-agent/metadata_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		// Real IMDS echoes the requested TTL back as a response header; the SDK
+		// client parses it from there, not the body.
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	guard := func(w http.ResponseWriter, r *http.Request) bool {
+		if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return false
+		}
+		return true
+	}
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("i-0abc123"))
+		}
+	})
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("us-east-1a"))
+		}
+	})
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
+		}
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestFetchInstanceMetadata_OK(t *testing.T) {
+	srv := metadataStub(t, true)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest")
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	if got.InstanceID != "i-0abc123" || got.AZ != "us-east-1a" || got.AccountID != "123456789012" {
+		t.Errorf("metadata mismatch: %+v", got)
+	}
+}
+
+func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
+	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
+	srv := metadataStub(t, true)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err != nil {
+		t.Fatalf("v2 path should succeed: %v", err)
+	}
+}
+
+func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("t"))
+	})
+	mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err == nil {
+		t.Fatal("expected parse error on bad identity document")
+	}
+}

cmd/ecs-agent/netns.go

@@ -0,0 +1,216 @@
+package main
+
+import (
+	"fmt"
+	"hash/crc32"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// netnsRunDir is the iproute2 netns mount directory; a name here is the netns
+// path containerd joins via the OCI network namespace path.
+const netnsRunDir = "/var/run/netns"
+
+// Credentials path constants: each task netns gets a /30 veth into the host netns
+// from 169.254.172.0/22 and a host-routed /32 to the credential endpoint, keeping
+// the endpoint off the task ENI's VPC route. credTaskVeth is the task-side name
+// (unique per netns); the host side is per-task (credVethName).
+const (
+	credTaskVeth     = "ecsc0"
+	credEndpointCIDR = defaultCredEndpointIP + "/32"
+)
+
+// udhcpcScript is the config script busybox udhcpc execs to apply a lease. It
+// is the reason presence of the udhcpc binary is not enough to use it: Debian
+// and Ubuntu's busybox ship the applet without this script, and udhcpc then
+// takes the lease and exits nonzero having configured nothing.
+const udhcpcScript = "/usr/share/udhcpc/default.script"
+
+// netCmdRunner executes a host networking command. Abstracted so unit tests can
+// assert the ip/netns sequence without touching the kernel.
+type netCmdRunner interface {
+	Run(name string, args ...string) (string, error)
+}
+
+// execNetRunner runs real commands.
+type execNetRunner struct{}
+
+func (execNetRunner) Run(name string, args ...string) (string, error) {
+	out, err := exec.Command(name, args...).CombinedOutput()
+	return string(out), err
+}
+
+// taskNetns builds and tears down per-task awsvpc network namespaces. The ENI is
+// already hot-plugged into the VM (scheduler did that); the agent finds the NIC
+// by MAC, moves it into a task netns, and DHCPs the OVN-assigned ENI IP — the
+// same way the primary NIC obtains its address.
+type taskNetns struct {
+	run     netCmdRunner
+	nicWait time.Duration
+	poll    time.Duration
+	// Injected so the DHCP-client choice below is unit-testable on a host that
+	// has neither client installed.
+	lookPath   func(string) (string, error)
+	fileIsExec func(string) bool
+}
+
+func newTaskNetns(run netCmdRunner) *taskNetns {
+	return &taskNetns{
+		run:        run,
+		nicWait:    30 * time.Second,
+		poll:       time.Second,
+		lookPath:   exec.LookPath,
+		fileIsExec: func(p string) bool { fi, err := os.Stat(p); return err == nil && fi.Mode()&0o111 != 0 },
+	}
+}
+
+// dhcpStep returns the in-netns DHCP command that leases the task ENI's
+// OVN-assigned address.
+//
+// This image ships on two distributions and they do not agree on a DHCP client.
+// Alpine has busybox udhcpc; Ubuntu has no udhcpc at all, and `ip netns exec`
+// exits 1 — not 127 — when it cannot exec the binary, so a hardcoded udhcpc
+// fails as an opaque "network setup failed" with the task stuck in STOPPED.
+// Resolve the client at setup time instead, preferring udhcpc so Alpine keeps
+// its existing behaviour exactly, then falling back to dhcpcd. Same order and
+// same reasoning as dhcp_oneshot() in mulga-mgmt-net.sh.
+//
+// Both invocations are one-shot: udhcpc -q exits once it has the lease and -n
+// gives up rather than backgrounding, and dhcpcd -1 exits after configuring the
+// interface. Neither leaves a daemon behind in the netns.
+func (n *taskNetns) dhcpStep(name, iface string) ([]string, error) {
+	if _, err := n.lookPath("udhcpc"); err == nil && n.fileIsExec(udhcpcScript) {
+		return []string{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"}, nil
+	}
+	// Lease state is keyed by interface name, which is unique per task here:
+	// each task ENI is a separate hot-plug, so it lands on its own PCI slot and
+	// gets its own predictable name before being moved into the netns.
+	if _, err := n.lookPath("dhcpcd"); err == nil {
+		return []string{"ip", "netns", "exec", name, "dhcpcd", "-q", "-1", "-t", "20", iface}, nil
+	}
+	return nil, fmt.Errorf("no DHCP client for %s: need udhcpc (with %s) or dhcpcd", iface, udhcpcScript)
+}
+
+func netnsName(taskID string) string { return "ecs-" + taskID }
+func netnsPathFor(taskID string) string {
+	return netnsRunDir + "/" + netnsName(taskID)
+}
+
+// Setup creates the task netns, moves the ENI NIC (matched by MAC) into it, and
+// brings it up with a DHCP lease. It returns the netns path for containerd. On
+// any failure it tears the partial netns down so a retry starts clean.
+func (n *taskNetns) Setup(taskID, mac string) (string, error) {
+	iface, err := n.waitForNIC(mac)
+	if err != nil {
+		return "", err
+	}
+	name := netnsName(taskID)
+	if _, err := n.run.Run("ip", "netns", "add", name); err != nil {
+		return "", fmt.Errorf("netns add %s: %w", name, err)
+	}
+	hostVeth := credVethName(taskID)
+	hostIP, taskIP := credSubnet(taskID)
+	dhcp, err := n.dhcpStep(name, iface)
+	if err != nil {
+		_ = n.Teardown(taskID)
+		return "", err
+	}
+	steps := [][]string{
+		{"ip", "link", "set", iface, "netns", name},
+		{"ip", "-n", name, "link", "set", "lo", "up"},
+		{"ip", "-n", name, "link", "set", iface, "up"},
+		dhcp,
+		// Credentials path: a veth into the host netns plus a host-routed /32 to the
+		// credential endpoint, so the container reaches 169.254.170.2 without it
+		// leaking onto the task ENI's VPC route.
+		{"ip", "link", "add", hostVeth, "type", "veth", "peer", "name", credTaskVeth, "netns", name},
+		{"ip", "addr", "add", hostIP + "/30", "dev", hostVeth},
+		{"ip", "link", "set", hostVeth, "up"},
+		{"ip", "-n", name, "addr", "add", taskIP + "/30", "dev", credTaskVeth},
+		{"ip", "-n", name, "link", "set", credTaskVeth, "up"},
+		{"ip", "-n", name, "route", "add", credEndpointCIDR, "via", hostIP},
+	}
+	for _, s := range steps {
+		if _, err := n.run.Run(s[0], s[1:]...); err != nil {
+			_ = n.Teardown(taskID)
+			return "", fmt.Errorf("netns setup %v: %w", s, err)
+		}
+	}
+	return netnsPathFor(taskID), nil
+}
+
+// credVethName is the host-side veth name for a task's credential path. Derived
+// from the taskID and kept within the 15-char interface-name limit.
+func credVethName(taskID string) string {
+	return fmt.Sprintf("ecsv%08x", crc32.ChecksumIEEE([]byte(taskID)))
+}
+
+// credSubnet derives a unique /30 in 169.254.172.0/22 for a task's credential
+// veth, returning the host-side and task-side addresses. The host side is the
+// container's gateway to the credential endpoint.
+func credSubnet(taskID string) (hostIP, taskIP string) {
+	slot := crc32.ChecksumIEEE([]byte(taskID)) % 256 // 256 /30s in the /22
+	third := 172 + (slot*4)/256
+	base := (slot * 4) % 256
+	hostIP = fmt.Sprintf("169.254.%d.%d", third, base+1)
+	taskIP = fmt.Sprintf("169.254.%d.%d", third, base+2)
+	return hostIP, taskIP
+}
+
+// Teardown deletes the task netns (releasing the moved NIC). Idempotent enough
+// for the stop path: a missing netns is reported but not fatal to the caller.
+func (n *taskNetns) Teardown(taskID string) error {
+	name := netnsName(taskID)
+	// Deleting the netns destroys the task-side veth and its host peer; remove the
+	// host side explicitly too so a leaked half from a partial setup is reaped.
+	_, _ = n.run.Run("ip", "link", "del", credVethName(taskID))
+	if _, err := n.run.Run("ip", "netns", "del", name); err != nil {
+		return fmt.Errorf("netns del %s: %w", name, err)
+	}
+	return nil
+}
+
+// waitForNIC polls until a NIC with the given MAC appears in the host netns,
+// returning its interface name. The hot-plug pipeline is async, so the device
+// may lag the assign by a few seconds.
+func (n *taskNetns) waitForNIC(mac string) (string, error) {
+	want := strings.ToLower(strings.TrimSpace(mac))
+	deadline := time.Now().Add(n.nicWait)
+	for {
+		iface, err := n.findNICByMAC(want)
+		if err == nil {
+			return iface, nil
+		}
+		if time.Now().After(deadline) {
+			return "", fmt.Errorf("nic with mac %s not found within %s", want, n.nicWait)
+		}
+		time.Sleep(n.poll)
+	}
+}
+
+// findNICByMAC parses `ip -o link` and returns the interface whose link/ether
+// matches mac.
+func (n *taskNetns) findNICByMAC(mac string) (string, error) {
+	out, err := n.run.Run("ip", "-o", "link")
+	if err != nil {
+		return "", fmt.Errorf("ip -o link: %w", err)
+	}
+	for line := range strings.SplitSeq(out, "\n") {
+		if !strings.Contains(strings.ToLower(line), mac) {
+			continue
+		}
+		// "<idx>: <ifname>: <flags> ... link/ether <mac> ..."
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		name := strings.TrimSuffix(strings.TrimSuffix(fields[1], ":"), "@")
+		if i := strings.IndexByte(name, '@'); i >= 0 {
+			name = name[:i]
+		}
+		return name, nil
+	}
+	return "", fmt.Errorf("no interface with mac %s", mac)
+}

cmd/ecs-agent/netns_test.go

@@ -0,0 +1,244 @@
+package main
+
+import (
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// fakeNetRunner records every command and serves canned `ip -o link` output. A
+// failOn substring forces the matching command to error so teardown paths run.
+type fakeNetRunner struct {
+	mu      sync.Mutex
+	calls   [][]string
+	linkOut string
+	failOn  string
+}
+
+func (f *fakeNetRunner) Run(name string, args ...string) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	cmd := append([]string{name}, args...)
+	f.calls = append(f.calls, cmd)
+	joined := strings.Join(cmd, " ")
+	if f.failOn != "" && strings.Contains(joined, f.failOn) {
+		return "", errors.New("boom: " + joined)
+	}
+	if name == "ip" && len(args) >= 2 && args[0] == "-o" && args[1] == "link" {
+		return f.linkOut, nil
+	}
+	return "", nil
+}
+
+func (f *fakeNetRunner) joined() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]string, len(f.calls))
+	for i, c := range f.calls {
+		out[i] = strings.Join(c, " ")
+	}
+	return out
+}
+
+func (f *fakeNetRunner) sawAny(sub string) bool {
+	for _, j := range f.joined() {
+		if strings.Contains(j, sub) {
+			return true
+		}
+	}
+	return false
+}
+
+const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:00:00:00 ...
+2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
+3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
+
+// withDHCPClients pins which DHCP clients the netns sees, so a test asserts the
+// branch it names rather than whatever happens to be installed on the machine
+// running it. Without this the DHCP step is host-dependent: an Alpine CI box and
+// an Ubuntu dev box would exercise different code from the same test.
+func withDHCPClients(n *taskNetns, present map[string]bool, scriptOK bool) {
+	n.lookPath = func(f string) (string, error) {
+		if present[f] {
+			return "/sbin/" + f, nil
+		}
+		return "", errors.New("not found")
+	}
+	n.fileIsExec = func(string) bool { return scriptOK }
+}
+
+// newTestNetns defaults to the Alpine arrangement — udhcpc plus its config
+// script — so the sequence assertions below keep describing that platform.
+func newTestNetns(f *fakeNetRunner) *taskNetns {
+	n := newTaskNetns(f)
+	n.nicWait = 50 * time.Millisecond
+	n.poll = 5 * time.Millisecond
+	withDHCPClients(n, map[string]bool{"udhcpc": true}, true)
+	return n
+}
+
+func TestNetns_FindNICByMAC(t *testing.T) {
+	n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+	iface, err := n.findNICByMAC("52:54:00:de:ad:01")
+	if err != nil || iface != "eth1" {
+		t.Fatalf("want eth1, got %q err=%v", iface, err)
+	}
+	if _, err := n.findNICByMAC("52:54:00:ff:ff:ff"); err == nil {
+		t.Fatal("want error for absent mac")
+	}
+}
+
+func TestNetns_SetupSequence(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	path, err := n.Setup("t-001", "52:54:00:DE:AD:01")
+	if err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	if path != netnsPathFor("t-001") {
+		t.Fatalf("want %q, got %q", netnsPathFor("t-001"), path)
+	}
+	hostIP, taskIP := credSubnet("t-001")
+	hostVeth := credVethName("t-001")
+	want := []string{
+		"ip netns add ecs-t-001",
+		"ip link set eth1 netns ecs-t-001",
+		"ip -n ecs-t-001 link set lo up",
+		"ip -n ecs-t-001 link set eth1 up",
+		"ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		"ip link add " + hostVeth + " type veth peer name ecsc0 netns ecs-t-001",
+		"ip addr add " + hostIP + "/30 dev " + hostVeth,
+		"ip -n ecs-t-001 addr add " + taskIP + "/30 dev ecsc0",
+		"ip -n ecs-t-001 route add 169.254.170.2/32 via " + hostIP,
+	}
+	for _, w := range want {
+		if !f.sawAny(w) {
+			t.Errorf("missing command %q in %v", w, f.joined())
+		}
+	}
+}
+
+func TestCredSubnet_UniquePer30(t *testing.T) {
+	hostA, taskA := credSubnet("task-a")
+	hostB, taskB := credSubnet("task-b")
+	if hostA == hostB {
+		t.Errorf("distinct tasks share a host IP: %s", hostA)
+	}
+	for _, ip := range []string{hostA, taskA, hostB, taskB} {
+		if !strings.HasPrefix(ip, "169.254.17") {
+			t.Errorf("cred IP %s outside 169.254.172.0/22", ip)
+		}
+	}
+	// host and task ends of one task sit in the same /30 (differ in last octet).
+	if hostA[:strings.LastIndexByte(hostA, '.')] != taskA[:strings.LastIndexByte(taskA, '.')] {
+		t.Errorf("host %s and task %s not in same /30", hostA, taskA)
+	}
+}
+
+func TestNetns_SetupNICNotFound(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:00:00:99"); err == nil {
+		t.Fatal("want timeout error for absent NIC")
+	}
+	if f.sawAny("netns add") {
+		t.Error("should not create netns when NIC never appears")
+	}
+}
+
+func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI, failOn: "udhcpc"}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when udhcpc fails")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown after step failure, got %v", f.joined())
+	}
+}
+
+// The image runs on two distributions with different DHCP clients, and picking
+// the wrong one fails as an opaque "network setup failed" with the task stuck
+// in STOPPED — `ip netns exec` exits 1, not 127, when it cannot exec the
+// binary. Each branch is asserted so that misresolution is a test failure.
+func TestNetns_DHCPStepPerPlatform(t *testing.T) {
+	tests := []struct {
+		name     string
+		present  map[string]bool
+		scriptOK bool
+		want     string
+		wantErr  bool
+	}{
+		{
+			name:     "alpine: udhcpc with its config script",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: true,
+			want:     "ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		},
+		{
+			// Ubuntu ships no udhcpc at all — the case that broke awsvpc tasks.
+			name:    "ubuntu: no udhcpc, falls back to dhcpcd",
+			present: map[string]bool{"dhcpcd": true},
+			want:    "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			// Debian/Ubuntu busybox ships the applet without the script, so
+			// presence of the binary alone must not select it.
+			name:     "udhcpc present but config script missing: still dhcpcd",
+			present:  map[string]bool{"udhcpc": true, "dhcpcd": true},
+			scriptOK: false,
+			want:     "ip netns exec ecs-t-001 dhcpcd -q -1 -t 20 eth1",
+		},
+		{
+			name:    "neither client: explicit error, not a silent bad command",
+			present: map[string]bool{},
+			wantErr: true,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+			withDHCPClients(n, tc.present, tc.scriptOK)
+			got, err := n.dhcpStep("ecs-t-001", "eth1")
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("want error with no DHCP client, got %v", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("dhcpStep: %v", err)
+			}
+			if joined := strings.Join(got, " "); joined != tc.want {
+				t.Errorf("want %q, got %q", tc.want, joined)
+			}
+		})
+	}
+}
+
+// A missing DHCP client must not leave the half-built netns behind for the next
+// attempt to trip over, same as any other failed setup step.
+func TestNetns_SetupTearsDownWhenNoDHCPClient(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	withDHCPClients(n, map[string]bool{}, false)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when no DHCP client is available")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown, got %v", f.joined())
+	}
+}
+
+func TestNetns_Teardown(t *testing.T) {
+	f := &fakeNetRunner{}
+	n := newTestNetns(f)
+	if err := n.Teardown("t-001"); err != nil {
+		t.Fatalf("teardown: %v", err)
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected netns del, got %v", f.joined())
+	}
+}

cmd/ecs-agent/reconcile.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// reconcile re-syncs the gateway with what this instance is actually running
+// after an agent restart. It lists the runtime's containers and, for each task
+// in this cluster, either re-adopts it (still running: re-register IAM
+// credentials, re-attach the exit-wait, refresh RUNNING) or reports it STOPPED
+// and clears its leftovers. The returned set seeds pollAssignments so the
+// gateway's re-delivered assignments for these tasks are acked but not re-run.
+//
+// Reporting the dead ones is the load-bearing half. The exit-wait is cancelled
+// when the agent shuts down, so a container that dies while the agent is down
+// has nobody to report its exit; the gateway holds the task RUNNING forever,
+// its service never replaces it, and the target never recovers.
+//
+// It is best-effort: a nil runner or a List error returns an empty set and the
+// agent falls back to its pre-reconcile behaviour (poll + run).
+func (a *Agent) reconcile(ctx context.Context) map[string]bool {
+	adopted := map[string]bool{}
+	if a.runner == nil {
+		return adopted
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: reconcile list failed", "err", err)
+		return adopted
+	}
+
+	// Group this cluster's containers back into their tasks, running or not. The
+	// Assign is reconstructed from labels — enough to re-register creds and tear
+	// down netns.
+	type task struct {
+		as      *bus.Assign
+		running []ctrruntime.Container
+		dead    []ctrruntime.Container
+	}
+	tasks := map[string]*task{}
+	for _, c := range containers {
+		if c.Labels[labelClusterName] != a.id.ClusterName {
+			continue
+		}
+		taskID := c.Labels[labelTaskID]
+		if taskID == "" {
+			continue
+		}
+		t := tasks[taskID]
+		if t == nil {
+			t = &task{as: &bus.Assign{
+				TaskID:        taskID,
+				ClusterName:   a.id.ClusterName,
+				CredID:        c.Labels[labelCredID],
+				TaskRoleARN:   c.Labels[labelTaskRoleARN],
+				ENIMacAddress: c.Labels[labelENIMac],
+			}}
+			tasks[taskID] = t
+		}
+		if c.Running {
+			t.running = append(t.running, c)
+		} else {
+			t.dead = append(t.dead, c)
+		}
+	}
+
+	for taskID, t := range tasks {
+		// A task with nothing running is over, whatever the gateway still believes.
+		// Mixed running/dead is left to the running path: the exit-wait re-attached
+		// below reports the individual container when it is reaped.
+		if len(t.running) == 0 {
+			a.reportStoppedTask(ctx, t.as, t.dead)
+			adopted[taskID] = true
+			continue
+		}
+
+		a.registerTaskCreds(t.as)
+		statuses := make([]bus.ContainerStatus, 0, len(t.running))
+		for _, c := range t.running {
+			name := c.Labels[labelContainerName]
+			go a.waitContainer(ctx, t.as, name, c.ID)
+			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
+		}
+		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
+		adopted[taskID] = true
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
+	}
+	return adopted
+}
+
+// reportStoppedTask clears a task's leftover containers and tells the gateway it
+// is STOPPED, so the scheduler stops counting a task this instance is not
+// running and its service can place a replacement.
+func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
+	statuses := make([]bus.ContainerStatus, 0, len(dead))
+	for _, c := range dead {
+		if err := a.runner.Remove(ctx, c.ID); err != nil {
+			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	a.reportTaskState(as, bus.TaskStatusStopped, "Container exited while the agent was not running", statuses)
+	slog.Info("ecs-agent: reported stopped task", "task", as.TaskID, "containers", len(dead))
+}

cmd/ecs-agent/reconcile_test.go

@@ -0,0 +1,250 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+const testTaskRole = "arn:aws:iam::111122223333:role/task"
+
+// adoptedContainer builds a runtime.Container as the reconciler would discover it,
+// stamping only the mulga.ecs.* labels that are non-empty.
+func adoptedContainer(taskID, name, cluster, credID, role, mac string, running bool) ctrruntime.Container {
+	labels := map[string]string{
+		labelTaskID:        taskID,
+		labelContainerName: name,
+		labelClusterName:   cluster,
+	}
+	if credID != "" {
+		labels[labelCredID] = credID
+	}
+	if role != "" {
+		labels[labelTaskRoleARN] = role
+	}
+	if mac != "" {
+		labels[labelENIMac] = mac
+	}
+	return ctrruntime.Container{ID: containerID(taskID, name), Labels: labels, Running: running}
+}
+
+func waitedFor(t *testing.T, rt *ctrruntime.FakePuller, id string) {
+	t.Helper()
+	deadline := time.After(time.Second)
+	for !slices.Contains(rt.Waits(), id) {
+		select {
+		case <-deadline:
+			t.Fatalf("exit-wait never re-attached for %s; waited=%v", id, rt.Waits())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+}
+
+// A running, correctly-labeled container is re-adopted: its credentials are
+// re-registered, its RUNNING state refreshed, and its exit-wait re-attached.
+func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if !adopted["t-001"] {
+		t.Fatalf("t-001 not adopted; got %v", adopted)
+	}
+	if got := a.cred.roles["t-001"]; got != testTaskRole {
+		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
+	}
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-001 = %d, want 1", n)
+	}
+	waitedFor(t, rt, containerID("t-001", "web"))
+}
+
+// A task whose container died while the agent was down must be reported STOPPED,
+// not silently skipped. The exit-wait is cancelled on agent shutdown, so nothing
+// else ever reports that exit: skipping it leaves the gateway holding the task
+// RUNNING forever, its service never places a replacement, and the load
+// balancer target stays unhealthy with no path back.
+func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusStopped); n != 1 {
+		t.Errorf("STOPPED reports for t-dead = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("t-dead must not be reported RUNNING, got %d such reports", n)
+	}
+	// Seeded so a re-delivered assignment for the dead task is acked, not re-run.
+	if !adopted["t-dead"] {
+		t.Errorf("t-dead not seeded into the adopted set; got %v", adopted)
+	}
+	if want := containerID("t-dead", "web"); !slices.Contains(rt.Removed, want) {
+		t.Errorf("leftover container %s not removed; removed=%v", want, rt.Removed)
+	}
+}
+
+// A task with one live and one dead container is still running: the live half is
+// re-adopted and the re-attached exit-wait reports the dead one when it is
+// reaped, so reconcile must not declare the whole task STOPPED.
+func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-mixed", "web", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", true),
+			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	a.reconcile(context.Background())
+
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-mixed = %d, want 1", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusStopped); n != 0 {
+		t.Errorf("t-mixed must not be reported STOPPED while a container still runs, got %d", n)
+	}
+}
+
+// pollAssignments seeded with an adopted task acks its re-delivered assignment
+// but does not re-run it, while a genuinely new task still dispatches.
+func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
+	mk := func(id string) bus.Assign {
+		return bus.Assign{
+			AccountID: "123456789012", ClusterName: "default", InstanceID: "i-abc123", TaskID: id,
+			Containers: []bus.AssignContainer{{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"}}},
+		}
+	}
+	adopted, fresh := mk("t-001"), mk("t-002")
+	cp := &fakeCP{pollReplies: [][]bus.Assign{
+		{adopted, fresh},
+		{adopted, fresh},
+		nil,
+	}}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{"t-001": true})
+
+	deadline := time.After(time.Second)
+	for countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning) < 1 {
+		select {
+		case <-deadline:
+			t.Fatal("fresh task t-002 was never dispatched")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("adopted task t-001 was re-run %d times, want 0", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("fresh task t-002 dispatched %d times, want 1", n)
+	}
+	var t1Acked bool
+	for _, ack := range cp.acks() {
+		for _, id := range ack {
+			if id == "t-001" {
+				t1Acked = true
+			}
+		}
+	}
+	if !t1Acked {
+		t.Errorf("adopted task t-001 was never acked; acks=%v", cp.acks())
+	}
+}
+
+// Containers belonging to another cluster are ignored entirely: this agent
+// speaks for its own cluster only, and must neither adopt nor report them.
+//
+// A stopped container of this cluster IS handled — see
+// TestReconcile_ReportsTaskWhoseContainerDied. It lands in the adopted set so
+// its re-delivered assignment is acked rather than re-run, after reconcile has
+// reported the task STOPPED.
+func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-other", "web", "other", "", "", "", true),
+			adoptedContainer("t-other-dead", "web", "other", "", "", "", false),
+			adoptedContainer("t-ok", "web", "default", "", "", "", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if len(adopted) != 1 || !adopted["t-ok"] {
+		t.Fatalf("adopted = %v, want only t-ok", adopted)
+	}
+	for _, id := range []string{"t-other", "t-other-dead"} {
+		if n := countStatus(cp.taskStates(), id, bus.TaskStatusStopped); n != 0 {
+			t.Errorf("reported %d STOPPED states for another cluster's task %s, want 0", n, id)
+		}
+	}
+}
+
+// A nil runner or a List error degrades to an empty set without panicking.
+func TestReconcile_DegradesGracefully(t *testing.T) {
+	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil, nil)
+	if got := noRunner.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("nil runner: adopted = %v, want empty", got)
+	}
+
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
+	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	if got := listErr.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("list error: adopted = %v, want empty", got)
+	}
+}
+
+// taskLabels emits the optional cred/role/MAC labels only when set.
+func TestTaskLabels_OmitsEmptyOptionalLabels(t *testing.T) {
+	full := taskLabels(&bus.Assign{
+		TaskID: "t1", ClusterName: "c", CredID: "x", TaskRoleARN: testTaskRole, ENIMacAddress: "mac",
+	}, "web")
+	for k, want := range map[string]string{
+		labelCredID: "x", labelTaskRoleARN: testTaskRole, labelENIMac: "mac",
+	} {
+		if full[k] != want {
+			t.Errorf("full labels[%q] = %q, want %q", k, full[k], want)
+		}
+	}
+
+	bare := taskLabels(&bus.Assign{TaskID: "t1", ClusterName: "c"}, "web")
+	if len(bare) != 3 {
+		t.Errorf("bare labels = %v, want only taskID/containerName/clusterName", bare)
+	}
+	for _, k := range []string{labelCredID, labelTaskRoleARN, labelENIMac} {
+		if _, ok := bare[k]; ok {
+			t.Errorf("bare labels unexpectedly carry %q", k)
+		}
+	}
+}

cmd/ecs-agent/registration.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// identity is the container instance's stable identity, assembled at boot from
+// IMDS (account, instance, AZ) and config (cluster). It is the join key the
+// scheduler uses to track this instance.
+type identity struct {
+	AccountID    string
+	ClusterName  string
+	InstanceID   string
+	AZ           string
+	Hostname     string
+	Capacity     bus.InstanceCapacity
+	AgentVersion string
+}
+
+// registrar registers the host as a container instance through the gateway.
+type registrar struct {
+	cp controlPlane
+	id identity
+}
+
+func newRegistrar(cp controlPlane, id identity) *registrar {
+	return &registrar{cp: cp, id: id}
+}
+
+// Register calls the gateway's RegisterContainerInstance. The scheduler records
+// (or refreshes) the container instance on receipt.
+func (r *registrar) Register() error {
+	if err := r.cp.Register(r.id); err != nil {
+		return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
+	}
+	return nil
+}

cmd/ecs-agent/registration_test.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// fakeCP is a test double for the gateway control-plane. It records registers,
+// task-state reports and GPU reports, and replays a scripted queue of poll
+// responses.
+type fakeCP struct {
+	mu sync.Mutex
+
+	registers    int
+	states       []bus.TaskState
+	gpuReports   []gpuReport
+	pollAcks     [][]string
+	pollStopAcks [][]string
+	pollReplies  [][]bus.Assign
+	stopReplies  [][]bus.StopDirective
+	pollCalls    int
+
+	registerErr bool
+	submitErr   bool
+}
+
+// gpuReport is one recorded ReportTaskGPU call.
+type gpuReport struct {
+	cluster    string
+	task       string
+	containers []handlers_ecs.ContainerGPUReport
+}
+
+var _ controlPlane = (*fakeCP)(nil)
+
+func (f *fakeCP) Register(_ identity) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.registerErr {
+		return errors.New("register boom")
+	}
+	f.registers++
+	return nil
+}
+
+func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.submitErr {
+		return errors.New("submit boom")
+	}
+	f.states = append(f.states, st)
+	return nil
+}
+
+func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.pollAcks = append(f.pollAcks, append([]string(nil), ackAssigns...))
+	f.pollStopAcks = append(f.pollStopAcks, append([]string(nil), ackStops...))
+	var out []bus.Assign
+	if f.pollCalls < len(f.pollReplies) {
+		out = f.pollReplies[f.pollCalls]
+	}
+	var stops []bus.StopDirective
+	if f.pollCalls < len(f.stopReplies) {
+		stops = f.stopReplies[f.pollCalls]
+	}
+	f.pollCalls++
+	return out, stops, nil
+}
+
+func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
+	return nil
+}
+
+func (f *fakeCP) gpuReportsFor(taskID string) []gpuReport {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []gpuReport
+	for _, r := range f.gpuReports {
+		if r.task == taskID {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
+func (f *fakeCP) registerCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.registers
+}
+
+func (f *fakeCP) taskStates() []bus.TaskState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]bus.TaskState(nil), f.states...)
+}
+
+func (f *fakeCP) acks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollAcks...)
+}
+
+func (f *fakeCP) stopAcks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollStopAcks...)
+}
+
+func testIdentity() identity {
+	return identity{
+		AccountID:    "123456789012",
+		ClusterName:  "default",
+		InstanceID:   "i-abc123",
+		AZ:           "us-east-1a",
+		Hostname:     "ecs-host",
+		Capacity:     bus.InstanceCapacity{CPU: 2048, MemoryMiB: 4096},
+		AgentVersion: "test",
+	}
+}
+
+// testIdentityWithGPUs is testIdentity seeded with discovered GPU UUIDs, so
+// newAgent's ledger has real devices to pin from.
+func testIdentityWithGPUs(uuids ...string) identity {
+	id := testIdentity()
+	id.Capacity.GPU = len(uuids)
+	id.Capacity.GPUIDs = uuids
+	return id
+}
+
+func TestRegistrar_Register(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("registers = %d, want 1", cp.registerCount())
+	}
+}
+
+func TestRegistrar_RegisterError(t *testing.T) {
+	cp := &fakeCP{registerErr: true}
+	if err := newRegistrar(cp, testIdentity()).Register(); err == nil {
+		t.Fatal("expected register error")
+	}
+}
+
+func TestHeartbeater_Beat(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newHeartbeater(cp, testIdentity(), time.Second).beat(); err != nil {
+		t.Fatalf("beat: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("beat should re-register once, got %d", cp.registerCount())
+	}
+}
+
+func TestHeartbeater_DefaultInterval(t *testing.T) {
+	h := newHeartbeater(&fakeCP{}, testIdentity(), 0)
+	if h.interval != defaultHeartbeat {
+		t.Errorf("interval = %v, want default", h.interval)
+	}
+}
+
+func TestHeartbeater_RunStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	done := make(chan struct{})
+	go func() { h.Run(ctx); close(done) }()
+
+	time.Sleep(25 * time.Millisecond)
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("Run did not stop on context cancel")
+	}
+	if cp.registerCount() == 0 {
+		t.Error("expected at least one re-register before cancel")
+	}
+}

cmd/ecs-agent/runtime/containerd.go

@@ -0,0 +1,118 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/core/remotes"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/distribution/reference"
+)
+
+// ecsNamespace isolates ecs-agent images/containers in containerd, mirroring the
+// "moby"/"k8s.io" convention other runtimes use.
+const ecsNamespace = "ecs"
+
+// servingProbeTimeout bounds the boot-time daemon liveness check.
+const servingProbeTimeout = 3 * time.Second
+
+// containerdPuller drives a real containerd daemon over its unix socket.
+type containerdPuller struct {
+	client *containerd.Client
+}
+
+var _ ImagePuller = (*containerdPuller)(nil)
+
+// New connects to the containerd daemon at socket (e.g.
+// "/run/containerd/containerd.sock") and returns an ImagePuller. The containerd
+// client dials lazily, so New actively probes the daemon with IsServing and
+// returns an error if it is unreachable — otherwise a dead socket would only
+// surface on the first pull, and the agent's boot-time "containerd unavailable"
+// log would never fire.
+func New(socket string) (Runtime, error) {
+	client, err := containerd.New(socket, containerd.WithDefaultNamespace(ecsNamespace))
+	if err != nil {
+		return nil, fmt.Errorf("connect containerd %s: %w", socket, err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), servingProbeTimeout)
+	defer cancel()
+	serving, err := client.IsServing(ctx)
+	if err != nil || !serving {
+		_ = client.Close()
+		return nil, fmt.Errorf("containerd %s not serving: %w", socket, err)
+	}
+	return &containerdPuller{client: client}, nil
+}
+
+// Pull resolves credentials via r, then pulls and unpacks the image.
+func (p *containerdPuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+
+	ref, err := normalizeRef(spec.Ref)
+	if err != nil {
+		return Image{}, err
+	}
+
+	resolver, err := newDockerResolver(ctx, ref, r)
+	if err != nil {
+		return Image{}, err
+	}
+
+	img, err := p.client.Pull(ctx, ref,
+		containerd.WithPullUnpack,
+		containerd.WithResolver(resolver),
+	)
+	if err != nil {
+		return Image{}, fmt.Errorf("pull %s: %w", ref, err)
+	}
+
+	target := img.Target()
+	return Image{
+		Ref:    ref,
+		Digest: target.Digest.String(),
+		Size:   target.Size,
+	}, nil
+}
+
+// normalizeRef expands a task-definition image string into the fully-qualified
+// reference containerd's resolver requires: a bare "nginx:alpine" becomes
+// "docker.io/library/nginx:alpine" and an untagged ref gets ":latest". Without
+// this, containerd parses the bare ref as "dummy://nginx:alpine" and rejects it.
+func normalizeRef(ref string) (string, error) {
+	named, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return "", fmt.Errorf("parse image ref %q: %w", ref, err)
+	}
+	return reference.TagNameOnly(named).String(), nil
+}
+
+// Close releases the containerd client connection.
+func (p *containerdPuller) Close() error {
+	if p.client == nil {
+		return nil
+	}
+	return p.client.Close()
+}
+
+// newDockerResolver builds a containerd remote resolver whose per-host
+// credentials come from r (the ECR token resolver). Credentials are fetched once
+// for ref and reused for every host containerd asks about during the pull.
+func newDockerResolver(ctx context.Context, ref string, r Resolver) (remotes.Resolver, error) {
+	creds := func(string) (string, string, error) { return "", "", nil }
+	if r != nil {
+		user, pass, _, err := r.Authorize(ctx, ref)
+		if err != nil {
+			return nil, fmt.Errorf("authorize %s: %w", ref, err)
+		}
+		creds = func(string) (string, string, error) { return user, pass, nil }
+	}
+
+	authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(creds))
+	return docker.NewResolver(docker.ResolverOptions{
+		Hosts: docker.ConfigureDefaultRegistries(docker.WithAuthorizer(authorizer)),
+	}), nil
+}