feat(quota): add unwired owner-side quota override cleanup (#1240)

Pin that account teardown removes the usage counter but leaves the
per-account quota override, so a reused account ID inherits raised or
Unlimited limits. Add an unwired, idempotent CleanupAccount on the quota
service: a present override key is removed by a revision-guarded delete
and reported deleted with its revision; a missing key is reported only as
observed-absent-at-scan.

.githooks/pre-commit

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-branch=$(git rev-parse --abbrev-ref HEAD)
-
-# Only enforce preflight on dev and main
-if [ "$branch" != "dev" ] && [ "$branch" != "main" ]; then
-  exit 0
-fi
-
-# Only run preflight if Go files were staged
-if git diff --cached --name-only | grep -q '\.go$'; then
-  echo "Go files staged on $branch — running make preflight..."
-  make preflight
-else
-  echo "No Go files staged, skipping preflight."
-fi

.github/CODEOWNERS

@@ -0,0 +1,15 @@
+# CODEOWNERS — GitHub PR review enforcement
+#
+# Format: <path pattern>  <owner> [<owner> …]
+# Order matters — last matching pattern wins.
+#
+# Set the matching branch protection rule on `dev` and `main` to
+# "Require review from Code Owners" or this file is advisory only.
+
+# Network layer is governed by its invariant tests.
+# Changes here must respect those clauses
+# see spinifex/domains/network/invariants/
+# mulga docs/development/proposals/region-local-architecture/0006-spinifex-network-layer-contract.md (proposal, not accepted)
+/spinifex/domains/network/          @joshsiv-mulga
+/spinifex/runtime/roles/vpcd/ @joshsiv-mulga
+/spinifex/domains/network/invariants/ @joshsiv-mulga

.github/ISSUE_TEMPLATE/1-bug.yml

@@ -0,0 +1,59 @@
+name: Bug report
+description: Something is not working as it should.
+labels: ["bug"]
+body:
+  - type: dropdown
+    id: component
+    attributes:
+      label: Component
+      options:
+        - EC2 / instances
+        - EBS / volumes
+        - S3 / object store
+        - VPC / networking
+        - IAM / authentication
+        - EKS
+        - API gateway
+        - Web UI
+        - CLI / daemon
+        - Observability / telemetry
+        - Build / CI / packaging
+        - Documentation
+        - Other / not sure
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: Spinifex version
+      description: Output of `spinifex --version`, or the commit SHA you are on.
+      placeholder: v1.0.0, or commit abc1234
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened
+      description: What you saw, what you expected instead, and how to reproduce it.
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs and output
+      description: Relevant log lines, error messages or command output.
+      render: shell
+
+  - type: textarea
+    id: platform
+    attributes:
+      label: Platform
+      description: Where you are running it, and how you deployed it.
+      placeholder: |
+        OS: Debian 13
+        Kernel: 6.12
+        Arch: amd64
+        Deployment: single node, systemd

.github/ISSUE_TEMPLATE/2-feature.yml

@@ -0,0 +1,11 @@
+name: Feature request
+description: Suggest something new.
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: proposal
+    attributes:
+      label: What would you like to see?
+      description: What you are trying to do, and what stops you today.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/3-docs.yml

@@ -0,0 +1,20 @@
+name: Documentation
+description: Something in the docs is wrong, missing or unclear.
+labels: ["documentation"]
+body:
+  - type: input
+    id: location
+    attributes:
+      label: Where
+      description: Link to the page, or the path to the file in this repository.
+      placeholder: https://docs.mulgadc.com/..., or README.md
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: What is wrong
+      description: What is inaccurate, missing or confusing, and what it should say instead.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Documentation
+    url: https://docs.mulgadc.com
+    about: Guides, configuration reference and API compatibility notes.

.github/actions/e2e-analyze/README.md

@@ -70,7 +70,7 @@ When the report format changes intentionally, regenerate the snapshots:
 go test ./.github/actions/e2e-analyze/ -update
 ```
 
-## Per-failure bundles (Stage 2)
+## Per-failure bundles
 
 After parsing, the action materialises a bundle directory per failure
 under `<log-dir>/analysis/NN-<TestName>/`. The report links to it from

.github/actions/e2e-analyze/analyze.go

@@ -1,7 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1 of docs/development/improvements/e2e-go-failure-analysis.md:
-// cluster failures from go-junit-report's junit-*.xml by error signature,
+// Cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -74,8 +73,14 @@ type Failure struct {
 	// `test-<suite>.start` file written by the workflow before the suite
 	// runs) can recompute StartAt without re-parsing the XML.
 	OffsetFromSuiteStart time.Duration
+	// SuiteSpan is the suite's own wall-clock span, set only when the suite
+	// ran its tests in parallel. Overlapping tests make the cumulative
+	// offset above pure fiction — it lands minutes past the real failure —
+	// so consumers widen the journal window to the suite instead of
+	// slicing a plausible-looking window around the wrong minute.
+	SuiteSpan time.Duration
 	// BundlePath is the relative path (from log-dir) to the per-failure
-	// bundle written by Stage 2's WriteBundles. Empty when bundles were
+	// bundle written by WriteBundles. Empty when bundles were
 	// not generated (e.g. unit tests that call Render directly).
 	BundlePath string
 }
@@ -86,9 +91,18 @@ type SuiteReport struct {
 	Label     string // short name, e.g. "single"
 	Total     int
 	FailCount int
-	Root      *Failure    // nil if no failures
-	Cascades  []Failure   // everything else that failed in this suite
-	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// Unresolved counts testcases go-junit-report could not find an outcome
+	// for. They are not failures and the suite can be green with hundreds.
+	Unresolved int
+	Root       *Failure    // nil if no failures
+	Cascades   []Failure   // everything else that failed in this suite
+	Buckets    [][]Failure // failures grouped by signature (root's bucket first)
+	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
+	// the XML was produced — i.e. when the suite finished, not when it began.
+	EndedAt time.Time
+	// TopLevelTotal is the summed duration of the parentless tests. Longer than
+	// the suite's wall clock means they overlapped.
+	TopLevelTotal time.Duration
 }
 
 // Report is the full analysis across every junit file.
@@ -120,6 +134,29 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
+// nonFailurePatterns match content a test emits AFTER the assertion that failed
+// — t.Cleanup, harness diagnostics, or a t.Logf on a passing branch — which
+// extractErrorLine's keep-the-last rule would otherwise pick over the assertion.
+// Go emits t.Logf and t.Errorf identically, so anchor every entry: a fragment
+// loose enough to match an assertion blanks out the line the report exists to
+// show. `\bas expected\b` unanchored also matches "did not fail as expected".
+var nonFailurePatterns = []*regexp.Regexp{
+	regexp.MustCompile(`^DB instance \S+ is gone$`),
+	regexp.MustCompile(`^db diagnostics \S+:`),
+	regexp.MustCompile(`^\S+: management bridge \S+:\d+ refused as expected$`),
+}
+
+// isNonFailureLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for, or reports an expected outcome.
+func isNonFailureLine(content string) bool {
+	for _, re := range nonFailurePatterns {
+		if re.MatchString(content) {
+			return true
+		}
+	}
+	return false
+}
+
 // noiseReplacers normalise dynamic identifiers so that "ten failures, same
 // bug" bucket together. Pattern ordering matters: longer/more-specific
 // patterns must precede the catch-alls.
@@ -207,11 +244,14 @@ func extractErrorLine(body string) string {
 		if m := goFileLineRe.FindStringSubmatch(l); m != nil {
 			// Keep the LAST occurrence: t.Log / harness traces emit
 			// progress lines from the same file:line pattern before the
-			// failing assertion finally fires.
-			goFileLine = strings.TrimSpace(m[1])
+			// failing assertion finally fires. Cleanup runs after it, though,
+			// so its lines are last of all and have to be skipped.
+			if content := strings.TrimSpace(m[1]); !isNonFailureLine(content) {
+				goFileLine = content
+			}
 			continue
 		}
-		if fileHintRe.MatchString(l) {
+		if fileHintRe.MatchString(l) && !isNonFailureLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -255,11 +295,27 @@ func extractFileHint(body string) string {
 	if m := errorTraceRe.FindStringSubmatch(body); m != nil {
 		return m[1]
 	}
-	all := fileHintRe.FindAllString(body, -1)
-	if len(all) == 0 {
-		return ""
+	// Scanned line by line rather than across the body so a cleanup line's own
+	// file:line does not become the site the report points a reader at.
+	hint, filtered := "", ""
+	for _, raw := range strings.Split(body, "\n") {
+		l := strings.TrimSpace(raw)
+		all := fileHintRe.FindAllString(l, -1)
+		if len(all) == 0 {
+			continue
+		}
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isNonFailureLine(strings.TrimSpace(m[1])) {
+			filtered = all[len(all)-1]
+			continue
+		}
+		hint = all[len(all)-1]
 	}
-	return all[len(all)-1]
+	// A body whose only file:line was allowlisted still gets a pointer: coarse
+	// beats none, and extractErrorLine has the same last-resort shape.
+	if hint == "" {
+		return filtered
+	}
+	return hint
 }
 
 // signature normalises an error line so different runs of the same root
@@ -314,6 +370,47 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 	return leaf
 }
 
+// computeRolledUpSet returns the testcase names whose failure is already
+// reported by a failing descendant. go-junit-report gives such a parent a
+// testcase of its own, and its body is the parent's t.Log output rather than an
+// assertion — a cleanup line like "DB instance … is gone", which passing tests
+// emit too, reads to extractErrorLine exactly like a failure site.
+func computeRolledUpSet(cases []junitTC) map[string]bool {
+	rolledUp := make(map[string]bool, len(cases))
+	for _, tc := range cases {
+		if tc.Failure == nil && !isRealError(tc.Error) {
+			continue
+		}
+		// Mark every ancestor: the failure belongs to the deepest case that
+		// carries it, not to the tests it is nested under.
+		name := tc.Name
+		for {
+			i := strings.LastIndex(name, "/")
+			if i <= 0 {
+				break
+			}
+			name = name[:i]
+			rolledUp[name] = true
+		}
+	}
+	return rolledUp
+}
+
+// topLevelTotal sums the durations of the tests with no parent, which are the
+// units t.Parallel overlaps. Compared against the suite's wall clock it says
+// whether the suite ran its tests concurrently — see ApplySuiteStartFiles.
+// suite.Time cannot answer that: go-junit-report sets it to the sum of every
+// testcase, parents included, so it is never less than any subset of itself.
+func topLevelTotal(cases []junitTC) time.Duration {
+	total := 0.0
+	for _, tc := range cases {
+		if !strings.Contains(tc.Name, "/") {
+			total += tc.Time
+		}
+	}
+	return time.Duration(total * float64(time.Second))
+}
+
 // parseStartTime turns a junit timestamp attr (RFC3339) into a time.Time;
 // zero value if unparseable.
 func parseStartTime(s string) time.Time {
@@ -327,6 +424,18 @@ func parseStartTime(s string) time.Time {
 	return t
 }
 
+// noResultMessage is go-junit-report's marker for a testcase it saw start and
+// never saw finish. On cell-20 it accounted for 93 of the 120 testcases and 24
+// of the 25 reported failures, none of which had failed.
+const noResultMessage = "No test result found"
+
+// isRealError separates a genuine <error> from the parser's no-result marker.
+// A parent must not be treated as rolled up by a child whose outcome is merely
+// unknown, or the parent's own failure is dropped along with it.
+func isRealError(e *junitFailure) bool {
+	return e != nil && strings.TrimSpace(e.Message) != noResultMessage
+}
+
 // suiteLabel turns "junit-single.xml" into "single".
 func suiteLabel(path string) string {
 	base := filepath.Base(path)
@@ -360,6 +469,9 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 		// summing every testcase double-counts. Pre-compute the leaf set so
 		// only leaves contribute to the suite's running offset.
 		isLeaf := computeLeafSet(suite.Cases)
+		rolledUp := computeRolledUpSet(suite.Cases)
+		rep.EndedAt = start
+		rep.TopLevelTotal += topLevelTotal(suite.Cases)
 		cumul := 0.0
 		for _, tc := range suite.Cases {
 			rep.Total++
@@ -369,6 +481,17 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 			case tc.Failure != nil:
 				body = tc.Failure.Body
 			case tc.Error != nil:
+				// go-junit-report emits this when a `=== RUN` line has no
+				// matching result line. It means the outcome is unknown, not
+				// that the test errored, and it attaches the test's own log
+				// output — which then reads as an assertion.
+				if !isRealError(tc.Error) {
+					rep.Unresolved++
+					if leaf {
+						cumul += tc.Time
+					}
+					continue
+				}
 				body = tc.Error.Body
 			default:
 				if leaf {
@@ -377,11 +500,12 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 				continue
 			}
 
-			// Skip the synthetic parent-test "Failed" stub that go-junit-report
-			// emits for any Go test whose subtests failed. It has an empty body
-			// and would otherwise outrank real failures by XML order.
+			// Skip the parent-test entry go-junit-report emits for any Go test
+			// whose subtests failed: it is a rollup of failures already counted
+			// below it, and it would outrank them by XML order. An empty body is
+			// the same stub in the case where the parent logged nothing.
 			trimmed := strings.TrimSpace(stripANSI(body))
-			if trimmed == "" {
+			if trimmed == "" || rolledUp[tc.Name] {
 				if leaf {
 					cumul += tc.Time
 				}
@@ -463,15 +587,36 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }
 
-// Render writes the markdown report. Format is documented in
-// docs/development/improvements/e2e-go-failure-analysis.md (Stage 1).
+// Render writes the markdown report.
+// writeUnresolved names the testcases whose outcome the parser could not read.
+// Worth surfacing because a suite that leaves most of its subtests unresolved
+// is reporting on far less than its test count suggests.
+func writeUnresolved(b *strings.Builder, s SuiteReport) {
+	if s.Unresolved == 0 {
+		return
+	}
+	fmt.Fprintf(b, "ℹ️ %d of %d testcases have no result line in the log, so `go-junit-report` "+
+		"could not read their outcome. Not failures — but this suite reports on %d tests, not %d.\n\n",
+		s.Unresolved, s.Total, s.Total-s.Unresolved, s.Total)
+}
+
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
 
-	totalFail := 0
+	totalFail, totalTests := 0, 0
 	for _, s := range r.Suites {
 		totalFail += s.FailCount
+		totalTests += s.Total
+	}
+
+	// A cell that dies before the suites run emits junit with no testcases at
+	// all. Reporting that as clean turns "nothing ran" into "nothing wrong", so
+	// name it and point above the suites.
+	if totalTests == 0 {
+		b.WriteString("⚠️ No tests ran — no testcases in any junit file. " +
+			"The failure is upstream of the suites (provisioning, bootstrap, or reimage); check the job log.\n")
+		return b.String()
 	}
 	if totalFail == 0 {
 		b.WriteString("✅ No failures across any suite.\n")
@@ -481,17 +626,29 @@ func Render(r Report) string {
 	for _, s := range r.Suites {
 		if s.FailCount == 0 {
 			fmt.Fprintf(&b, "### Suite `%s`: ✅ pass (%d tests)\n\n", s.Label, s.Total)
+			writeUnresolved(&b, s)
 			continue
 		}
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
+		writeUnresolved(&b, s)
 
 		if s.Root != nil {
-			b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			// XML order is completion order, so "earliest" only means earliest
+			// in time for a suite whose tests ran one after another.
+			if s.Root.SuiteSpan > 0 {
+				b.WriteString("**First non-cascade failure** (the suite ran in parallel, so this is report order, not time order)\n\n")
+			} else {
+				b.WriteString("**Root cause (earliest non-cascade)**\n\n")
+			}
 			fmt.Fprintf(&b, "- Test: `%s`\n", s.Root.Name)
-			if !s.Root.StartAt.IsZero() {
+			switch {
+			case s.Root.SuiteSpan > 0:
+				fmt.Fprintf(&b, "- Start: unknown, the suite ran in parallel (duration %.1fs)\n",
+					s.Root.Duration)
+			case !s.Root.StartAt.IsZero():
 				fmt.Fprintf(&b, "- Start: %s (duration %.1fs)\n",
 					s.Root.StartAt.Format("15:04:05"), s.Root.Duration)
-			} else {
+			default:
 				fmt.Fprintf(&b, "- Duration: %.1fs\n", s.Root.Duration)
 			}
 			if s.Root.Error != "" {

.github/actions/e2e-analyze/analyze_test.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
 // updateGolden lets `go test ./.github/actions/e2e-analyze/ -update` rewrite
@@ -124,6 +125,62 @@ func TestExtractErrorLine_FallsBackToLastTestLine(t *testing.T) {
 	}
 }
 
+// A test's cleanup logs after it has failed, so its lines are the last in the
+// body — and the cleanup line here is one passing tests emit too.
+func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
+	body := `    failure_test.go:42: DB instance rds-e2e-failure-1 reached status available
+    failure_test.go:60: EventuallyErr: condition not met within 4m0s: rds-e2e-failure-1 status=available want=failed
+    rds.go:364: db diagnostics rds-e2e-failure-1: no VM to capture a console from (<nil>)
+    main_test.go:325: DB instance rds-e2e-failure-1 is gone`
+	got := extractErrorLine(body)
+	if !strings.HasPrefix(got, "EventuallyErr:") {
+		t.Errorf("extractErrorLine = %q, want the assertion at failure_test.go:60", got)
+	}
+	if hint := extractFileHint(body); hint != "failure_test.go:60" {
+		t.Errorf("extractFileHint = %q, want failure_test.go:60", hint)
+	}
+}
+
+// A subtest looping over two subjects calls t.Errorf for the first and t.Logf
+// for the second, so the success line is last in the body. Nightly run
+// 34383800404 reported that log line as the failure.
+func TestExtractErrorLine_SkipsSuccessLoggedAfterTheFailure(t *testing.T) {
+	body := `    isolation_test.go:80: rds-e2e-iso-a-1788981456: management bridge 10.15.8.10:5432 could not be tested: want connection refused, got dial tcp 10.15.8.10:5432: i/o timeout
+    isolation_test.go:84: rds-e2e-iso-b-1788981456: management bridge 10.15.8.11:5432 refused as expected`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "could not be tested") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// "as expected" reads the same in an affirmation and in its negation, so an
+// unanchored pattern suppresses the assertion it was meant to outrank and
+// promotes a harness progress line in its place.
+func TestExtractErrorLine_KeepsNegatedExpectationAssertions(t *testing.T) {
+	body := `    isolation_test.go:60: waiting for the db instance to become available
+    isolation_test.go:80: rds-e2e-iso-a: management bridge 10.15.8.10:5432 was not refused as expected: got a connection`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "was not refused as expected") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// Every candidate allowlisted leaves extractFileHint with nothing to return,
+// and unlike extractErrorLine it has no last-resort branch. A coarse pointer
+// beats sending the reader to no file at all.
+func TestExtractFileHint_FallsBackWhenEveryCandidateFiltered(t *testing.T) {
+	body := `    isolation_test.go:84: rds-e2e-iso-b: management bridge 10.15.8.11:5432 refused as expected`
+	if hint := extractFileHint(body); hint != "isolation_test.go:84" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:84", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -157,6 +214,165 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+// go-junit-report emits <error message="No test result found"> for a `=== RUN`
+// with no matching result line, and attaches the test's own log output. On a
+// baremetal cell that was 93 of 120 testcases and 24 of 25 reported failures,
+// none of which had failed: "ALB internal: 20/20 successful" read as an error.
+func TestParseFile_NoResultFoundIsNotAFailure(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="3" failures="1" errors="2">
+  <testcase name="TestLB/Internal_ALB" time="0.0"><error message="No test result found"><![CDATA[
+    lb_test.go:88: ALB internal: 20/20 successful, 2 unique
+]]></error></testcase>
+  <testcase name="TestLB/Internal_NLB" time="0.0"><error message="No test result found"></error></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (no-result entries are not failures)", sr.FailCount)
+	}
+	if sr.Unresolved != 2 {
+		t.Fatalf("Unresolved = %d, want 2", sr.Unresolved)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the real failure as root, got %+v", sr.Root)
+	}
+}
+
+// An <error> that is not the parser's no-result marker is a genuine error and
+// must still be reported.
+func TestParseFile_KeepsARealError(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="1" failures="0" errors="1">
+  <testcase name="TestX" time="0.5"><error message="panic"><![CDATA[
+    foo_test.go:10: panic: runtime error: index out of range
+]]></error></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 || sr.Unresolved != 0 {
+		t.Fatalf("FailCount = %d, Unresolved = %d, want 1 and 0", sr.FailCount, sr.Unresolved)
+	}
+}
+
+func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
+	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
+	// "DB instance … is gone" — which passing tests emit too — reads exactly
+	// like an assertion site. Its subtest holds the real failure.
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="2">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    main_test.go:325: DB instance rds-e2e-backup-1 is gone
+]]></failure></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (the parent rollup should be skipped)", sr.FailCount)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the subtest as root, got %+v", sr.Root)
+	}
+}
+
+// A parent that fails on its own after its subtests passed is a real failure,
+// and dropping it would lose the only record of it.
+func TestParseFile_KeepsParentWithNoFailingSubtest(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1">
+  <testcase name="TestX" time="1.5"><failure message="Failed"><![CDATA[
+    foo_test.go:42:
+        Messages: the parent asserted after its subtests
+]]></failure></testcase>
+  <testcase name="TestX/Passing" time="0.5"></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX" {
+		t.Fatalf("expected the parent as root, got %+v", sr.Root)
+	}
+}
+
+// suiteFixture writes a junit file plus its .start sidecar into a temp log dir
+// and returns the parsed, start-file-corrected report for one suite.
+func suiteFixture(t *testing.T, start string, xml []byte) SuiteReport {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "test-x.start"), []byte(start), 0o644); err != nil {
+		t.Fatalf("write start file: %v", err)
+	}
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	rep := Report{Suites: []SuiteReport{sr}}
+	ApplySuiteStartFiles(&rep, dir)
+	return rep.Suites[0]
+}
+
+// Parallel tests overlap, so summing the ones before a failure puts its start
+// minutes past where it really was — and the journal slice cut from that window
+// misses the failure entirely. The tell is the top-level tests taking longer in
+// total than the suite's own wall clock.
+func TestApplySuiteStartFiles_DropsTheOffsetWhenTheSuiteRanInParallel(t *testing.T) {
+	// timestamp is when go-junit-report wrote the XML, i.e. the suite's end.
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestSlow" time="100.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 2*time.Minute {
+		t.Errorf("SuiteSpan = %s, want 2m0s to mark the suite parallel", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 0, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start %s rather than a summed offset", s.Root.StartAt, want)
+	}
+}
+
+// The sequential case must keep its offsets: they are the only thing that makes
+// a per-failure journal slice narrow enough to read.
+func TestApplySuiteStartFiles_KeepsTheOffsetWhenTheSuiteRanSequentially(t *testing.T) {
+	s := suiteFixture(t, "2026-08-03T02:00:00Z", []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="2" failures="1" timestamp="2026-08-03T02:02:00Z">
+  <testcase name="TestFirst" time="30.0"></testcase>
+  <testcase name="TestX" time="90.0"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`))
+	if s.Root == nil {
+		t.Fatal("expected a root failure")
+	}
+	if s.Root.SuiteSpan != 0 {
+		t.Errorf("SuiteSpan = %s, want 0 for a sequential suite", s.Root.SuiteSpan)
+	}
+	if want := time.Date(2026, 8, 3, 2, 0, 30, 0, time.UTC); !s.Root.StartAt.Equal(want) {
+		t.Errorf("StartAt = %s, want the suite start plus 30s of preceding tests %s", s.Root.StartAt, want)
+	}
+}
+
 func TestParseFile_CascadeFallbackWhenAllCascades(t *testing.T) {
 	// If every failure is a cascade marker, the earliest one becomes root
 	// so the report isn't blank.
@@ -189,3 +405,25 @@ func TestRender_NoFailuresIsClean(t *testing.T) {
 		t.Errorf("expected zero-failure banner, got:\n%s", out)
 	}
 }
+
+// A cell that fails in provisioning writes junit with no testcases. Rendering
+// the clean banner there reports "nothing ran" as "nothing wrong".
+func TestRender_NoTestsRanIsNotClean(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		suites []SuiteReport
+	}{
+		{"empty junit file", []SuiteReport{{File: "junit-baremetal.xml", Label: "baremetal", Total: 0, FailCount: 0}}},
+		{"no junit files at all", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			out := Render(Report{Title: "E2E failure analysis", Suites: tc.suites})
+			if strings.Contains(out, "No failures across any suite") {
+				t.Errorf("clean banner rendered when no test ran, got:\n%s", out)
+			}
+			if !strings.Contains(out, "No tests ran") {
+				t.Errorf("expected a no-tests-ran warning, got:\n%s", out)
+			}
+		})
+	}
+}

.github/actions/e2e-analyze/bundle.go

@@ -1,7 +1,6 @@
 package main
 
-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// for each failure surfaced by Stage 1, materialise a per-failure bundle
+// Stage 2: for each failure surfaced by Stage 1, materialise a per-failure bundle
 // directory containing the failure summary, a time-window slice of the
 // daemon journal, a testcase-window slice of the suite's stdout log, and
 // any per-test artifacts the harness already dumped at run-time.
@@ -18,9 +17,9 @@ import (
 )
 
 // bundlePad is the per-failure window padding either side of the
-// [StartAt, StartAt+Duration] testcase span. Five seconds matches the
-// plan doc: long enough to catch the daemon's pre-test setup and
-// post-test cleanup chatter, short enough to keep the slice focused.
+// [StartAt, StartAt+Duration] testcase span. Five seconds catches the
+// daemon's pre-test setup and post-test cleanup chatter while keeping
+// the slice focused.
 const bundlePad = 5 * time.Second
 
 // nameSanitiseRe replaces characters unsafe in directory names with `_`.
@@ -59,21 +58,57 @@ func ApplySuiteStartFiles(rep *Report, logDir string) {
 			fmt.Fprintf(os.Stderr, "::warning::e2e-analyze: parse %s: %v\n", startPath, err)
 			continue
 		}
-		if s.Root != nil {
-			s.Root.StartAt = ts.Add(s.Root.OffsetFromSuiteStart)
-		}
-		for ci := range s.Cascades {
-			s.Cascades[ci].StartAt = ts.Add(s.Cascades[ci].OffsetFromSuiteStart)
-		}
-		for bi := range s.Buckets {
-			for fi := range s.Buckets[bi] {
-				f := &s.Buckets[bi][fi]
-				f.StartAt = ts.Add(f.OffsetFromSuiteStart)
+		// The start file is the first wall clock the analyzer has, so it is also
+		// the first chance to tell a parallel suite from a sequential one — and
+		// a parallel one's per-test offsets are fiction, since overlapping tests
+		// never started at the sum of the ones before them.
+		span := suiteSpanIfParallel(s, ts)
+		for _, f := range collectFailures(s) {
+			f.SuiteSpan = span
+			if span > 0 {
+				f.StartAt = ts
+				continue
 			}
+			f.StartAt = ts.Add(f.OffsetFromSuiteStart)
 		}
 	}
 }
 
+// suiteSpanIfParallel returns the suite's wall-clock span when its top-level
+// tests overlapped, and zero when they ran one after another. Consumers read a
+// non-zero span as "no per-test offset is derivable from this suite".
+func suiteSpanIfParallel(s *SuiteReport, start time.Time) time.Duration {
+	if s.EndedAt.IsZero() || s.TopLevelTotal == 0 {
+		return 0
+	}
+	wall := s.EndedAt.Sub(start)
+	// The 5% margin keeps per-testcase rounding from calling a sequential
+	// suite parallel; real overlap runs to several times the wall clock.
+	if wall <= 0 || s.TopLevelTotal <= time.Duration(float64(wall)*1.05) {
+		return 0
+	}
+	return wall
+}
+
+// collectFailures returns every failure in the report by pointer. Root, cascades
+// and buckets hold independent copies of the same failures, so a field that has
+// to reach the bundle writer must be set on all three.
+func collectFailures(s *SuiteReport) []*Failure {
+	var all []*Failure
+	if s.Root != nil {
+		all = append(all, s.Root)
+	}
+	for ci := range s.Cascades {
+		all = append(all, &s.Cascades[ci])
+	}
+	for bi := range s.Buckets {
+		for fi := range s.Buckets[bi] {
+			all = append(all, &s.Buckets[bi][fi])
+		}
+	}
+	return all
+}
+
 // WriteBundles materialises per-failure bundles under <logDir>/analysis/.
 // Mutates rep in-place so each Failure.BundlePath holds the relative
 // path the renderer will link from the summary.
@@ -144,7 +179,11 @@ func writeOneBundle(logDir, base, suiteLabel string, f *Failure) {
 
 	writeFailureSummary(dir, suiteLabel, f)
 
-	if !f.StartAt.IsZero() {
+	// A parallel suite offers no derivable per-test window, and the narrowing is
+	// the whole point of the slice: cutting one from the wrong minute leaves out
+	// the lines that explain the failure, and widening it to the suite just
+	// copies the artifact's own journal into every bundle.
+	if !f.StartAt.IsZero() && f.SuiteSpan == 0 {
 		start := f.StartAt.Add(-bundlePad)
 		end := f.StartAt.Add(time.Duration(f.Duration * float64(time.Second))).Add(bundlePad)
 		journalSrc := filepath.Join(logDir, "spinifex-journal.log")
@@ -189,7 +228,11 @@ func writeFailureSummary(dir, suiteLabel string, f *Failure) {
 	var b strings.Builder
 	fmt.Fprintf(&b, "Suite:    %s\n", suiteLabel)
 	fmt.Fprintf(&b, "Test:     %s\n", f.Name)
-	if !f.StartAt.IsZero() {
+	switch {
+	case f.SuiteSpan > 0:
+		b.WriteString("Start:    unknown, the suite ran in parallel; " +
+			"no journal window is derivable — read ../../spinifex-journal.log\n")
+	case !f.StartAt.IsZero():
 		fmt.Fprintf(&b, "Start:    %s\n", f.StartAt.Format(time.RFC3339))
 	}
 	fmt.Fprintf(&b, "Duration: %.3fs\n", f.Duration)

.github/actions/e2e-analyze/main.go

@@ -45,7 +45,7 @@ func main() {
 		report.Suites = append(report.Suites, sr)
 	}
 
-	// Stage 2: prefer authoritative per-suite start times written by the
+	// Prefer authoritative per-suite start times written by the
 	// workflow over junit-report's conversion-time `timestamp` attribute,
 	// then materialise per-failure bundles so Render can link them.
 	if logDir != "" {

.github/actions/e2e-analyze/slice.go

@@ -1,7 +1,6 @@
 package main
 
-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// per-failure log slicers used by bundle.go to materialise time-window /
+// Per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.
 
 import (

.github/actions/junit-report/README.md

@@ -8,7 +8,7 @@ Actions job summary and emits one `::error` annotation per failed test case.
 ```yaml
 - name: Publish test summary
   if: always()
-  uses: ./spinifex/.github/actions/junit-report   # or your own checkout path
+  uses: $/.github/actions/junit-report
   with:
     junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
     title: "Go E2E results"

.github/actions/junit-to-es/action.yml

@@ -0,0 +1,43 @@
+name: JUnit to Elasticsearch
+description: >-
+  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
+  so per-test pass rates outlive the 90-day artifact retention. Call it once
+  per job right after the JUnit XML is produced (with `if: always()`), next to
+  the junit-report action that renders the same files into the job summary.
+
+inputs:
+  junit-glob:
+    description: Shell glob matching the JUnit XML files to ship.
+    required: true
+  cell:
+    description: >-
+      Cell label stored as labels.ci_cell — the same value the job puts in
+      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
+      Without it, results from different topologies collapse into one bucket.
+    required: true
+  es-sink:
+    description: host:port of the Elasticsearch sink.
+    required: false
+    default: "192.168.1.13:9200"
+  index:
+    description: Target index.
+    required: false
+    default: mulga-ci-test-results
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        JUNIT_GLOB: ${{ inputs.junit-glob }}
+        JUNIT_CELL: ${{ inputs.cell }}
+        MULGA_ES_SINK: ${{ inputs.es-sink }}
+        ES_INDEX: ${{ inputs.index }}
+        RUN_ID: ${{ github.run_id }}
+        RUN_ATTEMPT: ${{ github.run_attempt }}
+        WORKFLOW: ${{ github.workflow }}
+        BRANCH: ${{ github.head_ref || github.ref_name }}
+        SHA: ${{ github.sha }}
+        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Ship per-test JUnit results to the Elasticsearch sink.
+
+One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
+mulga-ci-runs summary says which suite failed; this says which test, with the
+failure text, so a per-test pass-rate trend outlives artifact retention.
+
+Inputs (env):
+    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
+                    a leg that never ran produces none.
+    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
+    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
+                    there, so no credentials are involved.
+    ES_INDEX        Target index; defaults to mulga-ci-test-results.
+    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+
+# Go test output carries colour codes into the JUnit CDATA (harness.Phase
+# emits them), which are noise in a stored failure message.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+# Enough failure text to recognise a failure; the artifact log stays the full
+# record.
+_MAX_MESSAGE = 4000
+
+# go-junit-report emits this when a `=== RUN` line has no matching result line,
+# which means the log it was fed was truncated or filtered. It is a parse gap,
+# not a test outcome, so it is stored under its own status and left out of every
+# pass rate rather than counted as a failure.
+_NO_RESULT = "No test result found"
+
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+
+def _truncate(s: str) -> str:
+    s = s.strip()
+    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
+
+
+def _suite_from_path(path: str) -> str:
+    base = os.path.basename(path)
+    if base.endswith(".xml"):
+        base = base[:-4]
+    if base.startswith("junit-"):
+        base = base[len("junit-") :]
+    return base
+
+
+def _cases(path: str) -> list[dict]:
+    """Parse one JUnit file into leaf test cases."""
+    try:
+        root = ET.parse(path).getroot()
+    except ET.ParseError as exc:
+        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
+        return []
+
+    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
+    # several. .iter() handles both.
+    out: list[dict] = []
+    for s in root.iter("testsuite"):
+        for tc in s.findall("testcase"):
+            status = "pass"
+            fail_type = ""
+            message = ""
+            for tag in ("failure", "error"):
+                node = tc.find(tag)
+                if node is not None:
+                    status = "fail" if tag == "failure" else "error"
+                    fail_type = tag
+                    message = _strip_ansi(
+                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
+                    )
+                    if (node.get("message") or "").strip() == _NO_RESULT:
+                        status = "unknown"
+                    break
+            if status == "pass" and tc.find("skipped") is not None:
+                status = "skip"
+            out.append(
+                {
+                    "name": tc.get("name", ""),
+                    "classname": tc.get("classname", ""),
+                    "time": float(tc.get("time", 0) or 0),
+                    "status": status,
+                    "fail_type": fail_type,
+                    "message": message,
+                }
+            )
+
+    # Drop container cases: Go reports a parent t.Run as a rollup of its
+    # children, so keeping both double-counts one outcome. Same rule the job
+    # summary renderer applies.
+    names = {c["name"] for c in out}
+    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
+    return [c for c in out if c["name"] not in containers]
+
+
+def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
+    full = case["name"]
+    top, _, sub = full.partition("/")
+    doc = {
+        "@timestamp": ts,
+        "suite": suite,
+        "status": case["status"],
+        "duration_seconds": case["time"],
+        "artifact_url": ci["artifact_url"],
+        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
+        "test": {
+            "name": top,
+            "subtest": sub,
+            "full_name": full,
+            "classname": case["classname"],
+        },
+        "ci": {
+            "workflow": ci["workflow"],
+            "branch": ci["branch"],
+            "sha": ci["sha"],
+            "run_attempt": ci["run_attempt"],
+        },
+    }
+    if case["fail_type"]:
+        doc["failure"] = {
+            "type": case["fail_type"],
+            "message": _truncate(case["message"]),
+        }
+    return doc
+
+
+def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
+    # Deterministic so a replayed post overwrites instead of double-counting.
+    # run_attempt is part of it because a re-run is a distinct result.
+    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
+
+
+def _bulk(sink: str, index: str, body: str) -> int:
+    req = urllib.request.Request(
+        f"http://{sink}/{index}/_bulk",
+        data=body.encode("utf-8"),
+        headers={"Content-Type": "application/x-ndjson"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            payload = json.loads(resp.read().decode("utf-8"))
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
+        return 1
+
+    if payload.get("errors"):
+        for item in payload.get("items", []):
+            err = next(iter(item.values())).get("error")
+            if err:
+                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
+                return 1
+    return 0
+
+
+def main() -> int:
+    pattern = os.environ.get("JUNIT_GLOB", "").strip()
+    if not pattern:
+        print("::error::junit-to-es: JUNIT_GLOB is empty")
+        return 1
+    cell = os.environ.get("JUNIT_CELL", "").strip()
+    if not cell:
+        print("::error::junit-to-es: JUNIT_CELL is empty")
+        return 1
+    sink = os.environ.get("MULGA_ES_SINK", "").strip()
+    if not sink:
+        print("::error::junit-to-es: MULGA_ES_SINK is empty")
+        return 1
+    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
+
+    ci = {
+        "run_id": os.environ.get("RUN_ID", ""),
+        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
+        "workflow": os.environ.get("WORKFLOW", ""),
+        "branch": os.environ.get("BRANCH", ""),
+        "sha": os.environ.get("SHA", ""),
+        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
+    }
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    lines: list[str] = []
+    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
+    for path in sorted(glob.glob(pattern)):
+        suite = _suite_from_path(path)
+        for case in _cases(path):
+            doc = _doc(cell, suite, case, ts, ci)
+            counts[case["status"]] = counts.get(case["status"], 0) + 1
+            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
+            lines.append(json.dumps(meta))
+            lines.append(json.dumps(doc))
+
+    if not lines:
+        # Not an error on its own: the leg may have been skipped, or died
+        # before any test ran. The run summary doc records that separately.
+        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
+        return 0
+
+    rc = _bulk(sink, index, "\n".join(lines) + "\n")
+    if rc == 0:
+        print(
+            f"junit-to-es: posted {sum(counts.values())} test docs "
+            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
+            f"{counts['fail']} fail, {counts['error']} error, "
+            f"{counts['skip']} skip, {counts['unknown']} unknown"
+        )
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/actions/otel-job-id/action.yml

@@ -0,0 +1,52 @@
+name: Resolve OTEL job id
+description: >-
+  Append ci.job_id to SPINIFEX_OTEL_ATTRS so telemetry emitted by this job can
+  be pivoted straight back to its GitHub job log. Must run before the step that
+  bakes SPINIFEX_OTEL_ATTRS into a VM's telemetry.env, and requires
+  `actions: read` on the calling job.
+
+inputs:
+  job-name:
+    description: >-
+      Rendered name of the calling job, exactly as it appears in the Actions UI.
+      github.job is only the YAML key and no context exposes the numeric id, so
+      the id has to be looked up by name.
+    required: true
+  token:
+    description: Token used for the Actions jobs API lookup.
+    required: false
+    default: ${{ github.token }}
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve CI job id (best-effort)
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+        JOB_NAME: ${{ inputs.job-name }}
+        REPO: ${{ github.repository }}
+        RUN_ID: ${{ github.run_id }}
+      run: |
+        # Never fail the caller: on any lookup error or name mismatch ci.job_id
+        # is simply omitted and the run proceeds with the labels it already has.
+        set +e
+
+        if [ -z "${SPINIFEX_OTEL_ATTRS:-}" ]; then
+          echo "SPINIFEX_OTEL_ATTRS is unset or empty — telemetry is disabled, nothing to label"
+          exit 0
+        fi
+
+        # env.JOB_NAME rather than shell interpolation into the jq filter: job
+        # names carry quotes, slashes and non-ASCII separators that would
+        # otherwise break the expression.
+        JOB_ID=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}/jobs" --paginate \
+          --jq '.jobs[] | select(.name == env.JOB_NAME) | .id' 2>/dev/null | head -n1)
+
+        if [ -n "$JOB_ID" ]; then
+          echo "SPINIFEX_OTEL_ATTRS=${SPINIFEX_OTEL_ATTRS},ci.job_id=${JOB_ID}" >> "$GITHUB_ENV"
+          echo "resolved ci.job_id=${JOB_ID} for job '${JOB_NAME}'"
+        else
+          echo "::warning::could not resolve numeric job id for '${JOB_NAME}' — continuing without ci.job_id"
+        fi
+        exit 0

.github/assets/architecture.svg

@@ -1,225 +1,114 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="900" height="620" viewBox="0 0 900 620">
-  <defs>
-    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
-      <stop offset="0%" style="stop-color:#0a0a0a"/>
-      <stop offset="100%" style="stop-color:#0d1117"/>
-    </linearGradient>
-    <linearGradient id="bar-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#161b22"/>
-      <stop offset="45%" style="stop-color:#1f2028"/>
-      <stop offset="100%" style="stop-color:#161b22"/>
-    </linearGradient>
-    <linearGradient id="client-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#3fb95022"/>
-      <stop offset="100%" style="stop-color:#3fb95008"/>
-    </linearGradient>
-    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#DC262633"/>
-      <stop offset="100%" style="stop-color:#7F1D1D14"/>
-    </linearGradient>
-    <linearGradient id="nats-grad" x1="0%" y1="0%" x2="100%" y2="0%">
-      <stop offset="0%" style="stop-color:#F87171"/>
-      <stop offset="50%" style="stop-color:#DC2626"/>
-      <stop offset="100%" style="stop-color:#7F1D1D"/>
-    </linearGradient>
-  </defs>
-
-  <rect width="900" height="620" fill="url(#bg)" rx="12"/>
-
-  <!-- Title bar -->
-  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ARCHITECTURE</text>
-  <text x="24" y="52" font-family="'Segoe UI', system-ui, sans-serif" font-size="16" fill="#f5f5f5" font-weight="500">AWS SDK → Gateway → NATS → Service daemons</text>
-  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">stateless · message-driven · horizontally scalable</text>
-
-  <!-- ═══ CLIENT LAYER ════════════════════════════════════ -->
-  <rect x="16" y="72" width="868" height="78" rx="8" fill="url(#client-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.5"/>
-  <text x="32" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" font-weight="700" letter-spacing="1">CLIENTS</text>
-  <text x="32" y="108" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">Standard AWS tooling — unmodified, no shims, no custom SDK</text>
-
-  <g transform="translate(32, 120)" font-family="'SF Mono', ui-monospace, monospace" font-size="10">
-    <rect x="0" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="43" y="15" fill="#7ee787" text-anchor="middle">aws-cli</text>
-
-    <rect x="94" y="0" width="96" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="142" y="15" fill="#7ee787" text-anchor="middle">Terraform</text>
-
-    <rect x="198" y="0" width="76" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="236" y="15" fill="#7ee787" text-anchor="middle">boto3</text>
-
-    <rect x="282" y="0" width="98" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="331" y="15" fill="#7ee787" text-anchor="middle">aws-sdk-go</text>
-
-    <rect x="388" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="431" y="15" fill="#7ee787" text-anchor="middle">kubectl</text>
-
-    <rect x="482" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="525" y="15" fill="#7ee787" text-anchor="middle">Packer</text>
-
-    <rect x="576" y="0" width="120" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
-    <text x="636" y="15" fill="#7ee787" text-anchor="middle">your application</text>
-  </g>
-
-  <!-- ═══ HTTPS + SigV4 connector ═══════════════════════════ -->
-  <line x1="450" y1="150" x2="450" y2="178" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
-  <rect x="390" y="154" width="120" height="18" rx="9" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
-  <text x="450" y="166" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#F87171" text-anchor="middle">HTTPS · SigV4 · :9999</text>
-
-  <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
-  <rect x="16" y="178" width="868" height="68" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
-  <text x="32" y="200" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY</text>
-  <text x="32" y="215" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
-
-  <g transform="translate(32, 224)" font-family="'SF Mono', ui-monospace, monospace" font-size="9">
-    <rect x="0" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="39" y="11" fill="#FCA5A5" text-anchor="middle">TLS 1.3</text>
-
-    <rect x="86" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="125" y="11" fill="#FCA5A5" text-anchor="middle">SigV4 auth</text>
-
-    <rect x="172" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="221" y="11" fill="#FCA5A5" text-anchor="middle">policy eval</text>
-
-    <rect x="278" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="327" y="11" fill="#FCA5A5" text-anchor="middle">req routing</text>
-
-    <rect x="384" y="0" width="88" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="428" y="11" fill="#FCA5A5" text-anchor="middle">rate-limit</text>
-
-    <rect x="480" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="529" y="11" fill="#FCA5A5" text-anchor="middle">audit log</text>
-  </g>
-
-  <!-- ═══ NATS BUS ═══════════════════════════════════════ -->
-  <line x1="450" y1="246" x2="450" y2="272" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
-  <rect x="16" y="272" width="868" height="34" rx="6" fill="url(#nats-grad)" opacity="0.92"/>
-  <text x="32" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#f5f5f5" font-weight="700" letter-spacing="1.5">NATS BUS</text>
-  <text x="140" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#fee2e2" opacity="0.95">ec2.*   ebs.*   s3.*   vpc.*   iam.*</text>
-  <text x="876" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#fee2e2" text-anchor="end" opacity="0.9">queue groups · fan-out · request/reply</text>
-
-  <!-- ═══ DAEMON FANOUT CONNECTORS ═══════════════════════ -->
-  <g stroke="#DC2626" stroke-width="0.9" stroke-dasharray="3,3" opacity="0.55" fill="none">
-    <line x1="100" y1="306" x2="100" y2="328"/>
-    <line x1="275" y1="306" x2="275" y2="328"/>
-    <line x1="450" y1="306" x2="450" y2="328"/>
-    <line x1="625" y1="306" x2="625" y2="328"/>
-    <line x1="800" y1="306" x2="800" y2="328"/>
-  </g>
-
-  <!-- ═══ SERVICE DAEMONS ROW ═══════════════════════════ -->
-  <!-- EC2 -->
-  <g transform="translate(28, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ec2.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">RunInstances</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">TerminateInstances</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">QMP · cloud-init</text>
-  </g>
-
-  <!-- EBS -->
-  <g transform="translate(204, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ebs.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">CreateVolume</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">AttachVolume</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">snapshots · WAL</text>
-  </g>
-
-  <!-- S3 -->
-  <g transform="translate(380, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">s3.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">PutObject · GetObject</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Multipart · presigned</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">erasure-coded</text>
-  </g>
-
-  <!-- VPC -->
-  <g transform="translate(556, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">vpc.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Subnets · SGs · RTs</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">ALB · NAT · IGW</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">OVN · OVS</text>
-  </g>
-
-  <!-- IAM -->
-  <g transform="translate(732, 328)">
-    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">iam.* daemon</text>
-    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
-    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Users · Roles · Keys</text>
-    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Policy evaluation</text>
-    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">audit trail</text>
-  </g>
-
-  <!-- ═══ DAEMON → BACKEND CONNECTORS ═══════════════════ -->
-  <g stroke="#71717a" stroke-width="0.8" stroke-dasharray="2,3" opacity="0.55" fill="none">
-    <line x1="100" y1="410" x2="100" y2="440"/>
-    <line x1="276" y1="410" x2="276" y2="440"/>
-    <line x1="452" y1="410" x2="452" y2="440"/>
-    <line x1="628" y1="410" x2="628" y2="440"/>
-    <line x1="804" y1="410" x2="804" y2="440"/>
-  </g>
-
-  <!-- ═══ BACKEND RESOURCES BAR ═════════════════════════ -->
-  <rect x="16" y="440" width="868" height="26" rx="5" fill="url(#bar-grad)"/>
-  <text x="450" y="458" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle" letter-spacing="1.5">BACKEND RESOURCES · local hardware · no cloud dependency</text>
-
-  <!-- ═══ BACKEND ROW ═══════════════════════════════════ -->
-  <!-- QEMU/KVM -->
-  <g transform="translate(28, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">hypervisor</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">host kernel · virtio</text>
-  </g>
-
-  <!-- Viperblock -->
-  <g transform="translate(204, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Viperblock</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">block storage</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">NVMe WAL · NBD</text>
-  </g>
-
-  <!-- Predastore -->
-  <g transform="translate(380, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Predastore</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">object storage</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">Reed–Solomon</text>
-  </g>
-
-  <!-- OVN/OVS -->
-  <g transform="translate(556, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">OVN / OVS</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">SDN fabric</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">overlays · flow rules</text>
-  </g>
-
-  <!-- IAM store -->
-  <g transform="translate(732, 474)">
-    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
-    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Identity store</text>
-    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">users · keys · rules</text>
-    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">local · audit-logged</text>
-  </g>
-
-  <!-- ═══ FOOTER NOTES ══════════════════════════════════ -->
-  <g transform="translate(16, 558)">
-    <rect width="868" height="48" rx="6" fill="#0a0a0a" stroke="#27272a" stroke-width="0.8"/>
-    <text x="24" y="20" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
-      <tspan fill="#F87171" font-weight="600">Queue groups</tspan> load-balance ops across daemon replicas.
-      <tspan fill="#F87171" font-weight="600"> Fan-out</tspan> (no queue group) dispatches describe calls to every node.
-    </text>
-    <text x="24" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
-      Daemons are <tspan fill="#F87171" font-weight="600">stateless</tspan> — scale horizontally by starting more.
-      No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.
-    </text>
-  </g>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="770" viewBox="0 0 1020 770" role="img" aria-label="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus; stateless ec2/ebs/s3/vpc/iam daemons answer and drive local backends (QEMU/KVM, Viperblock, Predastore, OVN/OVS, identity store).">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0.0" y="0.0" width="1020.0" height="770.0" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28.0" y="42.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ARCHITECTURE</text>
+<text x="992.0" y="42.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">stateless · message-driven · horizontally scalable</text>
+<text x="28.0" y="72.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">AWS SDK → Gateway → NATS → service daemons</text>
+<rect x="28.0" y="92.0" width="964.0" height="92.0" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48.0" y="119.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">CLIENTS</text>
+<text x="48.0" y="140.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#5B636C" font-weight="400" text-anchor="start">Standard AWS tooling, unmodified. No shims, no custom SDK.</text>
+<rect x="46.0" y="153.0" width="68.2" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="80.1" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli</text>
+<rect x="123.2" y="153.0" width="81.4" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="163.9" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Terraform</text>
+<rect x="213.6" y="153.0" width="55.0" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="241.1" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">boto3</text>
+<rect x="277.6" y="153.0" width="88.0" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="321.6" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-sdk-go</text>
+<rect x="374.6" y="153.0" width="68.2" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="408.7" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">kubectl</text>
+<rect x="451.8" y="153.0" width="61.6" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="482.6" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Packer</text>
+<rect x="522.4" y="153.0" width="74.8" height="22.0" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="559.8" y="167.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">your app</text>
+<line x1="510.0" y1="186.0" x2="510.0" y2="192.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="429.9" y="191.0" width="160.3" height="22.0" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.4"/>
+<text x="510.0" y="205.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="400" text-anchor="middle" fill-opacity="0.9">HTTPS · SigV4 · :9999</text>
+<line x1="510.0" y1="212.0" x2="510.0" y2="220.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28.0" y="224.0" width="964.0" height="94.0" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="48.0" y="251.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="700" text-anchor="start" letter-spacing="1.5">AWS GATEWAY</text>
+<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/runtime/roles/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
+<rect x="46.0" y="287.0" width="68.2" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="80.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">TLS 1.3</text>
+<rect x="123.2" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="167.2" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">SigV4 auth</text>
+<rect x="220.2" y="287.0" width="94.6" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="267.5" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">policy eval</text>
+<rect x="323.8" y="287.0" width="94.6" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="371.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">req routing</text>
+<rect x="427.4" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.4" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">rate-limit</text>
+<rect x="524.4" y="287.0" width="81.4" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="565.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">audit log</text>
+<line x1="510.0" y1="320.0" x2="510.0" y2="332.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28.0" y="334.0" width="964.0" height="52.0" rx="12" fill="url(#nats)" fill-opacity="1.0" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<rect x="28.0" y="334.0" width="964.0" height="52.0" rx="12" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55"/>
+<text x="50.0" y="365.6" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">NATS BUS</text>
+<text x="178.0" y="364.6" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#F5F3EE" font-weight="400" text-anchor="start" fill-opacity="0.92">ec2.*  ebs.*  s3.*  vpc.*  iam.*</text>
+<text x="972.0" y="363.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F5F3EE" font-weight="400" text-anchor="end" fill-opacity="0.9">queue groups · fan-out · request/reply</text>
+<line x1="119.6" y1="388.0" x2="119.6" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="314.8" y1="388.0" x2="314.8" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="510.0" y1="388.0" x2="510.0" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="705.2" y1="388.0" x2="705.2" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="900.4" y1="388.0" x2="900.4" y2="408.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="28.0" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="44.0" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">ec2.* daemon</text>
+<line x1="44.0" y1="448.0" x2="195.2" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="44.0" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">RunInstances</text>
+<text x="44.0" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">TerminateInstances</text>
+<text x="44.0" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">QMP · cloud-init</text>
+<rect x="223.2" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="239.2" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">ebs.* daemon</text>
+<line x1="239.2" y1="448.0" x2="390.4" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="239.2" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">CreateVolume</text>
+<text x="239.2" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">AttachVolume</text>
+<text x="239.2" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">snapshots · WAL</text>
+<rect x="418.4" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="434.4" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">s3.* daemon</text>
+<line x1="434.4" y1="448.0" x2="585.6" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="434.4" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">PutObject · GetObject</text>
+<text x="434.4" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Multipart · presigned</text>
+<text x="434.4" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">erasure-coded</text>
+<rect x="613.6" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="629.6" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">vpc.* daemon</text>
+<line x1="629.6" y1="448.0" x2="780.8" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="629.6" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Subnets · SGs · RTs</text>
+<text x="629.6" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">ALB · NAT · IGW</text>
+<text x="629.6" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">OVN · OVS</text>
+<rect x="808.8" y="412.0" width="183.2" height="104.0" rx="11" fill="#EE4B12" fill-opacity="0.045" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="824.8" y="438.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="700" text-anchor="start">iam.* daemon</text>
+<line x1="824.8" y1="448.0" x2="976.0" y2="448.0" stroke="#EE4B12" stroke-opacity="0.22" stroke-width="1"/>
+<text x="824.8" y="468.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Users · Roles · Keys</text>
+<text x="824.8" y="486.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">Policy evaluation</text>
+<text x="824.8" y="502.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">audit trail</text>
+<line x1="119.6" y1="516.0" x2="119.6" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="314.8" y1="516.0" x2="314.8" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="510.0" y1="516.0" x2="510.0" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="705.2" y1="516.0" x2="705.2" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<line x1="900.4" y1="516.0" x2="900.4" y2="540.0" stroke="#5B636C" stroke-width="1.3" stroke-opacity="0.35" stroke-dasharray="2 5"/>
+<rect x="28.0" y="530.0" width="964.0" height="30.0" rx="9" fill="#1E242B" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="510.0" y="548.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle" letter-spacing="1">BACKEND RESOURCES  ·  local hardware  ·  no cloud dependency</text>
+<rect x="28.0" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="119.6" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">QEMU / KVM</text>
+<text x="119.6" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">hypervisor</text>
+<text x="119.6" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">host kernel · virtio</text>
+<rect x="223.2" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="314.8" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Viperblock</text>
+<text x="314.8" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">block storage</text>
+<text x="314.8" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">NVMe WAL · NBD</text>
+<rect x="418.4" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="510.0" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Predastore</text>
+<text x="510.0" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">object storage</text>
+<text x="510.0" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Reed-Solomon</text>
+<rect x="613.6" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="705.2" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">OVN / OVS</text>
+<text x="705.2" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">SDN fabric</text>
+<text x="705.2" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">overlays · flow rules</text>
+<rect x="808.8" y="572.0" width="183.2" height="84.0" rx="11" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="900.4" y="602.7" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13.5" fill="#F5F3EE" font-weight="700" text-anchor="middle">Identity store</text>
+<text x="900.4" y="624.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="middle">users · keys · rules</text>
+<text x="900.4" y="642.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">local · audit-logged</text>
+<rect x="28.0" y="670.0" width="964.0" height="72.0" rx="11" fill="#181D23" fill-opacity="0.6" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48.0" y="697.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400"><tspan fill="#EE4B12" font-weight="700">Queue groups</tspan><tspan fill="#5B636C"> load-balance ops across daemon replicas. </tspan><tspan fill="#EE4B12" font-weight="700">Fan-out</tspan><tspan fill="#5B636C"> (no queue group) dispatches describe calls to every node.</tspan></text>
+<text x="48.0" y="720.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400"><tspan fill="#5B636C">Daemons are </tspan><tspan fill="#EE4B12" font-weight="700">stateless</tspan><tspan fill="#5B636C">: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.</tspan></text>
+</svg>

.github/assets/design-architecture.svg

@@ -51,7 +51,7 @@
   <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
   <rect x="16" y="154" width="868" height="104" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
   <text x="32" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY · port 9999</text>
-  <text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/services/awsgw/awsgw.go</text>
+  <text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/runtime/roles/awsgw/awsgw.go</text>
 
   <!-- Two-column feature bullets -->
   <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">

.github/assets/diagrams/ecs-topology.svg

@@ -0,0 +1,102 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">ECS · EC2 LAUNCH TYPE ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">container instances from spinifex-ecs-node AMI · awsvpc + ALB</text>
+
+  <!-- Internet -->
+  <rect x="120" y="52" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="220" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+
+  <!-- connector to ALB -->
+  <line x1="220" y1="92" x2="220" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="216,108 224,108 220,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- ALB -->
+  <g transform="translate(90, 116)">
+    <rect width="260" height="60" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">Application Load Balancer</text>
+    <line x1="16" y1="32" x2="244" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787">target group · target_type = ip</text>
+  </g>
+
+  <!-- Spinifex Gateway (control plane) -->
+  <g transform="translate(560, 110)">
+    <rect width="316" height="72" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">Spinifex Gateway :9999</text>
+    <text x="296" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">control plane</text>
+    <line x1="16" y1="32" x2="300" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">ECS agent registers · TLS + SigV4 (IMDS creds)</text>
+    <text x="20" y="63" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">task role creds → container at 169.254.170.2</text>
+  </g>
+
+  <!-- ALB fan-out to subnets (traffic) -->
+  <line x1="220" y1="176" x2="220" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="650" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="220" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="650" y1="200" x2="650" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="435" y="195" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">balances to each task's ENI</text>
+
+  <!-- agent path: subnets up to gateway -->
+  <line x1="718" y1="228" x2="718" y2="182" stroke="#F87171" stroke-width="1.1" stroke-dasharray="3,3" opacity="0.6"/>
+  <polygon points="714,186 722,186 718,178" fill="#F87171" opacity="0.8"/>
+
+  <!-- subnet_a -->
+  <g transform="translate(60, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- container instance -->
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="225" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(520, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="103" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">image pull</text>
+    <text x="225" y="117" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">registry / ECR</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="410" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">No ECS launch API — you add capacity by booting EC2 instances from the spinifex-ecs-node AMI (the console's "Provision capacity" wraps RunInstances)</text>
+  <text x="450" y="426" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">The service keeps the desired task count running and registers each task's ENI IP with the ALB target group</text>
+</svg>

.github/assets/diagrams/eks-topology.svg

@@ -0,0 +1,96 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="cp-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">EKS · MANAGED KUBERNETES ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">control plane VM + NLB · workers from eks-node AMI</text>
+
+  <!-- Internet / kubectl -->
+  <rect x="330" y="52" width="240" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="450" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet · kubectl + aws eks</text>
+
+  <!-- connector to control plane -->
+  <line x1="450" y1="92" x2="450" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="446,108 454,108 450,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- Control plane (Spinifex-managed) -->
+  <g transform="translate(290, 116)">
+    <rect width="320" height="76" rx="8" fill="url(#cp-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">EKS Control Plane</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">Spinifex-managed</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="87" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">API NLB (public)</text>
+    <rect x="165" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="232" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">apiserver VM</text>
+  </g>
+
+  <!-- authn note -->
+  <rect x="624" y="138" width="172" height="32" rx="6" fill="#0a0a0a" stroke="#71717a" stroke-width="0.6" stroke-opacity="0.5"/>
+  <text x="710" y="151" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">authn: API · access entries</text>
+  <text x="710" y="163" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">aws eks get-token (IAM → RBAC)</text>
+
+  <!-- fan-out to node-group subnets -->
+  <line x1="450" y1="192" x2="450" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="670" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="230" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="670" y1="210" x2="670" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="450" y="206" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">managed node group · workers join the control plane</text>
+
+  <!-- subnet_a -->
+  <g transform="translate(108, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- worker node -->
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(472, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">ECR</text>
+    <text x="248" y="107" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">pull</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="404" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">You provide the VPC, subnets, and IAM roles · Spinifex provisions the control plane, node-group VMs, and security groups</text>
+  <text x="450" y="420" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Node groups always boot the registered eks-node image · workers need egress (IGW / NAT) to pull images</text>
+</svg>

.github/assets/diagrams/exoscale-single-node.svg

@@ -0,0 +1,79 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single Exoscale instance: the instance sits on the zone&#x27;s public network with one interface, eth0, holding its own address and the default route. Manual Elastic IPs are delivered to eth0 and routed by the host into OVN through the spx-nat transit veth, and an Exoscale block storage volume backs /var/lib/spinifex.">
+<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — SINGLE NODE</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · Elastic IPs</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One Exoscale instance, single Spinifex node</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
+<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
+<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1  ·  PUBLIC NETWORK 91.92.116.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">gateway 91.92.116.1 · security group open to all</text>
+<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">standard.huge</text>
+<text x="212" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01  ·  8 vCPU / 32 GiB  ·  Ubuntu 26.04</text>
+<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
+<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="68" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">eth0 — PUBLIC INTERFACE</text>
+<text x="482" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">the only NIC</text>
+<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">91.92.116.250/23  ·  holds the default route</text>
+<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · API :9999 · SSH · host masquerade</text>
+<rect x="518" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="538" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">ELASTIC IPs — MANUAL</text>
+<text x="932" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">attached to the instance</text>
+<text x="538" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">89.145.163.1/32  →  delivered to eth0</text>
+<text x="538" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">the host routes each /32 into OVN, never owns it</text>
+<line x1="735" y1="406" x2="735" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
+<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24  ·  EIP/32 routed in  ·  masquerade out eth0 for the rest</text>
+<line x1="510" y1="464" x2="510" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
+<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
+<rect x="86" y="530" width="93.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="132.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw :9999</text>
+<rect x="188.0" y="530" width="58.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="217.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="255.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="277.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="308.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="330.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">NATS</text>
+<rect x="361.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="404.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="456.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="499.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="551.0" y="530" width="79.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="590.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">northstar</text>
+<rect x="639.0" y="530" width="72.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="675.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">ui :3000</text>
+<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM (nested) · OVN/OVS · Predastore · Viperblock underneath</text>
+<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
+<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
+<rect x="86" y="644" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a  172.31.0.0/16</text>
+<text x="300" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  172.31.0.4  →  89.145.163.1</text>
+<rect x="86" y="686" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b  10.0.0.0/16</text>
+<text x="300" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  10.0.1.9  ·  private, egress only</text>
+<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK STORAGE VOLUME</text>
+<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GiB+</text>
+<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">virtio-blk  →  /dev/vdb</text>
+<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
+<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">found by-id, defaults,nofail</text>
+<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
+<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest&#x27;s public address is a manual Exoscale Elastic IP, delivered to eth0 addressed to itself — no private/public pair to translate.</text>
+<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">The host routes it into OVN, so inbound and outbound are the same address and the guest sees its private IP only.</text>
+</svg>

.github/assets/diagrams/exoscale-three-node.svg

@@ -0,0 +1,123 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three Exoscale instances in one zone, spread by an anti-affinity group. Each has a public eth0 carrying its own Elastic IPs, a private-network eth1 for the cluster mesh, its own block storage volume and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west over the private network. This topology is not yet tested.">
+<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — THREE NODES</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">anti-affinity group · one zone · not yet tested</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three Exoscale instances, redundant multi-node install</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
+<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
+<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="440" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1  ·  PUBLIC NETWORK + PRIVATE NETWORK</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one security group · one anti-affinity group</text>
+<rect x="48" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="64" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
+<text x="332" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="64" y1="272" x2="332" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="64" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="78" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.116.250</text>
+<text x="78" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="64" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="78" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.11</text>
+<text x="78" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="64" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="78" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.163.1</text>
+<text x="78" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="64" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="89.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="124.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="152.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="190.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="212.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="64" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="86.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="117.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="159.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="210.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="252.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="64" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2…  172.31.0.4 → 89.145.163.1</text>
+<rect x="64" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<rect x="360" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="376" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
+<text x="644" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="376" y1="272" x2="644" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="376" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="390" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.117.14</text>
+<text x="390" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="376" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="390" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.12</text>
+<text x="390" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="376" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="390" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.160.107</text>
+<text x="390" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="376" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="401.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="436.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="464.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="502.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="524.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="376" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="398.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="429.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="522.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="564.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="376" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="388" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91…  172.31.0.9 → 89.145.160.107</text>
+<rect x="376" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="388" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<rect x="672" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="688" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
+<text x="956" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="688" y1="272" x2="956" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="688" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.116.88</text>
+<text x="702" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="688" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.13</text>
+<text x="702" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="688" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="702" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.162.131</text>
+<text x="702" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="688" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="713.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="748.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="776.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="814.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="836.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="688" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="710.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="741.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="783.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="834.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="876.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="688" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="700" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4…  172.31.1.6 → 89.145.162.131</text>
+<rect x="688" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="700" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<line x1="510" y1="628" x2="510" y2="644" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="646" width="964" height="70" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, OVER THE EXOSCALE PRIVATE NETWORK</text>
+<text x="972" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no Elastic IP is involved · MTU 1500</text>
+<rect x="46" y="684" width="124.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="108.0" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Geneve UDP 6081</text>
+<rect x="179.0" y="684" width="137.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="247.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">NATS JetStream R3</text>
+<rect x="325.6" y="684" width="144.4" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="397.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">predastore RS(2,1)</text>
+<rect x="479.0" y="684" width="151.2" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="554.6" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVN Northbound raft</text>
+<rect x="639.2" y="684" width="56.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="667.2" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVSDB</text>
+<text x="28" y="744" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes&#x27; private-network interfaces — it never touches eth0 or an Elastic IP.</text>
+<text x="28" y="762" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Each node carries the Elastic IPs of its own guests. Moving one between nodes is an Exoscale API call, not yet automated.</text>
+<text x="28" y="780" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one; the anti-affinity group keeps them on different hypervisors.</text>
+</svg>

.github/assets/diagrams/oci-single-node.svg

@@ -0,0 +1,80 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single OCI instance: the instance sits inside the VCN public subnet 10.200.0.0/23; its primary VNIC carries the default route and the cluster mesh, its secondary VNIC is the br-wan bridge holding registered secondary private IPs paired with reserved public IPs; guests reach them through the spx-nat transit veth, and an iSCSI block volume backs /var/lib/spinifex.">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — SINGLE NODE</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · real public addresses</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One OCI instance, single Spinifex node</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
+<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
+<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16  ·  PUBLIC SUBNET 10.200.0.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
+<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">BM.Standard.E2.64</text>
+<text x="260" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01  ·  FAULT-DOMAIN-1</text>
+<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
+<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="68" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 0 — PRIMARY</text>
+<text x="482.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">enp0s9</text>
+<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.0.252/23  ·  holds the default route</text>
+<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · ovn-encap-ip · NATS · API :9999</text>
+<rect x="518.0" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="538.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 1 — SECONDARY</text>
+<text x="932.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">br-wan (Linux bridge)</text>
+<text x="538.0" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.1.233/32  +  ip rule → table 200</text>
+<text x="538.0" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">secondary private IPs ⇄ reserved public IPs</text>
+<line x1="735.0" y1="406" x2="735.0" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
+<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24  ·  nft chain spinifex-nat-egress masquerades out br-wan</text>
+<line x1="510.0" y1="464" x2="510.0" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
+<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
+<rect x="86" y="530" width="109.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="140.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw :9999</text>
+<rect x="204.01999999999998" y="530" width="74.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="241.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="287.94" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="318.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="358.22" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="388.9" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS</text>
+<rect x="428.5" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="479.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="539.7" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="590.8" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="650.9000000000001" y="530" width="95.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="698.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">northstar</text>
+<rect x="755.2800000000001" y="530" width="88.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="799.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">ui :3000</text>
+<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM · OVN/OVS · Predastore · Viperblock underneath</text>
+<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
+<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
+<rect x="86" y="644" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a  10.0.0.0/16</text>
+<text x="310" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  10.0.1.4  →  150.230.13.131</text>
+<rect x="86" y="686" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b  172.31.0.0/16</text>
+<text x="310" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  172.31.4.9  ·  private, egress only</text>
+<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK VOLUME — iSCSI</text>
+<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GB+</text>
+<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">169.254.2.2:3260  →  /dev/sdb</text>
+<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
+<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">mounted by UUID, defaults,_netdev,nofail</text>
+<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
+<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest's public address is a real OCI reserved public IP, attached to a secondary private IP on br-wan.</text>
+<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Inbound and outbound are therefore the same address, and OCI's per-VNIC source check passes.</text>
+</svg>

.github/assets/diagrams/oci-three-node.svg

@@ -0,0 +1,116 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three OCI instances: all three sit inside the VCN public subnet 10.200.0.0/23, one per fault domain. Each carries a primary VNIC for the cluster mesh and a br-wan secondary VNIC holding its reserved public addresses, its own iSCSI block volume, and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west.">
+<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
+<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — THREE NODES</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">three fault domains · one region · one API endpoint</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three OCI instances, redundant multi-node install</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
+<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
+<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="464" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16  ·  PUBLIC SUBNET 10.200.0.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
+<rect x="48.0" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="66.0" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
+<text x="327.3333333333333" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-1</text>
+<text x="66.0" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="66.0" y1="292" x2="327.3333333333333" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="66.0" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.252</text>
+<text x="66.0" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="66.0" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.233/32</text>
+<text x="66.0" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="66.0" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="91.5" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="123.0" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="151.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="186.2" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="208.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="66.0" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="88.4" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="116.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="157.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="204.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="245.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="66.0" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76.0" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2…  10.0.1.4 → 150.230.13.131</text>
+<rect x="66.0" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76.0" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<line x1="345.3333333333333" y1="364" x2="361.3333333333333" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
+<rect x="361.3333333333333" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="379.3333333333333" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
+<text x="640.6666666666666" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-2</text>
+<text x="379.3333333333333" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="379.3333333333333" y1="292" x2="640.6666666666666" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="379.3333333333333" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.118</text>
+<text x="379.3333333333333" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="379.3333333333333" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.96/32</text>
+<text x="379.3333333333333" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="379.3333333333333" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="404.8" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="436.3333333333333" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="464.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="499.5333333333333" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="521.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="379.3333333333333" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="401.7" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="430.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="518.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="559.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="379.3333333333333" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="389.3333333333333" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91…  10.0.1.9 → 137.23.9.125</text>
+<rect x="379.3333333333333" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="389.3333333333333" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<line x1="658.6666666666666" y1="364" x2="674.6666666666666" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
+<rect x="674.6666666666666" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="692.6666666666666" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
+<text x="954.0" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-3</text>
+<text x="692.6666666666666" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
+<line x1="692.6666666666666" y1="292" x2="954.0" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<text x="692.6666666666666" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0  10.200.0.41</text>
+<text x="692.6666666666666" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
+<text x="692.6666666666666" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1  br-wan  10.200.1.12/32</text>
+<text x="692.6666666666666" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
+<rect x="692.6666666666666" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="718.2" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
+<rect x="749.6666666666666" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="778.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
+<rect x="812.8666666666667" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="835.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
+<rect x="692.6666666666666" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="715.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
+<rect x="743.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="784.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
+<rect x="831.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="872.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
+<rect x="692.6666666666666" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702.6666666666666" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4…  10.0.2.6 → 150.230.4.88</text>
+<rect x="692.6666666666666" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702.6666666666666" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
+<rect x="48" y="524" width="924" height="104" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="68" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, INSIDE THE VCN</text>
+<text x="952" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no public address is involved</text>
+<rect x="66" y="574" width="136.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="134.2" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">Geneve UDP 6081</text>
+<rect x="211.3" y="574" width="149.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="286.3" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS JetStream R3</text>
+<rect x="370.24" y="574" width="156.8" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="448.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore RS(2,1)</text>
+<rect x="536.0" y="574" width="163.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="617.8" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVN Northbound raft</text>
+<rect x="708.58" y="574" width="68.1" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="742.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVSDB</text>
+<text x="66" y="612" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes' primary VNICs — it never leaves the subnet and never touches br-wan.</text>
+<rect x="28" y="668" width="964" height="46" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.55"/>
+<text x="48" y="688" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13" fill="#F5F3EE" font-weight="700" text-anchor="start">A guest that moves takes its address with it.</text>
+<text x="48" y="705" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">The reserved public IP follows the secondary private IP to whichever node's br-wan the guest lands on — the tenant sees no change.</text>
+<text x="28" y="756" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one: NATS JetStream keeps quorum, predastore RS(2,1)</text>
+<text x="28" y="774" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">keeps the data readable, and the OVN Northbound raft keeps a leader. One node per fault domain is what makes that real.</text>
+</svg>

.github/assets/images/cisco-ucs-platform-benchmark/guest-storage-throughput.svg

@@ -0,0 +1,44 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Guest block storage throughput (Viperblock)</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">fio, 4 jobs, iodepth 32, direct I/O, per-guest sequential runs — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
+<line x1="74" y1="305.0" x2="734" y2="305.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="309.0" font-size="12" text-anchor="end" fill="#6b7280">100</text>
+<line x1="74" y1="224.0" x2="734" y2="224.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="228.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
+<line x1="74" y1="143.0" x2="734" y2="143.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="147.0" font-size="12" text-anchor="end" fill="#6b7280">300</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
+<rect x="111.4" y="81.4" width="45.4" height="304.6" rx="2.5" fill="#2563eb"/>
+<text x="134.1" y="75.4" font-size="11" text-anchor="middle" fill="#1f2937">376</text>
+<rect x="159.8" y="198.1" width="45.4" height="187.9" rx="2.5" fill="#0d9488"/>
+<text x="182.5" y="192.1" font-size="11" text-anchor="middle" fill="#1f2937">232</text>
+<rect x="208.2" y="299.3" width="45.4" height="86.7" rx="2.5" fill="#f59e0b"/>
+<text x="230.9" y="293.3" font-size="11" text-anchor="middle" fill="#1f2937">107</text>
+<text x="184.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU</text>
+<rect x="331.4" y="85.5" width="45.4" height="300.5" rx="2.5" fill="#2563eb"/>
+<text x="354.1" y="79.5" font-size="11" text-anchor="middle" fill="#1f2937">371</text>
+<rect x="379.8" y="202.9" width="45.4" height="183.1" rx="2.5" fill="#0d9488"/>
+<text x="402.5" y="196.9" font-size="11" text-anchor="middle" fill="#1f2937">226</text>
+<rect x="428.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
+<text x="450.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
+<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1</text>
+<rect x="551.4" y="84.7" width="45.4" height="301.3" rx="2.5" fill="#2563eb"/>
+<text x="574.1" y="78.7" font-size="11" text-anchor="middle" fill="#1f2937">372</text>
+<rect x="599.8" y="201.3" width="45.4" height="184.7" rx="2.5" fill="#0d9488"/>
+<text x="622.5" y="195.3" font-size="11" text-anchor="middle" fill="#1f2937">228</text>
+<rect x="648.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
+<text x="670.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
+<text x="624.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">128K random read</text>
+<rect x="216.8" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
+<text x="235.8" y="455" font-size="12.5" fill="#1f2937">128K mixed read</text>
+<rect x="352.3" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
+<text x="371.3" y="455" font-size="12.5" fill="#1f2937">16K random read</text>
+</svg>

.github/assets/images/cisco-ucs-platform-benchmark/network-goodput-vs-streams.svg

@@ -0,0 +1,35 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Network throughput: overlay vs physical fabric</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">iperf3 TCP receiver goodput, median across node pairs — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">20</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">25</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">Gbit/s</text>
+<text x="140.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">1</text>
+<text x="272.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">2</text>
+<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">4</text>
+<text x="536.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">8</text>
+<text x="668.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">16</text>
+<path d="M140.0,164.4 L272.0,160.5 L404.0,160.5 L536.0,160.5 L668.0,160.5" fill="none" stroke="#2563eb" stroke-width="3"/>
+<circle cx="140.0" cy="164.4" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="272.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="404.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="536.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<circle cx="668.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
+<text x="664.0" y="149.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#2563eb">23.48</text>
+<path d="M140.0,255.1 L272.0,222.1 L404.0,222.1 L536.0,222.1 L668.0,228.5" fill="none" stroke="#0d9488" stroke-width="3"/>
+<circle cx="140.0" cy="255.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="272.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="404.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="536.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<circle cx="668.0" cy="228.5" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
+<text x="664.0" y="217.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#0d9488">22.43</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<text x="404.0" y="430.0" font-size="12.5" text-anchor="middle" fill="#6b7280">parallel TCP streams</text>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">Host physical fabric (25GbE)</text>
+<rect x="304.4" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
+<text x="323.4" y="455" font-size="12.5" fill="#1f2937">Guest overlay (OVN)</text>
+</svg>

.github/assets/images/cisco-ucs-platform-benchmark/predastore-s3-read-by-client.svg

@@ -0,0 +1,39 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
+<rect width="760" height="470" fill="#ffffff"/>
+<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Predastore S3 read throughput by client</text>
+<text x="74" y="49" font-size="12.5" fill="#6b7280">1 GiB objects, same presigned-curl client on every path — Spinifex v1.20.0</text>
+<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
+<line x1="74" y1="278.0" x2="734" y2="278.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="282.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
+<line x1="74" y1="170.0" x2="734" y2="170.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="174.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
+<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
+<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">600</text>
+<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
+<rect x="102.0" y="202.4" width="51.5" height="183.6" rx="2.5" fill="#2563eb"/>
+<text x="127.8" y="196.4" font-size="11" text-anchor="middle" fill="#1f2937">340</text>
+<rect x="156.5" y="136.5" width="51.5" height="249.5" rx="2.5" fill="#f59e0b"/>
+<text x="182.2" y="130.5" font-size="11" text-anchor="middle" fill="#1f2937">462</text>
+<text x="156.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">Host</text>
+<rect x="267.1" y="170.5" width="51.5" height="215.5" rx="2.5" fill="#2563eb"/>
+<text x="292.8" y="164.5" font-size="11" text-anchor="middle" fill="#1f2937">399</text>
+<rect x="321.5" y="174.9" width="51.5" height="211.1" rx="2.5" fill="#f59e0b"/>
+<text x="347.2" y="168.9" font-size="11" text-anchor="middle" fill="#1f2937">391</text>
+<text x="321.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU guest</text>
+<rect x="432.1" y="102.5" width="51.5" height="283.5" rx="2.5" fill="#2563eb"/>
+<text x="457.8" y="96.5" font-size="11" text-anchor="middle" fill="#1f2937">525</text>
+<rect x="486.5" y="197.0" width="51.5" height="189.0" rx="2.5" fill="#f59e0b"/>
+<text x="512.2" y="191.0" font-size="11" text-anchor="middle" fill="#1f2937">350</text>
+<text x="486.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1 guest</text>
+<rect x="597.0" y="106.8" width="51.5" height="279.2" rx="2.5" fill="#2563eb"/>
+<text x="622.8" y="100.8" font-size="11" text-anchor="middle" fill="#1f2937">517</text>
+<rect x="651.5" y="196.5" width="51.5" height="189.5" rx="2.5" fill="#f59e0b"/>
+<text x="677.2" y="190.5" font-size="11" text-anchor="middle" fill="#1f2937">351</text>
+<text x="651.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2 guest</text>
+<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
+<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
+<text x="93" y="455" font-size="12.5" fill="#1f2937">Single-stream GET</text>
+<rect x="224.1" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
+<text x="243.1" y="455" font-size="12.5" fill="#1f2937">Chunked/concurrent GET</text>
+</svg>

.github/assets/platform.svg

@@ -1,144 +1,84 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="900" height="560" viewBox="0 0 900 560">
-  <defs>
-    <linearGradient id="pf-bg" x1="0%" y1="0%" x2="100%" y2="100%">
-      <stop offset="0%" style="stop-color:#0a0a0a"/>
-      <stop offset="100%" style="stop-color:#0d1117"/>
-    </linearGradient>
-    <linearGradient id="hw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#52525b;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#52525b;stop-opacity:0.06"/>
-    </linearGradient>
-    <linearGradient id="os-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#a855f7;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#a855f7;stop-opacity:0.05"/>
-    </linearGradient>
-    <linearGradient id="spx-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0.28"/>
-      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0.1"/>
-    </linearGradient>
-    <linearGradient id="app-grad" x1="0%" y1="0%" x2="0%" y2="100%">
-      <stop offset="0%" style="stop-color:#3fb950;stop-opacity:0.18"/>
-      <stop offset="100%" style="stop-color:#3fb950;stop-opacity:0.05"/>
-    </linearGradient>
-  </defs>
-
-  <rect width="900" height="560" fill="url(#pf-bg)" rx="12"/>
-
-  <!-- Subtle mesh dots background -->
-  <g opacity="0.12" fill="#7F1D1D">
-    <circle cx="40" cy="40" r="1"/><circle cx="120" cy="30" r="1"/><circle cx="220" cy="50" r="1"/>
-    <circle cx="320" cy="25" r="1"/><circle cx="440" cy="45" r="1"/><circle cx="560" cy="30" r="1"/>
-    <circle cx="680" cy="50" r="1"/><circle cx="780" cy="30" r="1"/><circle cx="860" cy="45" r="1"/>
-    <circle cx="60" cy="530" r="1"/><circle cx="160" cy="520" r="1"/><circle cx="260" cy="540" r="1"/>
-    <circle cx="380" cy="525" r="1"/><circle cx="500" cy="540" r="1"/><circle cx="620" cy="520" r="1"/>
-    <circle cx="740" cy="540" r="1"/><circle cx="840" cy="525" r="1"/>
-  </g>
-
-  <!-- Title -->
-  <text x="450" y="32" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" text-anchor="middle" letter-spacing="2.8">THE MULGA PLATFORM</text>
-  <text x="450" y="58" font-family="'Segoe UI', system-ui, sans-serif" font-size="20" font-weight="600" fill="#f5f5f5" text-anchor="middle">AWS-style workflows, on your own stack</text>
-
-  <!-- Layer 4: Applications (top) -->
-  <g transform="translate(60, 95)">
-    <rect width="780" height="88" rx="10" fill="url(#app-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
-    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" letter-spacing="1.5" font-weight="600">LAYER 4 · APPLICATIONS &amp; TOOLING</text>
-    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">Your workloads — unmodified AWS tooling</text>
-
-    <g transform="translate(20, 60)" font-family="'Segoe UI', sans-serif" font-size="10.5">
-      <rect x="0" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="35" y="14" fill="#7ee787" text-anchor="middle">aws-cli</text>
-
-      <rect x="78" y="0" width="78" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="117" y="14" fill="#7ee787" text-anchor="middle">Terraform</text>
-
-      <rect x="164" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="208" y="14" fill="#7ee787" text-anchor="middle">AWS SDKs</text>
-
-      <rect x="260" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="304" y="14" fill="#7ee787" text-anchor="middle">Kubernetes</text>
-
-      <rect x="356" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="391" y="14" fill="#7ee787" text-anchor="middle">Packer</text>
-
-      <rect x="434" y="0" width="98" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
-      <text x="483" y="14" fill="#7ee787" text-anchor="middle">Your Services</text>
-    </g>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 188)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 3: Spinifex (highlighted) -->
-  <g transform="translate(60, 215)">
-    <rect width="780" height="140" rx="10" fill="url(#spx-grad)" stroke="#DC2626" stroke-width="1.5" stroke-opacity="0.75"/>
-    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" letter-spacing="1.5" font-weight="600">LAYER 3 · SPINIFEX · AWS ORCHESTRATION</text>
-    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">AWS Gateway → NATS → Service Daemons · SigV4, IAM, VPC-aware</text>
-
-    <g transform="translate(20, 66)" font-family="'Segoe UI', sans-serif">
-      <!-- EC2 -->
-      <rect x="0" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="59" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EC2</text>
-      <text x="59" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">QEMU/KVM VMs</text>
-      <text x="59" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">cloud-init · AMIs</text>
-
-      <!-- EBS -->
-      <rect x="126" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="185" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EBS</text>
-      <text x="185" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Viperblock</text>
-      <text x="185" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">WAL · NBD · snapshots</text>
-
-      <!-- S3 -->
-      <rect x="252" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="311" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">S3</text>
-      <text x="311" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Predastore</text>
-      <text x="311" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Reed–Solomon EC</text>
-
-      <!-- VPC -->
-      <rect x="378" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
-      <text x="437" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">VPC</text>
-      <text x="437" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">OVN / OVS</text>
-      <text x="437" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">subnets · SG · ALB</text>
-
-      <!-- IAM -->
-      <rect x="504" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
-      <text x="563" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">IAM</text>
-      <text x="563" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">SigV4 · policies</text>
-      <text x="563" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">users · roles · keys</text>
-
-      <!-- Control -->
-      <rect x="630" y="0" width="110" height="58" rx="6" fill="#27272a" fill-opacity="0.8" stroke="#71717a" stroke-opacity="0.6" stroke-width="1"/>
-      <text x="685" y="18" font-size="13" fill="#e4e4e7" text-anchor="middle" font-weight="600">Control</text>
-      <text x="685" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">NATS bus</text>
-      <text x="685" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">spx · daemons</text>
-    </g>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 360)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 2: OS -->
-  <g transform="translate(60, 387)">
-    <rect width="780" height="62" rx="10" fill="url(#os-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.5"/>
-    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" letter-spacing="1.5" font-weight="600">LAYER 2 · OPERATING SYSTEM</text>
-    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · headless autoinstall</text>
-  </g>
-
-  <!-- Down arrow -->
-  <g transform="translate(450, 454)" opacity="0.45">
-    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
-    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
-  </g>
-
-  <!-- Layer 1: Hardware -->
-  <g transform="translate(60, 481)">
-    <rect width="780" height="62" rx="10" fill="url(#hw-grad)" stroke="#71717a" stroke-width="1.2" stroke-opacity="0.6"/>
-    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" letter-spacing="1.5" font-weight="600">LAYER 1 · HARDWARE</text>
-    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Single node · multi-node cluster · edge appliance · air-gapped rack · commodity x86 / ARM</text>
-  </g>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="664" viewBox="0 0 1000 664" role="img" aria-label="The Spinifex stack: applications and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU/GPU hardware, deployable to Neocloud, on-prem or edge.">
+<rect x="0.0" y="0.0" width="1000.0" height="664.0" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<rect x="28.0" y="204.0" width="950.0" height="438.0" rx="16" fill="none" fill-opacity="1.0" stroke="#9AA1A8" stroke-width="1" stroke-opacity="0.12" stroke-dasharray="2 4"/>
+<rect x="25.0" y="201.0" width="950.0" height="438.0" rx="16" fill="none" fill-opacity="1.0" stroke="#9AA1A8" stroke-width="1" stroke-opacity="0.2" stroke-dasharray="2 4"/>
+<rect x="22.0" y="198.0" width="950.0" height="438.0" rx="16" fill="#181D23" fill-opacity="0.25" stroke="#9AA1A8" stroke-width="1.3" stroke-opacity="0.45" stroke-dasharray="2 4"/>
+<text x="36.0" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Apps</text>
+<rect x="136.0" y="34.0" width="836.0" height="54.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="66.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">Your infrastructure &amp; apps</text>
+<rect x="516.7" y="51.0" width="89.3" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.16" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<text x="561.4" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="middle">AI training</text>
+<rect x="613.0" y="51.0" width="76.7" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.16" stroke="none" stroke-width="1" stroke-opacity="1.0"/>
+<text x="651.4" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="middle">Inference</text>
+<rect x="696.7" y="51.0" width="79.0" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="736.2" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Web &amp; APIs</text>
+<rect x="782.7" y="51.0" width="72.7" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="819.1" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Databases</text>
+<rect x="862.4" y="51.0" width="91.6" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="908.2" y="64.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Batch &amp; data</text>
+<text x="36.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Tooling</text>
+<rect x="136.0" y="102.0" width="836.0" height="54.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="134.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">AWS tooling</text>
+<rect x="600.9" y="119.0" width="64.1" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="633.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">aws-cli</text>
+<rect x="672.0" y="119.0" width="76.7" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="710.4" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">Terraform</text>
+<rect x="755.7" y="119.0" width="64.1" height="20.0" rx="10" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="787.8" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.85">kubectl</text>
+<rect x="826.8" y="119.0" width="66.4" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="860.0" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">AWS SDKs</text>
+<rect x="900.2" y="119.0" width="53.8" height="20.0" rx="10" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1"/>
+<text x="927.1" y="132.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="middle">Packer</text>
+<text x="484.0" y="184.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">↓  AWS API · SigV4</text>
+<text x="624.0" y="184.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">RESPONSE  ↑</text>
+<text x="136.0" y="231.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="15" fill="#F5F3EE" font-weight="700" text-anchor="start">One portable stack.</text>
+<text x="136.0" y="249.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="start">Runs the same on a single node, cluster, edge, or air-gapped rack.</text>
+<rect x="752.3" y="222.0" width="72.4" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="788.5" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">NEOCLOUD</text>
+<rect x="832.7" y="222.0" width="66.1" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="865.7" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">ON-PREM</text>
+<rect x="906.8" y="222.0" width="47.2" height="22.0" rx="6" fill="none" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="930.4" y="236.5" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="500" text-anchor="middle">EDGE</text>
+<text x="152.0" y="295.6" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">AWS orchestration</text>
+<text x="956.0" y="293.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.9">Gateway → NATS → daemons</text>
+<rect x="150.0" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="164.0" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EC2</text>
+<text x="164.0" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Compute</text>
+<text x="164.0" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">QEMU / KVM</text>
+<rect x="422.7" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="436.7" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EBS</text>
+<text x="436.7" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Block storage</text>
+<text x="436.7" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">Viperblock</text>
+<rect x="695.3" y="312.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="709.3" y="333.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">S3</text>
+<text x="709.3" y="349.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Object storage</text>
+<text x="709.3" y="363.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">Predastore</text>
+<rect x="150.0" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="164.0" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">VPC</text>
+<text x="164.0" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Networking</text>
+<text x="164.0" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">OVN / OVS</text>
+<rect x="422.7" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="436.7" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">IAM</text>
+<text x="436.7" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Identity</text>
+<text x="436.7" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">SigV4 · policies</text>
+<rect x="695.3" y="382.0" width="262.7" height="60.0" rx="9" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="709.3" y="403.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="13" fill="#EE4B12" font-weight="600" text-anchor="start">EKS</text>
+<text x="709.3" y="419.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12.5" fill="#F3F1EC" font-weight="400" text-anchor="start">Kubernetes</text>
+<text x="709.3" y="433.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">K8s-conformant</text>
+<rect x="150.0" y="452.0" width="808.0" height="32.0" rx="8" fill="#14181D" fill-opacity="0.6" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0" stroke-dasharray="3 3"/>
+<text x="164.0" y="472.2" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">+ ALB · ECS · ECR · RDS and more of the core AWS services</text>
+<text x="944.0" y="471.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end">See all →</text>
+<text x="36.0" y="386.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#EE4B12" font-weight="500" text-anchor="start">Spinifex</text>
+<rect x="136.0" y="268.0" width="836.0" height="230.0" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="36.0" y="536.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">OS</text>
+<rect x="136.0" y="510.0" width="836.0" height="46.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="537.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">Operating system</text>
+<text x="952.0" y="536.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="end">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · Open vSwitch</text>
+<text x="36.0" y="596.7" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="500" text-anchor="start">Hardware</text>
+<rect x="136.0" y="568.0" width="413.0" height="50.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="156.0" y="597.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">CPU</text>
+<text x="533.0" y="596.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="end">Intel · AMD · ARM</text>
+<rect x="559.0" y="568.0" width="413.0" height="50.0" rx="12" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="579.0" y="597.9" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="700" text-anchor="start">GPU</text>
+<text x="956.0" y="596.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#EE4B12" font-weight="400" text-anchor="end">NVIDIA · AMD</text>
+</svg>

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -19,15 +19,10 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
-    ignore:
-      - dependency-name: "*"
-        update-types:
-          - "version-update:semver-minor"
-          - "version-update:semver-patch"
     groups:
       github-actions:
         patterns:
@@ -35,10 +30,10 @@ updates:
 
   - package-ecosystem: npm
     target-branch: dev
-    directory: /spinifex/services/spinifexui/frontend
+    directory: /spinifex/runtime/roles/spinifexui/frontend
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -50,3 +45,28 @@ updates:
       npm-dependencies:
         patterns:
           - "*"
+
+  # Base images are pinned by digest, so this is what moves them. The golang
+  # version is deliberate and changes with go.mod; only its digest moves here.
+  - package-ecosystem: docker
+    target-branch: dev
+    directories:
+      - /build
+      - /scripts
+      - /docs/terraform-workbooks/demo-app
+    schedule:
+      interval: weekly
+      day: sunday
+      timezone: Australia/Sydney
+    cooldown:
+      default-days: 3
+    ignore:
+      - dependency-name: golang
+        update-types:
+          - "version-update:semver-major"
+          - "version-update:semver-minor"
+          - "version-update:semver-patch"
+    groups:
+      docker-images:
+        patterns:
+          - "*"

.github/e2e-cells.json

@@ -0,0 +1,194 @@
+[
+  {
+    "cell": 1,
+    "install": "binary",
+    "topo": "single",
+    "net": "dhcp",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 2,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 3,
+    "install": "binary",
+    "topo": "single",
+    "net": "dhcp",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 8,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "dhcp",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 9,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 10,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "dhcp",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 17,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-pseudo",
+    "extra": "tofu-examples",
+    "suite": ""
+  },
+  {
+    "cell": 18,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "reboot-single",
+    "suite": ""
+  },
+  {
+    "cell": 19,
+    "install": "binary",
+    "topo": "single",
+    "net": "nat",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "nat-single",
+    "suite": ""
+  },
+  {
+    "cell": 25,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single-nvme",
+    "extra": "eks",
+    "suite": "eks"
+  },
+  {
+    "cell": 26,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "ecs",
+    "suite": "ecs"
+  },
+  {
+    "cell": 27,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "rds",
+    "suite": "rds",
+    "timeout_minutes": 120
+  },
+  {
+    "cell": 29,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "storagefault",
+    "timeout_minutes": 100
+  },
+  {
+    "cell": 30,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "nat",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "nat-multi",
+    "suite": ""
+  },
+  {
+    "cell": 31,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "instancerecovery",
+    "timeout_minutes": 105
+  },
+  {
+    "cell": 32,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "lbrecovery",
+    "timeout_minutes": 90
+  },
+  {
+    "cell": 33,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "kvquorum",
+    "timeout_minutes": 55
+  }
+]

.github/pull_request_template.md

@@ -0,0 +1,21 @@
+## Summary
+
+<!-- What does this PR do, and why? -->
+
+## Changes
+
+<!-- The notable changes, as bullets. -->
+
+## Account-scoped lifecycle impact
+
+<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
+
+- Lifecycle owner and resource family:
+- Owner-local ordering, recovery, retained-state, or force impact:
+- Cross-domain dependency or ownership impact:
+- Evidence added or rechecked:
+- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
+
+## Community Contribution License
+
+- [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.

.github/resolve-release-sibling-refs.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Resolve the tag of each sibling repo (viperblock, predastore, northstar,
+# bluebottle) a release build checks out. A release ships what go.mod requires,
+# so each sibling is built at the version go.mod names, never a branch tip. A
+# pseudo-version has no tag to check out, so it fails the release rather than
+# silently building something else.
+#
+# Takes the path to spinifex's go.mod and writes viperblock=<tag>,
+# predastore=<tag>, northstar=<tag> and bluebottle=<tag> to $GITHUB_OUTPUT for
+# the checkout steps to consume.
+set -euo pipefail
+
+gomod="${1:?usage: $0 <path to spinifex go.mod>}"
+: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
+
+for repo in viperblock predastore northstar bluebottle; do
+  module="github.com/mulgadc/${repo}"
+  version=$(awk -v m="$module" '$1 == m { print $2; exit }' "$gomod")
+  if [[ -z "$version" ]]; then
+    echo "::error::${gomod} does not require ${module}"
+    exit 1
+  fi
+  if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ || "$version" =~ [0-9]{14}-[0-9a-f]{12}$ ]]; then
+    echo "::error::${module} is required at ${version}, which is not a release tag; tag ${repo} and bump go.mod before releasing"
+    exit 1
+  fi
+  echo "${repo}: building against ${version}"
+  echo "${repo}=${version}" >>"$GITHUB_OUTPUT"
+done

.github/resolve-sibling-refs.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Resolve which branch of each sibling repo (viperblock, predastore, northstar,
+# bluebottle) the CI workspace should build against. A cross-repo change is developed on a branch
+# of the SAME name in each repo; build against that branch when it exists so the
+# set can be tested together, and fall back to dev otherwise. This mirrors the
+# resolution the e2e path already does in
+# mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
+# agree on how sibling branches are selected.
+#
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref>,
+# predastore=<ref>, northstar=<ref> and bluebottle=<ref> to $GITHUB_OUTPUT for
+# the checkout steps to consume.
+set -euo pipefail
+
+: "${BRANCH:?BRANCH must be set to the triggering branch name}"
+: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
+
+for repo in viperblock predastore northstar bluebottle; do
+  if git ls-remote --exit-code --heads \
+      "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
+    ref="$BRANCH"
+  else
+    ref="dev"
+  fi
+  echo "${repo}: building against ${ref}"
+  echo "${repo}=${ref}" >>"$GITHUB_OUTPUT"
+done

.github/scripts/capture-net-diag.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# capture-net-diag.sh — write a snapshot of network / OVN / OVS / IPsec
+# state into $1 (a directory the caller already created on the local host).
+# Designed to run on a spinifex node after an E2E suite, before tofu destroy
+# pulls the rug out. Every command is independent — `|| true` so a missing
+# binary on one path doesn't abort the rest of the capture.
+#
+# Output layout (under $1/net-diag/):
+#   openvswitch-ipsec.status         systemctl status (output-only)
+#   openvswitch-ipsec.journal        last 200 journal lines
+#   ipsec.openvswitch_conf           ovs-vsctl get Open_vSwitch . other_config
+#   ipsec.xfrm_state                 ip xfrm state list
+#   ipsec.xfrm_policy                ip xfrm policy list
+#   ovn.sb_show                      ovn-sbctl show
+#   ovn.nb_show                      ovn-nbctl show
+#   ovs.show                         ovs-vsctl show
+#   ovs.flows_br_int                 ovs-ofctl dump-flows br-int
+#   ovs.tunnels                      ovs-appctl ofproto/list-tunnels br-int
+#   net.links                        ip -s link show
+#   net.routes                       ip route show table all
+#   net.addrs                        ip -br addr show
+#   systemd.failed_units             systemctl list-units --state=failed
+set +e
+OUT="${1:?usage: capture-net-diag.sh <output-dir>}"
+mkdir -p "$OUT"
+
+run() {
+    local label="$1"; shift
+    {
+        echo "# $*"
+        "$@" 2>&1
+    } > "$OUT/$label" || true
+}
+
+run openvswitch-ipsec.status            sudo systemctl status openvswitch-ipsec.service --no-pager --full --lines=50
+run openvswitch-ipsec.journal           sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso -n 200
+run ipsec.openvswitch_conf              sudo ovs-vsctl get Open_vSwitch . other_config
+run ipsec.xfrm_state                    sudo ip xfrm state list
+run ipsec.xfrm_policy                   sudo ip xfrm policy list
+
+run ovn.sb_show                         sudo ovn-sbctl show
+run ovn.nb_show                         sudo ovn-nbctl show
+run ovn.sb_chassis                      sudo ovn-sbctl list chassis
+
+run_per_lr() {
+    local label="$1"; shift
+    local subcmd="$1"; shift
+    {
+        echo "# ovn-nbctl $subcmd <each-logical-router>"
+        local lrs
+        lrs="$(sudo ovn-nbctl --bare --columns=name list Logical_Router 2>/dev/null)"
+        for lr in $lrs; do
+            echo
+            echo "=== $lr ==="
+            sudo ovn-nbctl "$subcmd" "$lr" 2>&1
+        done
+    } > "$OUT/$label" || true
+}
+
+run_per_lr ovn.lr_routes                lr-route-list
+run_per_lr ovn.lr_policies              lr-policy-list
+run_per_lr ovn.lr_nat                   lr-nat-list
+
+run ovs.show                            sudo ovs-vsctl show
+run ovs.flows_br_int                    sudo ovs-ofctl dump-flows br-int
+run ovs.tunnels                         sudo ovs-appctl ofproto/list-tunnels br-int
+
+run net.links                           ip -s link show
+run net.routes                          ip route show table all
+run net.addrs                           ip -br addr show
+run net.neigh                           ip neigh show
+run net.bridge_fdb                      bridge fdb show
+run net.conntrack                       sudo conntrack -L
+
+run systemd.failed_units                sudo systemctl list-units --state=failed --no-pager
+
+# Background pcap (started by start-net-capture.sh, if present).
+if [ -f /tmp/spinifex-e2e-tcpdump.pid ]; then
+    PID="$(cat /tmp/spinifex-e2e-tcpdump.pid 2>/dev/null)"
+    sudo kill -INT "$PID" 2>/dev/null || true
+    sleep 1
+    if [ -f /tmp/spinifex-e2e-wan.pcap ]; then
+        sudo cp /tmp/spinifex-e2e-wan.pcap "$OUT/wan.pcap" 2>/dev/null || true
+        sudo chmod 0644 "$OUT/wan.pcap" 2>/dev/null || true
+    fi
+fi
+
+exit 0

.github/scripts/ci-fmt.sh

@@ -61,3 +61,37 @@ quiet_run() {
     return "$rc"
   fi
 }
+
+# Connection-layer failures from the libvirt tofu provider. The socket drops
+# when a hypervisor is carrying several environments at once, which is the
+# steady state here. A resource that failed to build says something else.
+CI_LIBVIRT_TRANSIENT='Unable to Connect to Libvirt|error connecting to libvirt|procedure interrupted|Cannot recv data|Cannot write data|End of file while reading data'
+
+# retry_run LABEL LOG_FILE -- CMD ARGS …
+# quiet_run with a bounded retry, taken only when that attempt's own output
+# names a libvirt connection failure. Every other failure is reported on the
+# first attempt, so a real bug never becomes a slow flake.
+retry_run() {
+  local label="$1" log="$2"; shift 2
+  local _attempts="${CI_RETRY_ATTEMPTS:-3}" _delay="${CI_RETRY_DELAY:-30}" _i _rc=1 _mark
+  for ((_i = 1; _i <= _attempts; _i++)); do
+    _mark=$(wc -c < "$log" 2>/dev/null || echo 0)
+    # rc must be read inside the else: an `if` whose condition fails and has no
+    # else exits 0, so reading $? after `fi` reports success for a failed run.
+    if "$@" >> "$log" 2>&1; then
+      if [[ $_i -gt 1 ]]; then ok "$label (libvirt recovered on attempt $_i)"; else ok "$label"; fi
+      return 0
+    else
+      _rc=$?
+    fi
+    tail -c "+$((_mark + 1))" "$log" | grep -Eq "$CI_LIBVIRT_TRANSIENT" || break
+    [[ $_i -eq $_attempts ]] && break
+    warn "$label: libvirt connection dropped (attempt $_i/$_attempts), retrying in ${_delay}s"
+    sleep "$_delay"
+  done
+  bad "$label (rc=$_rc) — last 80 lines of $(basename "$log"):"
+  group_open "tail $(basename "$log")"
+  tail -n 80 "$log"
+  group_close
+  return "$_rc"
+}

.github/scripts/e2e-run-summary.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Render one run-level table for the e2e nightly into the job summary.
+
+Reading a nightly meant opening nineteen jobs, and the Actions UI does not
+distinguish a cell that failed from one that was cancelled out from under it —
+a distinction every investigation of this suite has needed.
+
+Inputs (env):
+    GITHUB_TOKEN         Token with actions:read.
+    GITHUB_REPOSITORY    owner/repo.
+    GITHUB_RUN_ID        Run to summarise.
+    JUNIT_ROOT           Directory holding downloaded junit-cell-* artifacts.
+    GITHUB_STEP_SUMMARY  Appended to.
+
+Test counts come from the junit artifacts; result, duration and runner come
+from the jobs API, so a cell that died before writing junit still gets a row.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime
+
+API = "https://api.github.com"
+
+# A cell whose job produced no verdict is not a pass and not a failure. Keeping
+# them apart is the point of the table.
+VERDICT_ICON = {
+    "success": "✅",
+    "failure": "❌",
+    "cancelled": "🚫",
+    "skipped": "⏭️",
+    "timed_out": "⏱️",
+}
+
+
+def fetch_jobs(repo: str, run_id: str, token: str) -> list[dict]:
+    jobs: list[dict] = []
+    page = 1
+    while True:
+        url = f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}"
+        req = urllib.request.Request(
+            url,
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/vnd.github+json",
+                "X-GitHub-Api-Version": "2022-11-28",
+            },
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                payload = json.load(resp)
+        except (urllib.error.URLError, TimeoutError) as exc:
+            print(f"::warning::jobs API page {page} failed: {exc}", file=sys.stderr)
+            break
+        batch = payload.get("jobs", [])
+        jobs.extend(batch)
+        if len(batch) < 100:
+            break
+        page += 1
+    return jobs
+
+
+def duration_s(job: dict) -> int | None:
+    started, completed = job.get("started_at"), job.get("completed_at")
+    if not started or not completed:
+        return None
+    fmt = "%Y-%m-%dT%H:%M:%SZ"
+    try:
+        return int((datetime.strptime(completed, fmt) - datetime.strptime(started, fmt)).total_seconds())
+    except ValueError:
+        return None
+
+
+def human(seconds: int | None) -> str:
+    if seconds is None:
+        return "—"
+    if seconds < 0:
+        return "0s"
+    return f"{seconds // 60}m{seconds % 60:02d}s" if seconds >= 60 else f"{seconds}s"
+
+
+def junit_counts(root: str) -> dict[str, dict]:
+    """Map cell number to its aggregate junit counts and failing test names."""
+    per_cell: dict[str, dict] = {}
+    for path in glob.glob(os.path.join(root, "**", "junit-*.xml"), recursive=True):
+        cell = cell_from_artifact_path(path)
+        if cell is None:
+            continue
+        acc = per_cell.setdefault(cell, {"tests": 0, "failures": 0, "skipped": 0, "failing": []})
+        try:
+            root_el = ET.parse(path).getroot()
+        except ET.ParseError as exc:
+            print(f"::warning::unparseable junit {path}: {exc}", file=sys.stderr)
+            continue
+        suites = root_el.iter("testsuite") if root_el.tag != "testsuite" else [root_el]
+        for suite in suites:
+            acc["tests"] += int(suite.get("tests") or 0)
+            acc["failures"] += int(suite.get("failures") or 0) + int(suite.get("errors") or 0)
+            acc["skipped"] += int(suite.get("skipped") or 0)
+        for case in root_el.iter("testcase"):
+            if case.find("failure") is not None or case.find("error") is not None:
+                name = case.get("name") or "?"
+                if name not in acc["failing"]:
+                    acc["failing"].append(name)
+    return per_cell
+
+
+def cell_from_artifact_path(path: str) -> str | None:
+    """Recover the cell number from a junit-cell-<n>-<run_id> artifact directory."""
+    for part in path.split(os.sep):
+        if part.startswith("junit-cell-"):
+            rest = part[len("junit-cell-"):]
+            cell = rest.split("-", 1)[0]
+            return cell if cell.isdigit() else None
+    return None
+
+
+def cell_number(job_name: str) -> str | None:
+    """cell-19 (single/static/debian-13) -> 19."""
+    if not job_name.startswith("cell-"):
+        return None
+    cell = job_name[len("cell-"):].split(" ", 1)[0]
+    return cell if cell.isdigit() else None
+
+
+def main() -> int:
+    repo = os.environ.get("GITHUB_REPOSITORY", "")
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    token = os.environ.get("GITHUB_TOKEN", "")
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY", "")
+    junit_root = os.environ.get("JUNIT_ROOT", "")
+
+    if not (repo and run_id and token and summary_path):
+        print("::warning::e2e-run-summary: missing required environment", file=sys.stderr)
+        return 0
+
+    jobs = fetch_jobs(repo, run_id, token)
+    counts = junit_counts(junit_root) if junit_root and os.path.isdir(junit_root) else {}
+
+    rows = []
+    for job in jobs:
+        cell = cell_number(job.get("name", ""))
+        if cell is None:
+            continue
+        rows.append((int(cell), job))
+    rows.sort(key=lambda r: r[0])
+
+    lines = [f"## e2e nightly — run summary ({len(rows)} cells)", ""]
+    if not rows:
+        lines.append("No cell jobs found for this run.")
+        write(summary_path, lines)
+        return 0
+
+    lines += [
+        "| Cell | Topology | Result | Duration | Runner | Tests | Failed | Skipped |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+    ]
+
+    tally: dict[str, int] = {}
+    total_tests = total_failed = total_seconds = 0
+    failing_detail: list[tuple[int, list[str]]] = []
+
+    for cell, job in rows:
+        name = job.get("name", "")
+        # A cell dispatched to a reusable workflow reports as
+        # "cell-20 (baremetal/pxe/real-hw) / Bare-Metal Single-Node E2E".
+        topo = name.split("(", 1)[1].split(")", 1)[0] if "(" in name else "—"
+        verdict = job.get("conclusion") or job.get("status") or "unknown"
+        tally[verdict] = tally.get(verdict, 0) + 1
+        secs = duration_s(job)
+        total_seconds += secs or 0
+        c = counts.get(str(cell), {})
+        total_tests += c.get("tests", 0)
+        total_failed += c.get("failures", 0)
+        if c.get("failing"):
+            failing_detail.append((cell, c["failing"]))
+        lines.append(
+            f"| {cell} | {topo} | {VERDICT_ICON.get(verdict, '❔')} {verdict} | {human(secs)} "
+            f"| {job.get('runner_name') or '—'} | {c.get('tests', '—')} "
+            f"| {c.get('failures', '—')} | {c.get('skipped', '—')} |"
+        )
+
+    verdicts = ", ".join(f"{n} {v}" for v, n in sorted(tally.items(), key=lambda kv: -kv[1]))
+    lines += [
+        "",
+        f"**Cells:** {verdicts}",
+        f"**Tests:** {total_tests} run, {total_failed} failed",
+        f"**Cell wall clock:** {human(total_seconds)} across all cells",
+    ]
+
+    slowest = sorted(rows, key=lambda r: duration_s(r[1]) or 0, reverse=True)[:3]
+    if slowest:
+        lines.append(
+            "**Slowest:** " + ", ".join(f"cell {c} ({human(duration_s(j))})" for c, j in slowest)
+        )
+
+    if failing_detail:
+        lines += ["", "<details><summary>Failing tests</summary>", ""]
+        for cell, names in failing_detail:
+            lines.append(f"- **cell {cell}** — " + ", ".join(f"`{n}`" for n in names))
+        lines += ["", "</details>"]
+
+    write(summary_path, lines)
+    return 0
+
+
+def write(path: str, lines: list[str]) -> None:
+    with open(path, "a", encoding="utf-8") as fh:
+        fh.write("\n".join(lines) + "\n")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/ensure-libguestfs.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Prepare a runner to build a guest image with build-system-image.sh.
+#
+# Installs libguestfs + qemu-utils if they are missing and makes a host kernel
+# readable, so the libguestfs appliance can boot. Nothing here touches a host
+# block device: virt-customize works inside an isolated userspace appliance, so
+# this is safe on a shared runner.
+#
+# Callers must still export LIBGUESTFS_BACKEND=direct themselves — it has to be
+# set in the environment the build runs in, which is not this subshell.
+set -euo pipefail
+
+# Three nightly service cells share one ci-single runner and start together, so
+# concurrent apt-get runs are the normal case rather than the exception. Two of
+# them lost the lock outright before this guard existed. Skipping the install
+# when the tools are already present avoids most of the contention; the lock
+# timeout covers the rest, including apt runs started by anything else on the
+# box.
+if command -v virt-customize >/dev/null && command -v qemu-img >/dev/null; then
+    echo "libguestfs-tools and qemu-utils already present"
+else
+    APT_OPTS=(-o DPkg::Lock::Timeout=600)
+    sudo apt-get "${APT_OPTS[@]}" update -qq
+    sudo apt-get "${APT_OPTS[@]}" install -y --no-install-recommends \
+        libguestfs-tools qemu-utils
+fi
+
+# Debian/Ubuntu ship /boot/vmlinuz-* mode 0600 and the appliance needs to read a
+# host kernel. Tolerated on failure: a runner whose kernel is already readable
+# has no vmlinuz to chmod under some images.
+sudo chmod 0644 /boot/vmlinuz-* || true

.github/scripts/start-net-capture.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# start-net-capture.sh — kick off background tcpdump on br-wan.
+# Pair: capture-net-diag.sh stops it and copies the pcap into net-diag/.
+#
+# Filter: anything on br-wan involving the EIP pool (192.168.0.0/23) plus
+# ARP for the same range. Single ring buffer (-W 1 -G 0) → grows until stop;
+# capped at 64 MiB via -C 64. Snaplen 200 bytes (headers + a bit) to keep
+# size sane while preserving L2/L3/L4 + first payload bytes.
+set +e
+
+PCAP=/tmp/spinifex-e2e-wan.pcap
+PIDFILE=/tmp/spinifex-e2e-tcpdump.pid
+
+if [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE" 2>/dev/null)" 2>/dev/null; then
+    echo "[start-net-capture] tcpdump already running pid=$(cat "$PIDFILE")"
+    exit 0
+fi
+
+sudo rm -f "$PCAP" "$PIDFILE"
+
+if ! command -v tcpdump >/dev/null 2>&1; then
+    echo "[start-net-capture] tcpdump not installed — skipping" >&2
+    exit 0
+fi
+if ! ip link show br-wan >/dev/null 2>&1; then
+    echo "[start-net-capture] br-wan absent — skipping" >&2
+    exit 0
+fi
+
+sudo nohup tcpdump -i br-wan -s 200 -C 64 -W 1 -U \
+    -w "$PCAP" \
+    '(net 192.168.0.0/23) or (arp and (arp[24:4]&0xfffffe00=0xc0a80000))' \
+    >/tmp/spinifex-e2e-tcpdump.log 2>&1 &
+PID=$!
+echo "$PID" | sudo tee "$PIDFILE" >/dev/null
+disown
+
+sleep 1
+if ! kill -0 "$PID" 2>/dev/null; then
+    echo "[start-net-capture] tcpdump died — see /tmp/spinifex-e2e-tcpdump.log" >&2
+    cat /tmp/spinifex-e2e-tcpdump.log >&2 || true
+    sudo rm -f "$PIDFILE"
+    exit 1
+fi
+
+echo "[start-net-capture] tcpdump pid=$PID pcap=$PCAP"
+exit 0

.github/scripts/suite-sets.sh

@@ -0,0 +1,165 @@
+# shellcheck shell=bash
+# shellcheck disable=SC2034  # every variable here is consumed by the sourcing workflow
+# Source from a workflow `run:` block:
+#   . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+#
+# Which e2e suites each topology can run. Sourced by every workflow that drives
+# suites (e2e.yml, e2e-suites.yml, e2e-nightly.yml) so the lists cannot drift
+# apart — they previously disagreed three ways.
+#
+# Eligibility is a property of the suite, not of the workflow: a suite belongs
+# to a topology when all of its assertions are reachable there. lb is
+# multi-only for exactly that reason — on a single node every internet-facing
+# subtest skips for want of a peer, so a single-node lb run reports green
+# having exercised a fraction of the suite.
+
+# Suites runnable against a one-node environment.
+#
+# partialblock gates merges because it earned it, not because it is new: on one
+# node, same binary and workload, it fails against the pre-fix engine (1758 of
+# 8192 halves reverted a generation) and passes against the fixed one (0 of
+# 8192). A suite that has never been shown to go red for the reason it claims
+# does not belong here.
+# rds is single-only for the opposite reason to lb: v1 has no multi-AZ
+# behaviour, so a second node adds no assertion. Its DB VMs are capped at four
+# concurrent by the suite's own semaphore, which is what keeps it inside the
+# job's 50-minute budget alongside the suites above.
+# quota belongs to both topologies because the gate it asserts is gateway code
+# that runs identically on either, and because a quota left switched on would
+# cap every suite sharing the environment — so it must be proved to restore
+# itself wherever it runs, not only on one node.
+# storagefault runs last in both lists and belongs to both topologies, for
+# different reasons. On one node it is RS(1,0), so freezing the only predastore
+# is a total outage and the guest-survival assertion is unambiguous. On three it
+# is RS(2,1) with degraded writes on, so two must be frozen to break the shard
+# floor — and only there is a cross-node restart possible at all, which is the
+# case that takes a volume over from a node holding writes the backend never
+# received. It is last because it SIGSTOPs a cluster-wide
+# service: anything sharing the environment would see the same outage and report
+# a failure that says nothing about itself.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
+
+# Suites runnable against a multi-node environment.
+#
+# instancerecovery is multi-only and needs three: recovery is inert below that,
+# because there is no survivor to move a guest to that is not the node that
+# failed. Like storagefault it takes a node away and freezes a cluster-wide
+# service, so it runs last and owns its environment.
+#
+# lbrecovery is multi-only and needs three for two reasons rather than one: a
+# survivor to move the load balancer to, and a node for its backends that is not
+# the load balancer's, so taking that node away does not take the backends with
+# it. It runs beside instancerecovery at the end because it also takes a node
+# away and owns its environment while it does.
+#
+# kvquorum is multi-only and needs three, for the same reason as those two: a
+# surviving majority. It is what proves the control plane's own state is
+# replicated rather than sitting on one node, so it runs before the pair that
+# take a node away — a cluster whose buckets are already wrong should report
+# that, not report a recovery failing.
+E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
+
+# grep -xE alternation form, for narrowing a requested set to the eligible one.
+E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
+E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
+
+# What every e2e-nightly permutation cell must prove, deliberately narrower
+# than the two sets above: those cells have a 35-minute budget and exist to show
+# that every install / network / host-OS combination boots and serves. The
+# service suites are not deferred to the PR gate — eks, ecs and rds each run
+# nightly in their own dedicated cell, because each needs a guest AMI built
+# first and a service failure should not mask whether a host OS boots. These
+# are subsets, and e2e_suites_assert_subset below enforces that.
+#
+# storagefault has a dedicated cell for two reasons. It SIGSTOPs a cluster-wide
+# service, so in a permutation cell every suite sharing that environment would
+# report the outage as its own failure. And a full pass is ~63 minutes, nearly
+# all of it deliberate waiting, which is too long to add to the PR gate — where
+# it has never run, since the gate's default suite list does not name it.
+#
+# Its cell is three-node: the RS(2,1) shard floor and the cross-node takeover
+# after a failed seal do not exist on one node, and they are the assertions
+# worth a nightly.
+E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
+#
+# The multi-node nightly subset carries kvquorum's read-only half through
+# `multinode`, which sweeps every KV bucket rather than the three the daemon
+# owns. The node-loss half is too slow for a 35-minute cell and has its own.
+E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
+
+# How long a suite gets, for the same reason the lists above live here: the
+# timeout was written out separately in each workflow and drifted, so rds ran
+# 50m on the PR gate and 30m nightly and panicked there. A suite needing more
+# than the default declares it once, here.
+#   e2e_suite_timeout <suite>
+e2e_suite_timeout() {
+  case "$1" in
+    # rds serialises its DB VMs behind a 3 GiB semaphore, so its wall clock is
+    # what the queue costs, not what the tests do: single tests logged 11m, 22m
+    # and 26m waiting for budget. 17 tests at that concurrency is over an hour
+    # before anything goes wrong, and the first pass to run deep into the suite
+    # still had three left at 60m. The loaded graceful-reboot case adds an
+    # instance that holds its slot through a pgbench load as well as a create.
+    rds) echo "105m" ;;
+    # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
+    # creates first. One pass is ~20 minutes on the bare-metal cell and a
+    # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
+    diskperf) echo "180m" ;;
+    # storagefault is mostly deliberate waiting: one test alone holds predastore
+    # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
+    # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
+    storagefault) echo "90m" ;;
+    # instancerecovery is almost entirely waiting, and none of it is avoidable:
+    # a dead owner has to be seen stale twice, a guest has to boot, and the
+    # storage case waits for a pause measured at 4m47s and then holds four
+    # minutes to prove nothing moved. Three tests, each taking a node away and
+    # bringing it back, and the partition case adds a lease validity, a lease
+    # TTL and a boot on top of everything the other two wait for.
+    instancerecovery) echo "90m" ;;
+    # lbrecovery pays for a load balancer twice: once to stand one up with two
+    # backends placed around it, and once for the recovery itself — a stale
+    # owner, a lease, a boot, HAProxy's own start, and its health checks against
+    # the backends before it forwards anything. Then the node comes back and has
+    # to be shown not to answer for an address it no longer holds.
+    lbrecovery) echo "75m" ;;
+    # kvquorum audits every bucket in seconds and then spends the rest of its
+    # budget on a node stopping and starting: seven units down, a raft election,
+    # and the whole service stack back up with every stream caught up.
+    kvquorum) echo "40m" ;;
+    *) echo "30m" ;;
+  esac
+}
+
+# One matrix leg per suite, carrying the job budget that suite needs. The outer
+# timeout must always be later than the `go test` one inside it: go test reports
+# where it stopped, a cancelled job reports only "The operation was canceled".
+#   e2e_suite_matrix <suite-list> <floor-minutes> [allowance-minutes]
+e2e_suite_matrix() {
+  local list="$1" floor="$2" allowance="${3:-15}" suite budget
+  for suite in ${list}; do
+    budget="$(e2e_suite_timeout "${suite}")"
+    budget=$((${budget%m} + allowance))
+    if [ "${budget}" -lt "${floor}" ]; then budget="${floor}"; fi
+    printf '%s\t%s\n' "${suite}" "${budget}"
+  done | jq -cRs 'split("\n")|map(select(length>0)|split("\t")|{suite:.[0],timeout:(.[1]|tonumber)})'
+}
+
+# Fail loudly when a narrowed list names a suite its topology cannot run. This
+# is the guard that stops the lists drifting again: a suite added to a nightly
+# subset without also being declared eligible above stops the job here rather
+# than silently skipping half its assertions in CI.
+#   e2e_suites_assert_subset <label> <subset> <eligible-set>
+e2e_suites_assert_subset() {
+  local label="$1" subset="$2" eligible="$3" suite found eligible_suite
+  for suite in ${subset}; do
+    found=0
+    for eligible_suite in ${eligible}; do
+      if [ "${suite}" = "${eligible_suite}" ]; then found=1; break; fi
+    done
+    if [ "${found}" -ne 1 ]; then
+      echo "::error::${label} names suite '${suite}', which is not eligible for that topology (eligible: ${eligible}). Fix the list in .github/scripts/suite-sets.sh." >&2
+      return 1
+    fi
+  done
+  return 0
+}

.github/workflows/actions.lock

@@ -0,0 +1,137 @@
+# This file is machine-generated by `gh actions-lock`.
+# Do not edit by hand; run `gh actions-lock` to update.
+# Docs: https://gh.io/actions-lockfile
+version: 'v0.0.3'
+workflows:
+    '.github/workflows/apt-packages.yml':
+        - 'actions/checkout@v7.0.1'
+    '.github/workflows/dev-release.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+        - 'docker/setup-buildx-action@v4.4.1'
+    '.github/workflows/e2e-baremetal.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e-cloudvendor-nightly.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/upload-artifact@v7.0.2'
+        - 'hashicorp/setup-terraform@v4.0.1'
+    '.github/workflows/e2e-ddil.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e-nightly.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/download-artifact@v8.0.2'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e-suites.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/e2e.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/download-artifact@v8.0.2'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/gpu-e2e.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/image-scripts.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+    '.github/workflows/perf-3node.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+    '.github/workflows/publish-system-ami.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+    '.github/workflows/release.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/download-artifact@v8.0.2'
+        - 'actions/setup-go@v7.0.0'
+        - 'actions/upload-artifact@v7.0.2'
+        - 'docker/setup-buildx-action@v4.4.1'
+        - 'softprops/action-gh-release@v3.0.3'
+    '.github/workflows/segscan-oracle.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/setup-go@v7.0.0'
+    '.github/workflows/spinifex.yml':
+        - 'actions/cache@v6.1.0'
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-go@v7.0.0'
+        - 'golangci/golangci-lint-action@v9.3.0'
+    '.github/workflows/ui.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/setup-node@v7.1.0'
+        - 'pnpm/action-setup@v6.1.0'
+dependencies:
+    'actions/cache@v6.1.0':
+        ref: 'v6.1.0'
+        commit: 'sha1-55cc8345863c7cc4c66a329aec7e433d2d1c52a9'
+        owner_id: 44036562
+        repo_id: 215566462
+    'actions/checkout@v7.0.1':
+        ref: 'v7.0.1'
+        commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
+        owner_id: 44036562
+        repo_id: 197814629
+    'actions/create-github-app-token@v3.3.0':
+        ref: 'v3.3.0'
+        commit: 'sha1-64ab729a6294000f5aa4a4bc0b73d1f32bc45510'
+        owner_id: 44036562
+        repo_id: 642580244
+    'actions/download-artifact@v8.0.2':
+        ref: 'v8.0.2'
+        commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333'
+        owner_id: 44036562
+        repo_id: 192626254
+    'actions/setup-go@v7.0.0':
+        ref: 'v7.0.0'
+        commit: 'sha1-b7ad1dad31e06c5925ef5d2fc7ad053ef454303e'
+        owner_id: 44036562
+        repo_id: 192624594
+    'actions/setup-node@v7.1.0':
+        ref: 'v7.1.0'
+        commit: 'sha1-949feb2413d6458794dcd2491c4babbbce0c15c1'
+        owner_id: 44036562
+        repo_id: 189476904
+    'actions/upload-artifact@v7.0.2':
+        ref: 'v7.0.2'
+        commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9'
+        owner_id: 44036562
+        repo_id: 192625955
+    'docker/setup-buildx-action@v4.4.1':
+        ref: 'v4.4.1'
+        commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069'
+        owner_id: 5429470
+        repo_id: 288485773
+    'golangci/golangci-lint-action@v9.3.0':
+        ref: 'v9.3.0'
+        commit: 'sha1-ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a'
+        owner_id: 35628013
+        repo_id: 232539879
+    'hashicorp/setup-terraform@v4.0.1':
+        ref: 'v4.0.1'
+        commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
+        owner_id: 761456
+        repo_id: 258271226
+    'pnpm/action-setup@v6.1.0':
+        ref: 'v6.1.0'
+        commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
+        owner_id: 21320719
+        repo_id: 262289747
+    'softprops/action-gh-release@v3.0.3':
+        ref: 'v3.0.3'
+        commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
+        owner_id: 2242
+        repo_id: 204253808

.github/workflows/apt-packages.yml

@@ -0,0 +1,46 @@
+name: Apt Packages
+
+# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
+# list against each supported suite's real indices, so a renamed, dropped, or
+# ambiguously virtual package fails here rather than on a user's machine.
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/setup.sh
+      - scripts/lint-apt-packages.sh
+      - .github/workflows/apt-packages.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  apt_packages:
+    name: Resolve apt list (${{ matrix.image }})
+    runs-on: ubuntu-26.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - ubuntu:24.04
+          - ubuntu:26.04
+          - debian:13
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+
+      - name: Resolve runtime package list
+        run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}

.github/workflows/dev-release.yml

@@ -0,0 +1,185 @@
+name: Dev Prerelease
+
+# Publishes the dev channel that install.mulgadc.com already serves. setup.sh has
+# understood INSTALL_SPINIFEX_CHANNEL=dev since it was written and the install
+# Worker resolves that channel to the newest GitHub prerelease, but nothing has
+# ever produced one -- so /download/dev/amd64 returned 404. This is the producer.
+#
+# A separate workflow rather than conditionals in release.yml, because a dev build
+# is a different artifact and not a release with flags off: amd64 only, no ISO, no
+# AMI refresh, no bare-metal gate, and nothing that touches a self-hosted runner.
+on:
+  push:
+    branches: [dev]
+    # Nothing here changes the binary, and a prerelease per docs commit would
+    # churn the channel our contacts are installing from.
+    paths-ignore:
+      - "docs/**"
+      - "**/*.md"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+# Supersede rather than queue: on a run of merges to dev only the newest commit is
+# worth publishing, and the older build would land after it and become the channel.
+concurrency:
+  group: dev-release
+  cancel-in-progress: true
+
+jobs:
+  prerelease:
+    runs-on: ubuntu-26.04
+    steps:
+      # Full history and tags, because the tag name is git describe and a shallow
+      # clone has no tag to describe against.
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+          fetch-depth: 0
+
+      # dev, not each repository's default branch, which is what release.yml takes.
+      # A dev build of spinifex against main of the others is a pairing nobody runs.
+      - name: Checkout Predastore
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/predastore
+          ref: dev
+          path: predastore
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/viperblock
+          ref: dev
+          path: viperblock
+
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: dev
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: dev
+          path: bluebottle
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4.4.1
+
+      - name: Set up Go
+        uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      # The tag carries the describe string so the commit is recoverable from the
+      # version alone, and the -dev suffix is what release.yml's own convention
+      # already treats as a prerelease. VERSION must equal the tag: the install
+      # Worker derives the asset name from the tag it resolves.
+      - name: Resolve the version
+        id: version
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # Describing against a dev tag would nest it, growing the name every release.
+          TAG="$(git describe --tags --always --exclude '*-dev')-dev"
+          case "$TAG" in
+              v*) ;;
+              *) echo "::error::git describe gave '$TAG', which is not a v-prefixed version"; exit 1 ;;
+          esac
+          if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+              echo "::notice::$TAG already published; nothing to do"
+              echo "exists=true" >> "$GITHUB_OUTPUT"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "Publishing $TAG"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
+        if: steps.version.outputs.exists != 'true'
+        run: cd spinifex && make build-microvm-image
+
+      - name: Build amd64 Tarball
+        if: steps.version.outputs.exists != 'true'
+        env:
+          VERSION: ${{ steps.version.outputs.tag }}
+        run: cd spinifex && make distro-amd64 VERSION="$VERSION"
+
+      # --prerelease on create, never a later promote step. release.yml sets it in
+      # a job that runs after its verification gates, which leaves a window where
+      # the release is not flagged; GitHub's releases/latest excludes prereleases
+      # only, so in that window a dev build is what every customer curl installs.
+      - name: Publish the prerelease
+        if: steps.version.outputs.exists != 'true'
+        working-directory: spinifex
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.version.outputs.tag }}
+        run: |
+          set -euo pipefail
+          # A file, not an inline string: the notes are Markdown, and leading
+          # whitespace from indenting a shell literal turns them into code blocks.
+          cat > "$RUNNER_TEMP/notes.md" <<EOF
+          Automated dev-channel build of \`$GITHUB_SHA\`.
+
+          **Not a release.** amd64 only, no ISO and no system images, and it has not been through the release verification gates. Install it with:
+
+          \`\`\`
+          curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
+          \`\`\`
+          EOF
+          sed -i 's/^          //' "$RUNNER_TEMP/notes.md"
+          gh release create "$TAG" \
+              --repo "$GITHUB_REPOSITORY" \
+              --target "$GITHUB_SHA" \
+              --prerelease \
+              --title "$TAG" \
+              --notes-file "$RUNNER_TEMP/notes.md" \
+              "dist/spinifex-$TAG-linux-amd64.tar.gz" \
+              "dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
+              "scripts/setup.sh"
+
+      # The assertion that matters. A prerelease must never become the latest
+      # release, because latest is what an unqualified curl | bash installs -- so
+      # this checks the published state rather than trusting the flag we passed.
+      - name: Assert the stable channel is untouched
+        if: steps.version.outputs.exists != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.version.outputs.tag }}
+        run: |
+          set -euo pipefail
+          latest="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)"
+          if [ "$latest" = "$TAG" ]; then
+              echo "::error::$TAG became the latest release: every customer curl | bash would install it"
+              gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --prerelease
+              exit 1
+          fi
+          echo "latest is still $latest"
+          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
+          [ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
+
+      # Kept rather than unbounded: the Worker reads the first page of releases to
+      # resolve the channel, so enough prereleases would eventually push the real
+      # ones off it. Five leaves a few recent builds to roll back to.
+      - name: Prune old dev prereleases
+        if: steps.version.outputs.exists != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
+              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \
+              > stale.txt
+          while read -r tag; do
+              [ -n "$tag" ] || continue
+              echo "deleting $tag"
+              gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+          done < stale.txt

.github/workflows/e2e-baremetal.yml

@@ -1,7 +1,53 @@
 name: E2E Bare-Metal
 
 on:
+  # Mirrors the suite selection of workflow_call so a baseline capture can be
+  # driven by hand. A dispatch run sees only the inputs declared here, so
+  # omitting these would silently fall back to the correctness set.
   workflow_dispatch:
+    inputs:
+      suites:
+        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance run names its own so its numbers come off an otherwise idle cell"
+        required: false
+        type: string
+        default: "single cert lb"
+      suite-env:
+        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
+        required: false
+        type: string
+        default: ""
+      cell:
+        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
+        required: false
+        type: string
+        default: "20"
+  workflow_call:
+    inputs:
+      ref:
+        description: "spinifex ref to checkout/test (default: triggering ref)"
+        required: false
+        type: string
+        default: ""
+      otel-attrs:
+        description: "OTEL_RESOURCE_ATTRIBUTES value for this run, e.g. ci.run_id=123,ci.cell=20 (caller-specific, so it's an input rather than a workflow-level default)"
+        required: false
+        type: string
+        default: ""
+      suites:
+        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance caller names its own so its numbers come off an otherwise idle cell"
+        required: false
+        type: string
+        default: "single cert lb"
+      suite-env:
+        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
+        required: false
+        type: string
+        default: ""
+      cell:
+        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
+        required: false
+        type: string
+        default: "20"
 
 permissions:
   contents: read
@@ -9,31 +55,40 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
-    name: Bare-Metal Multi-Node E2E
-    runs-on: [ self-hosted, ci-baremetal ]
-    timeout-minutes: 120
+    name: Bare-Metal Single-Node E2E
+    runs-on: [self-hosted, ci-baremetal]
+    timeout-minutes: 240
     concurrency:
       group: e2e-baremetal-cluster
       cancel-in-progress: false
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
+      BM_CELL: ${{ inputs.cell || '20' }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
       - name: Clean Workspace
-        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
+          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7.0.1
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -49,6 +104,10 @@ jobs:
           ssh-keygen -y -f "${RUNNER_TEMP}/bm-ci" > "${RUNNER_TEMP}/bm-ci.pub"
 
       - name: Build Artifacts
+        env:
+          # The node runs pre-built binaries, so a suite absent here cannot be
+          # selected later. Same list as the run step for that reason.
+          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
         run: |
           mulga/scripts/baremetal/bm-build-artifacts.sh \
             "$GITHUB_WORKSPACE" \
@@ -64,7 +123,7 @@ jobs:
             INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
             SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
             VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
-            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
+            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
 
       - name: Write ePDU credentials
         run: |
@@ -91,6 +150,24 @@ jobs:
             echo "Ubuntu 26.04 cloud image cached ($(du -h "$UBUNTU_IMG" | cut -f1))"
           fi
 
+      - name: Install go-junit-report on runner
+        # Version from go.mod's tool directive, checked against go.sum.
+        run: GOWORK=off go install github.com/jstemmer/go-junit-report/v2
+
+      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
+      # AMI. Built here because libguestfs needs source and network, which the
+      # runner has and the PXE-booted node does not; bm-bootstrap.sh imports it.
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: contains(inputs.suites, 'eks')
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          # spinifex is the workspace root here, unlike the nightly where it is
+          # checked out into a subdirectory.
+          "${GITHUB_WORKSPACE}/.github/scripts/ensure-libguestfs.sh"
+          make build-eks-node-image
 
       - name: Run Bare-Metal E2E
         env:
@@ -99,9 +176,69 @@ jobs:
           BM_EPDU_ENV: ${{ runner.temp }}/epdu.env
           BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
           SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
+          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
+          BM_SUITE_ENV: ${{ inputs.suite-env }}
+          # Empty unless the eks build above ran, and bm-bootstrap.sh skips the
+          # import when unset, so the other suites are unaffected.
+          BM_EKS_IMAGE: ${{ contains(inputs.suites, 'eks') && '/tmp/eks-node-image-build/eks-node-alpine.raw' || '' }}
+          BM_EKS_MANIFEST: ${{ contains(inputs.suites, 'eks') && format('{0}/scripts/images/eks-node/manifest.conf', github.workspace) || '' }}
         run: |
+          mkdir -p "${ARTIFACT_DIR}"
+          set -o pipefail
+          . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
+          BM_TEST_TIMEOUT=""
+          for suite in ${BM_SUITES}; do
+            tmo="$(e2e_suite_timeout "${suite}")"
+            if [ -z "${BM_TEST_TIMEOUT}" ] || [ "${tmo%m}" -gt "${BM_TEST_TIMEOUT%m}" ]; then
+              BM_TEST_TIMEOUT="${tmo}"
+            fi
+          done
+          export BM_TEST_TIMEOUT
+          echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
           sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
             --squashfs  "${BM_ISO_CACHE}/spinifex.squashfs" \
-            --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+            --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
+            2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
+
+      - name: Generate JUnit report
+        if: always()
+        run: |
+          # suite-output.log, not test-baremetal.log. The harness streams the
+          # full suite output to the first and a grep-filtered summary to its
+          # stdout, and that filter anchors --- PASS at column 0, so every
+          # indented subtest result is dropped. Parsed from the filtered log,
+          # go-junit-report saw a === RUN with no result and emitted
+          # <error message="No test result found"> — 17 of 39 testcases on a
+          # green cell-28, 93 of 120 on cell-20.
+          LOG="${ARTIFACT_DIR}/suite-output.log"
+          [ -s "$LOG" ] || LOG="${ARTIFACT_DIR}/test-baremetal.log"
+          [ -f "$LOG" ] || exit 0
+          go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
+
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: $/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: ${{ env.BM_CELL }}
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: $/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
+
+      - uses: actions/upload-artifact@v7.0.2
+        if: always()
+        with:
+          name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/e2e-cloudvendor-nightly.yml

@@ -0,0 +1,287 @@
+name: E2E Cloud Vendor Nightly
+
+# Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per
+# ref, rather than by a `schedule:` here. GitHub registers a schedule only from
+# the default branch, so a cron in this file could never run `dev` and a feature
+# branch as separate nightlies, and it would always execute the default branch's
+# copy of this file -- leaving a fix to the workflow itself untestable until it
+# had already shipped. A dispatch takes both the file and the source from its ref.
+on:
+  workflow_dispatch:
+    inputs:
+      topologies:
+        description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
+        required: false
+      workbooks:
+        description: "Space-separated workbooks (blank = the driver's default list)"
+        required: false
+      source:
+        description: "tree (build this ref) or release (the published installer)"
+        required: false
+        default: tree
+      channel:
+        description: "With source=release: latest, or dev for the newest prerelease"
+        required: false
+        default: latest
+      keep_on_fail:
+        description: "Leave a failed topology up to inspect. It keeps billing."
+        required: false
+        type: boolean
+        default: false
+      auth:
+        description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal."
+        required: false
+        default: instance_principal
+        type: choice
+        options: [instance_principal, api_key]
+      no_external_pool:
+        description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal."
+        required: false
+        type: boolean
+        default: false
+      runner:
+        description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)"
+        required: false
+
+permissions:
+  contents: read
+
+# Every topology is built in one OCI compartment under one set of resource names,
+# so two runs at once would each see the other's instances and the sweep in
+# e2e-oci.sh could destroy a cluster the other run was still testing. Queue the
+# second rather than cancelling it: a dispatch is usually a verification run.
+concurrency:
+  group: e2e-oci
+  cancel-in-progress: false
+
+jobs:
+  oci:
+    # Hosted by default: this job only drives Terraform and SSH, and the work
+    # happens on OCI, so nothing here needs a hypervisor. Overridable because the
+    # argument for a per-provider self-hosted runner is control, not capacity --
+    # a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
+    # lifts the six-hour cap a hosted job carries.
+    runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
+    # Three topologies including a bare-metal build, each with its own install,
+    # formation and workbook pass. The sweep still runs if this fires.
+    timeout-minutes: 300
+    steps:
+      # Set here rather than in a job-level env block: the runner context is not
+      # available at job level, only from a step onwards, and a workflow that
+      # references it there is rejected as invalid before any job starts.
+      #
+      # Outside github.workspace on purpose. The checkout wipes untracked files, so
+      # on a persistent runner it would delete the Terraform state of a killed run
+      # before the next sweep could read it, leaving instances nothing can find.
+      - name: Put the Terraform state outside the checkout
+        run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - uses: actions/create-github-app-token@v3.3.0
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          # mulga alone. It is the only private repository of the six, and the
+          # five that build-install-artifacts.sh clones are public, so the builder
+          # needs no credential and this token is not widened to reach them.
+          repositories: mulga
+
+      # build-install-artifacts.sh lives in mulga. The workbooks and their driver
+      # are in spinifex, so only the one directory is needed here.
+      - uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/mulga
+          path: mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+
+      - uses: hashicorp/setup-terraform@v4.0.1
+        with:
+          terraform_wrapper: false
+
+      # Written to the runner's own filesystem and nowhere else. The OCI API
+      # credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in
+      # the environment, so no key file is ever created.
+      - name: Install the throwaway SSH key
+        env:
+          OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }}
+          OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }}
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.ssh
+          printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx
+          printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub
+          chmod 600 ~/.ssh/oci-spx
+          ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null
+
+      - name: Run the OCI topologies
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
+          OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
+          OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
+          OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
+          MULGA_ROOT: ${{ github.workspace }}/mulga
+          SPX_REF: ${{ github.ref_name }}
+          OCI_TOPOLOGIES: ${{ inputs.topologies }}
+          OCI_SOURCE: ${{ inputs.source }}
+          OCI_CHANNEL: ${{ inputs.channel }}
+          # Independent of the pool, deliberately. These were one switch, which made
+          # the combination we actually need -- an API key with the allocator running
+          # -- impossible to ask for, so public-IP allocation went untested.
+          OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
+          # Skipping the pool records the allocator and every public-address workbook
+          # as SKIPPED, so a green run with this set is not a claim about guest
+          # networking. It exists for a formation-only check, not as a way past a
+          # missing credential.
+          OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
+          # Only an API-key run needs one: an instance principal is the node's own
+          # certificate, so there is nothing to hand over. Unset is not an error, and
+          # the allocator gate then fails rather than passing a node that cannot
+          # authenticate.
+          OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }}
+          # vm-single-principal builds in its own tenancy. The dynamic group it needs
+          # is a tenancy-root resource, and the shared test tenancy grants us only
+          # compartment rights, so that topology can never pass there however correct
+          # the code is. Unset skips it rather than spending a provision to fail.
+          OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
+          OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
+          OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
+          OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
+          OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
+          OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
+          # Service limits are per tenancy, so the shape the other topologies use
+          # can have zero quota in this one. Ours has no E6 cores at all, and the
+          # apply then fails before anything about the principal is exercised.
+          OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
+          OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
+          # Through the environment, never interpolated into the script below: a
+          # dispatch input is attacker-controlled text as far as the shell is
+          # concerned, and a quote in it would otherwise end the string.
+          WORKBOOKS_INPUT: ${{ inputs.workbooks }}
+        run: |
+          set -uo pipefail
+          # WORKBOOKS is exported only when the dispatch named some: the harness
+          # distinguishes unset (the driver's default list) from empty (run none).
+          if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
+          ./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log"
+          exit "${PIPESTATUS[0]}"
+
+      # Before the sweep, which destroys the state the tables are read from, and
+      # always(): a run whose verdict is a failure is exactly the one somebody
+      # wants a table for. The successful path writes the same tables itself, so
+      # this step is for the run that did not get there.
+      #
+      # The markdown lives in the script rather than here. A heredoc in a YAML
+      # string is the one place in this repository nothing can lint or run.
+      - name: Publish the result tables
+        if: always()
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
+          OCI_TOPOLOGIES: ${{ inputs.topologies }}
+          OCI_SOURCE: ${{ inputs.source }}
+          OCI_CHANNEL: ${{ inputs.channel }}
+          SPX_REF: ${{ github.ref_name }}
+          # The same two the run step passes. The header states what was and was not
+          # tested, so a summary rendered without them would claim more than the run.
+          OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
+          OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
+        run: ./e2e-oci.sh --summary-only
+
+      # The sweep below deletes the per-topology state directories, so anything
+      # still only in one of them at that point is unrecoverable -- and a failing
+      # workbook writes its assertion output to exactly there. Copying into the
+      # artifact directory first is what makes a failure diagnosable by whoever
+      # reads the run page, rather than only on the runner that produced it.
+      - name: Collect the per-topology logs
+        if: always()
+        run: |
+          set -uo pipefail
+          dest="$RUNNER_TEMP/e2e-oci"
+          mkdir -p "$dest"
+          # The topologies this run drove, taken from the per-topology log the
+          # harness writes rather than from the dispatch input: a run that names
+          # none gets the harness's own default, and reading that back here would
+          # mean a second copy of the list to disagree with the first. A state
+          # directory left by an earlier run is then not collected, so the artifact
+          # cannot show a topology this run never attempted.
+          topologies=""
+          for log in "$dest"/*.log; do
+              [ -f "$log" ] || continue
+              name="${log##*/}"; name="${name%.log}"
+              [ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name"
+          done
+          echo "collecting:${topologies:- nothing}"
+          for topology in $topologies; do
+              for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \
+                          "$OCI_STATE_ROOT/.validate-$topology"/*.txt \
+                          "$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do
+                  [ -f "$file" ] || continue
+                  # The hook handles a private key and the one thing known about an
+                  # operator's executable is that it is not ours to publish.
+                  case "${file##*/}" in credential-hook.log) continue ;; esac
+                  cp -- "$file" "$dest/$topology-${file##*/}"
+              done
+          done
+          ls -l "$dest"
+
+      # always(), and before the artifact upload: an OCI bare-metal host left
+      # running by a cancelled or timed-out job is the most expensive way this
+      # workflow can fail. The sweep works from the Terraform state files, so it
+      # can only destroy what this run created.
+      - name: Sweep anything left behind
+        if: always() && !inputs.keep_on_fail
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
+          OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
+          OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
+          OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
+          # The sweep destroys by state file, so it needs whichever tenancy each
+          # topology built in. Without these a leftover principal deployment is
+          # swept against a tenancy that never held it and reported clean.
+          OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
+          OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
+          OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
+          OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
+          OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
+          OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep
+        run: ./e2e-oci.sh --sweep-only
+
+      # Explicit, not left to the runner's own cleanup: a hosted runner is
+      # discarded after the job, but a self-hosted one keeps $HOME, so the key
+      # would outlive the run that needed it and sit there between nightlies.
+      - name: Remove the throwaway SSH key
+        if: always()
+        run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub
+
+      # Named extensions rather than a directory glob: the artifact directory also
+      # holds the distro tarball and the test archive, which are hundreds of
+      # megabytes and reproducible from the ref. Everything here is a file the
+      # collect step above chose, so terraform.tfstate cannot reach an artifact.
+      - name: Upload the run
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: e2e-oci-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/e2e-oci.log
+            ${{ runner.temp }}/e2e-oci/*.log
+            ${{ runner.temp }}/e2e-oci/verdict.txt
+            ${{ runner.temp }}/e2e-oci/summary.md
+            ${{ runner.temp }}/e2e-oci/*.txt
+            ${{ runner.temp }}/e2e-oci/*.tsv
+            ${{ runner.temp }}/e2e-oci-sweep/*.log
+          retention-days: 14

.github/workflows/e2e-ddil.yml

@@ -1,8 +1,5 @@
 name: E2E DDIL
 
-# Phase 1 trigger: workflow_dispatch only. Nightly cron + on-main-merge
-# land in mulga-siv-5.8 once ≥4 scenarios are asserting. See
-# docs/development/improvements/ddil-e2e-test-harness.md §4 / §Dependencies.
 on:
   workflow_dispatch:
     inputs:
@@ -11,44 +8,49 @@ on:
         required: false
         default: ""
 
+# Shared with e2e-nightly.yml and e2e-suites.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
+# flight compete for their disks and their libvirt guests. Queued rather than
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 permissions:
   contents: read
+  actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   ddil:
     name: DDIL Real Multi-Node
-    runs-on: [ self-hosted, ci-multi ]
-    # Budget: ~45m go test + tofu provision/bootstrap/teardown on ci-multi.
-    # In Phase 1 the scenarios all t.Skip, so the real-world runtime is
-    # dominated by provisioning; the 60m ceiling leaves headroom for
-    # Phase 2/3 when assertions come online.
+    runs-on: [self-hosted, ci-multi]
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace
         run: sudo rm -rf "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      # Full spinifex checkout: the Go harness must run on the runner
-      # (outside the cluster) so its SSH path survives scenario partitions
-      # that drop peer-to-peer cluster traffic. Running inside the cluster
-      # would self-sever when Scenario C isolates a node.
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7.0.1
         name: Checkout Spinifex
         with:
           path: spinifex
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7.0.1
         name: Checkout Mulga (tofu-cluster)
         with:
           repository: mulgadc/mulga
@@ -76,6 +78,13 @@ jobs:
           echo "${BIN_DIR}" >> "${GITHUB_PATH}"
           "${BIN_DIR}/gotestsum" --version
 
+      # Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
+      # the VMs, so a telemetry doc points back at this job's log directly.
+      - name: Resolve CI job id
+        uses: $/.github/actions/otel-job-id
+        with:
+          job-name: DDIL Real Multi-Node
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -88,14 +97,13 @@ jobs:
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VMs provisioned"
 
       - name: Bootstrap via Binary Install
@@ -140,10 +148,6 @@ jobs:
             exit 1
           fi
 
-          # Guest SSH key: bootstrap-install.sh imports spinifex-key into EC2;
-          # the matching private material was generated on the primary node at
-          # ~/.ssh/spinifex-key. Pull it to the runner so the harness can SSH
-          # the witness via its MapPublicIpOnLaunch-assigned EIP.
           GUEST_KEY="${RUNNER_TEMP}/spinifex-key"
           scp -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
             "${SSH_USER}@${PRIMARY_IP}:~/.ssh/spinifex-key" "${GUEST_KEY}"
@@ -151,11 +155,6 @@ jobs:
 
           mkdir -p "${RUNNER_TEMP}/ddil-results"
           cd "${GITHUB_WORKSPACE}/spinifex"
-          # gotestsum: human-readable PASS/FAIL/SKIP lines in Actions stdout,
-          # plus machine-readable jsonfile + JUnit XML artifacts. The JUnit
-          # file feeds the junit-report composite action below so failures
-          # become summary table rows + workflow annotations.
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           banner "DDIL scenarios"
           DDIL_NODES="${NODE_IPS}" \
@@ -176,19 +175,25 @@ jobs:
 
       - name: Publish test summary
         if: always()
-        uses: ./spinifex/.github/actions/junit-report
+        uses: $/.github/actions/junit-report
         with:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: $/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
+          cell: ddil
+
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          # Pull spinifex-* journals + QEMU process listings from every
-          # node into the artifact bundle so failures can be triaged
-          # without re-provisioning the cluster. Best-effort — a missing
-          # node should not mask the original test failure.
           set +e
           INVENTORY=$(tofu output -json inventory)
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -197,7 +202,6 @@ jobs:
           LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs"
           mkdir -p "${LOG_DIR}"
 
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip')
           for ip in "${NODE_IPS[@]}"; do
@@ -210,7 +214,7 @@ jobs:
 
       - name: Upload DDIL Results
         if: always()
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: ddil-results-${{ github.run_id }}
           path: ${{ runner.temp }}/ddil-results
@@ -221,9 +225,18 @@ jobs:
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
-          # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
-          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: tofu-log-ddil-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 30

.github/workflows/e2e-suites.yml

@@ -0,0 +1,933 @@
+# E2E per-suite blocks. Each suite appears as its own node in the Actions
+# graph:
+#
+#   single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
+#                     └─ single_teardown (if: always)
+#   multi_provision  ─┬─ multi_suite  (matrix: the multi-node-eligible suites)
+#                     └─ multi_teardown (if: always)
+#
+# Both matrices come from .github/scripts/suite-sets.sh, the one definition of
+# which suites a topology can run, shared with e2e.yml and e2e-nightly.yml.
+#
+# Constraint: tofu state is runner-filesystem-local, so *_provision and
+# *_teardown must execute on the same self-hosted runner (one runner per label
+# today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does
+# NOT re-checkout mulga (that would wipe local state). Suite-matrix legs reach
+# the already-provisioned VM over its WAN IP using the key path the provision
+# job emits; they share the runner home dir, so the key file persists.
+#
+# This file does NOT post the gating `e2e` commit status — it runs alongside
+# e2e.yml for validation. Once green across a few manual runs, fold the
+# triggers from e2e.yml here and retire the loop.
+
+name: E2E (suite blocks)
+
+on:
+  workflow_dispatch:
+    inputs:
+      single_suites:
+        description: "Single-node suites to run as matrix legs (empty = every single-node-eligible suite, per .github/scripts/suite-sets.sh)"
+        required: false
+        default: ""
+      multi_suites:
+        description: "Multi-node suites to run as matrix legs (empty = every multi-node-eligible suite, per .github/scripts/suite-sets.sh)"
+        required: false
+        default: ""
+      run_multinode:
+        description: "Also run the multinode env block"
+        type: boolean
+        required: false
+        default: true
+      test_run:
+        description: "Optional -test.run regex (empty = all tests in suite)"
+        required: false
+        default: ""
+
+# Shared with e2e-nightly.yml and e2e-ddil.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
+# flight compete for their disks and their libvirt guests. Queued rather than
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  actions: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Single-node env
+  # ---------------------------------------------------------------------------
+  single_provision:
+    name: Single env · provision
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 40
+    outputs:
+      wan_ip: ${{ steps.vm.outputs.wan_ip }}
+      ssh_key: ${{ steps.vm.outputs.ssh_key }}
+      suites: ${{ steps.suites.outputs.list }}
+      matrix: ${{ steps.suites.outputs.matrix }}
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM; sleep 3; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3.3.0
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      # Default and allow-list both come from the shared definition, so this
+      # workflow cannot drift from e2e.yml about what a single node can run.
+      - name: Compute suite matrix
+        id: suites
+        env:
+          REQUESTED: ${{ inputs.single_suites }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
+          e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
+          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
+          # 45 is the floor this job has always had, so resolving each leg's
+          # budget can only raise it — storagefault and rds ask for more.
+          JSON=$(e2e_suite_matrix "${LIST}" 45)
+          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
+
+      - name: Reclaim docker build cache
+        run: |
+          docker container prune -f || true
+          docker image prune -f || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
+      # ci.job_id names the provision job, not the suite job that later drives
+      # the env: the suite legs share this env and its baked telemetry.env, so
+      # the provisioner is the only job the daemon telemetry can belong to.
+      - name: Resolve CI job id
+        uses: $/.github/actions/otel-job-id
+        with:
+          job-name: Single env · provision
+
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Provision VM
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "VM provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve VM SSH details
+        id: vm
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: contains(steps.suites.outputs.list, 'eks')
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          banner "build eks-node AMI (libguestfs — no host devices)"
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          group_open "make build-eks-node-image"
+          make build-eks-node-image
+          group_close
+          ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
+
+      - name: Import eks-node AMI on VM
+        if: contains(steps.suites.outputs.list, 'eks')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/eks-node/manifest.conf)"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "import eks-node AMI to VM image store"
+          scp "${SSH_OPTS[@]}" \
+            /tmp/eks-node-image-build/eks-node-alpine.raw \
+            "$VM":/tmp/eks-node-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/eks-node-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
+          ok "eks-node AMI imported"
+
+      - name: Install kubectl on VM
+        if: contains(steps.suites.outputs.list, 'eks')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "install kubectl on VM (eks data-plane subtest)"
+          ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
+          set -euo pipefail
+          KVER=v1.32.5
+          case "$(uname -m)" in
+            x86_64)  ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+            *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
+          esac
+          cd /tmp
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
+          echo "$(cat kubectl.sha256)  kubectl" | sha256sum -c -
+          sudo install -m 0755 kubectl /usr/local/bin/kubectl
+          kubectl version --client
+          EOF
+          ok "kubectl installed on VM"
+
+      - name: Build E2E test binaries on runner
+        working-directory: spinifex/tests/e2e
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # teardown always builds: the run-scoped sweep binary runs in the
+          # teardown job, it is never a selectable suite leg.
+          SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
+          banner "build ${SUITES_BUILD// /, }"
+          group_open "go test -c"
+          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
+          group_close
+          for f in _bin/*.test; do
+            ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
+          done
+
+      - name: Ship test binaries to VM
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
+          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
+          ok "shipped binaries to $VM"
+
+      - name: Start packet capture on VM br-wan
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+            "$VM":/tmp/start-net-capture.sh
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  single_suite:
+    name: Single env · ${{ matrix.suite }}
+    needs: single_provision
+    runs-on: [self-hosted, ci-single]
+    # Per leg, from suite-sets.sh: one number for every suite would have to be
+    # storagefault's, and every other leg would then hang for 105 minutes.
+    timeout-minutes: ${{ matrix.timeout }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
+    env:
+      WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
+      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
+    steps:
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Install go-junit-report
+        run: |
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Run ${{ matrix.suite }}
+        id: e2e
+        run: |
+          set +e
+          SUITE="${{ matrix.suite }}"
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
+          VM_HOST="tf-user@${WAN_IP}"
+          TEST_RUN_FLAG=""
+          [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          cleanup_remote() {
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
+              "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
+              2>/dev/null || true
+          }
+          trap cleanup_remote EXIT INT TERM
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
+             R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
+             printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"${SUITE}\" \"\$R\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
+                    | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
+             exit \${PIPESTATUS[0]}"
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect suite artifacts from VM
+        if: always()
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          true
+
+      - name: Convert test log to JUnit XML
+        if: always()
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always()
+        uses: $/.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "E2E ${{ matrix.suite }} results"
+
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: $/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: $/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E ${{ matrix.suite }} failure analysis"
+
+      - uses: actions/upload-artifact@v7.0.2
+        if: always()
+        with:
+          name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Fail job if suite failed
+        if: always()
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
+            exit 1
+          fi
+
+  single_teardown:
+    name: Single env · teardown
+    needs: [single_provision, single_suite]
+    if: always()
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 20
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-single-diag-${{ github.run_id }}
+      WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
+      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
+    steps:
+      # Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
+      # Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
+      # again would wipe the local tofu state and orphan the VM.
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - name: Capture network/IPsec diagnostics on VM
+        if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+            "$VM":/tmp/capture-net-diag.sh 2>/dev/null
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" 2>/dev/null
+          ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "90 min ago"' \
+            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
+          mkdir -p "${ARTIFACT_DIR}/lb-console"
+          ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+            | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
+          true
+
+      - uses: actions/upload-artifact@v7.0.2
+        if: always()
+        with:
+          name: e2e-single-diag-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+      # Reclaim any resource leaked by a crashed/interrupted suite before the VM
+      # is destroyed. Scoped to this run's tag, best-effort, never fails the job.
+      - name: Sweep leaked run resources on VM
+        if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          banner "sweep run-tagged resources (run ${{ github.run_id }})"
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
+             SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+             SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+             ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
+            2>&1 || true
+          true
+
+      - name: Teardown VM
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          if [ -n "${WAN_IP}" ] && [ -n "${SSH_KEY}" ]; then
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
+              tf-user@${WAN_IP} \
+              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
+              2>/dev/null || true
+          fi
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 2; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: tofu-log-single-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  # ---------------------------------------------------------------------------
+  # Multinode env
+  # ---------------------------------------------------------------------------
+  multi_provision:
+    name: Multi env · provision
+    if: ${{ inputs.run_multinode }}
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 45
+    outputs:
+      node1_ip: ${{ steps.cluster.outputs.node1_ip }}
+      ssh_key: ${{ steps.cluster.outputs.ssh_key }}
+      nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
+      suites: ${{ steps.suites.outputs.list }}
+      matrix: ${{ steps.suites.outputs.matrix }}
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
+      # See single_provision above — daemon telemetry is baked in here, at
+      # provision time, before the multi_suite matrix exists.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) echo "reap $sig pid=$pid"; kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 3; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3.3.0
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      - name: Reclaim docker build cache
+        run: |
+          docker container prune -f || true
+          docker image prune -f || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
+      # See the matching note on single_provision.
+      - name: Resolve CI job id
+        uses: $/.github/actions/otel-job-id
+        with:
+          job-name: Multi env · provision
+
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      # Mirrors single_provision's step: the multi leg previously ran multinode
+      # alone, silently dropping the lb and cert coverage that only a
+      # multi-node env can complete.
+      - name: Compute suite matrix
+        id: suites
+        env:
+          REQUESTED: ${{ inputs.multi_suites }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
+          e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
+          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
+          # 60 is the floor this job has always had; multi-node legs carry a
+          # heavier setup than single-node ones, which is why it is higher.
+          JSON=$(e2e_suite_matrix "${LIST}" 60)
+          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
+
+      - name: Build multinode binaries on runner
+        working-directory: spinifex/tests/e2e
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # teardown always builds: the run-scoped sweep binary runs in the
+          # teardown job, it is never a selectable suite leg.
+          SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
+          banner "build ${SUITES_BUILD// /, }"
+          group_open "go test -c"
+          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
+          group_close
+          ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)"
+
+      - name: Ship test binaries to node1
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
+          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
+          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
+          ok "shipped binaries to $VM"
+
+      - name: Start packet capture on all nodes
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            scp "${SSH_OPTS[@]}" \
+              "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+              tf-user@"$ip":/tmp/start-net-capture.sh
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
+          done
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  multi_suite:
+    name: Multi env · ${{ matrix.suite }}
+    needs: multi_provision
+    runs-on: [self-hosted, ci-multi]
+    # See the note on single_suite: the budget is the suite's, not the job's.
+    timeout-minutes: ${{ matrix.timeout }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
+    env:
+      NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
+      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
+      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
+    steps:
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Install go-junit-report
+        run: |
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Run ${{ matrix.suite }}
+        id: e2e
+        run: |
+          set +e
+          SUITE="${{ matrix.suite }}"
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
+          VM_HOST="tf-user@${NODE1_IP}"
+          TEST_RUN_FLAG=""
+          [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          cleanup_remote() {
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
+              "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
+              2>/dev/null || true
+          }
+          trap cleanup_remote EXIT INT TERM
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
+             printf '\n━━ suite %s ━━\n' \"${SUITE}\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+               SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
+                    | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
+             exit \${PIPESTATUS[0]}"
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect suite artifacts from node1
+        if: always()
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
+          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          true
+
+      - name: Convert test log to JUnit XML
+        if: always()
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always()
+        uses: $/.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "Multi-Node E2E ${{ matrix.suite }} results"
+
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: $/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: $/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
+
+      - uses: actions/upload-artifact@v7.0.2
+        if: always()
+        with:
+          name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Fail job if suite failed
+        if: always()
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
+            exit 1
+          fi
+
+  multi_teardown:
+    name: Multi env · teardown
+    needs: [multi_provision, multi_suite]
+    if: always() && inputs.run_multinode
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 20
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log
+      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
+      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
+    steps:
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      # Reclaim leaked run-tagged resources before the cluster is destroyed.
+      - name: Sweep leaked run resources on node1
+        if: ${{ needs.multi_provision.result == 'success' && needs.multi_provision.outputs.node1_ip != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${{ needs.multi_provision.outputs.node1_ip }}
+          banner "sweep run-tagged resources (run ${{ github.run_id }})"
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
+             SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+             SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+             ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
+            2>&1 || true
+          true
+
+      - name: Teardown cluster
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          if [ -n "${NODES_CSV}" ] && [ -n "${SSH_KEY}" ]; then
+            IFS=',' read -ra NODE_LIST <<< "${NODES_CSV}"
+            for ip in "${NODE_LIST[@]}"; do
+              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
+                tf-user@"$ip" \
+                'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
+                2>/dev/null || true
+            done
+          fi
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 2; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: tofu-log-multi-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+  e2e_suites_status:
+    name: E2E suite blocks · summary
+    if: always()
+    needs: [single_suite, multi_suite]
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    steps:
+      - name: Summarize
+        run: |
+          echo "## E2E suite-block results" >> "$GITHUB_STEP_SUMMARY"
+          echo "single_suite: ${{ needs.single_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "multi_suite:  ${{ needs.multi_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
+          case "${{ needs.single_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
+          case "${{ needs.multi_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac

.github/workflows/gpu-e2e.yml

@@ -0,0 +1,307 @@
+# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
+# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
+# exposure).
+#
+# ---------------------------------------------------------------------------
+# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
+#
+#   1. Register ONE self-hosted GitHub Actions runner on a banksia host with
+#      the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
+#      NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
+#      other runner the `ci-gpu` label — a second labelled runner would
+#      let two GPU jobs race the single RTX A1000. With exactly one runner
+#      carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
+#      serialization; the `concurrency:` block below queues same-group runs
+#      when that runner is already busy with a live job.
+#   2. Drop the wattle SSH private key on that runner's filesystem and wire
+#      it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
+#      contents). Optionally override `vars.WATTLE_SSH_HOST` /
+#      `vars.WATTLE_SSH_USER` if they differ from the defaults below.
+#   3. Confirm the runner user can drive wattle's control plane over SSH. The
+#      job calls `spx` (read-only checks) and the `aws` CLI with the already-
+#      configured `spinifex` profile directly — no wrapper scripts, no install
+#      on wattle (tf-user is already in the `spinifex` group there):
+#        ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
+#   4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
+#      block CI by touching a sentinel file in the RUNNER's home directory
+#      (NOT on wattle):
+#        touch ~/.ci-gpu-paused     # blocks CI
+#        rm ~/.ci-gpu-paused        # resumes CI
+#      or by setting the repo/environment variable `CI_GPU_PAUSED=true`
+#      (no runner filesystem access needed). Either one causes this workflow
+#      to skip cleanly (job succeeds, all work steps skipped) instead of
+#      queuing behind manual work indefinitely.
+#
+# ---------------------------------------------------------------------------
+
+name: GPU E2E
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_run:
+        description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+# Single dedicated runner (see setup note 1 above) + this group together
+# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
+# else to land (unique label) and, if it somehow overlaps in scheduling,
+# queues here rather than cancelling/stomping the in-flight run.
+concurrency:
+  group: ci-gpu
+  cancel-in-progress: false
+
+env:
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
+  WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
+  WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+
+jobs:
+  gpu_e2e:
+    name: GPU E2E (wattle)
+    runs-on: [self-hosted, ci-gpu]
+    timeout-minutes: 75
+    env:
+      ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
+    steps:
+      - name: Manual-work mutex gate
+        id: gate
+        env:
+          REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
+        run: |
+          set -euo pipefail
+          PAUSED=false
+          if [ -f "$HOME/.ci-gpu-paused" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
+          fi
+          if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
+          fi
+          echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
+
+      - name: Paused — nothing to do
+        if: steps.gate.outputs.paused == 'true'
+        run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
+
+      - name: Clean Workspace
+        if: steps.gate.outputs.paused != 'true'
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v7.0.1
+        if: steps.gate.outputs.paused != 'true'
+
+      - uses: actions/setup-go@v7.0.0
+        if: steps.gate.outputs.paused != 'true'
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install wattle SSH key
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
+          echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
+          echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
+
+      - name: Healthcheck — describe-regions
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
+            'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
+
+      - name: Ensure base GPU AMI present (import-if-absent)
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$BASE" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            ID=$(present)
+            if [ -n "$ID" ] && [ "$ID" != "None" ]; then
+              echo "base GPU AMI already present: $BASE = $ID"; exit 0
+            fi
+            echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
+            sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
+            for i in 1 2 3 4 5 6; do
+              ID=$(present)
+              [ -n "$ID" ] && [ "$ID" != "None" ] && break
+              sleep 5
+            done
+            [ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
+            echo "imported base GPU AMI: $BASE = $ID"
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      - name: Preflight — verify GPU AMIs
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$1" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            ID=$(present "$BASE")
+            if [ -z "$ID" ] || [ "$ID" = "None" ]; then
+              echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
+              exit 1
+            fi
+            echo "required: $BASE = $ID"
+            for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
+              OID=$(present "$OPT")
+              if [ -z "$OID" ] || [ "$OID" = "None" ]; then
+                echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
+              else
+                echo "opt-in: $OPT = $OID"
+              fi
+            done
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      - name: Build GPU e2e test binary on runner
+        if: steps.gate.outputs.paused != 'true'
+        working-directory: tests/e2e
+        run: |
+          set -euo pipefail
+          make e2e-build SUITES_BUILD=gpu
+          ls -la _bin/gpu.test
+
+      - name: Install go-junit-report on runner
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          # Version from go.mod's tool directive, checked against go.sum.
+          GOWORK=off go install github.com/jstemmer/go-junit-report/v2
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Ship GPU e2e test binary to wattle
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
+          scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
+
+      - name: Run GPU e2e suite on wattle
+        id: e2e
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
+          ssh -tt ${{ env.ssh_opts }} "$VM" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
+             exit \${PIPESTATUS[0]}"
+          E2E_RC=$?
+          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect artifacts from wattle
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Convert test log to JUnit XML
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log)
+            SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: $/.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "GPU E2E results"
+
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.paused != 'true'
+        continue-on-error: true
+        uses: $/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: gpu
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: $/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "GPU E2E failure analysis"
+
+      - uses: actions/upload-artifact@v7.0.2
+        if: always() && steps.gate.outputs.paused != 'true'
+        with:
+          name: gpu-e2e-artifacts-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Cleanup — terminate leftover GPU instances, release GPU
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          REMOTE='
+            set +e
+            GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
+              --query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
+            if [ -z "$GPU_TYPES" ]; then
+              echo "no GPU instance types advertised; nothing to reconcile"
+              exit 0
+            fi
+            IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
+              --filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
+              --query "Reservations[].Instances[].InstanceId" --output text)
+            if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
+              echo "no leftover GPU instances — GPU already released"
+              exit 0
+            fi
+            echo "terminating leftover GPU instance(s): $IDS"
+            AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
+          '
+          ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
+
+      - name: Fail job if GPU suite failed
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
+            exit 1
+          fi

.github/workflows/image-scripts.yml

@@ -0,0 +1,60 @@
+name: Image Scripts
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths:
+      - scripts/images/**
+      - images/**
+      - .github/workflows/image-scripts.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  image_scripts:
+    name: Image Scripts
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+
+      - name: Run image script tests + shellcheck
+        run: make test-images
+
+  eks_mkosi_image:
+    name: Build mkosi EKS image
+    # A fork PR's code must never run on a persistent self-hosted runner: it
+    # shares disk, caches and credentials with the release jobs.
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: [self-hosted, ci-single]
+    concurrency:
+      group: eks-mkosi-image-${{ github.ref }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+
+      - name: Set up Go
+        uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: "go.mod"
+          # Self-hosted: the module + build cache already persists on local
+          # disk, and cache-save would re-upload hundreds of MB for nothing.
+          cache: false
+
+      - name: Build the image
+        run: make build-eks-node-mkosi-image

.github/workflows/perf-3node.yml

@@ -0,0 +1,415 @@
+name: Predastore 3-Node Performance
+
+on:
+  workflow_dispatch:
+    inputs:
+      perf_preset:
+        description: "Warp workload preset: smoke (30s) or compare (2m)"
+        required: false
+        default: compare
+        type: choice
+        options:
+          - smoke
+          - compare
+      cluster_ref:
+        description: "Git ref to build the cluster from, e.g. v1.18.0. Empty uses this branch."
+        required: false
+        default: ""
+        type: string
+
+# Shared with e2e.yml and e2e-nightly.yml: this job provisions on the same
+# hypervisors they do, so a run in flight there and here would compete for
+# disks and libvirt guests. Queued, not cancelled, same as those workflows.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+
+jobs:
+  perf_3node:
+    name: Predastore Throughput (3-node)
+    # ci-multi is a systemd runner on the LAN, not the ephemeral pool. A pool
+    # VM sits behind the geneve overlay at MTU 1408, which would put the
+    # runner's own network path inside the thing being measured.
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-perf3node-${{ github.run_id }}.log
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=perf-3node
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 3
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      # ci-multi and ci-single resolve to the same physical host, so the
+      # nvme path cell-23 uses for its throughput run is already there.
+      # Pointing TMPDIR at it keeps warp's payloads off a small /tmp tmpfs.
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3.3.0
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      # Same-named branch when one exists, dev otherwise. Matches how the
+      # e2e path picks siblings, so this measures the same predastore
+      # commit a cross-repo change would be built and tested against.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - uses: actions/checkout@v7.0.1
+        id: predastore
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The cluster's ref is a knob and the harness's is not. Measuring an old
+      # release with a new harness is the only way its number can be set beside
+      # a current one; rebuilding both would change the ruler and the thing.
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        env:
+          CLUSTER_REF: ${{ inputs.cluster_ref }}
+        run: |
+          ./build-install-artifacts.sh "${CLUSTER_REF:-$GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
+
+      # The libvirt provider dials the hypervisors by bare hostname. A pool VM
+      # sits inside a VPC and cannot resolve them; a LAN runner already can, so
+      # this is self-gating on the lookup rather than on the runner's labels.
+      - name: Map hypervisor names
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          if getent hosts banksia >/dev/null; then
+            echo "hypervisor names already resolve"
+          else
+            sudo tee -a /etc/hosts < network/hosts >/dev/null
+            echo "added $(grep -c '^[0-9]' network/hosts) entries to /etc/hosts"
+          fi
+          # Pinned rather than StrictHostKeyChecking=no: these hops reach the
+          # hypervisors themselves, so an unverified key is the one place in
+          # this job where accepting anything would matter.
+          if ssh-keygen -F banksia >/dev/null 2>&1; then
+            echo "hypervisor host keys already known"
+          else
+            install -d -m 700 ~/.ssh
+            cat network/known_hosts >> ~/.ssh/known_hosts
+            chmod 600 ~/.ssh/known_hosts
+            echo "pinned $(grep -c '^[a-z]' network/known_hosts) host keys"
+          fi
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          REGION=$(echo "$INVENTORY" | jq -r '.region')
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "ssh_user=${SSH_USER}" >> "$GITHUB_OUTPUT"
+          echo "region=${REGION}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      # PERF_EXTERNAL_HOSTS wants host:port, and the gate listens on 8443 on
+      # every node. run-info.txt from this run is then comparable with the
+      # nightly's loopback one, since the harness itself never changes.
+      - name: Resolve PERF_EXTERNAL_HOSTS
+        id: perf_hosts
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          HOSTS=""
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            if [ -z "$HOSTS" ]; then HOSTS="$ip:8443"; else HOSTS="$HOSTS,$ip:8443"; fi
+          done
+          echo "hosts=${HOSTS}" >> "$GITHUB_OUTPUT"
+
+      # A slow bootstrap should fail here, legibly, rather than inside warp
+      # with a wall of connection-refused errors. Bounded at 5 minutes per
+      # gate so a genuinely dead node still fails the job in finite time.
+      - name: Wait for gates to answer on :8443
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra GATES <<< "${{ steps.perf_hosts.outputs.hosts }}"
+          for gate in "${GATES[@]}"; do
+            echo "waiting for https://$gate"
+            up=0
+            for i in $(seq 1 60); do
+              if curl -k -sS --max-time 5 -o /dev/null "https://$gate/"; then
+                up=1
+                break
+              fi
+              sleep 5
+            done
+            [ "$up" -eq 1 ] || { echo "::error::gate $gate never answered HTTPS on :8443"; exit 1; }
+            echo "gate $gate is up"
+          done
+
+      # Warp reads whole objects, so no workload here touches a ranged read and
+      # the throughput figures stay healthy while ranged GETs are broken. This
+      # probe is the only thing in CI that looks at them.
+      - name: Ranged GET probe
+        env:
+          GATE: ${{ steps.cluster.outputs.node1_ip }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::$SK"
+          B="ranged-probe-${GITHUB_RUN_ID}"
+          AWSX=(--region "$CLUSTER_REGION" --endpoint-url "https://${GATE}:8443" --no-verify-ssl)
+          export AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK"
+          aws s3 mb "s3://$B" "${AWSX[@]}"
+          trap 'aws s3 rb "s3://$B" --force "${AWSX[@]}" >/dev/null 2>&1 || true' EXIT
+          # Two sizes: read amplification makes a boundary-crossing range cost
+          # the whole object, so the fault shows as time tracking object size.
+          for mb in 64 256; do
+            head -c $((mb*1024*1024)) /dev/urandom > "${TMPDIR}/probe-${mb}.bin"
+            aws s3 cp "${TMPDIR}/probe-${mb}.bin" "s3://$B/probe-${mb}.bin" --only-show-errors "${AWSX[@]}"
+            rm -f "${TMPDIR}/probe-${mb}.bin"
+          done
+          t() {
+            curl -sk --aws-sigv4 "aws:amz:${CLUSTER_REGION}:s3" -u "$AK:$SK" \
+              -H "Range: bytes=0-$1" -o /dev/null -w '%{time_total}' \
+              --url "https://${GATE}:8443/${B}/probe-${2}.bin"
+          }
+          # 4194304 is the last byte inside the first 4 MiB block; one more
+          # crosses into the second and leaves the single-shard fast path.
+          IN64=$(t 4194303 64);  OUT64=$(t 4194304 64)
+          IN256=$(t 4194303 256); OUT256=$(t 4194304 256)
+          printf '%-10s in-block=%ss  crossing=%ss\n' "64MiB" "$IN64" "$OUT64"
+          printf '%-10s in-block=%ss  crossing=%ss\n' "256MiB" "$IN256" "$OUT256"
+          # The 4x object-size ratio is the signal, and it is compared against
+          # the crossing reads alone so a slow runner cancels out of both sides.
+          awk -v a="$OUT64" -v b="$OUT256" 'BEGIN{
+            r = (a > 0) ? b/a : 0
+            printf "crossing-range cost ratio 256MiB/64MiB = %.2f (object size ratio is 4.00)\n", r
+            if (r > 2.0) {
+              print "::error::a boundary-crossing ranged GET scales with object size, so it is reading the whole object"
+              exit 1
+            }
+          }'
+          echo "ranged GET does not scale with object size"
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark, so this asserts the same headroom cell-23 does.
+      # tmpfs is rejected outright: that would measure RAM, not the gates.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      - name: Install warp
+        working-directory: predastore
+        run: make warp-install
+
+      # PERF_EXTERNAL_HOSTS skips profile selection, port offsetting, and
+      # cluster start/stop, and measures these three gates directly. Same
+      # workloads and analysis as the nightly loopback run, so the numbers
+      # from the two are directly comparable.
+      #
+      # The cluster's credentials are read and used inside this one step, the
+      # way e2e-ddil.yml does it: a step-level env: block is echoed verbatim
+      # into the run log, and a shell variable is not.
+      - name: Run 3-node throughput benchmark
+        working-directory: predastore
+        env:
+          PERF_EXTERNAL_HOSTS: ${{ steps.perf_hosts.outputs.hosts }}
+          PERF_CONFIGS: spinifex-3node
+          PERF_PRESET: ${{ inputs.perf_preset || 'compare' }}
+          # What is deployed, not what the harness was checked out at. With a
+          # cluster_ref those differ, and run-info.txt records the cluster.
+          PERF_EXTERNAL_SHA: ${{ inputs.cluster_ref || steps.predastore.outputs.commit }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          # spx admin init mints a random pair at bootstrap and writes it to
+          # node1's ~/.aws/credentials. The benchmark's built-in key is the one
+          # the loopback configs trust, which this cluster has never heard of.
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          PERF_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          PERF_SECRET_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          if [ -z "$PERF_ACCESS_KEY" ] || [ -z "$PERF_SECRET_KEY" ]; then
+            echo "::error::could not read spinifex AWS credentials from ${NODE1_IP}"
+            exit 1
+          fi
+          echo "::add-mask::$PERF_SECRET_KEY"
+          mkdir -p "$PERF_RESULTS_ROOT"
+          PERF_ACCESS_KEY="$PERF_ACCESS_KEY" \
+          PERF_SECRET_KEY="$PERF_SECRET_KEY" \
+          PERF_REGION="$CLUSTER_REGION" \
+            make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only, matching cell-23: the baseline comparison and any
+      # regression gate are future work once run-to-run spread is known from
+      # real bare-metal runs.
+      - uses: actions/upload-artifact@v7.0.2
+        if: always()
+        with:
+          name: predastore-perf-3node-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "${PERF_RESULTS_ROOT}"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 2
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: tofu-log-perf3node-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/publish-system-ami.yml

@@ -0,0 +1,170 @@
+name: Publish System AMIs
+
+# Builds the system AMIs (libguestfs, on the runner) and publishes each
+# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
+# iso.mulgadc.com/system-ami/. Six images ship:
+#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node       Alpine ECS container instance
+#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
+#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
+# Operators then install them on any cluster with:
+#   spx admin images import --name spinifex-eks-node
+#   spx admin images import --name spinifex-ecs-node
+#   spx admin images import --name spinifex-eks-node-gpu
+#   spx admin images import --name spinifex-ecs-node-gpu
+#   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
+#
+# Publishing overwrites the mirror artifact, so it runs only from release.yml
+# once a tag has passed its gates, with siblings at the tags go.mod requires.
+# The `variant` input scopes which images build — the Ubuntu GPU legs are 16G
+# and much slower (NVIDIA driver download + DKMS build).
+
+on:
+  workflow_call:
+    inputs:
+      variant:
+        description: "Which images to build (all|cpu|gpu|rds)"
+        type: string
+        default: all
+      dry_run:
+        description: "Build + convert only; skip the R2 upload"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: publish-eks-ami
+  cancel-in-progress: false
+
+jobs:
+  matrix:
+    name: Resolve image matrix
+    runs-on: [self-hosted, ci-single]
+    outputs:
+      images: ${{ steps.gen.outputs.images }}
+    steps:
+      - name: Select images for variant
+        id: gen
+        env:
+          VARIANT: ${{ inputs.variant }}
+        run: |
+          set -euo pipefail
+          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
+          case "${VARIANT}" in
+            cpu) items="${cpu}" ;;
+            gpu) items="${gpu}" ;;
+            rds) items="${rds}" ;;
+            *)   items="${cpu},${gpu},${rds}" ;;
+          esac
+          echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    name: Build + publish ${{ matrix.name }}
+    needs: matrix
+    runs-on: [self-hosted, ci-single]
+    strategy:
+      fail-fast: false
+      # ci-single is one runner and each GPU build needs ~16G of scratch disk;
+      # serialise the legs so they never contend for /_temp or the mirror.
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.matrix.outputs.images) }}
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      - name: Setup Go
+        uses: actions/setup-go@v7.0.0
+        with:
+          go-version-file: "spinifex/go.mod"
+          # Self-hosted runner: the Go module + build cache already persists on
+          # local disk between runs.
+          cache: false
+
+      - name: Setup Go Workspace
+        # Idempotent: the ci-single runner reuses one workspace across matrix
+        # legs and runs, and checkout never scrubs the root go.work. Remove any
+        # stale file so each leg re-inits cleanly instead of failing on
+        # "go.work already exists".
+        run: |
+          rm -f go.work go.work.sum
+          go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+
+      - name: Install build + upload tooling
+        run: |
+          set -euo pipefail
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          if ! command -v aws &>/dev/null; then
+            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+            unzip -q /tmp/awscliv2.zip -d /tmp/aws-install
+            sudo /tmp/aws-install/aws/install
+            rm -rf /tmp/awscliv2.zip /tmp/aws-install
+          fi
+
+      - name: Build + publish ${{ matrix.name }} to R2
+        working-directory: spinifex
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
+          R2_BUCKET: spinifex-iso
+          DRY_RUN: ${{ inputs.dry_run }}
+          MANIFEST: ${{ matrix.manifest }}
+          # Ubuntu GPU images are 16G — a RAM-backed /tmp cannot hold the qcow2 +
+          # expanded raw, so point the build scratch at the runner's on-disk
+          # _temp. Harmless for the small Alpine legs too.
+          SYSTEM_IMAGE_BUILD_DIR: ${{ runner.temp }}/system-image-build
+        run: |
+          set -euo pipefail
+          # libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
+          # direct backend = no libvirt on the runner.
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          mkdir -p "${SYSTEM_IMAGE_BUILD_DIR}"
+          FLAGS=(--build)
+          if [[ "${DRY_RUN}" == "true" ]]; then
+            FLAGS+=(--dry-run)
+          fi
+          ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"

.github/workflows/release.yml

@@ -5,36 +5,57 @@ on:
     tags: [ "v*" ]
 
 permissions:
-  contents: write
+  contents: read
 
 jobs:
   distro-amd64:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
+        uses: docker/setup-buildx-action@v4.4.1
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7.0.0
         with:
-          go-version: '1.26.3'
+          go-version-file: spinifex/go.mod
+          cache: false
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -45,7 +66,7 @@ jobs:
         run: cd spinifex && make distro-amd64 VERSION="$VERSION"
 
       - name: Upload amd64 Artifacts
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: distro-amd64
           path: |
@@ -54,27 +75,47 @@ jobs:
             spinifex/scripts/setup.sh
 
   distro-arm64:
-    runs-on: ubuntu-24.04-arm
+    runs-on: ubuntu-26.04-arm
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
+        uses: docker/setup-buildx-action@v4.4.1
 
       - name: Build arm64 Tarball
         env:
@@ -82,7 +123,7 @@ jobs:
         run: cd spinifex && make distro-arm64 VERSION="$VERSION"
 
       - name: Upload arm64 Artifacts
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: distro-arm64
           path: |
@@ -91,22 +132,24 @@ jobs:
 
   release:
     needs: [ distro-amd64, distro-arm64 ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
     steps:
       - name: Download amd64 Artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         with:
           name: distro-amd64
           path: artifacts
 
       - name: Download arm64 Artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         with:
           name: distro-arm64
           path: artifacts
 
       - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v3
+        uses: softprops/action-gh-release@v3.0.3
         with:
           draft: true
           files: |
@@ -116,7 +159,7 @@ jobs:
           generate_release_notes: true
 
       - name: Upload Tarball for Verify Job
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v7.0.2
         with:
           name: release-assets
           path: artifacts/
@@ -124,6 +167,13 @@ jobs:
   verify:
     needs: release
     runs-on: [ self-hosted, ci-single ]
+    # Shared with the e2e workflows: this job provisions on the same runner
+    # pools and the same hypervisors, so two in flight compete for their disks
+    # and their libvirt guests. Job-level, so the build jobs are not queued
+    # behind a nightly that has nothing to do with them.
+    concurrency:
+      group: e2e-hypervisors
+      cancel-in-progress: false
     timeout-minutes: 25
     env:
       TOFU_LOG: /tmp/release-verify-${{ github.run_id }}.log
@@ -131,24 +181,34 @@ jobs:
       REPO: ${{ github.repository }}
     steps:
       - name: Clean Workspace
-        run: rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
+      # Only the shared shell helpers: this job builds nothing from source, but
+      # it drives the same libvirt provider the e2e cells retry around.
+      - uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts
+          path: spinifex
+
       - name: Download Release Assets
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v8.0.2
         with:
           name: release-assets
           path: release-assets
@@ -156,11 +216,12 @@ jobs:
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
 
       - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
@@ -169,8 +230,33 @@ jobs:
           TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
           ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" "$SETUP_SH" "$TARBALL"
 
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
+          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
+
+  baremetal:
+    needs: release
+    permissions:
+      contents: read
+      statuses: write
+    uses: ./.github/workflows/e2e-baremetal.yml
+    secrets: inherit
+
+  promote:
+    needs: [ verify, baremetal ]
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+    env:
+      TAG: ${{ github.ref_name }}
+      REPO: ${{ github.repository }}
+    steps:
       - name: Promote Release
-        if: success()
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -182,43 +268,58 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
+  # Refresh the system AMIs on the R2 mirror only once the release is out of
+  # draft. Shares the ci-single runner with `iso`, so it queues behind it.
+  system-images:
+    needs: promote
+    permissions:
+      contents: read
+    uses: ./.github/workflows/publish-system-ami.yml
+    with:
+      variant: all
+    secrets: inherit
+
+  # Annotate the (still-draft) release if either gate failed.
+  mark-failed:
+    needs: [ verify, baremetal ]
+    if: failure()
+    runs-on: ubuntu-26.04
+    permissions:
+      contents: write
+    env:
+      TAG: ${{ github.ref_name }}
+      REPO: ${{ github.repository }}
+    steps:
       - name: Mark Release Failed
-        if: failure()
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
           NOTES_FILE=$(mktemp)
-          printf 'Verification failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
+          printf 'Verification failed (tofu install or bare-metal boot) — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
           gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
           rm -f "$NOTES_FILE"
 
-      - name: Teardown
-        if: always()
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
-          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
-          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-
   iso:
     needs: release
     runs-on: [ self-hosted, ci-single ]
-    timeout-minutes: 90
+    permissions:
+      contents: write
+    # ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
+    # upload gate then waits for baremetal to finish, so the job must outlast it.
+    timeout-minutes: 180
     steps:
       - name: Clean Workspace
         run: |
           ROOTFS="${GITHUB_WORKSPACE}/mulga/scripts/iso-builder/.cache/rootfs"
-          # Mirror make clean's umount sequence unconditionally — lazy unmount
-          # detaches immediately even if the mount is busy, so rm -rf is safe.
           sudo umount -l -R "${ROOTFS}" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/dev/pts" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/sys" 2>/dev/null || true
           sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
-          sudo rm -rf "$GITHUB_WORKSPACE"/*
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v3
+      - uses: actions/create-github-app-token@v3.3.0
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
@@ -226,25 +327,45 @@ jobs:
           repositories: mulga
 
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           ref: ${{ github.ref }}
 
+      - name: Resolve sibling release tags from go.mod
+        id: siblings
+        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
       - name: Checkout Mulga iso-builder
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -255,13 +376,13 @@ jobs:
         run: sudo make -C mulga/scripts/iso-builder clean
 
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@v7.0.0
         with:
           go-version-file: spinifex/go.mod
           cache: false
 
       - name: Configure Go workspace
-        run: go work init ./spinifex ./predastore ./viperblock
+        run: go work init ./spinifex ./predastore ./viperblock ./northstar ./bluebottle
 
       - name: Install ISO builder dependencies
         run: |
@@ -289,24 +410,25 @@ jobs:
             SPINIFEX_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spx" \
             INSTALLER_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spinifex-installer"
 
-      - name: Wait for verify
+      - name: Wait for verify + baremetal
         id: wait-verify
         if: steps.build-iso.outcome == 'success'
         continue-on-error: true
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          echo "Waiting for verify job to complete..."
+          echo "Waiting for verify + baremetal jobs to complete..."
           while true; do
-            CONCLUSION=$(gh api \
-              "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
-              --jq '.jobs[] | select(.name == "verify") | .conclusion // empty')
-            if [ -n "$CONCLUSION" ]; then
-              echo "Verify conclusion: $CONCLUSION"
-              [ "$CONCLUSION" = "success" ] && exit 0
+            JOBS=$(gh api \
+              "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" --paginate)
+            VERIFY=$(echo "$JOBS" | jq -r '.jobs[] | select(.name == "verify") | .conclusion // empty')
+            BAREMETAL=$(echo "$JOBS" | jq -r '.jobs[] | select(.name | startswith("baremetal")) | .conclusion // empty')
+            if [ -n "$VERIFY" ] && [ -n "$BAREMETAL" ]; then
+              echo "verify=$VERIFY baremetal=$BAREMETAL"
+              { [ "$VERIFY" = "success" ] && [ "$BAREMETAL" = "success" ]; } && exit 0
               exit 1
             fi
-            echo "Verify still running, retrying in 30s..."
+            echo "Still waiting (verify=${VERIFY:-pending} baremetal=${BAREMETAL:-pending}), retry 30s..."
             sleep 30
           done
 
@@ -327,14 +449,13 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
           CF_R2_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT_URL }}
-          VERSION: ${{ github.ref_name }}
         run: |
           aws --endpoint-url="${CF_R2_ENDPOINT_URL}" \
-            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso" \
-                  "s3://spinifex-iso/spinifex-${VERSION}.iso"
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex.iso" \
+                  "s3://spinifex-iso/spinifex.iso"
           aws --endpoint-url="${CF_R2_ENDPOINT_URL}" \
-            s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso.sha256" \
-                  "s3://spinifex-iso/spinifex-${VERSION}.iso.sha256"
+            s3 cp "mulga/scripts/iso-builder/dist/spinifex.iso.sha256" \
+                  "s3://spinifex-iso/spinifex.iso.sha256"
 
       - name: Mark ISO failed
         if: always() && (steps.build-iso.outcome == 'failure' || steps.upload.outcome == 'failure')

.github/workflows/segscan-oracle.yml

@@ -0,0 +1,95 @@
+# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only,
+# deliberately NOT wired into spinifex.yml's pull_request path.
+#
+# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
+# scripts/segscan checked out alongside spinifex (it is a separate Go module,
+# `package main`, and cannot be imported — see internal/testkit/segscanoracle).
+# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
+# every PR; adding a second-repo checkout there would make every spinifex PR
+# depend on mulga main being healthy and pay that checkout cost on every run,
+# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger
+# This never gates a PR before merge, matching how e2e.yml's own
+# "front gate" job works.
+name: Segscan Storage Oracle
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  segscan_oracle:
+    name: Segscan Oracle
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      - uses: actions/create-github-app-token@v3.3.0
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Mulga (scripts/segscan only)
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/segscan
+          path: mulga
+
+      - name: Setup Go
+        uses: actions/setup-go@v7.0.0
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+
+      - name: Run Segscan Oracle Test
+        env:
+          SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan
+        run: make -C spinifex test-segscan-oracle

.github/workflows/spinifex.yml

@@ -23,124 +23,347 @@ permissions:
 jobs:
   build_and_test:
     name: Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
           fetch-depth: 2
 
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Setup Go
-        uses: actions/setup-go@v6
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
+      - name: Setup Go
+        uses: actions/setup-go@v7.0.0
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6.1.0
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6.1.0
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+
+      # The EBS provider contract suite needs a real implementation on each
+      # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which
+      # shares no code with viperblock, and nbdkit plus viperblock's plugin
+      # back viperblockd, the provider production actually uses. Without them
+      # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries
+      # the headers and nbdkit.pc that cgo needs to build that plugin, and
+      # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that
+      # judges the published export by something outside this codebase.
+      - name: Install conformance tools
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin
+
+      - name: Build viperblock nbdkit plugin
+        run: make -C viperblock go_build_nbd
 
       - name: Build Spinifex
         run: make -C spinifex build
 
+      - name: Manifest check + drift lint
+        run: make -C spinifex manifest-check manifest-lint
+
       - name: Run Tests with Coverage
-        run: make -C spinifex test-cover
+        env:
+          SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
+          VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so
+        run: make -C spinifex test-cover GOTESTFLAGS=-count=1
 
       - name: Run Action Tests
-        run: make -C spinifex test-actions
+        run: make -C spinifex test-actions GOTESTFLAGS=-count=1
 
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
+      # Compiled output stays valid whatever the tests said, and skipping the
+      # save on a red run would leave the next one rebuilding it, so this is
+      # gated on the branch rather than on success.
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6.1.0
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+
   race_detection:
     name: Race Detection
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
 
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Setup Go
-        uses: actions/setup-go@v6
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
+      - name: Setup Go
+        uses: actions/setup-go@v7.0.0
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6.1.0
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6.1.0
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Tests with Race Detector
-        run: make -C spinifex test-race
+        run: make -C spinifex test-race GOTESTFLAGS=-count=1
+
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6.1.0
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
 
   lint_and_security:
-    name: Lint and Security
-    runs-on: ubuntu-24.04
+    name: Lint + Security + Integration
+    runs-on: ubuntu-26.04
 
     steps:
       - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
 
       - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/viperblock
-          ref: dev
+          ref: ${{ steps.siblings.outputs.viperblock }}
           path: viperblock
 
       - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
           repository: mulgadc/predastore
-          ref: dev
+          ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Setup Go
-        uses: actions/setup-go@v6
+      - name: Checkout Northstar
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
+      - name: Setup Go
+        uses: actions/setup-go@v7.0.0
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6.1.0
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6.1.0
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Lint
-        uses: golangci/golangci-lint-action@v9
+        uses: golangci/golangci-lint-action@v9.3.0
         with:
-          version: v2.12.2
+          version: v2.14.0
           working-directory: spinifex
 
       - name: Govulncheck
         run: make -C spinifex govulncheck
+
+      - name: Run Integration Tests
+        if: ${{ !cancelled() }}
+        run: make -C spinifex test-integration
+
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6.1.0
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}

.github/workflows/ui.yml

@@ -6,14 +6,14 @@ on:
       - main
       - dev
     paths:
-      - spinifex/services/spinifexui/frontend/**
+      - spinifex/runtime/roles/spinifexui/frontend/**
       - .github/workflows/ui.yml
   pull_request:
     branches:
       - main
       - dev
     paths:
-      - spinifex/services/spinifexui/frontend/**
+      - spinifex/runtime/roles/spinifexui/frontend/**
       - .github/workflows/ui.yml
   workflow_dispatch:
 
@@ -23,28 +23,28 @@ permissions:
 jobs:
   ui_build_and_test:
     name: UI Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
     defaults:
       run:
-        working-directory: spinifex/services/spinifexui/frontend
+        working-directory: spinifex/runtime/roles/spinifexui/frontend
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7.0.1
         with:
-          sparse-checkout: spinifex/services/spinifexui/frontend
+          sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6
+        uses: pnpm/action-setup@v6.1.0
         with:
-          package_json_file: spinifex/services/spinifexui/frontend/package.json
+          package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 
       - name: Setup Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7.1.0
         with:
           cache: "pnpm"
-          cache-dependency-path: spinifex/services/spinifexui/frontend/pnpm-lock.yaml
-          node-version-file: spinifex/services/spinifexui/frontend/.nvmrc
+          cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
+          node-version-file: spinifex/runtime/roles/spinifexui/frontend/.nvmrc
 
       - name: Install Dependencies
         run: pnpm install --frozen-lockfile
@@ -57,3 +57,15 @@ jobs:
 
       - name: Build Frontend
         run: pnpm build
+
+      # spx embeds the committed dist/, so a bundle that differs from a clean
+      # build of this source would ship JavaScript CI never built. Porcelain
+      # rather than git diff, so added and deleted chunks count too.
+      - name: Check for generated file drift
+        run: |
+          drift=$(git status --porcelain -- src/routeTree.gen.ts src/lib/cn-tables.gen.ts dist)
+          if [ -n "$drift" ]; then
+            echo "::error::Generated files differ from a clean build. Run pnpm build and commit the result."
+            echo "$drift"
+            exit 1
+          fi

.gitignore

@@ -19,7 +19,7 @@ bin/
 dist/
 build/microvm/vmlinuz
 build/microvm/initramfs.cpio.gz
-!spinifex/services/spinifexui/frontend/dist/
+!spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/
 *.test
 *.out
@@ -34,16 +34,11 @@ Thumbs.db
 # Environment
 .env
 .venv/
+.cache/
+
+# VScode
+.vscode/
 
 # Test reports
 *-report.txt
-
-# Internal docs (moved to private mulga repo)
-docs/INTERNAL.md
-docs/PRIVATE.md
-docs/DEVNOTES.md
-docs/PLAN.md
-docs/TODO.md
-docs/development/
-docs/internal/
-scripts/iac/aws/spinifex-test.pem
+/rds-agent

.golangci.yml

@@ -1,5 +1,11 @@
 version: "2"
 
+run:
+  allow-serial-runners: true
+
+issues:
+  max-same-issues: 10
+
 formatters:
   enable:
     - gofmt
@@ -14,24 +20,29 @@ linters:
     - bidichk # checks for dangerous unicode bidi character sequences
     - bodyclose # checks whether HTTP response body is closed successfully
     - canonicalheader # checks whether net/http.Header uses canonical header keys
-    - decorder # checks declaration order of types, consts, vars, and funcs
+    - depguard # blocks imports of packages with a preferred replacement
     - durationcheck # checks for two durations multiplied together
     - embeddedstructfieldcheck # checks embedded sync.Mutex/RWMutex in structs
     - errcheck # checks for unchecked errors in type assertions and I/O
     - errchkjson # checks types passed to json encoding functions, flags omittable error checks
     - errname # checks that sentinel errors are prefixed with Err, error types suffixed with Error
+    - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forcetypeassert # flags unchecked type assertions that can panic
+    - forbidigo # blocks forbidden identifiers
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
+    - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
     - gosec # inspects source code for security problems
     - govet # reports suspicious constructs (Printf alignment, shadow, etc.)
+    - iface # detects interface pollution — duplicate interface declarations
     - ineffassign # detects assignments to existing variables that are not used
+    - iotamixing # flags const blocks that mix iota with non-iota declarations
     - makezero # finds slice declarations with non-zero initial length followed by append
     - mirror # catches wrong mirror patterns in bytes/strings usage
-    - musttag # checks structs passed to marshal/unmarshal have proper tags
+    - misspell # finds commonly misspelled English words
     - modernize # modernizes code to use newer Go idioms (go fix analyzers)
+    - musttag # checks structs passed to marshal/unmarshal have proper tags
     - nakedret # finds naked returns in functions
     - nilerr # catches code that returns nil even after checking err != nil
     - nilnesserr # catches returning a different nil error variable after err != nil check
@@ -43,21 +54,64 @@ linters:
     - recvcheck # checks for receiver type consistency (pointer vs value)
     - sloglint # enforces consistent log/slog usage (key-value pairs, no globals)
     - staticcheck # extensive static analysis checks
+    - testifylint # checks common testify mistakes (assert.Len over assert.Equal on len, etc.)
+    - tparallel # detects inappropriate usage of t.Parallel()
     - unconvert # removes unnecessary type conversions
     - usestdlibvars # detects hardcoded values that should use stdlib constants
     - usetesting # catches os.TempDir() etc. in tests — should use t.TempDir()
-    - wastedassign # finds assignments to variables that are never subsequently read
     - whitespace # detects leading and trailing whitespace in functions
 
   settings:
+    depguard:
+      rules:
+        main:
+          deny:
+            - pkg: log$
+              desc: Use log/slog instead, see https://go.dev/blog/slog
+            - pkg: math/rand$
+              desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
+
+        # The control plane reaches block storage through the EBS provider, not
+        # by importing the engine. viperblock.New starts a chunk uploader, so
+        # every construction is a writer against a volume viperblockd serves.
+        # No domain or legacy handler may import viperblock, in any form, tests
+        # included. A types-only import counts: one became an engine import.
+        ec2-control-plane:
+          files:
+            - '**/spinifex/domains/**'
+            - '**/spinifex/handlers/**'
+          deny:
+            - pkg: github.com/mulgadc/viperblock
+              desc: Use the providers/ebs EBSProvider interface; the control plane must not construct or name viperblock types.
+
     embeddedstructfieldcheck:
       forbid-mutex: true
 
     errcheck:
-      check-type-assertions: true
+      check-type-assertions: true # replaces forcetypeassert
       exclude-functions:
         - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
-        - (*github.com/kdomanski/iso9660.ImageWriter).Cleanup # deferred cleanup, errors not actionable
+
+    forbidigo:
+      analyze-types: true
+      forbid:
+        - pattern: '^panic$'
+          msg: panic terminates the process; return an error or explicitly justify the panic.
+        - pattern: '\.JetStream$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
+        - pattern: '^nats\.(JetStreamContext|KeyValue.*|KeyWatcher|Err(KeyNotFound|KeyExists|BucketNotFound|NoKeysFound|StreamNotFound))$'
+          pkg: '^github\.com/nats-io/nats\.go$'
+          msg: legacy nats KV types/sentinels are superseded; use github.com/nats-io/nats.go/jetstream.
+
+    gosec:
+      excludes:
+        - G104 # unchecked errors — errcheck is stricter
+        - G116 # bidi chars — bidichk is more thorough
+        - G117 # use of net/http serve functions (we handle TLS separately)
+        - G703 # path traversal taint analysis — false positives on config file loading
+        - G704 # SSRF taint analysis — false positives on resolver-derived endpoint URLs
+        - G705 # XSS taint analysis — false positives on internal error responses
 
     govet:
       enable-all: true
@@ -66,41 +120,12 @@ linters:
         - shadow # too noisy to retrofit — all 50 findings are err re-declarations
         - unusedwrite # false positives on struct field assignments before marshal/send
 
-    staticcheck:
-      checks:
-        - all
-        - -ST1000 # package comment
-        - -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
-        - -ST1012 # error var naming — errname covers this + error type suffixes
-        - -ST1013 # HTTP status constants — usestdlibvars is broader
-        - -ST1016 # receiver name consistency (too noisy to retrofit)
-        - -ST1020 # godoc on exported types — not enforcing yet
-        - -ST1021 # godoc on exported type groups — not enforcing yet
-        - -ST1022 # godoc on exported vars/consts — not enforcing yet
-        - -SA1019 # deprecated usage (handle separately)
-        - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
-        - -SA9005 # ineffective break (staticcheck false positives)
-
-    gosec:
-      excludes:
-        - G104 # unchecked errors — errcheck is stricter
-        - G116 # bidi chars — bidichk is more thorough
-        - G117 # use of net/http serve functions (we handle TLS separately)
-        - G703 # path traversal taint analysis — false positives on config file loading
-        - G705 # XSS taint analysis — false positives on internal error responses
-        - G706 # log injection taint analysis — false positives on structured slog usage
-
     modernize:
       disable:
+        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
-    perfsprint:
-      concat-loop: false
-      err-error: false # don't flag err.Error()
-      errorf: false # don't convert fmt.Errorf to errors.New
-      strconcat: false
-
     nakedret:
       max-func-lines: 0
 
@@ -108,10 +133,32 @@ linters:
       require-explanation: true
       require-specific: true
 
+    perfsprint:
+      concat-loop: false
+      err-error: false # don't flag err.Error()
+      errorf: false # don't convert fmt.Errorf to errors.New
+      strconcat: false
+
     reassign:
       patterns:
         - ".*"
 
+    staticcheck:
+      checks:
+        - all
+        - -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
+        - -ST1012 # error var naming — errname covers this + error type suffixes
+        - -ST1013 # HTTP status constants — usestdlibvars is broader
+        - -ST1016 # receiver name consistency (too noisy to retrofit)
+        - -SA4006 # value overwritten — ineffassign catches strictly more of these
+        - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
+
+    testifylint:
+      disable:
+        - encoded-compare # misidentifies non-JSON strings when identifiers contain "JSON"
+        - float-compare # false positives
+        - require-error # assert-on-error is deliberate where a test continues to collect further failures
+
   exclusions:
     presets:
       - std-error-handling
@@ -122,6 +169,7 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
+          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions
@@ -134,3 +182,8 @@ linters:
         linters: [gosec] # subprocess execution — legitimate in VM orchestration
       - text: "G304"
         linters: [gosec] # file path from variable — necessary for config loading
+
+      # aws-sdk-go v1 is deprecated wholesale (EOL 2025-07-31); the v1->v2
+      # migration is tracked separately, so suppress its SA1019 noise for now.
+      - text: "aws-sdk-go is deprecated"
+        linters: [staticcheck]

CONTRIBUTING.md

@@ -0,0 +1,97 @@
+# How to Contribute
+
+We welcome contributions to Spinifex!
+
+Report security vulnerabilities privately through [SECURITY.md](SECURITY.md), not as a public issue.
+
+**AWS parity is the design rule.** When Spinifex's behaviour is in question, the answer is what AWS does. If you believe Spinifex diverges from AWS, state what AWS does and show it with a test or a reproduction against a running cluster.
+
+## Understanding the Codebase
+
+Every AWS call enters through a SigV4-authenticated gateway, is published to NATS, and is answered by the daemon that claims it. Start with these:
+
+| Doc | Covers |
+|---|---|
+| [`docs/DESIGN.md`](docs/DESIGN.md) | Architecture, request flow, daemons, storage integration, file layout. |
+| [`docs/COMMANDS.md`](docs/COMMANDS.md) | The `spx admin` CLI and the systemd units it manages. |
+| [`docs/coverage/`](docs/coverage/README.md) | Which AWS API operations each service implements. |
+| [`scripts/README.md`](scripts/README.md) | Install, dev-environment, and image-build scripts. |
+| [`docs/PACKAGE_BOUNDARY_MIGRATION.md`](docs/PACKAGE_BOUNDARY_MIGRATION.md) | ADR-0001 implementation inventory and incremental source moves. |
+
+User-facing docs are published at [docs.mulgadc.com](https://docs.mulgadc.com).
+
+## Development Setup
+
+These steps build Spinifex from source and run it as a single-node cluster, following the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
+
+### Prerequisites
+
+- **Ubuntu 26.04** or **Debian 13**, on x86_64 or aarch64
+- A host with hardware virtualisation (KVM) and `sudo` access
+- The host's WAN interface enslaved to a Linux bridge named `br-wan`, with the host IP, default route and DHCP on the bridge rather than the bare NIC (see [VPC Networking → Bridge Setup](https://docs.mulgadc.com/docs/vpc-networking#bridge-setup-physical-network-wiring))
+
+Check the bridge before continuing:
+
+```bash
+ip -br link show br-wan   # bridge exists and is UP
+ip route                  # default route's dev is br-wan
+```
+
+### Install and Run
+
+```bash
+mkdir -p ~/Development/mulga && cd ~/Development/mulga
+git clone https://github.com/mulgadc/spinifex.git
+sudo make -C spinifex quickinstall        # system packages, Go 1.27, AWS CLI v2
+export PATH=$PATH:/usr/local/go/bin
+
+cd spinifex
+./scripts/clone-deps.sh                   # sibling repos: viperblock, predastore, northstar, bluebottle
+./scripts/dev-install.sh                  # build, install, init a one-node cluster, start services
+
+AWS_PROFILE=spinifex aws ec2 describe-instance-types
+```
+
+A list of instance types means the cluster is up.
+
+`make preflight` also needs `shellcheck` and `golangci-lint`
+
+```bash
+sudo apt-get install -y shellcheck
+curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.14.0
+```
+
+The web console additionally needs Node.js 24 and `pnpm`.
+
+## Development Workflow
+
+| Command | Use it to |
+|---|---|
+| `make deploy` | Rebuild, install `spx`, and restart `spinifex.target` after a code change. |
+| `make reinstall` | Re-run `dev-install.sh` after changing systemd units, helper scripts, or logrotate config. |
+| `./scripts/reset-dev-env.sh` | Wipe all cluster state and reinstall from scratch, keeping your network topology. |
+| `make test` | Run unit tests (`make test-race` for the race detector). |
+| `make test-integration` | Run the real gateway router against embedded NATS JetStream. |
+| `make preflight` | Run the pre-commit gate: lint, `govulncheck`, coverage, and manifest and script checks. Must pass. |
+
+Follow the services with `journalctl -u 'spinifex-*' -f`.
+
+The web console lives in `spinifex/runtime/roles/spinifexui/frontend/` (`pnpm install`, `pnpm dev`, `pnpm lint`, `pnpm test`). `spx` embeds its checked-in `dist/`, so run `make build-ui` and commit the rebuilt `dist/` with your change.
+
+## Creating a Pull Request
+
+1. Fork [mulgadc/spinifex](https://github.com/mulgadc/spinifex) and clone your fork into the same parent directory as the sibling repositories, branching off `dev`:
+
+   ```bash
+   cd ~/Development/mulga
+   git clone git@github.com:YOUR_USERNAME/spinifex.git && cd spinifex
+   git remote add upstream git@github.com:mulgadc/spinifex.git
+   git fetch upstream && git checkout -b feat/my-feature upstream/dev
+   ```
+
+2. Make your change, then verify it:
+   - `make preflight` passes.
+   - `go mod tidy` leaves only intended changes in `go.mod` and `go.sum`, with none of the `clone-deps.sh` `replace` lines.
+   - Runtime changes are exercised on your dev cluster with `make deploy`.
+
+3. Commit using [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, `docs:`, `chore:`), push, and open a pull request against `dev`, filling in the template.

INSTALL.md

@@ -1,3 +0,0 @@
-# Spinifex Installation
-
-For documentation on using and installing Spinifex see [https://docs.mulgadc.com](https://docs.mulgadc.com/)

Makefile

@@ -1,22 +1,22 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
+# GOFIPS140 is a global build-config change: it alters every package's action ID,
+# stdlib included, so a target that omits it shares no build-cache entry with one
+# that sets it. Exported once here rather than per-recipe so the two can't drift.
+export GOFIPS140 := v1.0.0
+
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
   GO_ARCH := amd64
   AWS_ARCH := x86_64
-  # On x86, we can run ARM VMs via emulation
-  QEMU_PACKAGES := qemu-system-x86 qemu-system-arm
 else ifeq ($(ARCH),aarch64)
   GO_ARCH := arm64
   AWS_ARCH := aarch64
-  # On ARM, we can run x86 VMs via emulation
-  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
 else ifeq ($(ARCH),arm64)
   GO_ARCH := arm64
   AWS_ARCH := aarch64
-  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
 else
   $(error Unsupported architecture: $(ARCH). Only x86_64 and aarch64/arm64 are supported.)
 endif
@@ -49,35 +49,83 @@ else
   _SECQ  = 2>&1 | tee
 endif
 
-build: go_build build-installer build-lb-agent
+build: go_build build-installer build-lb-agent generate-aws-model-coverage
 
 # Build spinifex-ui frontend (requires pnpm)
 build-ui:
 	@echo -e "\n....Building spinifex-ui frontend...."
-	cd spinifex/services/spinifexui/frontend && pnpm build
+	cd spinifex/runtime/roles/spinifexui/frontend && pnpm build
 
 # GO commands
-VERSION ?= $(shell git describe --tags --always --dirty)
+VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
 COMMIT  ?= $(shell git rev-parse --short HEAD)
-LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSION) -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=$(COMMIT)
+LDFLAGS := -s -w -X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=$(VERSION) -X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=$(COMMIT)
 
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
 
 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
-build-system-image: ## Build a system image from manifest (use IMAGE=lb)
+build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
+	@echo -e "\n....Building ecs-agent (static)"
+	CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
+build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
+	@echo -e "\n....Building spx-loadgen"
+	go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
+
+build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
 	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
 endif
-	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf
+	@if [ -f scripts/images/$(IMAGE).conf ]; then \
+		./scripts/build-system-image.sh scripts/images/$(IMAGE).conf $(if $(IMPORT),--import); \
+	elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \
+		./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf $(if $(IMPORT),--import); \
+	else \
+		echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \
+		exit 1; \
+	fi
+
+build-eks-node-image: ## Build the unified eks-node AMI (K3s server+agent; role at first boot; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=eks-node
+
+import-eks-node-image: ## Build + register the eks-node AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=eks-node IMPORT=1
+
+publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
+	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
+
+# The Ubuntu/systemd EKS image, as opposed to build-eks-node-image's Alpine one.
+# Composition lives in mkosi-build.sh's image_profiles table, so the image name
+# is the whole interface here.
+build-eks-node-mkosi-image: ## Build the Ubuntu mkosi EKS node image (server+agent; role at first boot)
+	./scripts/mkosi-build.sh --image spinifex-eks-node
+
+build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=ecs-agent
+
+import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
+
+build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + PostgreSQL 18 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-postgres
+
+import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
+
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
 
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
@@ -99,65 +147,187 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
 
-go_run:
-	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
-	$(GOPATH)/bin/$(GO_PROJECT_NAME)
-
-# Preflight — runs the same checks as GitHub Actions (lint + vuln + tests).
-# Use this before committing to catch CI failures locally.
+# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
+# integration and race tests skipped to keep quick, they run in CI.
+#
+# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
+# most of the machine idle, so pairing them cuts a cold run by about a quarter.
+# -Otarget keeps each gate's output in one block instead of interleaving them.
+# manifest-check manifest-lint test-package-check removed, can add back later
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check lint govulncheck test-cover diff-coverage test-race test-harness
+	@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts manifest-check manifest-lint
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
+# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
+# ship on every node (unlike scripts/images/, kept in preflight: a wrong
+# restart decision here is an outage, not asset churn).
+#
+# NODE_SCRIPTS are the four that setup.sh installs into
+# /usr/local/share/spinifex on every node. They are covered here for the same
+# reason: between them they install the software, wire OVN, erase the state and
+# remove the install, so a shellcheck warning in one is an outage or a wrong
+# deletion, not a style note.
+NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
+
+test-build-scripts:
+	@echo -e "\n....Running build/scripts/**/*_test.sh...."
+	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running scripts/*_test.sh...."
+	@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
+	@echo "  test-build-scripts ok"
+
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
+#
+# diskperf/ is here for the same reason: its baseline comparison and probe
+# accounting decide whether the disk-performance gate passes, and that
+# arithmetic has to be covered off-cluster or the gate is trusted untested.
 test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... ./tests/e2e/diskperf/... $(_RACEQ)
+
+# In-process integration tier: the real gateway router against embedded NATS
+# JetStream, with only the daemon-side NATS subjects stubbed.
+AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
+AWS_MODEL_CONFORMANCE_MODE ?= fail
+AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
+# -count=1 is load-bearing: the conformance report is written by the test binary,
+# so a cached pass skips the run, leaves no report, and the cat below fails the
+# target. It also keeps the conformance gate honest — a cached result would mean
+# the check never ran against this commit. The generated request sweep adds
+# about 15s, most of it EC2 fan-outs waiting out their window for instances
+# no daemon-lite owns.
+test-integration:
+	@echo -e "\n....Running in-process integration tests...."
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 120s ./tests/integration/... $(_RACEQ)
+	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
+	@$(MAKE) --no-print-directory generate-aws-model-coverage
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
+
+generate-aws-model-coverage:
+	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR) -json $(AWS_MODEL_OPERATION_COVERAGE_DIR)/coverage.json
+
+aws-model-coverage: generate-aws-model-coverage
+	@go run ./cmd/aws-model-coverage
+
+# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
+# checked out alongside spinifex (see internal/testkit/segscanoracle), which
+# is not the default local or CI layout, so this is a separate target from
+# test-integration rather than folded into it. Skips itself when segscan's
+# source isn't found.
+test-segscan-oracle:
+	@echo -e "\n....Running segscan storage oracle test...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
 
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
-# of services/suites/fixtures + on-disk path existence. Subject content
-# vs source is enforced separately in Bead 5 drift lint.
+# of services/suites/fixtures + on-disk path existence.
 manifest-check:
 	@echo -e "\n....Checking service-interfaces.yaml...."
 	@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
 
+# Drift guards: direct-create fixture lint + NATS subject lint,
+# ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
+# drift beyond the baseline.
+manifest-lint:
+	@echo -e "\n....Linting manifest drift (fixtures + subjects)...."
+	@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root .
+
+# Accept current drift into the baseline. Run after an intentional change.
+manifest-lint-update:
+	@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . -update
+
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
+	LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/...
+
+# Empty locally, where reusing a cached result between runs is the point.
+# CI passes -count=1 so a green run means the suite executed against that commit
+# rather than replaying results the persisted build cache carried over.
+GOTESTFLAGS ?=
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
 test-actions:
 	@echo -e "\n....Running action tests...."
-	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
+	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 60s ./.github/actions/...
+
+# Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
+# helpers baked into system images. Kept out of `preflight` (a dedicated CI
+# job gates it on scripts/images/**+images/mkosi.profiles/** changes instead)
+# so image-asset churn doesn't run on every Go contributor's commit.
+#
+# images/mkosi.profiles/ ships scripts with no .sh suffix (mkosi's own
+# mkosi.*.chroot lifecycle hooks, and wrapper binaries like vllm-serve under
+# mkosi.extra/) so a plain `-name '*.sh'` glob silently skips exactly the
+# files most worth linting. Sources there are found by shebang instead of
+# name; this also means a profile that ships no scripts at all yields an
+# empty match rather than an error.
+test-images:
+	@echo -e "\n....Running scripts/images/**/*_test.sh...."
+	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running images/mkosi.profiles/**/*_test.sh...."
+	@for t in $$(find images/mkosi.profiles -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
+	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo -e "\n....Running shellcheck over images/mkosi.profiles/** shell sources...."
+	@srcs="$$(grep -rlIE '^#!/bin/(bash|sh)' images/mkosi.profiles 2>/dev/null | sort)"; \
+	if [ -n "$$srcs" ]; then \
+		shellcheck -S warning $$srcs; \
+	else \
+		echo "  (no shell sources found under images/mkosi.profiles)"; \
+	fi
+	@echo "  test-images ok"
 
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
+# Check that newly added test files use an external test package
+test-package-check:
+	@QUIET=$(QUIET) scripts/check-test-package.sh
+
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
-	$(MAKE) easyjson
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
 
-run:
-	$(MAKE) go_build
-	$(MAKE) go_run
+# Fuzz every Fuzz* target for FUZZTIME each; -fuzz takes one target per run.
+# Plain `go test` already replays the seeds and testdata/fuzz on preflight.
+FUZZTIME ?= 30s
+fuzz:
+	@set -e; grep -rlE '^func Fuzz\w+\(' --include='*_test.go' spinifex cmd internal | while read -r file; do \
+		for name in $$(grep -oE '^func Fuzz\w+' "$$file" | cut -d' ' -f2); do \
+			echo -e "\n....Fuzzing $$name ($$(dirname $$file)) for $(FUZZTIME)...."; \
+			LOG_IGNORE=1 go test -run='^$$' -fuzz="^$$name$$" -fuzztime=$(FUZZTIME) ./$$(dirname $$file); \
+		done; \
+	done
 
 # Fast iteration: build + install binary + restart all services.
 # Microvm artifacts are reinstalled when they already exist on disk — the rule's
@@ -166,6 +336,20 @@ run:
 # `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
+		if ! sudo /usr/local/bin/spx admin preflight; then \
+			echo ""; \
+			echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
+			echo "  grants don't match the binary just installed. Remediate with:"; \
+			echo "    scripts/update-nodes.sh   (from the mulga root)"; \
+			echo "  or:"; \
+			echo "    make reinstall"; \
+			echo "  Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \
+			exit 1; \
+		fi; \
+	else \
+		echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
+	fi
 	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
 		$(MAKE) install-microvm; \
 	else \
@@ -181,24 +365,23 @@ reinstall:
 
 clean:
 	rm -f ./bin/$(GO_PROJECT_NAME)
-	rm -rf spinifex/services/spinifexui/frontend/dist
+	rm -rf spinifex/runtime/roles/spinifexui/frontend/dist
 
 install-system:
 	@echo -e "\n....Installing system dependencies for $(ARCH)...."
-	@echo "QEMU packages: $(QEMU_PACKAGES)"
 	sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		-o Dpkg::Options::="--force-confdef" \
 		-o Dpkg::Options::="--force-confold" \
-		nbdkit nbdkit-plugin-dev pkg-config $(QEMU_PACKAGES) qemu-utils qemu-kvm \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
 		ovmf qemu-efi-aarch64 \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.3 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -218,24 +401,26 @@ install-aws:
 quickinstall: install-system install-go install-aws
 	@echo -e "\n✅ Quickinstall complete for $(ARCH)."
 	@echo "   Please ensure /usr/local/go/bin is in your PATH."
-	@echo "   Installed: Go ($(GO_ARCH)), AWS CLI ($(AWS_ARCH)), QEMU ($(QEMU_PACKAGES))"
 
-# Lint all Go code via golangci-lint (replaces check-format, vet, gosec, staticcheck)
 lint:
 	@echo "Running golangci-lint..."
-	$(_Q)golangci-lint run ./...
+	$(_Q)scripts/run-gate.sh golangci-lint golangci-lint run ./...
 	@echo "  golangci-lint ok"
 
-# Auto-fix all linter issues that have fixers
 fix:
 	golangci-lint run --fix ./...
 
-# Govulncheck — dependency vulnerability scanning (not covered by golangci-lint)
 govulncheck:
 	@echo "Running govulncheck..."
-	$(_Q)go tool govulncheck ./...
+	$(_Q)scripts/run-gate.sh govulncheck go tool govulncheck ./...
 	@echo "  govulncheck ok"
 
+# NilAway — advisory nil-panic analysis. Not in preflight due to false positives
+nilaway:
+	@echo "Running nilaway..."
+	$(_Q)scripts/run-gate.sh nilaway go tool nilaway -include-pkgs=github.com/mulgadc/spinifex -exclude-test-files ./...
+	@echo "  nilaway ok"
+
 # Build release tarballs — use distro-ARCH for single arch, distro for both
 distro: distro-amd64 distro-arm64
 	@echo ""
@@ -279,26 +464,8 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-# Ansible dev lifecycle (experimental, parallel to dev-*.sh / reset-dev-env.sh).
-# See scripts/ansible/README.md and docs/development/improvements/ansible-dev-lifecycle.md.
-ansible-dev-preflight:
-	cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml
-
-ansible-dev-teardown:
-	cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml
-
-ansible-dev-install:
-	cd scripts/ansible && ansible-playbook playbooks/dev-install.yml
-
-ansible-dev-reset:
-	cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml
-
-ansible-dev-deploy:
-	cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml
-
-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race test-actions test-harness manifest-check diff-coverage bench run \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
-	lint fix govulncheck \
-	distro distro-amd64 distro-arm64 distro-clean \
-	ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy
+	lint fix govulncheck nilaway \
+	distro distro-amd64 distro-arm64 distro-clean

README.md

@@ -1,197 +1,144 @@
 <p align="center">
-  <img src=".github/assets/banner.svg" alt="Spinifex — AWS-Compatible Infrastructure Stack" width="900">
+  <img src=".github/assets/banner.svg" alt="Spinifex: the open, AWS-compatible cloud you run yourself. EC2, EBS, S3, VPC, and IAM on your infrastructure, at the edge, or on a Neocloud." width="900">
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-Reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-docs.mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/Site-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
 </p>
 
+
 <p align="center">
-  <a href="#what-is-spinifex">What is Spinifex?</a> ·
-  <a href="#the-platform">Platform</a> ·
-  <a href="#core-components">Components</a> ·
-  <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#installation">Installation</a> ·
-  <a href="https://docs.mulgadc.com">Docs</a>
+  <a href="#try-it-live">Live Demo</a> ·
+  <a href="#services">Services</a> ·
+  <a href="#deploy-anywhere">Deploy</a> ·
+  <a href="#install">Install</a> ·
+  <a href="#architecture">Architecture</a>
 </p>
 
 ---
 
-# Spinifex: An Open Source AWS-Compatible Stack for Bare-Metal, Edge, and On-Prem Deployments
+# Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex developed by [Mulga Defense Corporation](https://mulgadc.com) is an open source infrastructure platform that brings the core services of AWS—like EC2, VPC, EBS, and S3—to environments where running in the cloud isn't an option. Whether you're deploying to edge sites, private data-centers, or need to operate in low-connectivity or highly contested environments, Spinifex gives you AWS-style workflows on your own hardware.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line.
 
-## What is Spinifex?
+- **Your AWS tooling, unchanged.** The AWS CLI, SDKs, and Terraform work as they are.
+- **Full control of the stack.** Your hardware, data, and keys, with no external control plane, running through disconnection.
+- **Open core under AGPL-3.0.** Auditable, yours to keep, with no lock-in.
 
-Spinifex replicates essential AWS primitives—virtual machines, block volumes, and object storage—using lightweight, self-contained components that are easy to deploy and integrate.
+You change *where* your software runs, not *what* it is.
 
-It's designed for developers and operators who need:
+## Try It Live
 
-- Cloud-like infrastructure without the cloud
-- Minimal dependencies, full control
-- Drop-in compatibility with tools like the AWS CLI, SDKs, and Terraform
-- A secure cloud environment you control and own the entire hardware, network and software stack
+Get a real Spinifex endpoint in under a minute, with no card and no install. The sandbox is yours for 72 hours, with access keys, the web console, and EC2, EBS, S3, VPC, IAM, EKS, and RDS ready to call. Point your existing AWS CLI or Terraform straight at it.
 
-You can run Spinifex on a few servers in a rack, a field site, or anywhere centralized cloud services aren't feasible.
+**[Create your sandbox →](https://mulgadc.com/signup)**
 
-## The Platform
+## Same Commands, Your Cloud
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-images
+aws ec2 run-instances --image-id ami-xxxxxxxx --instance-type t3.micro
+aws s3 mb s3://my-bucket
+```
+
+For Terraform, start from the [example workbooks](https://docs.mulgadc.com/docs/nginx-webserver).
+
+## Why Not LocalStack or Floci?
+
+[LocalStack](https://github.com/localstack/localstack) and [Floci](https://github.com/floci-io/floci) are emulators for local development and CI testing. An emulator returns a plausible response; Spinifex boots the machine. A call launches a real KVM guest, attaches a replicated Viperblock volume, or writes an erasure-coded object into Predastore, so Spinifex can run your production workloads on hardware you own. Test against an emulator, then run in production on Spinifex.
+
+## Services
+
+The platform serves **392 AWS API operations** across the services below. See exactly which ones in the [coverage report](https://docs.mulgadc.com/coverage).
+
+| Service | What it is | Built on | Status |
+| --- | --- | --- | --- |
+| **EC2** | Compute | QEMU/KVM, with spot instances and GPU passthrough | Available |
+| **EBS** | Block storage | [Viperblock](https://github.com/mulgadc/viperblock): replicated, with a WAL on local NVMe | Available |
+| **S3** | Object storage | [Predastore](https://github.com/mulgadc/predastore): Reed-Solomon erasure coding | Available |
+| **VPC** | Networking | OVN, with security groups | Available |
+| **IAM** | Identity & auth | SigV4 across every service | Available |
+| **ALB / NLB** | Load balancers | HAProxy (L7), nginx `stream` (L4), with ACM certificates | Available |
+| **EKS** | Kubernetes | k3s, with HA control planes and managed node groups | Available |
+| **ECR** | Container registry | Predastore, deduplicated per account | Available |
+| **ECS** | Container service | Spinifex instances | Available |
+| **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
+| **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
+| **Lambda** | Serverless functions | | Q1 2027 |
+
+Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+
+## Deploy Anywhere
 
 <p align="center">
-  <img src=".github/assets/platform.svg" alt="The Mulga Platform — Hardware to Applications" width="900">
+  <img src=".github/assets/platform.svg" alt="The Spinifex stack: your apps and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU and GPU hardware, deployable to Neocloud, on-premise, or the edge." width="900">
 </p>
 
-From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
+- **Neocloud:** lift and shift onto partner GPU capacity (H100 / H200 / B200). Change an endpoint, keep the software.
+- **On-premise:** a multi-node HA cluster in your own data centre, so the stack, the data, and the jurisdiction stay yours.
+- **Edge:** air-gapped sites, vehicles, vessels, factories, and clinics, with compute next to the data.
 
-## Core Components
+## Install
 
-### Spinifex (Compute Service – EC2 Alternative)
+Spinifex runs on Ubuntu 26.04 or Debian 13. The full guides are at [docs.mulgadc.com](https://docs.mulgadc.com).
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
 
-- EC2-like VM management on bare metal
-- Launches with cloud-init metadata support
-- Works with standard AWS tooling
-
-### Viperblock (Block Storage – EBS Alternative)
-
-[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It's built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
-
-- Fast, durable virtual disks
-- Replication for resilience
-- Exposed over NBD or embedded in VMs
-- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
-- In memory read/write block cache for blazing performance.
-
-### Predastore (Object Storage – S3-Compatible)
-
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
-
-- S3-compatible API and auth
-- Multipart uploads, streaming reads/writes
-- Data redundancy with Reed-Solomon encoding
-
-## Architecture at a Glance
-
-<p align="center">
-  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture — AWS Gateway, NATS bus, service daemons" width="900">
-</p>
-
-Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless — scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
-
-## Key Features
-
-AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, or SDKs you already use.
-
-- Designed for Control – Run on your own terms, whether that's in a datacenter, remote site, or sensitive environment.
-- Minimal Dependencies – Each service is standalone and avoids complex orchestration layers.
-- Works Offline – No reliance on centralized cloud services or external networks.
-- Open Source – Licensed under AGPL 3.0. Fork it, extend it, or deploy it as-is.
-
-## Installation
-
-Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
-
-### Bare Metal ISO
-
-The recommended installation is a [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) for bare-metal hardware.
+**Bare metal (recommended):** download the [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) and follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb).
 
 ```bash
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
-
-
-### Single Node Install
-
-The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
-
->*Prerequisite:* Linux bridge for networking.
-
-Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
+**Single node:**
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
-
 sudo /usr/local/share/spinifex/setup-ovn.sh --management
-
 sudo spx admin init --node node1 --nodes 1
-
 sudo systemctl start spinifex.target
-
 export AWS_PROFILE=spinifex
-
 aws ec2 describe-instance-types
 ```
 
-### Development Setup
+**From source:** see the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
-
-### Component Repositories
-
-Spinifex coordinates these independent components:
-
-- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
-- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
-
-Each component can be developed independently. See component-specific documentation for focused development guides.
-
-## Spinifex UI
-
-Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+## Web Console
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
 </p>
 
-The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+Spinifex ships with a built-in web console for your instances, volumes, buckets, networks, and IAM. See [Launching the Web UI](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui).
 
-```bash
-open https://YOUR_NODE_IP:3000
-```
+## Architecture
 
-- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
-- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+<p align="center">
+  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
+</p>
 
-For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless, so you scale horizontally by starting more. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Development Philosophy
+## How We Build Spinifex
 
-### Built by Engineers, For Engineers
+Spinifex is built by infrastructure engineers with deep AWS experience, including former AWS team members, drawing on decades of combined work across AWS, enterprise infrastructure, and edge computing. Engineers architect and implement it, with **Claude Code** accelerating parts of the work.
 
-Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
+| Claude Code assists with | Engineers own |
+| --- | --- |
+| Boilerplate AWS API structures and handlers | Architecture and scalability decisions |
+| Development guides and API documentation | Authentication, encryption, and threat modelling |
+| Test case generation and validation scenarios | Real-world performance tuning and benchmarking |
+| Large-scale restructuring and refactoring | Deployment strategy and production operations |
 
-**Real-World Experience:**
+## Trademarks
 
-- Production AWS service development and operations
-- Large-scale infrastructure deployment and management
-- Edge computing and resource-constrained environments
-- Enterprise security and compliance requirements
-
-### AI-Assisted Development
-
-While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
-
-**How We Use Claude Code:**
-
-- **Code Generation**: Boilerplate AWS API structures and handlers
-- **Documentation**: Comprehensive development guides and API documentation
-- **Testing**: Test case generation and validation scenarios
-- **Refactoring**: Large-scale code restructuring and optimization
-
-**What Remains Human-Driven:**
-
-- **Architecture Decisions**: Core system design and scalability choices
-- **Security Implementation**: Authentication, encryption, and threat modeling
-- **Performance Optimization**: Real-world performance tuning and benchmarking
-- **Production Operations**: Deployment strategies and operational procedures
-
-This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
+"AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available.

SECURITY.md

@@ -0,0 +1,24 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please do not open a public issue for a security vulnerability.
+
+Report it privately, either way works:
+
+- [Open a private security advisory](https://github.com/mulgadc/spinifex/security/advisories/new) on this repository.
+- Email <security@mulgadc.com>.
+
+Please include enough for us to reproduce it: the version or commit, the configuration involved, and the steps that trigger it. If you have a proof of concept, include that too.
+
+## What Happens Next
+
+We aim to acknowledge a report within three business days. We will confirm whether we can reproduce it, agree a fix and a disclosure timeline with you, and credit you in the advisory unless you would rather we did not.
+
+## Scope
+
+Spinifex is one part of the [Mulga](https://mulgadc.com) stack. If a vulnerability spans more than one component, report it once against whichever repository you found it in and we will handle the rest.
+
+## Supported Versions
+
+Fixes land on the latest release and on `main`. Older releases are not patched.

architecture/instancestate/allowlist_test.go

@@ -0,0 +1,92 @@
+package instancestate
+
+// allowance is one recorded production consumer of raw instance state.
+type allowance struct {
+	File, Symbol string
+	Inv          string // inventory item in docs/PACKAGE_BOUNDARY_MIGRATION.md
+}
+
+// allowlist mirrors the ADR-0007 slice 0 inventory in
+// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: delete an entry as soon
+// as its use is gone, and never add one to make a new consumer pass.
+var allowlist = []allowance{
+	{File: "spinifex/daemon/daemon.go", Symbol: "daemon.LocalStatePath", Inv: "INV-01"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "daemon.ReadLocalState", Inv: "INV-01"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "daemon.MarshalLocalState", Inv: "INV-01"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "daemon.WriteLocalStateBytes", Inv: "INV-01"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-02"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.InitTerminatedInstanceBucket", Inv: "INV-02"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.WriteNodeMarkerBestEffort", Inv: "INV-03"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
+	{File: "spinifex/daemon/instance_record_repair.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
+	{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-04"},
+
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadState", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-05"},
+	{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-05"},
+
+	{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-06"},
+	{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
+	{File: "spinifex/runtime/compute/vm/shutdown.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
+	{File: "spinifex/runtime/compute/vm/orphan_qemu_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
+	{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
+	{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-06"},
+	{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-06"},
+
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-07"},
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-07"},
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-07"},
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-07"},
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-07"},
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-07"},
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-07"},
+	{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-07"},
+
+	{File: "spinifex/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-08"},
+	{File: "spinifex/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-08"},
+
+	{File: "spinifex/daemon/instance_recovery.go", Symbol: "vm.InstanceRecord", Inv: "INV-10"},
+	{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-10"},
+	{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ClaimRecoverableInstance", Inv: "INV-10"},
+	{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ReleaseRecoveredInstance", Inv: "INV-10"},
+	{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.AbandonRecovery", Inv: "INV-10"},
+	{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-10"},
+
+	{File: "spinifex/daemon/dns_reconcile.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-11"},
+	{File: "spinifex/daemon/dns_reconcile.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-11"},
+	{File: "spinifex/daemon/dns_reconcile.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-11"},
+
+	{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "vm.InstanceRecord", Inv: "INV-12"},
+	{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-12"},
+	{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstanceRecords", Inv: "INV-12"},
+
+	{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalState", Inv: "INV-13"},
+	{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalStatePath", Inv: "INV-13"},
+	{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.ReadLocalState", Inv: "INV-13"},
+	{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-13"},
+	{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-13"},
+
+	{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.InstanceRecord", Inv: "INV-14"},
+	{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.VMFromRecord", Inv: "INV-14"},
+	{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-14"},
+
+	{File: "spinifex/domains/network/reconcile/intent.go", Symbol: `literal "spinifex-instance-state"`, Inv: "INV-15"},
+	{File: "spinifex/domains/network/reconcile/intent.go", Symbol: "vm.InstanceRecord", Inv: "INV-15"},
+
+	{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-16"},
+	{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-16"},
+	{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "vm.InstanceRecord", Inv: "INV-16"},
+
+	{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.InstanceRecord", Inv: "INV-17"},
+	{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.VMFromRecord", Inv: "INV-17"},
+
+	{File: "spinifex/domains/admission/quota/records.go", Symbol: "vm.InstanceRecord", Inv: "INV-18"},
+}