fix: write MariaDB's startup spelling for time_zone into the option file

mariadbd has no time_zone startup option — the name only exists for SET — so
the resolved parameter group aborted the server while it parsed defaults, before
it opened the datadir. No pidfile was ever written and the reconciler failed
every instance at the 20-minute engine-health deadline.

ParameterSpec gains an option-file spelling, used only when rendering the
include. time_zone is the one setting that needs it, as default_time_zone, which
resolves @@global.time_zone to the same value. The customer-facing name is
unchanged: the API still reports time_zone and a live apply still issues
SET GLOBAL time_zone. Names read back out of a generated file return to the
catalog's namespace first, so a startup spelling cannot read as an unknown name
and report a pending restart nothing could clear.

setup.sh now feeds the catalog's resolved defaults to the image's own mariadbd
at build time, so a name the server refuses fails the build rather than every
instance built from it.

.github/ISSUE_TEMPLATE/1-bug.yml

@@ -1,59 +0,0 @@
-name: Bug report
-description: Something is not working as it should.
-labels: ["bug"]
-body:
-  - type: dropdown
-    id: component
-    attributes:
-      label: Component
-      options:
-        - EC2 / instances
-        - EBS / volumes
-        - S3 / object store
-        - VPC / networking
-        - IAM / authentication
-        - EKS
-        - API gateway
-        - Web UI
-        - CLI / daemon
-        - Observability / telemetry
-        - Build / CI / packaging
-        - Documentation
-        - Other / not sure
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: Spinifex version
-      description: Output of `spinifex --version`, or the commit SHA you are on.
-      placeholder: v1.0.0, or commit abc1234
-    validations:
-      required: true
-
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: What happened
-      description: What you saw, what you expected instead, and how to reproduce it.
-    validations:
-      required: true
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs and output
-      description: Relevant log lines, error messages or command output.
-      render: shell
-
-  - type: textarea
-    id: platform
-    attributes:
-      label: Platform
-      description: Where you are running it, and how you deployed it.
-      placeholder: |
-        OS: Debian 13
-        Kernel: 6.12
-        Arch: amd64
-        Deployment: single node, systemd

.github/ISSUE_TEMPLATE/2-feature.yml

@@ -1,11 +0,0 @@
-name: Feature request
-description: Suggest something new.
-labels: ["enhancement"]
-body:
-  - type: textarea
-    id: proposal
-    attributes:
-      label: What would you like to see?
-      description: What you are trying to do, and what stops you today.
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/3-docs.yml

@@ -1,20 +0,0 @@
-name: Documentation
-description: Something in the docs is wrong, missing or unclear.
-labels: ["documentation"]
-body:
-  - type: input
-    id: location
-    attributes:
-      label: Where
-      description: Link to the page, or the path to the file in this repository.
-      placeholder: https://docs.mulgadc.com/..., or README.md
-    validations:
-      required: true
-
-  - type: textarea
-    id: problem
-    attributes:
-      label: What is wrong
-      description: What is inaccurate, missing or confusing, and what it should say instead.
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Documentation
-    url: https://docs.mulgadc.com
-    about: Guides, configuration reference and API compatibility notes.

.github/actions/e2e-analyze/analyze.go

@@ -91,12 +91,9 @@ type SuiteReport struct {
 	Label     string // short name, e.g. "single"
 	Total     int
 	FailCount int
-	// Unresolved counts testcases go-junit-report could not find an outcome
-	// for. They are not failures and the suite can be green with hundreds.
-	Unresolved int
-	Root       *Failure    // nil if no failures
-	Cascades   []Failure   // everything else that failed in this suite
-	Buckets    [][]Failure // failures grouped by signature (root's bucket first)
+	Root      *Failure    // nil if no failures
+	Cascades  []Failure   // everything else that failed in this suite
+	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
 	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
 	// the XML was produced — i.e. when the suite finished, not when it began.
 	EndedAt time.Time
@@ -134,22 +131,21 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
-// nonFailurePatterns match content a test emits AFTER the assertion that failed
-// — t.Cleanup, harness diagnostics, or a t.Logf on a passing branch — which
-// extractErrorLine's keep-the-last rule would otherwise pick over the assertion.
-// Go emits t.Logf and t.Errorf identically, so anchor every entry: a fragment
-// loose enough to match an assertion blanks out the line the report exists to
-// show. `\bas expected\b` unanchored also matches "did not fail as expected".
-var nonFailurePatterns = []*regexp.Regexp{
+// teardownPatterns match the content a test logs from t.Cleanup or from the
+// harness's on-failure diagnostics — i.e. after it has already failed. They are
+// the LAST file:line lines in the failure body, so extractErrorLine's
+// keep-the-last rule picks them over the assertion unless they are skipped.
+// Anchored deliberately: a fragment loose enough to match an assertion would
+// blank out the one line the report exists to show.
+var teardownPatterns = []*regexp.Regexp{
 	regexp.MustCompile(`^DB instance \S+ is gone$`),
 	regexp.MustCompile(`^db diagnostics \S+:`),
-	regexp.MustCompile(`^\S+: management bridge \S+:\d+ refused as expected$`),
 }
 
-// isNonFailureLine reports whether a candidate error line was emitted after the
-// failure it would otherwise be mistaken for, or reports an expected outcome.
-func isNonFailureLine(content string) bool {
-	for _, re := range nonFailurePatterns {
+// isTeardownLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for.
+func isTeardownLine(content string) bool {
+	for _, re := range teardownPatterns {
 		if re.MatchString(content) {
 			return true
 		}
@@ -246,12 +242,12 @@ func extractErrorLine(body string) string {
 			// progress lines from the same file:line pattern before the
 			// failing assertion finally fires. Cleanup runs after it, though,
 			// so its lines are last of all and have to be skipped.
-			if content := strings.TrimSpace(m[1]); !isNonFailureLine(content) {
+			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
 				goFileLine = content
 			}
 			continue
 		}
-		if fileHintRe.MatchString(l) && !isNonFailureLine(l) {
+		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -297,23 +293,15 @@ func extractFileHint(body string) string {
 	}
 	// Scanned line by line rather than across the body so a cleanup line's own
 	// file:line does not become the site the report points a reader at.
-	hint, filtered := "", ""
+	hint := ""
 	for _, raw := range strings.Split(body, "\n") {
 		l := strings.TrimSpace(raw)
-		all := fileHintRe.FindAllString(l, -1)
-		if len(all) == 0 {
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
 			continue
 		}
-		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isNonFailureLine(strings.TrimSpace(m[1])) {
-			filtered = all[len(all)-1]
-			continue
+		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
+			hint = all[len(all)-1]
 		}
-		hint = all[len(all)-1]
-	}
-	// A body whose only file:line was allowlisted still gets a pointer: coarse
-	// beats none, and extractErrorLine has the same last-resort shape.
-	if hint == "" {
-		return filtered
 	}
 	return hint
 }
@@ -378,7 +366,7 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 func computeRolledUpSet(cases []junitTC) map[string]bool {
 	rolledUp := make(map[string]bool, len(cases))
 	for _, tc := range cases {
-		if tc.Failure == nil && !isRealError(tc.Error) {
+		if tc.Failure == nil && tc.Error == nil {
 			continue
 		}
 		// Mark every ancestor: the failure belongs to the deepest case that
@@ -424,18 +412,6 @@ func parseStartTime(s string) time.Time {
 	return t
 }
 
-// noResultMessage is go-junit-report's marker for a testcase it saw start and
-// never saw finish. On cell-20 it accounted for 93 of the 120 testcases and 24
-// of the 25 reported failures, none of which had failed.
-const noResultMessage = "No test result found"
-
-// isRealError separates a genuine <error> from the parser's no-result marker.
-// A parent must not be treated as rolled up by a child whose outcome is merely
-// unknown, or the parent's own failure is dropped along with it.
-func isRealError(e *junitFailure) bool {
-	return e != nil && strings.TrimSpace(e.Message) != noResultMessage
-}
-
 // suiteLabel turns "junit-single.xml" into "single".
 func suiteLabel(path string) string {
 	base := filepath.Base(path)
@@ -481,17 +457,6 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 			case tc.Failure != nil:
 				body = tc.Failure.Body
 			case tc.Error != nil:
-				// go-junit-report emits this when a `=== RUN` line has no
-				// matching result line. It means the outcome is unknown, not
-				// that the test errored, and it attaches the test's own log
-				// output — which then reads as an assertion.
-				if !isRealError(tc.Error) {
-					rep.Unresolved++
-					if leaf {
-						cumul += tc.Time
-					}
-					continue
-				}
 				body = tc.Error.Body
 			default:
 				if leaf {
@@ -588,18 +553,6 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 }
 
 // Render writes the markdown report.
-// writeUnresolved names the testcases whose outcome the parser could not read.
-// Worth surfacing because a suite that leaves most of its subtests unresolved
-// is reporting on far less than its test count suggests.
-func writeUnresolved(b *strings.Builder, s SuiteReport) {
-	if s.Unresolved == 0 {
-		return
-	}
-	fmt.Fprintf(b, "ℹ️ %d of %d testcases have no result line in the log, so `go-junit-report` "+
-		"could not read their outcome. Not failures — but this suite reports on %d tests, not %d.\n\n",
-		s.Unresolved, s.Total, s.Total-s.Unresolved, s.Total)
-}
-
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
@@ -626,11 +579,9 @@ func Render(r Report) string {
 	for _, s := range r.Suites {
 		if s.FailCount == 0 {
 			fmt.Fprintf(&b, "### Suite `%s`: ✅ pass (%d tests)\n\n", s.Label, s.Total)
-			writeUnresolved(&b, s)
 			continue
 		}
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
-		writeUnresolved(&b, s)
 
 		if s.Root != nil {
 			// XML order is completion order, so "earliest" only means earliest

.github/actions/e2e-analyze/analyze_test.go

@@ -141,46 +141,6 @@ func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
 	}
 }
 
-// A subtest looping over two subjects calls t.Errorf for the first and t.Logf
-// for the second, so the success line is last in the body. Nightly run
-// 34383800404 reported that log line as the failure.
-func TestExtractErrorLine_SkipsSuccessLoggedAfterTheFailure(t *testing.T) {
-	body := `    isolation_test.go:80: rds-e2e-iso-a-1788981456: management bridge 10.15.8.10:5432 could not be tested: want connection refused, got dial tcp 10.15.8.10:5432: i/o timeout
-    isolation_test.go:84: rds-e2e-iso-b-1788981456: management bridge 10.15.8.11:5432 refused as expected`
-	got := extractErrorLine(body)
-	if !strings.Contains(got, "could not be tested") {
-		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
-	}
-	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
-		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
-	}
-}
-
-// "as expected" reads the same in an affirmation and in its negation, so an
-// unanchored pattern suppresses the assertion it was meant to outrank and
-// promotes a harness progress line in its place.
-func TestExtractErrorLine_KeepsNegatedExpectationAssertions(t *testing.T) {
-	body := `    isolation_test.go:60: waiting for the db instance to become available
-    isolation_test.go:80: rds-e2e-iso-a: management bridge 10.15.8.10:5432 was not refused as expected: got a connection`
-	got := extractErrorLine(body)
-	if !strings.Contains(got, "was not refused as expected") {
-		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
-	}
-	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
-		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
-	}
-}
-
-// Every candidate allowlisted leaves extractFileHint with nothing to return,
-// and unlike extractErrorLine it has no last-resort branch. A coarse pointer
-// beats sending the reader to no file at all.
-func TestExtractFileHint_FallsBackWhenEveryCandidateFiltered(t *testing.T) {
-	body := `    isolation_test.go:84: rds-e2e-iso-b: management bridge 10.15.8.11:5432 refused as expected`
-	if hint := extractFileHint(body); hint != "isolation_test.go:84" {
-		t.Errorf("extractFileHint = %q, want isolation_test.go:84", hint)
-	}
-}
-
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -214,55 +174,6 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
-// go-junit-report emits <error message="No test result found"> for a `=== RUN`
-// with no matching result line, and attaches the test's own log output. On a
-// baremetal cell that was 93 of 120 testcases and 24 of 25 reported failures,
-// none of which had failed: "ALB internal: 20/20 successful" read as an error.
-func TestParseFile_NoResultFoundIsNotAFailure(t *testing.T) {
-	xml := []byte(`<?xml version="1.0"?>
-<testsuites><testsuite name="" tests="3" failures="1" errors="2">
-  <testcase name="TestLB/Internal_ALB" time="0.0"><error message="No test result found"><![CDATA[
-    lb_test.go:88: ALB internal: 20/20 successful, 2 unique
-]]></error></testcase>
-  <testcase name="TestLB/Internal_NLB" time="0.0"><error message="No test result found"></error></testcase>
-  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
-    foo_test.go:10:
-        Messages: real assertion message
-]]></failure></testcase>
-</testsuite></testsuites>`)
-	sr, err := ParseFile("junit-x.xml", xml)
-	if err != nil {
-		t.Fatalf("ParseFile: %v", err)
-	}
-	if sr.FailCount != 1 {
-		t.Fatalf("FailCount = %d, want 1 (no-result entries are not failures)", sr.FailCount)
-	}
-	if sr.Unresolved != 2 {
-		t.Fatalf("Unresolved = %d, want 2", sr.Unresolved)
-	}
-	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
-		t.Fatalf("expected the real failure as root, got %+v", sr.Root)
-	}
-}
-
-// An <error> that is not the parser's no-result marker is a genuine error and
-// must still be reported.
-func TestParseFile_KeepsARealError(t *testing.T) {
-	xml := []byte(`<?xml version="1.0"?>
-<testsuites><testsuite name="" tests="1" failures="0" errors="1">
-  <testcase name="TestX" time="0.5"><error message="panic"><![CDATA[
-    foo_test.go:10: panic: runtime error: index out of range
-]]></error></testcase>
-</testsuite></testsuites>`)
-	sr, err := ParseFile("junit-x.xml", xml)
-	if err != nil {
-		t.Fatalf("ParseFile: %v", err)
-	}
-	if sr.FailCount != 1 || sr.Unresolved != 0 {
-		t.Fatalf("FailCount = %d, Unresolved = %d, want 1 and 0", sr.FailCount, sr.Unresolved)
-	}
-}
-
 func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
 	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
 	// "DB instance … is gone" — which passing tests emit too — reads exactly

.github/actions/junit-to-es/action.yml

@@ -1,43 +0,0 @@
-name: JUnit to Elasticsearch
-description: >-
-  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
-  so per-test pass rates outlive the 90-day artifact retention. Call it once
-  per job right after the JUnit XML is produced (with `if: always()`), next to
-  the junit-report action that renders the same files into the job summary.
-
-inputs:
-  junit-glob:
-    description: Shell glob matching the JUnit XML files to ship.
-    required: true
-  cell:
-    description: >-
-      Cell label stored as labels.ci_cell — the same value the job puts in
-      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
-      Without it, results from different topologies collapse into one bucket.
-    required: true
-  es-sink:
-    description: host:port of the Elasticsearch sink.
-    required: false
-    default: "192.168.1.13:9200"
-  index:
-    description: Target index.
-    required: false
-    default: mulga-ci-test-results
-
-runs:
-  using: composite
-  steps:
-    - shell: bash
-      env:
-        JUNIT_GLOB: ${{ inputs.junit-glob }}
-        JUNIT_CELL: ${{ inputs.cell }}
-        MULGA_ES_SINK: ${{ inputs.es-sink }}
-        ES_INDEX: ${{ inputs.index }}
-        RUN_ID: ${{ github.run_id }}
-        RUN_ATTEMPT: ${{ github.run_attempt }}
-        WORKFLOW: ${{ github.workflow }}
-        BRANCH: ${{ github.head_ref || github.ref_name }}
-        SHA: ${{ github.sha }}
-        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-      run: |
-        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-"""Ship per-test JUnit results to the Elasticsearch sink.
-
-One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
-mulga-ci-runs summary says which suite failed; this says which test, with the
-failure text, so a per-test pass-rate trend outlives artifact retention.
-
-Inputs (env):
-    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
-                    a leg that never ran produces none.
-    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
-    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
-                    there, so no credentials are involved.
-    ES_INDEX        Target index; defaults to mulga-ci-test-results.
-    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
-"""
-
-from __future__ import annotations
-
-import glob
-import json
-import os
-import re
-import sys
-import urllib.error
-import urllib.request
-import xml.etree.ElementTree as ET
-from datetime import datetime, timezone
-
-# Go test output carries colour codes into the JUnit CDATA (harness.Phase
-# emits them), which are noise in a stored failure message.
-_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
-
-# Enough failure text to recognise a failure; the artifact log stays the full
-# record.
-_MAX_MESSAGE = 4000
-
-# go-junit-report emits this when a `=== RUN` line has no matching result line,
-# which means the log it was fed was truncated or filtered. It is a parse gap,
-# not a test outcome, so it is stored under its own status and left out of every
-# pass rate rather than counted as a failure.
-_NO_RESULT = "No test result found"
-
-
-def _strip_ansi(s: str) -> str:
-    return _ANSI_RE.sub("", s)
-
-
-def _truncate(s: str) -> str:
-    s = s.strip()
-    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
-
-
-def _suite_from_path(path: str) -> str:
-    base = os.path.basename(path)
-    if base.endswith(".xml"):
-        base = base[:-4]
-    if base.startswith("junit-"):
-        base = base[len("junit-") :]
-    return base
-
-
-def _cases(path: str) -> list[dict]:
-    """Parse one JUnit file into leaf test cases."""
-    try:
-        root = ET.parse(path).getroot()
-    except ET.ParseError as exc:
-        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
-        return []
-
-    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
-    # several. .iter() handles both.
-    out: list[dict] = []
-    for s in root.iter("testsuite"):
-        for tc in s.findall("testcase"):
-            status = "pass"
-            fail_type = ""
-            message = ""
-            for tag in ("failure", "error"):
-                node = tc.find(tag)
-                if node is not None:
-                    status = "fail" if tag == "failure" else "error"
-                    fail_type = tag
-                    message = _strip_ansi(
-                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
-                    )
-                    if (node.get("message") or "").strip() == _NO_RESULT:
-                        status = "unknown"
-                    break
-            if status == "pass" and tc.find("skipped") is not None:
-                status = "skip"
-            out.append(
-                {
-                    "name": tc.get("name", ""),
-                    "classname": tc.get("classname", ""),
-                    "time": float(tc.get("time", 0) or 0),
-                    "status": status,
-                    "fail_type": fail_type,
-                    "message": message,
-                }
-            )
-
-    # Drop container cases: Go reports a parent t.Run as a rollup of its
-    # children, so keeping both double-counts one outcome. Same rule the job
-    # summary renderer applies.
-    names = {c["name"] for c in out}
-    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
-    return [c for c in out if c["name"] not in containers]
-
-
-def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
-    full = case["name"]
-    top, _, sub = full.partition("/")
-    doc = {
-        "@timestamp": ts,
-        "suite": suite,
-        "status": case["status"],
-        "duration_seconds": case["time"],
-        "artifact_url": ci["artifact_url"],
-        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
-        "test": {
-            "name": top,
-            "subtest": sub,
-            "full_name": full,
-            "classname": case["classname"],
-        },
-        "ci": {
-            "workflow": ci["workflow"],
-            "branch": ci["branch"],
-            "sha": ci["sha"],
-            "run_attempt": ci["run_attempt"],
-        },
-    }
-    if case["fail_type"]:
-        doc["failure"] = {
-            "type": case["fail_type"],
-            "message": _truncate(case["message"]),
-        }
-    return doc
-
-
-def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
-    # Deterministic so a replayed post overwrites instead of double-counting.
-    # run_attempt is part of it because a re-run is a distinct result.
-    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
-
-
-def _bulk(sink: str, index: str, body: str) -> int:
-    req = urllib.request.Request(
-        f"http://{sink}/{index}/_bulk",
-        data=body.encode("utf-8"),
-        headers={"Content-Type": "application/x-ndjson"},
-        method="POST",
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=60) as resp:
-            payload = json.loads(resp.read().decode("utf-8"))
-    except (urllib.error.URLError, OSError) as exc:
-        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
-        return 1
-
-    if payload.get("errors"):
-        for item in payload.get("items", []):
-            err = next(iter(item.values())).get("error")
-            if err:
-                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
-                return 1
-    return 0
-
-
-def main() -> int:
-    pattern = os.environ.get("JUNIT_GLOB", "").strip()
-    if not pattern:
-        print("::error::junit-to-es: JUNIT_GLOB is empty")
-        return 1
-    cell = os.environ.get("JUNIT_CELL", "").strip()
-    if not cell:
-        print("::error::junit-to-es: JUNIT_CELL is empty")
-        return 1
-    sink = os.environ.get("MULGA_ES_SINK", "").strip()
-    if not sink:
-        print("::error::junit-to-es: MULGA_ES_SINK is empty")
-        return 1
-    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
-
-    ci = {
-        "run_id": os.environ.get("RUN_ID", ""),
-        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
-        "workflow": os.environ.get("WORKFLOW", ""),
-        "branch": os.environ.get("BRANCH", ""),
-        "sha": os.environ.get("SHA", ""),
-        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
-    }
-    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    lines: list[str] = []
-    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
-    for path in sorted(glob.glob(pattern)):
-        suite = _suite_from_path(path)
-        for case in _cases(path):
-            doc = _doc(cell, suite, case, ts, ci)
-            counts[case["status"]] = counts.get(case["status"], 0) + 1
-            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
-            lines.append(json.dumps(meta))
-            lines.append(json.dumps(doc))
-
-    if not lines:
-        # Not an error on its own: the leg may have been skipped, or died
-        # before any test ran. The run summary doc records that separately.
-        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
-        return 0
-
-    rc = _bulk(sink, index, "\n".join(lines) + "\n")
-    if rc == 0:
-        print(
-            f"junit-to-es: posted {sum(counts.values())} test docs "
-            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
-            f"{counts['fail']} fail, {counts['error']} error, "
-            f"{counts['skip']} skip, {counts['unknown']} unknown"
-        )
-    return rc
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/assets/images/cisco-ucs-platform-benchmark/guest-storage-throughput.svg

@@ -1,44 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
-<rect width="760" height="470" fill="#ffffff"/>
-<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Guest block storage throughput (Viperblock)</text>
-<text x="74" y="49" font-size="12.5" fill="#6b7280">fio, 4 jobs, iodepth 32, direct I/O, per-guest sequential runs — Spinifex v1.20.0</text>
-<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
-<line x1="74" y1="305.0" x2="734" y2="305.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="309.0" font-size="12" text-anchor="end" fill="#6b7280">100</text>
-<line x1="74" y1="224.0" x2="734" y2="224.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="228.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
-<line x1="74" y1="143.0" x2="734" y2="143.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="147.0" font-size="12" text-anchor="end" fill="#6b7280">300</text>
-<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
-<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
-<rect x="111.4" y="81.4" width="45.4" height="304.6" rx="2.5" fill="#2563eb"/>
-<text x="134.1" y="75.4" font-size="11" text-anchor="middle" fill="#1f2937">376</text>
-<rect x="159.8" y="198.1" width="45.4" height="187.9" rx="2.5" fill="#0d9488"/>
-<text x="182.5" y="192.1" font-size="11" text-anchor="middle" fill="#1f2937">232</text>
-<rect x="208.2" y="299.3" width="45.4" height="86.7" rx="2.5" fill="#f59e0b"/>
-<text x="230.9" y="293.3" font-size="11" text-anchor="middle" fill="#1f2937">107</text>
-<text x="184.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU</text>
-<rect x="331.4" y="85.5" width="45.4" height="300.5" rx="2.5" fill="#2563eb"/>
-<text x="354.1" y="79.5" font-size="11" text-anchor="middle" fill="#1f2937">371</text>
-<rect x="379.8" y="202.9" width="45.4" height="183.1" rx="2.5" fill="#0d9488"/>
-<text x="402.5" y="196.9" font-size="11" text-anchor="middle" fill="#1f2937">226</text>
-<rect x="428.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
-<text x="450.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
-<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1</text>
-<rect x="551.4" y="84.7" width="45.4" height="301.3" rx="2.5" fill="#2563eb"/>
-<text x="574.1" y="78.7" font-size="11" text-anchor="middle" fill="#1f2937">372</text>
-<rect x="599.8" y="201.3" width="45.4" height="184.7" rx="2.5" fill="#0d9488"/>
-<text x="622.5" y="195.3" font-size="11" text-anchor="middle" fill="#1f2937">228</text>
-<rect x="648.2" y="321.2" width="45.4" height="64.8" rx="2.5" fill="#f59e0b"/>
-<text x="670.9" y="315.2" font-size="11" text-anchor="middle" fill="#1f2937">80</text>
-<text x="624.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2</text>
-<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
-<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
-<text x="93" y="455" font-size="12.5" fill="#1f2937">128K random read</text>
-<rect x="216.8" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
-<text x="235.8" y="455" font-size="12.5" fill="#1f2937">128K mixed read</text>
-<rect x="352.3" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
-<text x="371.3" y="455" font-size="12.5" fill="#1f2937">16K random read</text>
-</svg>

.github/assets/images/cisco-ucs-platform-benchmark/network-goodput-vs-streams.svg

@@ -1,35 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
-<rect width="760" height="470" fill="#ffffff"/>
-<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Network throughput: overlay vs physical fabric</text>
-<text x="74" y="49" font-size="12.5" fill="#6b7280">iperf3 TCP receiver goodput, median across node pairs — Spinifex v1.20.0</text>
-<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">20</text>
-<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">25</text>
-<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">Gbit/s</text>
-<text x="140.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">1</text>
-<text x="272.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">2</text>
-<text x="404.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">4</text>
-<text x="536.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">8</text>
-<text x="668.0" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">16</text>
-<path d="M140.0,164.4 L272.0,160.5 L404.0,160.5 L536.0,160.5 L668.0,160.5" fill="none" stroke="#2563eb" stroke-width="3"/>
-<circle cx="140.0" cy="164.4" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
-<circle cx="272.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
-<circle cx="404.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
-<circle cx="536.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
-<circle cx="668.0" cy="160.5" r="4.5" fill="#fff" stroke="#2563eb" stroke-width="2.5"/>
-<text x="664.0" y="149.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#2563eb">23.48</text>
-<path d="M140.0,255.1 L272.0,222.1 L404.0,222.1 L536.0,222.1 L668.0,228.5" fill="none" stroke="#0d9488" stroke-width="3"/>
-<circle cx="140.0" cy="255.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
-<circle cx="272.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
-<circle cx="404.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
-<circle cx="536.0" cy="222.1" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
-<circle cx="668.0" cy="228.5" r="4.5" fill="#fff" stroke="#0d9488" stroke-width="2.5"/>
-<text x="664.0" y="217.5" font-size="11.5" text-anchor="end" font-weight="600" fill="#0d9488">22.43</text>
-<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
-<text x="404.0" y="430.0" font-size="12.5" text-anchor="middle" fill="#6b7280">parallel TCP streams</text>
-<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
-<text x="93" y="455" font-size="12.5" fill="#1f2937">Host physical fabric (25GbE)</text>
-<rect x="304.4" y="444" width="13" height="13" rx="2.5" fill="#0d9488"/>
-<text x="323.4" y="455" font-size="12.5" fill="#1f2937">Guest overlay (OVN)</text>
-</svg>

.github/assets/images/cisco-ucs-platform-benchmark/predastore-s3-read-by-client.svg

@@ -1,39 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="760" height="470" viewBox="0 0 760 470" font-family="-apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif">
-<rect width="760" height="470" fill="#ffffff"/>
-<text x="74" y="30" font-size="19" font-weight="700" fill="#1f2937">Predastore S3 read throughput by client</text>
-<text x="74" y="49" font-size="12.5" fill="#6b7280">1 GiB objects, same presigned-curl client on every path — Spinifex v1.20.0</text>
-<line x1="74" y1="386.0" x2="734" y2="386.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="390.0" font-size="12" text-anchor="end" fill="#6b7280">0</text>
-<line x1="74" y1="278.0" x2="734" y2="278.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="282.0" font-size="12" text-anchor="end" fill="#6b7280">200</text>
-<line x1="74" y1="170.0" x2="734" y2="170.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="174.0" font-size="12" text-anchor="end" fill="#6b7280">400</text>
-<line x1="74" y1="62.0" x2="734" y2="62.0" stroke="#e5e7eb" stroke-width="1"/>
-<text x="64" y="66.0" font-size="12" text-anchor="end" fill="#6b7280">600</text>
-<text x="22" y="224.0" font-size="12.5" fill="#6b7280" transform="rotate(-90 22 224.0)" text-anchor="middle">MiB/s</text>
-<rect x="102.0" y="202.4" width="51.5" height="183.6" rx="2.5" fill="#2563eb"/>
-<text x="127.8" y="196.4" font-size="11" text-anchor="middle" fill="#1f2937">340</text>
-<rect x="156.5" y="136.5" width="51.5" height="249.5" rx="2.5" fill="#f59e0b"/>
-<text x="182.2" y="130.5" font-size="11" text-anchor="middle" fill="#1f2937">462</text>
-<text x="156.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">Host</text>
-<rect x="267.1" y="170.5" width="51.5" height="215.5" rx="2.5" fill="#2563eb"/>
-<text x="292.8" y="164.5" font-size="11" text-anchor="middle" fill="#1f2937">399</text>
-<rect x="321.5" y="174.9" width="51.5" height="211.1" rx="2.5" fill="#f59e0b"/>
-<text x="347.2" y="168.9" font-size="11" text-anchor="middle" fill="#1f2937">391</text>
-<text x="321.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">GPU guest</text>
-<rect x="432.1" y="102.5" width="51.5" height="283.5" rx="2.5" fill="#2563eb"/>
-<text x="457.8" y="96.5" font-size="11" text-anchor="middle" fill="#1f2937">525</text>
-<rect x="486.5" y="197.0" width="51.5" height="189.0" rx="2.5" fill="#f59e0b"/>
-<text x="512.2" y="191.0" font-size="11" text-anchor="middle" fill="#1f2937">350</text>
-<text x="486.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU1 guest</text>
-<rect x="597.0" y="106.8" width="51.5" height="279.2" rx="2.5" fill="#2563eb"/>
-<text x="622.8" y="100.8" font-size="11" text-anchor="middle" fill="#1f2937">517</text>
-<rect x="651.5" y="196.5" width="51.5" height="189.5" rx="2.5" fill="#f59e0b"/>
-<text x="677.2" y="190.5" font-size="11" text-anchor="middle" fill="#1f2937">351</text>
-<text x="651.5" y="408.0" font-size="13" text-anchor="middle" font-weight="600" fill="#1f2937">CPU2 guest</text>
-<line x1="74" y1="386" x2="734" y2="386" stroke="#1f2937" stroke-width="1.5"/>
-<rect x="74" y="444" width="13" height="13" rx="2.5" fill="#2563eb"/>
-<text x="93" y="455" font-size="12.5" fill="#1f2937">Single-stream GET</text>
-<rect x="224.1" y="444" width="13" height="13" rx="2.5" fill="#f59e0b"/>
-<text x="243.1" y="455" font-size="12.5" fill="#1f2937">Chunked/concurrent GET</text>
-</svg>

.github/e2e-cells.json

@@ -1,147 +0,0 @@
-[
-  {
-    "cell": 1,
-    "install": "binary",
-    "topo": "single",
-    "net": "dhcp",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 2,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 3,
-    "install": "binary",
-    "topo": "single",
-    "net": "dhcp",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 8,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "dhcp",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 9,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "static",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 10,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "dhcp",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 17,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-pseudo",
-    "extra": "tofu-examples",
-    "suite": ""
-  },
-  {
-    "cell": 18,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "reboot-single",
-    "suite": ""
-  },
-  {
-    "cell": 19,
-    "install": "binary",
-    "topo": "single",
-    "net": "nat",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "nat-single",
-    "suite": ""
-  },
-  {
-    "cell": 25,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single-nvme",
-    "extra": "eks",
-    "suite": "eks"
-  },
-  {
-    "cell": 26,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "ecs",
-    "suite": "ecs"
-  },
-  {
-    "cell": 27,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "rds",
-    "suite": "rds",
-    "timeout_minutes": 120
-  },
-  {
-    "cell": 29,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": "storagefault",
-    "timeout_minutes": 100
-  }
-]

.github/resolve-sibling-refs.sh

@@ -1,21 +1,20 @@
 #!/usr/bin/env bash
-# Resolve which branch of each sibling repo (viperblock, predastore, northstar,
-# bluebottle) the CI workspace should build against. A cross-repo change is developed on a branch
+# Resolve which branch of each sibling repo (viperblock, predastore) the CI
+# workspace should build against. A cross-repo change is developed on a branch
 # of the SAME name in each repo; build against that branch when it exists so the
 # set can be tested together, and fall back to dev otherwise. This mirrors the
 # resolution the e2e path already does in
 # mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
 # agree on how sibling branches are selected.
 #
-# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref>,
-# predastore=<ref>, northstar=<ref> and bluebottle=<ref> to $GITHUB_OUTPUT for
-# the checkout steps to consume.
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
+# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
 set -euo pipefail
 
 : "${BRANCH:?BRANCH must be set to the triggering branch name}"
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
 
-for repo in viperblock predastore northstar bluebottle; do
+for repo in viperblock predastore; do
   if git ls-remote --exit-code --heads \
       "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
     ref="$BRANCH"

.github/scripts/ci-fmt.sh

@@ -61,37 +61,3 @@ quiet_run() {
     return "$rc"
   fi
 }
-
-# Connection-layer failures from the libvirt tofu provider. The socket drops
-# when a hypervisor is carrying several environments at once, which is the
-# steady state here. A resource that failed to build says something else.
-CI_LIBVIRT_TRANSIENT='Unable to Connect to Libvirt|error connecting to libvirt|procedure interrupted|Cannot recv data|Cannot write data|End of file while reading data'
-
-# retry_run LABEL LOG_FILE -- CMD ARGS …
-# quiet_run with a bounded retry, taken only when that attempt's own output
-# names a libvirt connection failure. Every other failure is reported on the
-# first attempt, so a real bug never becomes a slow flake.
-retry_run() {
-  local label="$1" log="$2"; shift 2
-  local _attempts="${CI_RETRY_ATTEMPTS:-3}" _delay="${CI_RETRY_DELAY:-30}" _i _rc=1 _mark
-  for ((_i = 1; _i <= _attempts; _i++)); do
-    _mark=$(wc -c < "$log" 2>/dev/null || echo 0)
-    # rc must be read inside the else: an `if` whose condition fails and has no
-    # else exits 0, so reading $? after `fi` reports success for a failed run.
-    if "$@" >> "$log" 2>&1; then
-      if [[ $_i -gt 1 ]]; then ok "$label (libvirt recovered on attempt $_i)"; else ok "$label"; fi
-      return 0
-    else
-      _rc=$?
-    fi
-    tail -c "+$((_mark + 1))" "$log" | grep -Eq "$CI_LIBVIRT_TRANSIENT" || break
-    [[ $_i -eq $_attempts ]] && break
-    warn "$label: libvirt connection dropped (attempt $_i/$_attempts), retrying in ${_delay}s"
-    sleep "$_delay"
-  done
-  bad "$label (rc=$_rc) — last 80 lines of $(basename "$log"):"
-  group_open "tail $(basename "$log")"
-  tail -n 80 "$log"
-  group_close
-  return "$_rc"
-}

.github/scripts/e2e-run-summary.py

@@ -1,220 +0,0 @@
-#!/usr/bin/env python3
-"""Render one run-level table for the e2e nightly into the job summary.
-
-Reading a nightly meant opening nineteen jobs, and the Actions UI does not
-distinguish a cell that failed from one that was cancelled out from under it —
-a distinction every investigation of this suite has needed.
-
-Inputs (env):
-    GITHUB_TOKEN         Token with actions:read.
-    GITHUB_REPOSITORY    owner/repo.
-    GITHUB_RUN_ID        Run to summarise.
-    JUNIT_ROOT           Directory holding downloaded junit-cell-* artifacts.
-    GITHUB_STEP_SUMMARY  Appended to.
-
-Test counts come from the junit artifacts; result, duration and runner come
-from the jobs API, so a cell that died before writing junit still gets a row.
-"""
-
-from __future__ import annotations
-
-import glob
-import json
-import os
-import sys
-import urllib.error
-import urllib.request
-import xml.etree.ElementTree as ET
-from datetime import datetime
-
-API = "https://api.github.com"
-
-# A cell whose job produced no verdict is not a pass and not a failure. Keeping
-# them apart is the point of the table.
-VERDICT_ICON = {
-    "success": "✅",
-    "failure": "❌",
-    "cancelled": "🚫",
-    "skipped": "⏭️",
-    "timed_out": "⏱️",
-}
-
-
-def fetch_jobs(repo: str, run_id: str, token: str) -> list[dict]:
-    jobs: list[dict] = []
-    page = 1
-    while True:
-        url = f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}"
-        req = urllib.request.Request(
-            url,
-            headers={
-                "Authorization": f"Bearer {token}",
-                "Accept": "application/vnd.github+json",
-                "X-GitHub-Api-Version": "2022-11-28",
-            },
-        )
-        try:
-            with urllib.request.urlopen(req, timeout=30) as resp:
-                payload = json.load(resp)
-        except (urllib.error.URLError, TimeoutError) as exc:
-            print(f"::warning::jobs API page {page} failed: {exc}", file=sys.stderr)
-            break
-        batch = payload.get("jobs", [])
-        jobs.extend(batch)
-        if len(batch) < 100:
-            break
-        page += 1
-    return jobs
-
-
-def duration_s(job: dict) -> int | None:
-    started, completed = job.get("started_at"), job.get("completed_at")
-    if not started or not completed:
-        return None
-    fmt = "%Y-%m-%dT%H:%M:%SZ"
-    try:
-        return int((datetime.strptime(completed, fmt) - datetime.strptime(started, fmt)).total_seconds())
-    except ValueError:
-        return None
-
-
-def human(seconds: int | None) -> str:
-    if seconds is None:
-        return "—"
-    if seconds < 0:
-        return "0s"
-    return f"{seconds // 60}m{seconds % 60:02d}s" if seconds >= 60 else f"{seconds}s"
-
-
-def junit_counts(root: str) -> dict[str, dict]:
-    """Map cell number to its aggregate junit counts and failing test names."""
-    per_cell: dict[str, dict] = {}
-    for path in glob.glob(os.path.join(root, "**", "junit-*.xml"), recursive=True):
-        cell = cell_from_artifact_path(path)
-        if cell is None:
-            continue
-        acc = per_cell.setdefault(cell, {"tests": 0, "failures": 0, "skipped": 0, "failing": []})
-        try:
-            root_el = ET.parse(path).getroot()
-        except ET.ParseError as exc:
-            print(f"::warning::unparseable junit {path}: {exc}", file=sys.stderr)
-            continue
-        suites = root_el.iter("testsuite") if root_el.tag != "testsuite" else [root_el]
-        for suite in suites:
-            acc["tests"] += int(suite.get("tests") or 0)
-            acc["failures"] += int(suite.get("failures") or 0) + int(suite.get("errors") or 0)
-            acc["skipped"] += int(suite.get("skipped") or 0)
-        for case in root_el.iter("testcase"):
-            if case.find("failure") is not None or case.find("error") is not None:
-                name = case.get("name") or "?"
-                if name not in acc["failing"]:
-                    acc["failing"].append(name)
-    return per_cell
-
-
-def cell_from_artifact_path(path: str) -> str | None:
-    """Recover the cell number from a junit-cell-<n>-<run_id> artifact directory."""
-    for part in path.split(os.sep):
-        if part.startswith("junit-cell-"):
-            rest = part[len("junit-cell-"):]
-            cell = rest.split("-", 1)[0]
-            return cell if cell.isdigit() else None
-    return None
-
-
-def cell_number(job_name: str) -> str | None:
-    """cell-19 (single/static/debian-13) -> 19."""
-    if not job_name.startswith("cell-"):
-        return None
-    cell = job_name[len("cell-"):].split(" ", 1)[0]
-    return cell if cell.isdigit() else None
-
-
-def main() -> int:
-    repo = os.environ.get("GITHUB_REPOSITORY", "")
-    run_id = os.environ.get("GITHUB_RUN_ID", "")
-    token = os.environ.get("GITHUB_TOKEN", "")
-    summary_path = os.environ.get("GITHUB_STEP_SUMMARY", "")
-    junit_root = os.environ.get("JUNIT_ROOT", "")
-
-    if not (repo and run_id and token and summary_path):
-        print("::warning::e2e-run-summary: missing required environment", file=sys.stderr)
-        return 0
-
-    jobs = fetch_jobs(repo, run_id, token)
-    counts = junit_counts(junit_root) if junit_root and os.path.isdir(junit_root) else {}
-
-    rows = []
-    for job in jobs:
-        cell = cell_number(job.get("name", ""))
-        if cell is None:
-            continue
-        rows.append((int(cell), job))
-    rows.sort(key=lambda r: r[0])
-
-    lines = [f"## e2e nightly — run summary ({len(rows)} cells)", ""]
-    if not rows:
-        lines.append("No cell jobs found for this run.")
-        write(summary_path, lines)
-        return 0
-
-    lines += [
-        "| Cell | Topology | Result | Duration | Runner | Tests | Failed | Skipped |",
-        "| --- | --- | --- | --- | --- | --- | --- | --- |",
-    ]
-
-    tally: dict[str, int] = {}
-    total_tests = total_failed = total_seconds = 0
-    failing_detail: list[tuple[int, list[str]]] = []
-
-    for cell, job in rows:
-        name = job.get("name", "")
-        # A cell dispatched to a reusable workflow reports as
-        # "cell-20 (baremetal/pxe/real-hw) / Bare-Metal Single-Node E2E".
-        topo = name.split("(", 1)[1].split(")", 1)[0] if "(" in name else "—"
-        verdict = job.get("conclusion") or job.get("status") or "unknown"
-        tally[verdict] = tally.get(verdict, 0) + 1
-        secs = duration_s(job)
-        total_seconds += secs or 0
-        c = counts.get(str(cell), {})
-        total_tests += c.get("tests", 0)
-        total_failed += c.get("failures", 0)
-        if c.get("failing"):
-            failing_detail.append((cell, c["failing"]))
-        lines.append(
-            f"| {cell} | {topo} | {VERDICT_ICON.get(verdict, '❔')} {verdict} | {human(secs)} "
-            f"| {job.get('runner_name') or '—'} | {c.get('tests', '—')} "
-            f"| {c.get('failures', '—')} | {c.get('skipped', '—')} |"
-        )
-
-    verdicts = ", ".join(f"{n} {v}" for v, n in sorted(tally.items(), key=lambda kv: -kv[1]))
-    lines += [
-        "",
-        f"**Cells:** {verdicts}",
-        f"**Tests:** {total_tests} run, {total_failed} failed",
-        f"**Cell wall clock:** {human(total_seconds)} across all cells",
-    ]
-
-    slowest = sorted(rows, key=lambda r: duration_s(r[1]) or 0, reverse=True)[:3]
-    if slowest:
-        lines.append(
-            "**Slowest:** " + ", ".join(f"cell {c} ({human(duration_s(j))})" for c, j in slowest)
-        )
-
-    if failing_detail:
-        lines += ["", "<details><summary>Failing tests</summary>", ""]
-        for cell, names in failing_detail:
-            lines.append(f"- **cell {cell}** — " + ", ".join(f"`{n}`" for n in names))
-        lines += ["", "</details>"]
-
-    write(summary_path, lines)
-    return 0
-
-
-def write(path: str, lines: list[str]) -> None:
-    with open(path, "a", encoding="utf-8") as fh:
-        fh.write("\n".join(lines) + "\n")
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/scripts/ensure-libguestfs.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-# Prepare a runner to build a guest image with build-system-image.sh.
-#
-# Installs libguestfs + qemu-utils if they are missing and makes a host kernel
-# readable, so the libguestfs appliance can boot. Nothing here touches a host
-# block device: virt-customize works inside an isolated userspace appliance, so
-# this is safe on a shared runner.
-#
-# Callers must still export LIBGUESTFS_BACKEND=direct themselves — it has to be
-# set in the environment the build runs in, which is not this subshell.
-set -euo pipefail
-
-# Three nightly service cells share one ci-single runner and start together, so
-# concurrent apt-get runs are the normal case rather than the exception. Two of
-# them lost the lock outright before this guard existed. Skipping the install
-# when the tools are already present avoids most of the contention; the lock
-# timeout covers the rest, including apt runs started by anything else on the
-# box.
-if command -v virt-customize >/dev/null && command -v qemu-img >/dev/null; then
-    echo "libguestfs-tools and qemu-utils already present"
-else
-    APT_OPTS=(-o DPkg::Lock::Timeout=600)
-    sudo apt-get "${APT_OPTS[@]}" update -qq
-    sudo apt-get "${APT_OPTS[@]}" install -y --no-install-recommends \
-        libguestfs-tools qemu-utils
-fi
-
-# Debian/Ubuntu ship /boot/vmlinuz-* mode 0600 and the appliance needs to read a
-# host kernel. Tolerated on failure: a runner whose kernel is already readable
-# has no vmlinuz to chmod under some images.
-sudo chmod 0644 /boot/vmlinuz-* || true

.github/scripts/suite-sets.sh

@@ -24,88 +24,23 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-# quota belongs to both topologies because the gate it asserts is gateway code
-# that runs identically on either, and because a quota left switched on would
-# cap every suite sharing the environment — so it must be proved to restore
-# itself wherever it runs, not only on one node.
-# storagefault runs last in both lists and belongs to both topologies, for
-# different reasons. On one node it is RS(1,0), so freezing the only predastore
-# is a total outage and the guest-survival assertion is unambiguous. On three it
-# is RS(2,1) with degraded writes on, so two must be frozen to break the shard
-# floor — and only there is a cross-node restart possible at all, which is the
-# case that takes a volume over from a node holding writes the backend never
-# received. It is last because it SIGSTOPs a cluster-wide
-# service: anything sharing the environment would see the same outage and report
-# a failure that says nothing about itself.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert quota storagefault"
+E2E_SUITES_MULTI="multinode lb cert"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
 E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 
-# What every e2e-nightly permutation cell must prove, deliberately narrower
-# than the two sets above: those cells have a 35-minute budget and exist to show
-# that every install / network / host-OS combination boots and serves. The
-# service suites are not deferred to the PR gate — eks, ecs and rds each run
-# nightly in their own dedicated cell, because each needs a guest AMI built
-# first and a service failure should not mask whether a host OS boots. These
-# are subsets, and e2e_suites_assert_subset below enforces that.
-#
-# storagefault has a dedicated cell for two reasons. It SIGSTOPs a cluster-wide
-# service, so in a permutation cell every suite sharing that environment would
-# report the outage as its own failure. And a full pass is ~63 minutes, nearly
-# all of it deliberate waiting, which is too long to add to the PR gate — where
-# it has never run, since the gate's default suite list does not name it.
-#
-# Its cell is three-node: the RS(2,1) shard floor and the cross-node takeover
-# after a failed seal do not exist on one node, and they are the assertions
-# worth a nightly.
-E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
+# e2e-nightly runs a narrower set than the two sets above, deliberately: its
+# ~19 cells each have a 35-minute budget and exist to prove that every
+# install / network / host-OS permutation boots and serves. Suite breadth is
+# the PR gate's job, and it already runs on every push. These are subsets, and
+# e2e_suites_assert_subset below enforces that.
+E2E_SUITES_NIGHTLY_SINGLE="single cert"
 E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
 
-# How long a suite gets, for the same reason the lists above live here: the
-# timeout was written out separately in each workflow and drifted, so rds ran
-# 50m on the PR gate and 30m nightly and panicked there. A suite needing more
-# than the default declares it once, here.
-#   e2e_suite_timeout <suite>
-e2e_suite_timeout() {
-  case "$1" in
-    # rds serialises its DB VMs behind a 3 GiB semaphore, so its wall clock is
-    # what the queue costs, not what the tests do: single tests logged 11m, 22m
-    # and 26m waiting for budget. 17 tests at that concurrency is over an hour
-    # before anything goes wrong, and the first pass to run deep into the suite
-    # still had three left at 60m. The loaded graceful-reboot case adds an
-    # instance that holds its slot through a pgbench load as well as a create.
-    rds) echo "105m" ;;
-    # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
-    # creates first. One pass is ~20 minutes on the bare-metal cell and a
-    # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
-    diskperf) echo "180m" ;;
-    # storagefault is mostly deliberate waiting: one test alone holds predastore
-    # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
-    # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
-    storagefault) echo "90m" ;;
-    *) echo "30m" ;;
-  esac
-}
-
-# One matrix leg per suite, carrying the job budget that suite needs. The outer
-# timeout must always be later than the `go test` one inside it: go test reports
-# where it stopped, a cancelled job reports only "The operation was canceled".
-#   e2e_suite_matrix <suite-list> <floor-minutes> [allowance-minutes]
-e2e_suite_matrix() {
-  local list="$1" floor="$2" allowance="${3:-15}" suite budget
-  for suite in ${list}; do
-    budget="$(e2e_suite_timeout "${suite}")"
-    budget=$((${budget%m} + allowance))
-    if [ "${budget}" -lt "${floor}" ]; then budget="${floor}"; fi
-    printf '%s\t%s\n' "${suite}" "${budget}"
-  done | jq -cRs 'split("\n")|map(select(length>0)|split("\t")|{suite:.[0],timeout:(.[1]|tonumber)})'
-}
-
 # Fail loudly when a narrowed list names a suite its topology cannot run. This
 # is the guard that stops the lists drifting again: a suite added to a nightly
 # subset without also being declared eligible above stops the job here rather

.github/workflows/apt-packages.yml

@@ -33,6 +33,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
+        # accepted by detect_os but has never been exercised and does not resolve.
         image:
           - ubuntu:24.04
           - ubuntu:26.04

.github/workflows/e2e-baremetal.yml

@@ -1,26 +1,7 @@
 name: E2E Bare-Metal
 
 on:
-  # Mirrors the suite selection of workflow_call so a baseline capture can be
-  # driven by hand. A dispatch run sees only the inputs declared here, so
-  # omitting these would silently fall back to the correctness set.
   workflow_dispatch:
-    inputs:
-      suites:
-        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance run names its own so its numbers come off an otherwise idle cell"
-        required: false
-        type: string
-        default: "single cert lb"
-      suite-env:
-        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
-        required: false
-        type: string
-        default: ""
-      cell:
-        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
-        required: false
-        type: string
-        default: "20"
   workflow_call:
     inputs:
       ref:
@@ -33,21 +14,6 @@ on:
         required: false
         type: string
         default: ""
-      suites:
-        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance caller names its own so its numbers come off an otherwise idle cell"
-        required: false
-        type: string
-        default: "single cert lb"
-      suite-env:
-        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
-        required: false
-        type: string
-        default: ""
-      cell:
-        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
-        required: false
-        type: string
-        default: "20"
 
 permissions:
   contents: read
@@ -55,21 +21,25 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
+  # no default here before — this cell's telemetry never reached the sink at
+  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
     name: Bare-Metal Single-Node E2E
-    runs-on: [self-hosted, ci-baremetal]
-    timeout-minutes: 240
+    runs-on: [ self-hosted, ci-baremetal ]
+    timeout-minutes: 120
     concurrency:
       group: e2e-baremetal-cluster
       cancel-in-progress: false
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
-      BM_CELL: ${{ inputs.cell || '20' }}
-      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
-      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
+      # Caller-supplied (workflow_call input); empty for a manual
+      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
     steps:
       - name: Clean Workspace
         run: |
@@ -104,10 +74,6 @@ jobs:
           ssh-keygen -y -f "${RUNNER_TEMP}/bm-ci" > "${RUNNER_TEMP}/bm-ci.pub"
 
       - name: Build Artifacts
-        env:
-          # The node runs pre-built binaries, so a suite absent here cannot be
-          # selected later. Same list as the run step for that reason.
-          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
         run: |
           mulga/scripts/baremetal/bm-build-artifacts.sh \
             "$GITHUB_WORKSPACE" \
@@ -153,21 +119,6 @@ jobs:
       - name: Install go-junit-report on runner
         run: go install github.com/jstemmer/go-junit-report/v2@latest
 
-      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
-      # AMI. Built here because libguestfs needs source and network, which the
-      # runner has and the PXE-booted node does not; bm-bootstrap.sh imports it.
-      - name: Build eks-node AMI (libguestfs, runner)
-        if: contains(inputs.suites, 'eks')
-        run: |
-          set -euo pipefail
-          # direct backend = no libvirt. Set here rather than in the script:
-          # it has to be in the environment the build itself runs in.
-          export LIBGUESTFS_BACKEND=direct
-          # spinifex is the workspace root here, unlike the nightly where it is
-          # checked out into a subdirectory.
-          "${GITHUB_WORKSPACE}/.github/scripts/ensure-libguestfs.sh"
-          make build-eks-node-image
-
       - name: Run Bare-Metal E2E
         env:
           BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
@@ -175,25 +126,9 @@ jobs:
           BM_EPDU_ENV: ${{ runner.temp }}/epdu.env
           BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
           SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
-          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
-          BM_SUITE_ENV: ${{ inputs.suite-env }}
-          # Empty unless the eks build above ran, and bm-bootstrap.sh skips the
-          # import when unset, so the other suites are unaffected.
-          BM_EKS_IMAGE: ${{ contains(inputs.suites, 'eks') && '/tmp/eks-node-image-build/eks-node-alpine.raw' || '' }}
-          BM_EKS_MANIFEST: ${{ contains(inputs.suites, 'eks') && format('{0}/scripts/images/eks-node/manifest.conf', github.workspace) || '' }}
         run: |
           mkdir -p "${ARTIFACT_DIR}"
           set -o pipefail
-          . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
-          BM_TEST_TIMEOUT=""
-          for suite in ${BM_SUITES}; do
-            tmo="$(e2e_suite_timeout "${suite}")"
-            if [ -z "${BM_TEST_TIMEOUT}" ] || [ "${tmo%m}" -gt "${BM_TEST_TIMEOUT%m}" ]; then
-              BM_TEST_TIMEOUT="${tmo}"
-            fi
-          done
-          export BM_TEST_TIMEOUT
-          echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
           sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
@@ -204,40 +139,22 @@ jobs:
       - name: Generate JUnit report
         if: always()
         run: |
-          # suite-output.log, not test-baremetal.log. The harness streams the
-          # full suite output to the first and a grep-filtered summary to its
-          # stdout, and that filter anchors --- PASS at column 0, so every
-          # indented subtest result is dropped. Parsed from the filtered log,
-          # go-junit-report saw a === RUN with no result and emitted
-          # <error message="No test result found"> — 17 of 39 testcases on a
-          # green cell-28, 93 of 120 on cell-20.
-          LOG="${ARTIFACT_DIR}/suite-output.log"
-          [ -s "$LOG" ] || LOG="${ARTIFACT_DIR}/test-baremetal.log"
+          LOG="${ARTIFACT_DIR}/test-baremetal.log"
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: ${{ env.BM_CELL }}
-
       - name: Analyze e2e failures
         if: always()
         uses: ./.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
-          title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
+          title: "E2E failure analysis (cell 20)"
 
       - uses: actions/upload-artifact@v7
         if: always()
         with:
-          name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
+          name: e2e-artifacts-cell-20-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14

.github/workflows/e2e-ddil.yml

@@ -1,5 +1,7 @@
 name: E2E DDIL
 
+# This workflow is manually triggered while the DDIL scenarios are being
+# validated.
 on:
   workflow_dispatch:
     inputs:
@@ -8,31 +10,29 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml and e2e-suites.yml: every one of them provisions
-# many cells on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
-# is outside the group: one cluster per run is cheap enough to overlap.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
   contents: read
+  # actions: read lets the otel-job-id action look up this job's numeric id via
+  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   ddil:
     name: DDIL Real Multi-Node
-    runs-on: [self-hosted, ci-multi]
+    runs-on: [ self-hosted, ci-multi ]
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
+      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
+      # this workflow's telemetry so a DDIL run's partition-induced errors are
+      # never mistaken for a healthy multi-node run's in the sink.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace
@@ -103,7 +103,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VMs provisioned"
 
       - name: Bootstrap via Binary Install
@@ -180,16 +180,6 @@ jobs:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
-          cell: ddil
-
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster
@@ -228,7 +218,7 @@ jobs:
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
 

.github/workflows/e2e-suites.yml

@@ -43,22 +43,18 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml and e2e-ddil.yml: every one of them provisions
-# many cells on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
-# is outside the group: one cluster per run is cheap enough to overlap.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
   contents: read
+  # actions: read lets the otel-job-id action look up a provision job's numeric
+  # id via the Actions API — github.job is only the YAML key, no context
+  # exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
@@ -77,6 +73,12 @@ jobs:
       matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      # This job (not single_suite) is where "Bootstrap via Binary Install"
+      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
+      # suite matrix even exists — single_suite's legs share this one
+      # already-booted VM, so daemon telemetry can't be scoped any finer
+      # than "single env" per suite leg. ci.cell=single distinguishes it
+      # from multi_provision's VMs within the same run id.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
@@ -141,9 +143,10 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
           e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # 45 is the floor this job has always had, so resolving each leg's
-          # budget can only raise it — storagefault and rds ask for more.
-          JSON=$(e2e_suite_matrix "${LIST}" 45)
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Reclaim docker build cache
@@ -179,7 +182,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -308,16 +311,14 @@ jobs:
           retention-days: 14
 
   single_suite:
-    name: Single env · ${{ matrix.suite }}
+    name: Single env · suite
     needs: single_provision
     runs-on: [self-hosted, ci-single]
-    # Per leg, from suite-sets.sh: one number for every suite would have to be
-    # storagefault's, and every other leg would then hang for 105 minutes.
-    timeout-minutes: ${{ matrix.timeout }}
+    timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
+        suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
     env:
       WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
       SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
@@ -342,8 +343,6 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
-          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
-          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${WAN_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -361,7 +360,7 @@ jobs:
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -395,16 +394,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: single
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -517,7 +506,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -628,7 +617,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -670,9 +659,10 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
           e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # 60 is the floor this job has always had; multi-node legs carry a
-          # heavier setup than single-node ones, which is why it is higher.
-          JSON=$(e2e_suite_matrix "${LIST}" 60)
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Build multinode binaries on runner
@@ -722,15 +712,14 @@ jobs:
           retention-days: 14
 
   multi_suite:
-    name: Multi env · ${{ matrix.suite }}
+    name: Multi env · suite
     needs: multi_provision
     runs-on: [self-hosted, ci-multi]
-    # See the note on single_suite: the budget is the suite's, not the job's.
-    timeout-minutes: ${{ matrix.timeout }}
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
+        suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
     env:
       NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
       SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
@@ -756,8 +745,6 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
-          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
-          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${NODE1_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -776,7 +763,7 @@ jobs:
                SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -809,16 +796,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: multi
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -902,7 +879,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/e2e.yml

@@ -7,7 +7,6 @@ on:
       - dev
     paths:
       - "**/*.go"
-      - "!**/*_test.go"
       - go.mod
       - go.sum
       - Makefile
@@ -27,67 +26,20 @@ on:
 permissions:
   contents: read
   statuses: write
+  # actions: read lets the otel-job-id action look up a job's numeric id via
+  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # The inner timeout must fire before the outer one. `go test` reports where it
-  # stopped and the suite's own failure; a job cancelled on timeout reports only
-  # "The operation was canceled" and takes the artifacts with it.
-  #
-  # A job budget cannot be a literal here because the suite set is a dispatch
-  # input: storagefault asks for 90m and diskperf for 180m, both eligible on one
-  # node, so any fixed number is either wrong for them or wasteful for everyone
-  # else. Resolving it from suite-sets.sh keeps the one source of truth and
-  # leaves the default push set at exactly the 60m it has always had.
-  plan:
-    name: Resolve Job Budgets
-    runs-on: ubuntu-26.04
-    timeout-minutes: 5
-    outputs:
-      single_timeout: ${{ steps.budgets.outputs.single_timeout }}
-      multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          path: spinifex
-          sparse-checkout: .github/scripts
-
-      - name: Resolve Budgets
-        id: budgets
-        env:
-          # Provision, install and teardown around the suites, plus the headroom
-          # a multi-suite job needs for whatever ran before the one that hangs.
-          SETUP_ALLOWANCE_MIN: 15
-          # What the job has always had. Kept as a floor so resolving the budget
-          # can only ever raise it, never quietly shorten an existing run.
-          FLOOR_MIN: 60
-        run: |
-          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
-          # Only max() is a sound bound: the suites share one job, so a sum would
-          # demand hours for a set that has never taken them. Max is what makes
-          # the hanging suite panic on its own timeout first, which is the point.
-          budget() {
-            local want="${FLOOR_MIN}" s t
-            for s in $(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "$1"); do
-              t="$(e2e_suite_timeout "${s}")"
-              t=$(( ${t%m} + SETUP_ALLOWANCE_MIN ))
-              [ "${t}" -gt "${want}" ] && want="${t}"
-            done
-            echo "${want}"
-          }
-          single=$(budget "${E2E_SUITES_SINGLE_RE}")
-          multi=$(budget "${E2E_SUITES_MULTI_RE}")
-          echo "single_timeout=${single}" >> "$GITHUB_OUTPUT"
-          echo "multi_timeout=${multi}" >> "$GITHUB_OUTPUT"
-          echo "suites='${SUITES}' -> single ${single}m, multi ${multi}m"
-
   e2e_single:
     name: Single-Node E2E
     # The dispatch check gates on exactly the suites this job's Run step
@@ -99,9 +51,8 @@ jobs:
     # tear down. Push triggers keep running via the null-check on
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
-    needs: plan
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: ${{ fromJSON(needs.plan.outputs.single_timeout) }}
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -201,7 +152,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -406,10 +357,6 @@ jobs:
           # suite-sets.sh also carries the per-suite rationale.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_SINGLE_RE}" | tr '\n' ' ')
-          # Resolve each suite's timeout here, where suite-sets.sh is readable, and
-          # carry the pairs into the remote loop below — the VM has no checkout.
-          SUITE_TMOS=""
-          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -441,7 +388,7 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
-               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
+               case \$SUITE in rds) TMO=45m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
                  ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
@@ -517,16 +464,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: single
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -573,7 +510,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -602,9 +539,8 @@ jobs:
     # provisions the 3-node env when requested on its own, and the env stays
     # unprovisioned when none of them are asked for.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
-    needs: plan
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: ${{ fromJSON(needs.plan.outputs.multi_timeout) }}
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
@@ -700,7 +636,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -826,8 +762,6 @@ jobs:
           # Allow-list, not a deny-list — see the single-node job for why.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_MULTI_RE}" | tr '\n' ' ')
-          SUITE_TMOS=""
-          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -859,14 +793,13 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
-               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                  SPINIFEX_NODES='${NODES_CSV}' \
                  SPINIFEX_NODE_IPS='${NODES_CSV}' \
                  SPINIFEX_SSH_USER=tf-user \
                  SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -970,16 +903,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: multi
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -1029,7 +952,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/gpu-e2e.yml

@@ -56,7 +56,15 @@ concurrency:
   cancel-in-progress: false
 
 env:
+  # OTLP export to the central Elastic sink (wattle is the sink itself — see
+  # Part A.2). Repo variable overrides; carried for correlation even though
+  # the GPU suite's own daemon-side spans are emitted by wattle's already-
+  # configured obs-agent, not by this job.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
+  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
+  # note above), so the label only takes effect once wattle's obs-agent is
+  # given them; there is deliberately no ci.job_id lookup for the same reason.
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
   WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
   WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
@@ -70,6 +78,8 @@ jobs:
     env:
       ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
     steps:
+      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
+      # SSH to wattle so a paused run touches the box zero times.
       - name: Manual-work mutex gate
         id: gate
         env:
@@ -112,6 +122,8 @@ jobs:
           echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
           echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
 
+      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
+      # actually up before spending time building/shipping the test binary.
       - name: Healthcheck — describe-regions
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -119,6 +131,18 @@ jobs:
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
             'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
 
+      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
+      # test is the always-on baseline, so CI self-provisions it rather than
+      # depending on a manual import. `--name` selects the catalog entry (pulls
+      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
+      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
+      # falls back to the generic ami-<distro>-<version>-<arch>, which the
+      # tests/e2e/gpu filter would never match. Idempotent — skipped once
+      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
+      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
+      # `spx admin images import` writes local spinifex storage, so it needs
+      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
+      # HTTPS and don't.
       - name: Ensure base GPU AMI present (import-if-absent)
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -147,6 +171,11 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
+      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
+      # ensured it, so a miss here means the import silently failed. Hard-fail
+      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
+      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
+      # self-skip when absent, so only warn.
       - name: Preflight — verify GPU AMIs
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -176,6 +205,10 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
+      # Same build command CI's other e2e workflows use to cross-compile a
+      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
+      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
+      # toolchain / module cache off wattle entirely.
       - name: Build GPU e2e test binary on runner
         if: steps.gate.outputs.paused != 'true'
         working-directory: tests/e2e
@@ -198,6 +231,12 @@ jobs:
           ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
           scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
 
+      # Same invocation shape e2e.yml uses to run a pre-built suite binary
+      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
+      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
+      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
+      # the advertised gateway, so the override is required here. The harness
+      # should prefer spinifex.toml's advertised bridge.
       - name: Run GPU e2e suite on wattle
         id: e2e
         if: steps.gate.outputs.paused != 'true'
@@ -248,14 +287,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
-      - name: Ship test results to Elasticsearch
-        if: always() && steps.gate.outputs.paused != 'true'
-        continue-on-error: true
-        uses: ./.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: gpu
-
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
         uses: ./.github/actions/e2e-analyze
@@ -272,6 +303,14 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
+      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
+      # terminate any GPU-typed instance still pending/running (the Go
+      # suite's own t.Cleanup already does this on normal pass/fail exit;
+      # this is the backstop for a hard `-test.timeout` kill or a workflow
+      # cancellation, which skip deferred Go cleanups). Termination triggers
+      # gpuManager.Release() on wattle, returning the A1000 to the pool for
+      # the next queued run or manual test.
       - name: Cleanup — terminate leftover GPU instances, release GPU
         if: always() && steps.gate.outputs.paused != 'true'
         run: |

.github/workflows/image-scripts.yml

@@ -1,5 +1,9 @@
 name: Image Scripts
 
+# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
+# and mkosi assets baked into system images, not Go, so they get their own
+# path-filtered gate instead of running on every Go contributor's push (see
+# ui.yml for the same pattern applied to the frontend).
 on:
   push:
     branches:
@@ -33,25 +37,3 @@ jobs:
 
       - name: Run image script tests + shellcheck
         run: make test-images
-
-  eks_mkosi_image:
-    name: Build mkosi EKS image
-    runs-on: [self-hosted, ci-single]
-    concurrency:
-      group: eks-mkosi-image-${{ github.ref }}
-      cancel-in-progress: true
-
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v7
-
-      - name: Set up Go
-        uses: actions/setup-go@v7
-        with:
-          go-version-file: "go.mod"
-          # Self-hosted: the module + build cache already persists on local
-          # disk, and cache-save would re-upload hundreds of MB for nothing.
-          cache: false
-
-      - name: Build the image
-        run: make build-eks-node-mkosi-image

.github/workflows/perf-3node.yml

@@ -1,415 +0,0 @@
-name: Predastore 3-Node Performance
-
-on:
-  workflow_dispatch:
-    inputs:
-      perf_preset:
-        description: "Warp workload preset: smoke (30s) or compare (2m)"
-        required: false
-        default: compare
-        type: choice
-        options:
-          - smoke
-          - compare
-      cluster_ref:
-        description: "Git ref to build the cluster from, e.g. v1.18.0. Empty uses this branch."
-        required: false
-        default: ""
-        type: string
-
-# Shared with e2e.yml and e2e-nightly.yml: this job provisions on the same
-# hypervisors they do, so a run in flight there and here would compete for
-# disks and libvirt guests. Queued, not cancelled, same as those workflows.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
-permissions:
-  contents: read
-
-env:
-  GIT_BRANCH: ${{ github.ref_name }}
-  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
-
-jobs:
-  perf_3node:
-    name: Predastore Throughput (3-node)
-    # ci-multi is a systemd runner on the LAN, not the ephemeral pool. A pool
-    # VM sits behind the geneve overlay at MTU 1408, which would put the
-    # runner's own network path inside the thing being measured.
-    runs-on: [self-hosted, ci-multi]
-    timeout-minutes: 90
-    env:
-      TOFU_LOG: /tmp/e2e-tofu-perf3node-${{ github.run_id }}.log
-      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=perf-3node
-    steps:
-      - name: Reap Orphan Procs
-        run: |
-          set +e
-          UID_SELF=$(id -u)
-          WS="${GITHUB_WORKSPACE}"
-          reap() {
-            local sig="$1"
-            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
-              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
-              case "$cwd" in
-                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
-              esac
-            done
-          }
-          reap TERM
-          sleep 3
-          reap KILL
-          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
-          true
-
-      - name: Clean Workspace
-        run: |
-          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
-            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
-          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
-
-      # ci-multi and ci-single resolve to the same physical host, so the
-      # nvme path cell-23 uses for its throughput run is already there.
-      # Pointing TMPDIR at it keeps warp's payloads off a small /tmp tmpfs.
-      - name: Configure work directories
-        run: |
-          E2E_ROOT=/mnt/nvme/e2e
-          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
-            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
-          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
-          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
-
-      - uses: actions/create-github-app-token@v3
-        id: app-token
-        with:
-          client-id: ${{ secrets.APP_ID }}
-          private-key: ${{ secrets.APP_PRIVATE_KEY }}
-          repositories: mulga
-
-      - uses: actions/checkout@v7
-        with:
-          repository: mulgadc/mulga
-          token: ${{ steps.app-token.outputs.token }}
-          sparse-checkout: scripts/tofu-cluster
-          path: mulga
-
-      - uses: actions/checkout@v7
-        with:
-          path: spinifex
-
-      - uses: actions/setup-go@v7
-        with:
-          go-version-file: spinifex/go.mod
-          cache: false
-
-      - name: Re-enable Go toolchain auto-download
-        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
-
-      # Same-named branch when one exists, dev otherwise. Matches how the
-      # e2e path picks siblings, so this measures the same predastore
-      # commit a cross-repo change would be built and tested against.
-      - name: Resolve sibling refs
-        id: siblings
-        env:
-          BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
-        working-directory: spinifex
-
-      - uses: actions/checkout@v7
-        id: predastore
-        with:
-          repository: mulgadc/predastore
-          ref: ${{ steps.siblings.outputs.predastore }}
-          path: predastore
-
-      # The cluster's ref is a knob and the harness's is not. Measuring an old
-      # release with a new harness is the only way its number can be set beside
-      # a current one; rebuilding both would change the ruler and the thing.
-      - name: Build Install Artifacts
-        working-directory: mulga/scripts/tofu-cluster
-        env:
-          CLUSTER_REF: ${{ inputs.cluster_ref }}
-        run: |
-          ./build-install-artifacts.sh "${CLUSTER_REF:-$GIT_BRANCH}" \
-            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
-            "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar" \
-            --skip-e2e-build
-
-      # The libvirt provider dials the hypervisors by bare hostname. A pool VM
-      # sits inside a VPC and cannot resolve them; a LAN runner already can, so
-      # this is self-gating on the lookup rather than on the runner's labels.
-      - name: Map hypervisor names
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          set -euo pipefail
-          if getent hosts banksia >/dev/null; then
-            echo "hypervisor names already resolve"
-          else
-            sudo tee -a /etc/hosts < network/hosts >/dev/null
-            echo "added $(grep -c '^[0-9]' network/hosts) entries to /etc/hosts"
-          fi
-          # Pinned rather than StrictHostKeyChecking=no: these hops reach the
-          # hypervisors themselves, so an unverified key is the one place in
-          # this job where accepting anything would matter.
-          if ssh-keygen -F banksia >/dev/null 2>&1; then
-            echo "hypervisor host keys already known"
-          else
-            install -d -m 700 ~/.ssh
-            cat network/known_hosts >> ~/.ssh/known_hosts
-            chmod 600 ~/.ssh/known_hosts
-            echo "pinned $(grep -c '^[a-z]' network/known_hosts) host keys"
-          fi
-
-      - name: Provision VMs
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          set -euo pipefail
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          export TF_VAR_git_branch="${GIT_BRANCH}"
-          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
-          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
-          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
-          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
-          ok "3-node cluster provisioned"
-
-      - name: Bootstrap via Binary Install
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
-          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
-            "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
-            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
-
-      - name: Resolve cluster SSH details
-        id: cluster
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          INVENTORY=$(tofu output -json inventory)
-          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
-          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
-          SSH_KEY="${SSH_KEY/#\~/$HOME}"
-          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
-          REGION=$(echo "$INVENTORY" | jq -r '.region')
-          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
-          NODES_CSV=""
-          for i in $(seq 0 $((NODE_COUNT - 1))); do
-            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
-            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
-          done
-          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
-          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
-          echo "ssh_user=${SSH_USER}" >> "$GITHUB_OUTPUT"
-          echo "region=${REGION}" >> "$GITHUB_OUTPUT"
-          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
-
-      # PERF_EXTERNAL_HOSTS wants host:port, and the gate listens on 8443 on
-      # every node. run-info.txt from this run is then comparable with the
-      # nightly's loopback one, since the harness itself never changes.
-      - name: Resolve PERF_EXTERNAL_HOSTS
-        id: perf_hosts
-        run: |
-          set -euo pipefail
-          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
-          HOSTS=""
-          for ip in "${NODE_LIST[@]}"; do
-            [ -z "$ip" ] && continue
-            if [ -z "$HOSTS" ]; then HOSTS="$ip:8443"; else HOSTS="$HOSTS,$ip:8443"; fi
-          done
-          echo "hosts=${HOSTS}" >> "$GITHUB_OUTPUT"
-
-      # A slow bootstrap should fail here, legibly, rather than inside warp
-      # with a wall of connection-refused errors. Bounded at 5 minutes per
-      # gate so a genuinely dead node still fails the job in finite time.
-      - name: Wait for gates to answer on :8443
-        run: |
-          set -euo pipefail
-          IFS=',' read -ra GATES <<< "${{ steps.perf_hosts.outputs.hosts }}"
-          for gate in "${GATES[@]}"; do
-            echo "waiting for https://$gate"
-            up=0
-            for i in $(seq 1 60); do
-              if curl -k -sS --max-time 5 -o /dev/null "https://$gate/"; then
-                up=1
-                break
-              fi
-              sleep 5
-            done
-            [ "$up" -eq 1 ] || { echo "::error::gate $gate never answered HTTPS on :8443"; exit 1; }
-            echo "gate $gate is up"
-          done
-
-      # Warp reads whole objects, so no workload here touches a ranged read and
-      # the throughput figures stay healthy while ranged GETs are broken. This
-      # probe is the only thing in CI that looks at them.
-      - name: Ranged GET probe
-        env:
-          GATE: ${{ steps.cluster.outputs.node1_ip }}
-          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
-          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
-          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
-          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
-        run: |
-          set -euo pipefail
-          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
-          AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
-          SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
-          echo "::add-mask::$SK"
-          B="ranged-probe-${GITHUB_RUN_ID}"
-          AWSX=(--region "$CLUSTER_REGION" --endpoint-url "https://${GATE}:8443" --no-verify-ssl)
-          export AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK"
-          aws s3 mb "s3://$B" "${AWSX[@]}"
-          trap 'aws s3 rb "s3://$B" --force "${AWSX[@]}" >/dev/null 2>&1 || true' EXIT
-          # Two sizes: read amplification makes a boundary-crossing range cost
-          # the whole object, so the fault shows as time tracking object size.
-          for mb in 64 256; do
-            head -c $((mb*1024*1024)) /dev/urandom > "${TMPDIR}/probe-${mb}.bin"
-            aws s3 cp "${TMPDIR}/probe-${mb}.bin" "s3://$B/probe-${mb}.bin" --only-show-errors "${AWSX[@]}"
-            rm -f "${TMPDIR}/probe-${mb}.bin"
-          done
-          t() {
-            curl -sk --aws-sigv4 "aws:amz:${CLUSTER_REGION}:s3" -u "$AK:$SK" \
-              -H "Range: bytes=0-$1" -o /dev/null -w '%{time_total}' \
-              --url "https://${GATE}:8443/${B}/probe-${2}.bin"
-          }
-          # 4194304 is the last byte inside the first 4 MiB block; one more
-          # crosses into the second and leaves the single-shard fast path.
-          IN64=$(t 4194303 64);  OUT64=$(t 4194304 64)
-          IN256=$(t 4194303 256); OUT256=$(t 4194304 256)
-          printf '%-10s in-block=%ss  crossing=%ss\n' "64MiB" "$IN64" "$OUT64"
-          printf '%-10s in-block=%ss  crossing=%ss\n' "256MiB" "$IN256" "$OUT256"
-          # The 4x object-size ratio is the signal, and it is compared against
-          # the crossing reads alone so a slow runner cancels out of both sides.
-          awk -v a="$OUT64" -v b="$OUT256" 'BEGIN{
-            r = (a > 0) ? b/a : 0
-            printf "crossing-range cost ratio 256MiB/64MiB = %.2f (object size ratio is 4.00)\n", r
-            if (r > 2.0) {
-              print "::error::a boundary-crossing ranged GET scales with object size, so it is reading the whole object"
-              exit 1
-            }
-          }'
-          echo "ranged GET does not scale with object size"
-
-      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
-      # free-space watermark, so this asserts the same headroom cell-23 does.
-      # tmpfs is rejected outright: that would measure RAM, not the gates.
-      - name: Preflight
-        run: |
-          set -euo pipefail
-          mkdir -p "$TMPDIR"
-          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
-          case "$fstype" in
-            tmpfs|ramfs)
-              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
-              exit 1 ;;
-          esac
-          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
-               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
-          df -h "$TMPDIR"
-          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
-          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
-          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
-            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
-            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
-            exit 1
-          fi
-
-      - name: Install warp
-        working-directory: predastore
-        run: make warp-install
-
-      # PERF_EXTERNAL_HOSTS skips profile selection, port offsetting, and
-      # cluster start/stop, and measures these three gates directly. Same
-      # workloads and analysis as the nightly loopback run, so the numbers
-      # from the two are directly comparable.
-      #
-      # The cluster's credentials are read and used inside this one step, the
-      # way e2e-ddil.yml does it: a step-level env: block is echoed verbatim
-      # into the run log, and a shell variable is not.
-      - name: Run 3-node throughput benchmark
-        working-directory: predastore
-        env:
-          PERF_EXTERNAL_HOSTS: ${{ steps.perf_hosts.outputs.hosts }}
-          PERF_CONFIGS: spinifex-3node
-          PERF_PRESET: ${{ inputs.perf_preset || 'compare' }}
-          # What is deployed, not what the harness was checked out at. With a
-          # cluster_ref those differ, and run-info.txt records the cluster.
-          PERF_EXTERNAL_SHA: ${{ inputs.cluster_ref || steps.predastore.outputs.commit }}
-          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
-          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
-          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
-          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
-        run: |
-          set -euo pipefail
-          # spx admin init mints a random pair at bootstrap and writes it to
-          # node1's ~/.aws/credentials. The benchmark's built-in key is the one
-          # the loopback configs trust, which this cluster has never heard of.
-          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
-          PERF_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
-          PERF_SECRET_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
-          if [ -z "$PERF_ACCESS_KEY" ] || [ -z "$PERF_SECRET_KEY" ]; then
-            echo "::error::could not read spinifex AWS credentials from ${NODE1_IP}"
-            exit 1
-          fi
-          echo "::add-mask::$PERF_SECRET_KEY"
-          mkdir -p "$PERF_RESULTS_ROOT"
-          PERF_ACCESS_KEY="$PERF_ACCESS_KEY" \
-          PERF_SECRET_KEY="$PERF_SECRET_KEY" \
-          PERF_REGION="$CLUSTER_REGION" \
-            make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
-
-      # Report-only, matching cell-23: the baseline comparison and any
-      # regression gate are future work once run-to-run spread is known from
-      # real bare-metal runs.
-      - uses: actions/upload-artifact@v7
-        if: always()
-        with:
-          name: predastore-perf-3node-${{ github.run_id }}
-          path: ${{ env.PERF_RESULTS_ROOT }}
-          if-no-files-found: warn
-          retention-days: 90
-
-      - name: Clean up results
-        if: always()
-        run: rm -rf "${PERF_RESULTS_ROOT}"
-
-      - name: Teardown
-        if: always()
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          set +e
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          UID_SELF=$(id -u)
-          WS="${GITHUB_WORKSPACE}"
-          reap() {
-            local sig="$1"
-            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
-              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
-              case "$cwd" in
-                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
-              esac
-            done
-          }
-          reap TERM
-          sleep 2
-          reap KILL
-          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
-          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
-          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
-          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
-          ok "teardown complete"
-
-      - name: Upload tofu log
-        if: always()
-        uses: actions/upload-artifact@v7
-        with:
-          name: tofu-log-perf3node-${{ github.run_id }}
-          path: ${{ env.TOFU_LOG }}
-          if-no-files-found: ignore
-          retention-days: 14

.github/workflows/publish-system-ami.yml

@@ -17,11 +17,10 @@ name: Publish System AMIs
 #   spx admin images import --name spinifex-rds-postgres
 #   spx admin images import --name spinifex-rds-mariadb
 #
-# Publishing overwrites the mirror artifact, so it is a deliberate action, not
-# something every push should trigger: it runs manually, or from release.yml
-# once a tag has passed its gates. The `variant` input scopes which images
-# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
-# DKMS build), so publishing only the CPU AMIs stays cheap.
+# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
+# action, not something every push should trigger. The `variant` input scopes
+# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
+# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
@@ -39,16 +38,6 @@ on:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
         default: false
-  workflow_call:
-    inputs:
-      variant:
-        description: "Which images to build (all|cpu|gpu|rds)"
-        type: string
-        default: all
-      dry_run:
-        description: "Build + convert only; skip the R2 upload"
-        type: boolean
-        default: false
 
 permissions:
   contents: read
@@ -60,7 +49,7 @@ concurrency:
 jobs:
   matrix:
     name: Resolve image matrix
-    runs-on: [self-hosted, ci-single]
+    runs-on: [ self-hosted, ci-single ]
     outputs:
       images: ${{ steps.gen.outputs.images }}
     steps:
@@ -84,7 +73,7 @@ jobs:
   publish:
     name: Build + publish ${{ matrix.name }}
     needs: matrix
-    runs-on: [self-hosted, ci-single]
+    runs-on: [ self-hosted, ci-single ]
     strategy:
       fail-fast: false
       # ci-single is one runner and each GPU build needs ~16G of scratch disk;
@@ -112,26 +101,14 @@ jobs:
           ref: dev
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: dev
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: dev
-          path: bluebottle
-
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
           go-version-file: "spinifex/go.mod"
           # Self-hosted runner: the Go module + build cache already persists on
-          # local disk between runs.
+          # local disk between runs. setup-go's cache-save ("Post Setup Go")
+          # otherwise re-uploads hundreds of MB to the Actions cache backend,
+          # taking longer than the build itself.
           cache: false
 
       - name: Setup Go Workspace
@@ -141,7 +118,7 @@ jobs:
         # "go.work already exists".
         run: |
           rm -f go.work go.work.sum
-          go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+          go work init ./spinifex ./viperblock ./predastore
 
       - name: Install build + upload tooling
         run: |
@@ -158,6 +135,7 @@ jobs:
       - name: Build + publish ${{ matrix.name }} to R2
         working-directory: spinifex
         env:
+          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}

.github/workflows/release.yml

@@ -28,25 +28,13 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          path: bluebottle
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.27.0"
+          go-version: "1.26.5"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -85,18 +73,6 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          path: bluebottle
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -148,13 +124,6 @@ jobs:
   verify:
     needs: release
     runs-on: [ self-hosted, ci-single ]
-    # Shared with the e2e workflows: this job provisions on the same runner
-    # pools and the same hypervisors, so two in flight compete for their disks
-    # and their libvirt guests. Job-level, so the build jobs are not queued
-    # behind a nightly that has nothing to do with them.
-    concurrency:
-      group: e2e-hypervisors
-      cancel-in-progress: false
     timeout-minutes: 25
     env:
       TOFU_LOG: /tmp/release-verify-${{ github.run_id }}.log
@@ -181,13 +150,6 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      # Only the shared shell helpers: this job builds nothing from source, but
-      # it drives the same libvirt provider the e2e cells retry around.
-      - uses: actions/checkout@v7
-        with:
-          sparse-checkout: .github/scripts
-          path: spinifex
-
       - name: Download Release Assets
         uses: actions/download-artifact@v8
         with:
@@ -197,12 +159,11 @@ jobs:
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
       - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
@@ -215,9 +176,8 @@ jobs:
         if: always()
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
-          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
   baremetal:
@@ -247,17 +207,6 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
-  # Refresh the system AMIs on the R2 mirror only once the release is out of
-  # draft. Shares the ci-single runner with `iso`, so it queues behind it.
-  system-images:
-    needs: promote
-    permissions:
-      contents: read
-    uses: ./.github/workflows/publish-system-ami.yml
-    with:
-      variant: all
-    secrets: inherit
-
   # Annotate the (still-draft) release if either gate failed.
   mark-failed:
     needs: [ verify, baremetal ]
@@ -319,18 +268,6 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          path: bluebottle
-
       - name: Checkout Mulga iso-builder
         uses: actions/checkout@v7
         with:
@@ -349,7 +286,7 @@ jobs:
           cache: false
 
       - name: Configure Go workspace
-        run: go work init ./spinifex ./predastore ./viperblock ./northstar ./bluebottle
+        run: go work init ./spinifex ./predastore ./viperblock
 
       - name: Install ISO builder dependencies
         run: |

.github/workflows/segscan-oracle.yml

@@ -51,20 +51,6 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
@@ -87,7 +73,7 @@ jobs:
           go-version-file: "spinifex/go.mod"
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Run Segscan Oracle Test
         env:

.github/workflows/spinifex.yml

@@ -53,68 +53,14 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
+          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      # Restore runs everywhere; the matching save is gated to dev and main, so
-      # a PR reads the base branch's entry without spending a minute uploading
-      # one only that PR would ever read.
-      - name: Restore Go build cache
-        uses: actions/cache/restore@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
-
-      # The EBS provider contract suite needs a real implementation on each
-      # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which
-      # shares no code with viperblock, and nbdkit plus viperblock's plugin
-      # back viperblockd, the provider production actually uses. Without them
-      # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries
-      # the headers and nbdkit.pc that cgo needs to build that plugin, and
-      # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that
-      # judges the published export by something outside this codebase.
-      - name: Install conformance tools
-        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin
-
-      - name: Build viperblock nbdkit plugin
-        run: make -C viperblock go_build_nbd
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Build Spinifex
         run: make -C spinifex build
@@ -123,37 +69,62 @@ jobs:
         run: make -C spinifex manifest-check manifest-lint
 
       - name: Run Tests with Coverage
-        env:
-          SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
-          VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so
-        run: make -C spinifex test-cover GOTESTFLAGS=-count=1
+        run: make -C spinifex test-cover
 
       - name: Run Action Tests
-        run: make -C spinifex test-actions GOTESTFLAGS=-count=1
+        run: make -C spinifex test-actions
 
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
-      # Compiled output stays valid whatever the tests said, and skipping the
-      # save on a red run would leave the next one rebuilding it, so this is
-      # gated on the branch rather than on success.
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
-      - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+  # In-process integration tier (gateway.SetupRoutes over embedded NATS
+  # JetStream). Its own job so it runs in parallel with the unit tiers rather
+  # than adding to their wall time, and it still blocks a PR — unlike e2e.yml's
+  # "Integration (front gate)" job, which is push-triggered against main/dev on
+  # a self-hosted runner and so never gates a PR before merge.
+  integration_tests:
+    name: Integration Tests
+    runs-on: ubuntu-26.04
 
-      - name: Save Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
         with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Integration Tests
+        run: make -C spinifex test-integration
 
   race_detection:
     name: Race Detection
@@ -186,78 +157,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
+          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      # Restore runs everywhere; the matching save is gated to dev and main, so
-      # a PR reads the base branch's entry without spending a minute uploading
-      # one only that PR would ever read.
-      - name: Restore Go build cache
-        uses: actions/cache/restore@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Run Tests with Race Detector
-        run: make -C spinifex test-race GOTESTFLAGS=-count=1
-
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
-      - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
-
-      - name: Save Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
+        run: make -C spinifex test-race
 
   lint_and_security:
-    name: Lint + Security + Integration
+    name: Lint and Security
     runs-on: ubuntu-26.04
 
     steps:
@@ -265,7 +178,6 @@ jobs:
         uses: actions/checkout@v7
         with:
           path: spinifex
-          fetch-depth: 2
 
       - name: Resolve sibling refs
         id: siblings
@@ -288,82 +200,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
+          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      # Restore runs everywhere; the matching save is gated to dev and main, so
-      # a PR reads the base branch's entry without spending a minute uploading
-      # one only that PR would ever read.
-      - name: Restore Go build cache
-        uses: actions/cache/restore@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.13.0
+          version: v2.12.2
           working-directory: spinifex
 
       - name: Govulncheck
         run: make -C spinifex govulncheck
-
-      - name: Run Integration Tests
-        if: ${{ !cancelled() }}
-        run: make -C spinifex test-integration
-
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
-      - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
-
-      - name: Save Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}

.gitignore

@@ -34,11 +34,9 @@ Thumbs.db
 # Environment
 .env
 .venv/
-.cache/
-
-# VScode
-.vscode/
 
 # Test reports
 *-report.txt
-/rds-agent
+
+# Per-worktree golangci-lint cache
+/.cache/

.golangci.yml

@@ -26,7 +26,7 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forbidigo # blocks forbidden identifiers
+    - forbidigo # blocks the legacy JetStream/KV symbols
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
@@ -56,6 +56,7 @@ linters:
     - unconvert # removes unnecessary type conversions
     - usestdlibvars # detects hardcoded values that should use stdlib constants
     - usetesting # catches os.TempDir() etc. in tests — should use t.TempDir()
+    - wastedassign # finds assignments to variables that are never subsequently read
     - whitespace # detects leading and trailing whitespace in functions
 
   settings:
@@ -68,18 +69,6 @@ linters:
             - pkg: math/rand$
               desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
 
-        # The EC2 control plane reaches block storage through ebsprovider, not
-        # by importing the engine. viperblock.New starts a chunk uploader, so
-        # every construction is a writer against a volume viperblockd serves.
-        # Nothing under handlers/ may import viperblock, in any form, tests
-        # included. A types-only import counts: one became an engine import.
-        ec2-control-plane:
-          files:
-            - '**/spinifex/handlers/**'
-          deny:
-            - pkg: github.com/mulgadc/viperblock
-              desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
-
     embeddedstructfieldcheck:
       forbid-mutex: true
 
@@ -91,8 +80,6 @@ linters:
     forbidigo:
       analyze-types: true
       forbid:
-        - pattern: '^panic$'
-          msg: panic terminates the process; return an error or explicitly justify the panic.
         - pattern: '\.JetStream$'
           pkg: '^github\.com/nats-io/nats\.go$'
           msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -106,7 +93,6 @@ linters:
         - G116 # bidi chars — bidichk is more thorough
         - G117 # use of net/http serve functions (we handle TLS separately)
         - G703 # path traversal taint analysis — false positives on config file loading
-        - G704 # SSRF taint analysis — false positives on resolver-derived endpoint URLs
         - G705 # XSS taint analysis — false positives on internal error responses
 
     govet:
@@ -118,7 +104,6 @@ linters:
 
     modernize:
       disable:
-        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
@@ -150,7 +135,7 @@ linters:
         - -ST1020 # godoc on exported types — not enforcing yet
         - -ST1021 # godoc on exported type groups — not enforcing yet
         - -ST1022 # godoc on exported vars/consts — not enforcing yet
-        - -SA4006 # value overwritten — ineffassign catches strictly more of these
+        - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
         - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
 
     testifylint:
@@ -169,7 +154,6 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
-          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions

Makefile

@@ -1,10 +1,10 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
-# GOFIPS140 is a global build-config change: it alters every package's action ID,
-# stdlib included, so a target that omits it shares no build-cache entry with one
-# that sets it. Exported once here rather than per-recipe so the two can't drift.
-export GOFIPS140 := v1.0.0
+# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
+# path, not worktree path, so parallel worktrees of this module collide and
+# surface each other's issues. Scope it per worktree.
+export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
 
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
@@ -63,23 +63,19 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI
 
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
 
 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
 build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
 	@echo -e "\n....Building ecs-agent (static)"
-	CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
-
-build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
-	@echo -e "\n....Building spx-loadgen"
-	go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -103,12 +99,6 @@ import-eks-node-image: ## Build + register the eks-node AMI (requires a running
 publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
 	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
 
-# The Ubuntu/systemd EKS image, as opposed to build-eks-node-image's Alpine one.
-# Composition lives in mkosi-build.sh's image_profiles table, so the image name
-# is the whole interface here.
-build-eks-node-mkosi-image: ## Build the Ubuntu mkosi EKS node image (server+agent; role at first boot)
-	./scripts/mkosi-build.sh --image spinifex-eks-node
-
 build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
 	$(MAKE) build-system-image IMAGE=ecs-agent
 
@@ -149,59 +139,35 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 
 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
-#
-# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
-# most of the machine idle, so pairing them cuts a cold run by about a quarter.
-# -Otarget keeps each gate's output in one block instead of interleaving them.
 preflight:
-	@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
-# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
-# ship on every node (unlike scripts/images/, kept in preflight: a wrong
-# restart decision here is an outage, not asset churn).
-test-build-scripts:
-	@echo -e "\n....Running build/scripts/**/*_test.sh...."
-	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
-	done
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
-	@echo "  test-build-scripts ok"
-
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
-#
-# diskperf/ is here for the same reason: its baseline comparison and probe
-# accounting decide whether the disk-performance gate passes, and that
-# arithmetic has to be covered off-cluster or the gate is trusted untested.
 test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... ./tests/e2e/diskperf/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
 
 # In-process integration tier: the real gateway router against embedded NATS
 # JetStream, with only the daemon-side NATS subjects stubbed.
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
-AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
-# -count=1 is load-bearing: the conformance report is written by the test binary,
-# so a cached pass skips the run, leaves no report, and the cat below fails the
-# target. It also keeps the conformance gate honest — a cached result would mean
-# the check never ran against this commit.
+AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
-	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
 
 generate-aws-model-coverage:
-	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR) -json $(AWS_MODEL_OPERATION_COVERAGE_DIR)/coverage.json
+	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
+	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
 
 aws-model-coverage: generate-aws-model-coverage
-	@go run ./cmd/aws-model-coverage
+	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
 
 # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
 # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
@@ -232,30 +198,25 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
-
-# Empty locally, where reusing a cached result between runs is the point.
-# CI passes -count=1 so a green run means the suite executed against that commit
-# rather than replaying results the persisted build cache carried over.
-GOTESTFLAGS ?=
+	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
 test-actions:
 	@echo -e "\n....Running action tests...."
-	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 60s ./.github/actions/...
+	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
 # Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
 # helpers baked into system images. Kept out of `preflight` (a dedicated CI
@@ -294,10 +255,6 @@ test-images:
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
-# Check that newly added test files use an external test package
-test-package-check:
-	@QUIET=$(QUIET) scripts/check-test-package.sh
-
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
@@ -352,9 +309,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -437,7 +394,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -3,21 +3,19 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
 </p>
 
 <p align="center">
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS Compatibility</a> ·
-  <a href="#live-demo">Live Demo</a> ·
+  <a href="#aws-compatibility">AWS services</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#self-hosted-installation">Installation</a> ·
+  <a href="#installation">Installation</a> ·
   <a href="https://docs.mulgadc.com">Docs</a>
 </p>
 
@@ -31,9 +29,11 @@ Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
 
 Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Built for teams that need:
+Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
 
-- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+It's built for teams that need:
+
+- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
 - Full control of the stack: your hardware, your network, your data, your keys
 - A cloud that keeps running offline, through disconnection, with no external control plane
 - An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
@@ -48,69 +48,51 @@ You change *where* your software runs, not *what* it is.
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
-## Three Ways To Deploy
+## Three ways to deploy
 
-The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
+One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
+Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
+Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
 
 ### Edge
 
 Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
-## AWS Compatibility
+## AWS compatibility
 
-Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
+Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
 | **EC2** | Compute | Available |
-| **EBS** | Block Storage | Available |
-| **S3** | Object Storage | Available |
+| **EBS** | Block storage | Available |
+| **S3** | Object storage | Available |
 | **VPC** | Networking | Available |
-| **IAM** | Identity & Auth | Available |
-| **ALB / NLB** | Load Balancers | Available |
+| **IAM** | Identity & auth | Available |
+| **ALB / NLB** | Load balancers | Available |
 | **EKS** | Kubernetes | Available |
-| **ECR** | Container Registry | Available |
-| **ECS** | Container Service | Available |
+| **ECR** | Container registry | Available |
+| **ECS** | Container service | Available |
 | **RDS** | Databases | Available |
-| **Bedrock** | AI Deployment | Q4 2026 |
+| **Bedrock** | AI Deployment | Q3 2026 |
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
-## Live Demo
-
-Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
-
-* Live in under a minute. No card, no install.
-* Yours for 72 hours to test and experiment.
-* Real AWS-compatible endpoint, access keys and console.
-* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
-* Point your existing AWS CLI or Terraform straight at it.
-
-Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
-
 ## Core Components
 
-### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
+### Spinifex (Compute Service – EC2 Alternative)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
 
-The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
-
-- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
-- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
-- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
-- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
-- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
-- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
-- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
+- EC2-like VM management on bare metal
+- Launches with cloud-init metadata support
+- Works with standard AWS tooling
 
 ### Viperblock (Block Storage – EBS Alternative)
 
@@ -119,25 +101,17 @@ The managed services build on that same instance layer. An EKS node group, an EC
 - Fast, durable virtual disks
 - Replication for resilience
 - Exposed over NBD or embedded in VMs
-- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
-- In-memory read/write block cache
+- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
+- In memory read/write block cache for blazing performance.
 
 ### Predastore (Object Storage – S3-Compatible)
 
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
+[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
 
 - S3-compatible API and auth
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
-### Northstar (Authoritative DNS)
-
-[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
-
-- Authoritative DNS over UDP, TCP, and DNS-over-TLS
-- Service discovery for cluster components via SRV records
-- Zone files stored locally or in S3-compatible storage
-
 ## Architecture at a Glance
 
 <p align="center">
@@ -146,9 +120,18 @@ The managed services build on that same instance layer. An EKS node group, an EC
 
 Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Self-hosted Installation
+## Key Features
 
-Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
+- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
+- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
+- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
+- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
+- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
+- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
+
+## Installation
+
+Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
 
 ### Bare Metal ISO
 
@@ -158,11 +141,16 @@ The recommended installation is a [bootable x86 installer](https://iso.mulgadc.c
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
+
 
 ### Single Node Install
 
->*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
+The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
+
+>*Prerequisite:* Linux bridge for networking.
+
+Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -180,23 +168,36 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
+
+### Component Repositories
+
+Spinifex coordinates these independent components:
+
+- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
+- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
+
+Each component can be developed independently. See component-specific documentation for focused development guides.
 
 ## Spinifex UI
 
-Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
 </p>
 
-Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
+The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
 
-- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
-- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
+```bash
+open https://YOUR_NODE_IP:3000
+```
 
-For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
+- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+
+For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
 
 ## Development Philosophy
 
@@ -204,9 +205,16 @@ For the full walkthrough, covering first-time TLS certificate trust, login, and
 
 Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
 
+**Real-World Experience:**
+
+- Production AWS service development and operations
+- Large-scale infrastructure deployment and management
+- Edge computing and resource-constrained environments
+- Enterprise security and compliance requirements
+
 ### AI-Assisted Development
 
-Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
+While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
 
 **How We Use Claude Code:**
 
@@ -222,10 +230,12 @@ Spinifex is architected and implemented by experienced engineers, with **Claude
 - **Performance Optimization**: Real-world performance tuning and benchmarking
 - **Production Operations**: Deployment strategies and operational procedures
 
+This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
+
 ## Trademarks
 
 "AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

SECURITY.md

@@ -1,24 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-Please do not open a public issue for a security vulnerability.
-
-Report it privately, either way works:
-
-- [Open a private security advisory](https://github.com/mulgadc/spinifex/security/advisories/new) on this repository.
-- Email <security@mulgadc.com>.
-
-Please include enough for us to reproduce it: the version or commit, the configuration involved, and the steps that trigger it. If you have a proof of concept, include that too.
-
-## What Happens Next
-
-We aim to acknowledge a report within three business days. We will confirm whether we can reproduce it, agree a fix and a disclosure timeline with you, and credit you in the advisory unless you would rather we did not.
-
-## Scope
-
-Spinifex is one part of the [Mulga](https://mulgadc.com) stack. If a vulnerability spans more than one component, report it once against whichever repository you found it in and we will handle the rest.
-
-## Supported Versions
-
-Fixes land on the latest release and on `main`. Older releases are not patched.

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.27-bookworm AS builder
+FROM golang:1.26-bookworm AS builder
 
 ARG VERSION
 
@@ -16,11 +16,9 @@ COPY predastore/go.mod predastore/go.sum ./predastore/
 COPY viperblock/go.mod viperblock/go.sum ./viperblock/
 COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
 COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
-COPY northstar/go.mod northstar/go.sum ./northstar/
-COPY bluebottle/go.mod bluebottle/go.sum ./bluebottle/
 
 # Generate go.work (gitignored, so not in checkout)
-RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit ../northstar ../bluebottle
+RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit
 
 # Download dependencies (cached layer)
 RUN cd spinifex && go mod download
@@ -29,8 +27,6 @@ RUN cd spinifex && go mod download
 COPY spinifex/ ./spinifex/
 COPY predastore/ ./predastore/
 COPY viperblock/ ./viperblock/
-COPY northstar/ ./northstar/
-COPY bluebottle/ ./bluebottle/
 
 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \

build/scripts/nats-js-watchdog.sh

@@ -18,97 +18,22 @@
 # So a health check alone misses case 2 entirely. Both probes run here, and a
 # sustained failure of either restarts the service. The multi-sample window
 # avoids restarting during transient startup/quorum 503s.
-#
-# Three more guards sit around that decision, because "restart when the
-# probes fail" is not safe on its own:
-#
-#   - Age gate: spinifex-nats.service is Type=simple, so it reports active the
-#     instant the process forks, long before JetStream finishes replaying its
-#     state. Acting before a recovery budget has elapsed restarts a server
-#     that is still coming up, not one that has failed.
-#   - Cooldown: a restart that did not fix things must not be retried by the
-#     very next run, seconds later.
-#   - Escalation: a restart that has not fixed things after several attempts
-#     is not going to fix it on the next one either, and an unrunnable spx
-#     must never be read as proof of a JetStream failure at all.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
 SPX="${SPX_BIN:-/usr/local/bin/spx}"
-SAMPLES="${SAMPLES:-3}"           # consecutive failing samples required before acting
-SAMPLE_GAP="${SAMPLE_GAP:-2}"     # seconds between samples
-PROBE_TIMEOUT="${PROBE_TIMEOUT:-5}"  # seconds for one canary round-trip
+SAMPLES=3        # consecutive failing samples required before acting
+SAMPLE_GAP=2     # seconds between samples
+PROBE_TIMEOUT=5  # seconds for one canary round-trip
 
-# Recovery budget: on casuarina NATS needed 3m49s between "Started" and
-# JetStream actually accepting writes, and the bead recorded ~6min elsewhere.
-# 600s clears both with margin.
-NATS_MIN_AGE="${NATS_MIN_AGE:-600}"
-
-# Minimum time between restarts this watchdog performs, independent of the
-# age gate: closes the case where a restart did not fix things and the very
-# next run pulls the trigger again before the new process could recover.
-RESTART_COOLDOWN="${RESTART_COOLDOWN:-600}"
-
-# A restart that has not fixed things after this many attempts is not going
-# to fix it on the next one either, and looping costs more than stopping.
-RESTART_ESCALATE_AT="${RESTART_ESCALATE_AT:-3}"
-
-STATE_DIR="${STATE_DIR:-/run/spinifex}"
-COOLDOWN_STAMP="$STATE_DIR/nats-js-watchdog.last-restart"
-RESTART_COUNT_FILE="$STATE_DIR/nats-js-watchdog.restart-count"
-# Marks that the escalation message has already been printed for the current
-# latch, so a node that stays broken says so once instead of once per tick.
-ESCALATED_STAMP="$STATE_DIR/nats-js-watchdog.escalated"
-
-# Monotonic clock source, so a wall-clock step during boot (NTP correcting an
-# unset RTC) cannot manufacture a false age or cooldown. Overridable so tests
-# can stub a clock without faking /proc.
-UPTIME_FILE="${UPTIME_FILE:-/proc/uptime}"
-
-uptime_seconds() {
-    awk '{printf "%d", $1}' "$UPTIME_FILE"
-}
-
-# nats_age_seconds prints how long $SERVICE has been running. It reads
-# ExecMainStartTimestampMonotonic rather than is-active: that property goes
-# true the instant Type=simple forks, long before JetStream has replayed its
-# state, so it cannot express the recovery window this gate exists to
-# protect. A unit systemd has never started prints 0, which also covers a
-# stopped/activating unit the way the old is-active guard did.
-nats_age_seconds() {
-    start_usec=$(systemctl show --property=ExecMainStartTimestampMonotonic --value "$SERVICE" 2>/dev/null) || start_usec=""
-    case "$start_usec" in
-        ''|*[!0-9]*) echo 0; return ;;
-    esac
-    if [ "$start_usec" -eq 0 ]; then
-        echo 0
-        return
-    fi
-    echo $(( $(uptime_seconds) - start_usec / 1000000 ))
-}
-
-age=$(nats_age_seconds)
-if [ "$age" -lt "$NATS_MIN_AGE" ]; then
+# Only act on a running service: a stopped/activating unit is systemd's to manage,
+# and try-restart on it would either no-op or fight an in-flight (re)start.
+if ! systemctl is-active --quiet "$SERVICE"; then
     exit 0
 fi
 
-if [ -f "$COOLDOWN_STAMP" ]; then
-    last=$(cat "$COOLDOWN_STAMP" 2>/dev/null) || last=""
-    case "$last" in ''|*[!0-9]*) last=0 ;; esac
-    if [ $(( $(uptime_seconds) - last )) -lt "$RESTART_COOLDOWN" ]; then
-        exit 0
-    fi
-fi
-
-restart_count=$(cat "$RESTART_COUNT_FILE" 2>/dev/null) || restart_count=""
-case "$restart_count" in ''|*[!0-9]*) restart_count=0 ;; esac
-
-# The escalation count is read here but acted on only after the probes, below.
-# Checking it up front would return before the one line that clears it, so a
-# latched watchdog could never observe recovery and never re-arm itself.
-
 # curl is the probe transport; if it is unavailable, do nothing rather than
 # false-restart a healthy server.
 if ! command -v curl >/dev/null 2>&1; then
@@ -120,22 +45,13 @@ js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
-# Round-trips a canary key through the cluster-state bucket. Exit 0 means the
-# write succeeded and exit 1 means JetStream refused it; anything else
-# (spx's own exit 2, a missing binary, or one that cannot run at all) means
-# the probe observed nothing about JetStream and must never be read as
-# evidence of a failure.
+# Round-trips a canary key through the cluster-state bucket. Absent spx this
+# degrades to the health check alone rather than restarting on a missing tool.
 js_writable() {
     if [ ! -x "$SPX" ]; then
-        return 2
+        return 0
     fi
     "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
-    rc=$?
-    case "$rc" in
-        0) return 0 ;;
-        1) return 1 ;;
-        *) return 2 ;;
-    esac
 }
 
 failure=""
@@ -143,18 +59,6 @@ i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
         if js_writable; then
-            wrc=0
-        else
-            wrc=$?
-        fi
-        if [ "$wrc" -eq 0 ]; then
-            # Recovery: drop the history so a later failure gets the full
-            # restart budget again, and re-arm the escalation message.
-            rm -f "$RESTART_COUNT_FILE" "$ESCALATED_STAMP"
-            exit 0
-        fi
-        if [ "$wrc" -eq 2 ]; then
-            echo "nats-js-watchdog: js-probe inconclusive (config, connect, bucket-open, or spx itself failed); not evidence of a JetStream failure, taking no action" >&2
             exit 0
         fi
         failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
@@ -167,24 +71,5 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
-# The failure is real and current, so the restart history now means what it
-# says. Above the threshold another restart will not help, so stop acting —
-# but keep probing on later runs, because that is what notices a recovery.
-if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
-    if [ ! -f "$ESCALATED_STAMP" ]; then
-        mkdir -p "$STATE_DIR"
-        : > "$ESCALATED_STAMP"
-        echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2
-    fi
-    exit 0
-fi
-
 echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
-mkdir -p "$STATE_DIR"
-uptime_seconds > "$COOLDOWN_STAMP"
-restart_count=$((restart_count + 1))
-echo "$restart_count" > "$RESTART_COUNT_FILE"
-if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
-    echo "nats-js-watchdog: restart $restart_count of $RESTART_ESCALATE_AT before escalation; if $SERVICE is still failing after this, the watchdog will stop acting until $RESTART_COUNT_FILE is cleared" >&2
-fi
 systemctl try-restart "$SERVICE"

build/scripts/nats-js-watchdog_test.sh

@@ -1,197 +0,0 @@
-#!/bin/sh
-# Self-contained POSIX test for nats-js-watchdog.sh's restart-decision logic:
-# the age gate, cooldown, escalation, and js-probe exit-2 handling. No real
-# NATS or spx: systemctl, curl, and spx are all stubbed on PATH ahead of the
-# real ones, driven entirely by env vars this test controls.
-#
-# Run: sh build/scripts/nats-js-watchdog_test.sh
-set -eu
-
-SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
-SCRIPT="${SCRIPT_DIR}/nats-js-watchdog.sh"
-WORK=$(mktemp -d)
-trap 'rm -rf "${WORK}"' EXIT
-
-FAKE_BIN="${WORK}/bin"
-mkdir -p "${FAKE_BIN}"
-
-FAILS=0
-fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
-pass() { echo "ok: $*"; }
-
-# Fake systemctl: `show` reports a controllable ExecMainStartTimestampMonotonic,
-# and `try-restart` just records that it was called rather than touching a
-# real unit.
-cat > "${FAKE_BIN}/systemctl" <<'EOF'
-#!/bin/sh
-case "$1" in
-    show)
-        echo "${FAKE_NATS_START_USEC:-0}"
-        ;;
-    try-restart)
-        echo "try-restart $2" >> "${FAKE_RESTART_LOG:-/dev/null}"
-        ;;
-esac
-EOF
-chmod +x "${FAKE_BIN}/systemctl"
-
-# Fake curl: js_healthy() only cares about the exit status.
-cat > "${FAKE_BIN}/curl" <<'EOF'
-#!/bin/sh
-[ "${FAKE_HEALTHY:-1}" = "1" ]
-EOF
-chmod +x "${FAKE_BIN}/curl"
-
-# Fake spx: js-probe's whole contract from the watchdog's side is its exit
-# code (0 ok, 1 JetStream refused, 2 inconclusive).
-cat > "${FAKE_BIN}/spx" <<'EOF'
-#!/bin/sh
-exit "${FAKE_SPX_EXIT:-0}"
-EOF
-chmod +x "${FAKE_BIN}/spx"
-
-PATH="${FAKE_BIN}:${PATH}"
-export PATH
-
-# reset_env <name>: fresh per-case dirs and a default env that already clears
-# the age, cooldown, and escalation gates (old enough NATS, no cooldown
-# stamp, no restart history) and a healthy, single-sample probe, so a case
-# only has to override what it is actually testing.
-reset_env() {
-    CASE="${WORK}/$1"
-    mkdir -p "${CASE}/state"
-    UPTIME_FILE="${CASE}/uptime"
-    printf '700.00 0.00\n' > "${UPTIME_FILE}"
-    STATE_DIR="${CASE}/state"
-    FAKE_RESTART_LOG="${CASE}/restart.log"
-    FAKE_NATS_START_USEC=1000000   # 1s: 700s uptime - 1s start = 699s age, clears the 600s default gate
-    FAKE_HEALTHY=1
-    FAKE_SPX_EXIT=0
-    SAMPLES=1
-    SAMPLE_GAP=0
-    NATS_MIN_AGE=600
-    RESTART_COOLDOWN=600
-    RESTART_ESCALATE_AT=3
-    SPX_BIN="${FAKE_BIN}/spx"
-    STDOUT="${CASE}/stdout"
-    STDERR="${CASE}/stderr"
-    export UPTIME_FILE STATE_DIR FAKE_RESTART_LOG FAKE_NATS_START_USEC \
-        FAKE_HEALTHY FAKE_SPX_EXIT SAMPLES SAMPLE_GAP NATS_MIN_AGE \
-        RESTART_COOLDOWN RESTART_ESCALATE_AT SPX_BIN
-}
-
-# Run the watchdog, capturing its exit code (without tripping set -e) + streams.
-invoke() { rc=0; sh "${SCRIPT}" >"${STDOUT}" 2>"${STDERR}" || rc=$?; }
-
-restarted() { [ -s "${FAKE_RESTART_LOG}" ]; }
-
-# --- Case 1: NATS younger than the age gate -> no restart ---
-reset_env young
-FAKE_NATS_START_USEC=690000000  # 690s: 700s uptime - 690s start = 10s age, well under 600s
-export FAKE_NATS_START_USEC
-invoke
-[ "${rc}" -eq 0 ] && pass "young: exit 0" || fail "young: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "young: must not restart while still in the recovery window" || pass "young: no restart"
-
-# --- Case 2: cooldown stamp fresh -> no restart ---
-reset_env cooldown
-echo 650 > "${STATE_DIR}/nats-js-watchdog.last-restart"
-invoke
-[ "${rc}" -eq 0 ] && pass "cooldown: exit 0" || fail "cooldown: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "cooldown: must not restart again inside the cooldown window" || pass "cooldown: no restart"
-
-# --- Case 3: escalation threshold reached while still failing -> no restart,
-# loud log. The probe must fail: a healthy probe at the threshold is case 3b,
-# and it re-arms rather than giving up. ---
-reset_env escalated
-FAKE_SPX_EXIT=1
-export FAKE_SPX_EXIT
-echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-[ "${rc}" -eq 0 ] && pass "escalated: exit 0" || fail "escalated: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "escalated: must not restart past the escalation threshold" || pass "escalated: no restart"
-grep -q 'giving up rather than looping' "${STDERR}" \
-    && pass "escalated: loud log emitted" || fail "escalated: escalation log missing: $(cat "${STDERR}")"
-
-# Second identical run: the node is still broken, so the decision stands, but
-# repeating the message every timer tick is what buried the real signal for 42
-# hours on node1.
-invoke
-grep -q 'giving up rather than looping' "${STDERR}" \
-    && fail "escalated: message repeated on every run instead of once per latch" \
-    || pass "escalated: message not repeated"
-
-# --- Case 3b: escalation threshold reached but JetStream has recovered ->
-# the watchdog must re-arm itself. This is the production bug: the escalation
-# check used to run before the probe, so the clear below was unreachable and
-# the node stayed disarmed until reboot. ---
-reset_env escalated_recovered
-echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-: > "${STATE_DIR}/nats-js-watchdog.escalated"
-invoke
-[ "${rc}" -eq 0 ] && pass "escalated-recovered: exit 0" || fail "escalated-recovered: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "escalated-recovered: must not restart a recovered server" || pass "escalated-recovered: no restart"
-[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
-    && fail "escalated-recovered: restart count must clear once JetStream is writable again" \
-    || pass "escalated-recovered: restart count cleared"
-[ -f "${STATE_DIR}/nats-js-watchdog.escalated" ] \
-    && fail "escalated-recovered: escalation marker must clear so a later failure is reported" \
-    || pass "escalated-recovered: escalation marker cleared"
-[ -s "${STDERR}" ] \
-    && fail "escalated-recovered: a recovered node must be silent: $(cat "${STDERR}")" \
-    || pass "escalated-recovered: silent"
-
-# --- Case 3c: re-armed watchdog acts again. One healthy run clears the latch,
-# and the next genuine failure must restart -- proving it is armed, not just
-# quiet. ---
-reset_env escalated_rearm
-echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-FAKE_SPX_EXIT=1
-export FAKE_SPX_EXIT
-invoke
-restarted && pass "rearm: restarts again after recovery cleared the latch" \
-    || fail "rearm: expected a restart once re-armed: $(cat "${STDERR}")"
-[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "1" ] \
-    && pass "rearm: restart count restarts from 1" \
-    || fail "rearm: expected count 1, got $(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)"
-
-# --- Case 4: js-probe exits 2 (inconclusive) -> no restart ---
-reset_env inconclusive
-FAKE_SPX_EXIT=2
-export FAKE_SPX_EXIT
-invoke
-[ "${rc}" -eq 0 ] && pass "inconclusive: exit 0" || fail "inconclusive: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "inconclusive: exit 2 must never be read as a JetStream failure" || pass "inconclusive: no restart"
-grep -q 'inconclusive' "${STDERR}" \
-    && pass "inconclusive: reason logged" || fail "inconclusive: log missing: $(cat "${STDERR}")"
-
-# --- Case 5: genuine JetStream write failure -> restart happens ---
-reset_env genuine_failure
-FAKE_SPX_EXIT=1
-export FAKE_SPX_EXIT
-echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-[ "${rc}" -eq 0 ] && pass "genuine-failure: exit 0" || fail "genuine-failure: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && pass "genuine-failure: restart triggered" || fail "genuine-failure: expected a restart: $(cat "${STDERR}")"
-[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "3" ] \
-    && pass "genuine-failure: restart count incremented" || fail "genuine-failure: restart count not incremented"
-[ -f "${STATE_DIR}/nats-js-watchdog.last-restart" ] \
-    && pass "genuine-failure: cooldown stamp written" || fail "genuine-failure: cooldown stamp missing"
-
-# --- Case 6: healthy NATS -> no restart, and prior restart history clears ---
-reset_env healthy
-echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-[ "${rc}" -eq 0 ] && pass "healthy: exit 0" || fail "healthy: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "healthy: must not restart a healthy server" || pass "healthy: no restart"
-[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
-    && fail "healthy: restart count must clear once JetStream is confirmed writable" \
-    || pass "healthy: restart count cleared"
-
-if [ "${FAILS}" -eq 0 ]; then
-    echo "PASS: all nats-js-watchdog cases"
-    exit 0
-fi
-echo "FAILED: ${FAILS} case(s)"
-exit 1

build/scripts/wait-for-nats.sh

@@ -26,7 +26,6 @@ fi
 # Final fallback
 NATS_HOST="${NATS_HOST:-127.0.0.1}"
 
-# shellcheck disable=SC2034  # loop count only, index unused
 for i in $(seq 1 "$TIMEOUT"); do
     if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
         exit 0

build/systemd/regenerate-ssh-host-keys.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Regenerate SSH host keys
 ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key

build/systemd/spinifex-awsgw.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target

build/systemd/spinifex-daemon.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -39,13 +38,12 @@ PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
-DeviceAllow=/dev/vhost-net rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS
-# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall
+# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
+# ProtectKernelTunables=yes would otherwise keep read-only.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf
 
 # RG-9: hardening baseline
 ProtectHome=yes

build/systemd/spinifex-firewall.service

@@ -1,25 +0,0 @@
-# spinifex-unit-version: 1
-[Unit]
-Description=Spinifex host firewall (nftables input policy)
-Documentation=file:/etc/spinifex/firewall/spinifex.nft
-Wants=network-pre.target
-Before=network-pre.target shutdown.target
-DefaultDependencies=no
-ConditionPathExists=/etc/spinifex/firewall/peers.nft
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-StandardInput=null
-ProtectSystem=full
-ProtectHome=true
-ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
-ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
-TimeoutStopSec=10s
-
-# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD
-# rules with it. Remove the policy deliberately with
-# `nft delete table inet spinifex_filter`.
-
-[Install]
-WantedBy=sysinit.target

build/systemd/spinifex-guests.slice

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex guest VM tier
 Before=slices.target

build/systemd/spinifex-nats-watchdog.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog
 PartOf=spinifex.target
@@ -16,10 +15,6 @@ ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
 # cutting it off. Only the failing path runs that long; a healthy node exits on
 # the first sample.
 TimeoutStopSec=45
-# Same 34s worst case bounds the start side: without this the unit inherits
-# oneshot's infinite default, so a genuinely hung run would sit in
-# activating(start) forever and could not be retriggered by its timer.
-TimeoutStartSec=120
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog timer
 PartOf=spinifex.target

build/systemd/spinifex-nats.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS Message Bus
 PartOf=spinifex.target

build/systemd/spinifex-northstar.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Northstar (Authoritative DNS)
 PartOf=spinifex.target

build/systemd/spinifex-predastore.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -7,9 +6,9 @@ Wants=spinifex-nats.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
-# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
-# instead, and it can see whether a trace errored. A head ratio decides before
-# the error exists, so it discarded failures at the same rate as successes.
+# Guest block I/O arrives with no caller trace; sample those roots at 5% so
+# they don't drown request traces. Request-linked spans are always kept.
+Environment=MULGA_ROOT_TRACE_RATIO=0.05
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -18,9 +17,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
-# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
-# predastore, but After= already stops viperblock first (within its own 150s
-# budget); this is the backstop if that ordering is ever bypassed.
+# Matches viperblock's drain budget: viperblock's WAL flush writes into
+# predastore, and After= already stops viperblock first, but this is the
+# backstop if that ordering is ever bypassed (direct stop, future edit).
 TimeoutStopSec=30
 OOMScoreAdjust=-500
 # Legitimately the most memory-hungry infra service (S3 buffering/caching);

build/systemd/spinifex-qmp-collector.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex QMP Telemetry Collector (guest metrics)
 PartOf=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex graceful guest drain
 PartOf=spinifex.target
@@ -12,11 +11,11 @@ RemainAfterExit=yes
 ExecStart=/bin/true
 # ExecStop fires on ANY target stop, including the stop half of `systemctl
 # restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
-# of whether the host is actually going down. --unless-restarting drains on a
-# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it
-# skips only when a restart of the stack is already queued, so QEMU/nbdkit
-# survive (KillMode=process) and the daemon reconnects on the way back up.
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting
+# of whether the host is actually going down. --only-if-host-stopping drains
+# only when systemd itself is unwinding into shutdown.target; a plain restart
+# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
+# (KillMode=process) and the daemon reconnects to them on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-system.slice

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
 Before=slices.target

build/systemd/spinifex-ui.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target
@@ -24,7 +23,6 @@ MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
 Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
-Environment=SPINIFEX_UI_OCHRE_ENABLED=false
 
 # Filesystem access
 NoNewPrivileges=yes

build/systemd/spinifex-viperblock.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
@@ -24,10 +23,8 @@ MemoryMax=25%
 # way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
 # that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
 KillMode=process
-# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)
-# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must
-# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.
-TimeoutStopSec=150
+# Bounds this process's own WAL drain, which the plugin caps at 20s.
+TimeoutStopSec=30
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
@@ -51,8 +48,7 @@ ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
-# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's
-# QEMU /proc entries (different uid) orphan nbdkit reaping must see.
+ProtectProc=invisible
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK

build/systemd/spinifex-vpcd.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target

build/systemd/spinifex.slice

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex resource-bounded slice (host ceiling)
 Before=slices.target

build/systemd/spinifex.target

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target

cmd/aws-model-coverage/main.go

@@ -1,48 +1,14 @@
 package main
 
 import (
-	"flag"
 	"fmt"
 	"os"
 
-	"github.com/mulgadc/predastore/s3api"
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {
-	outputDir := flag.String("out", "", "write one publishable page per service into this directory")
-	jsonOutput := flag.String("json", "", "write a machine-readable coverage inventory to this file")
-	flag.Parse()
-
-	coverages, err := compareAll()
-	if err != nil {
-		fail(err)
-	}
-	if *outputDir == "" && *jsonOutput == "" {
-		fmt.Print(awsmodel.RenderCoverageSummary(coverages))
-		return
-	}
-	if *outputDir != "" {
-		if err := awsmodel.WritePages(*outputDir, coverages); err != nil {
-			fail(err)
-		}
-	}
-	if *jsonOutput != "" {
-		if err := awsmodel.WriteCoverageJSON(*jsonOutput, coverages); err != nil {
-			fail(err)
-		}
-	}
-}
-
-func fail(err error) {
-	fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
-	os.Exit(1)
-}
-
-func compareAll() ([]awsmodel.OperationCoverage, error) {
 	dispatch := gateway.AWSOperationInventory()
 	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
 	for _, service := range awsmodel.Services() {
@@ -55,19 +21,23 @@ func compareAll() ([]awsmodel.OperationCoverage, error) {
 				Stubbed:     inventory.Stubbed,
 				Unsupported: inventory.Unsupported,
 			}
-		// Predastore serves the S3 REST surface, and names the operations each
-		// of its routes answers, so the comparison is the same one.
 		case service == awsmodel.S3:
-			modelInventory = awsmodel.DispatchInventory{Registered: s3api.Operations()}
+			modelInventory = awsmodel.DispatchInventory{
+				Opaque: true,
+				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
+			}
 		default:
-			return nil, fmt.Errorf("no gateway inventory for %s", service)
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
+			os.Exit(1)
 		}
 
 		coverage, err := awsmodel.CompareOperations(service, modelInventory)
 		if err != nil {
-			return nil, err
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+			os.Exit(1)
 		}
 		coverages = append(coverages, coverage)
 	}
-	return coverages, nil
+
+	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
 }

cmd/dhcptest/main.go

@@ -24,8 +24,6 @@ import (
 	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/ecr-credential-provider/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 const (

cmd/ecs-agent/agent.go

@@ -30,13 +30,10 @@ type Agent struct {
 	cfg      config
 	id       identity
 	cp       controlPlane
+	puller   ctrruntime.ImagePuller
+	runner   ctrruntime.Runner
 	resolver ctrruntime.Resolver
 
-	// rt holds the container runtime, which is resolved by a loop rather than
-	// once at boot. Empty until the containerd socket answers, and emptied again
-	// if it stops answering.
-	rt *runtimeGate
-
 	reg *registrar
 	hb  *heartbeater
 
@@ -51,34 +48,26 @@ type Agent struct {
 	// nvidia-smi discovery.
 	gpu *gpuLedger
 
-	// dialRuntime connects to the container runtime. nil in unit tests, which
-	// drive the gate directly.
-	dialRuntime func() (ctrruntime.Runtime, error)
-
 	closers []func() error
 }
 
 // newAgent assembles an Agent from already-built seams. Tests use this directly
-// with fakes; New builds the production seams and delegates here. rt may be nil
-// when containerd is unavailable; the assign path then reports the task STOPPED
-// rather than crashing, and the watch loop keeps dialling for it.
-func newAgent(cfg config, id identity, cp controlPlane, rt ctrruntime.Runtime, resolver ctrruntime.Resolver) *Agent {
-	gate := newRuntimeGate(rt)
-	a := &Agent{
+// with fakes; New builds the production seams and delegates here. runner may be
+// nil when containerd is unavailable; the assign path then reports the task
+// STOPPED rather than crashing.
+func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImagePuller, runner ctrruntime.Runner, resolver ctrruntime.Resolver) *Agent {
+	return &Agent{
 		cfg:      cfg,
 		id:       id,
 		cp:       cp,
+		puller:   puller,
+		runner:   runner,
 		resolver: resolver,
-		rt:       gate,
+		reg:      newRegistrar(cp, id),
+		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
 		netns:    newTaskNetns(execNetRunner{}),
 		gpu:      newGPULedger(id.Capacity.GPUIDs),
 	}
-	// Registration reports the host's capacity only while a runtime is there to
-	// use it, so an agent that cannot run a container advertises none and takes
-	// no placement.
-	a.reg = newRegistrar(cp, id, gate.available)
-	a.hb = newHeartbeater(cp, id, cfg.Heartbeat, gate.available)
-	return a
 }
 
 // New builds an Agent from config: it resolves the host identity from IMDS,
@@ -100,7 +89,6 @@ func New(cfg config) (*Agent, error) {
 		ClusterName:  cfg.ClusterName,
 		InstanceID:   meta.InstanceID,
 		AZ:           meta.AZ,
-		InstanceType: meta.InstanceType,
 		Hostname:     host,
 		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
 		AgentVersion: version,
@@ -109,12 +97,13 @@ func New(cfg config) (*Agent, error) {
 	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
 	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
 
-	// A dial that fails here is not fatal and not final: the node starts the
-	// container daemon and the agent back to back, so the socket is often not
-	// accepting yet. Run's watch loop keeps dialling.
-	rt, perr := ctrruntime.New(cfg.ContainerdSocket)
-	if perr != nil {
-		slog.Warn("ecs-agent: containerd unavailable at boot, will keep retrying", "err", perr)
+	var puller ctrruntime.ImagePuller
+	var runner ctrruntime.Runner
+	if rt, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
+		slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
+	} else {
+		puller = rt
+		runner = rt
 	}
 
 	cp, err := newGatewayControlPlane(cfg, creds)
@@ -122,10 +111,12 @@ func New(cfg config) (*Agent, error) {
 		return nil, fmt.Errorf("build gateway control-plane: %w", err)
 	}
 
-	a := newAgent(cfg, id, cp, rt, resolver)
+	a := newAgent(cfg, id, cp, puller, runner, resolver)
 	a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
 		cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
-	a.dialRuntime = func() (ctrruntime.Runtime, error) { return ctrruntime.New(cfg.ContainerdSocket) }
+	if puller != nil {
+		a.closers = append(a.closers, puller.Close)
+	}
 	return a, nil
 }
 
@@ -146,12 +137,6 @@ func (a *Agent) Run(ctx context.Context) error {
 
 	go a.hb.Run(ctx)
 
-	// Keeps the runtime current for the life of the process: dials while there
-	// is none, and drops one that stops answering.
-	if a.dialRuntime != nil {
-		go a.rt.watch(ctx, a.dialRuntime, a.cfg.RuntimeRetry)
-	}
-
 	// Re-adopt containers still running from before this restart, then poll with
 	// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
 	go a.pollAssignments(ctx, a.reconcile(ctx))
@@ -171,7 +156,6 @@ func (a *Agent) Stop() error {
 			firstErr = err
 		}
 	}
-	a.rt.clear()
 	return firstErr
 }
 

cmd/ecs-agent/agent_test.go

@@ -12,7 +12,7 @@ func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
 	cp := &fakeCP{}
 	cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
 	puller := &ctrruntime.FakePuller{}
-	a := newAgent(cfg, testIdentity(), cp, puller, nil)
+	a := newAgent(cfg, testIdentity(), cp, puller, puller, nil)
 	a.closers = append(a.closers, puller.Close)
 
 	ctx, cancel := context.WithCancel(context.Background())

cmd/ecs-agent/config.go

@@ -33,7 +33,6 @@ type config struct {
 	ContainerdSocket string
 	Heartbeat        time.Duration
 	PollInterval     time.Duration
-	RuntimeRetry     time.Duration
 	CredEndpointIP   string
 	CredEndpointPort int
 }
@@ -76,12 +75,6 @@ func loadConfig(envFile string) config {
 			cfg.PollInterval = d
 		}
 	}
-	cfg.RuntimeRetry = defaultRuntimeRetry
-	if v := get("ECS_RUNTIME_RETRY"); v != "" {
-		if d, err := time.ParseDuration(v); err == nil && d > 0 {
-			cfg.RuntimeRetry = d
-		}
-	}
 	cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
 	if cfg.CredEndpointIP == "" {
 		cfg.CredEndpointIP = defaultCredEndpointIP

cmd/ecs-agent/controlplane.go

@@ -76,10 +76,6 @@ func (g *gatewayControlPlane) Register(id identity) error {
 			{Name: aws.String("MEMORY"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.MemoryMiB))},
 		},
 		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
-		// The built-in container-instance attributes AWS's own agent reports.
-		// The scheduler sizes the host's awsvpc ENI capacity from the instance
-		// type, so a registration without one is not ENI-gated.
-		Attributes: instanceAttributes(id),
 	}
 	if len(id.Capacity.GPUIDs) > 0 {
 		in.TotalResources = append(in.TotalResources, &ecs.Resource{

cmd/ecs-agent/credentials/imds.go

@@ -9,9 +9,9 @@ import (
 )
 
 // IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
-// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
-// which refetches once the real expiry passes. A failed refresh is an error
-// here, not an extended expiry. Safe for concurrent use.
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
+// (which refetches once the real expiry passes; ec2rolecreds also caps
+// Expires to at most an hour out). Safe for concurrent use.
 type IMDSProvider struct {
 	cache *aws.CredentialsCache
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -88,63 +88,6 @@ func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	}
 }
 
-// imdsStubFailAfterFirst serves creds on the first credential request and 500s
-// on every one after it, so a test can drive a refresh that fails against a
-// credential the cache has already stored.
-func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
-	t.Helper()
-	var hits atomic.Int32
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits.Add(1) > 1 {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-// A refresh that fails must surface as an error. Left to itself the SDK's
-// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
-// key with Expires pushed 5-15 minutes out, so the agent would sign requests
-// with a credential the control plane has already rejected — indefinitely,
-// since every subsequent refresh fails the same way.
-func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
-	srv := imdsStubFailAfterFirst(t, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		// Already expired, so the next Retrieve must go back to IMDS.
-		"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	first, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("first Retrieve: %v", err)
-	}
-
-	got, err := p.Retrieve(context.Background())
-	if err == nil {
-		t.Fatalf("refresh failure was reported as success: %+v", got)
-	}
-	if got.Expiration.After(first.Expiration) {
-		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
-	}
-}
-
 func TestRetrieve_CancelledContext(t *testing.T) {
 	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
 	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")

cmd/ecs-agent/ecrresolver.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"net/http"
-	"net/url"
 	"strings"
 	"sync"
 
@@ -61,25 +60,12 @@ func (e *ecrResolver) client() (*http.Client, error) {
 	return c, nil
 }
 
-// gatewayRegistryHost returns the host[:port] of the co-located ECR registry
-// from the gateway URL; empty/unparseable yields "" (matches no ref). The
-// registry shares the gateway listener, so its port is the gateway port.
-func gatewayRegistryHost(gatewayURL string) string {
-	u, err := url.Parse(gatewayURL)
-	if err != nil {
-		return ""
-	}
-	return u.Host
-}
-
-// isECRRef reports whether ref targets an ECR registry: an "<acct>.dkr.ecr."
-// host, or one equal to gatewayHost (the agent's own co-located registry
-// advertised as a bare host/IP). Other refs pull anonymously, no token.
-func isECRRef(ref, gatewayHost string) bool {
+// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
+// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
+// public registries (docker.io and friends) pull anonymously, so refs without
+// an ECR host must not drag in IMDS host credentials.
+func isECRRef(ref string) bool {
 	host, _, _ := strings.Cut(ref, "/")
-	if gatewayHost != "" && host == gatewayHost {
-		return true
-	}
 	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
 		return false // no registry host -> docker.io default
 	}
@@ -89,7 +75,7 @@ func isECRRef(ref, gatewayHost string) bool {
 // Authorize returns the ECR token's user/password and proxy endpoint for ref.
 // Non-ECR refs resolve to anonymous pull (empty credentials).
 func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
-	if !isECRRef(ref, gatewayRegistryHost(e.gatewayURL)) {
+	if !isECRRef(ref) {
 		return "", "", "", nil
 	}
 	c, err := e.creds.Retrieve(ctx)

cmd/ecs-agent/ecrresolver_test.go

@@ -115,44 +115,17 @@ func TestECRResolver_NonECRAnonymous(t *testing.T) {
 }
 
 func TestIsECRRef(t *testing.T) {
-	cases := []struct {
-		ref, gatewayHost string
-		want             bool
-	}{
-		{"nginx:alpine", "192.168.1.13:9999", false},
-		{"docker.io/library/nginx", "192.168.1.13:9999", false},
-		{"ghcr.io/org/app:1", "192.168.1.13:9999", false},
-		{"registry:5000/app:1", "192.168.1.13:9999", false},
-		{"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1", "192.168.1.13:9999", true},
-		{"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x", "192.168.1.13:9999", true},
-		// Co-located gateway registry advertised as a bare host/IP.
-		{"192.168.1.13:9999/bunyip:latest", "192.168.1.13:9999", true},
-		{"192.168.1.13:9999/bunyip:latest", "", false},
+	cases := map[string]bool{
+		"nginx:alpine":            false,
+		"docker.io/library/nginx": false,
+		"ghcr.io/org/app:1":       false,
+		"registry:5000/app:1":     false,
+		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
+		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
 	}
-	for _, c := range cases {
-		if got := isECRRef(c.ref, c.gatewayHost); got != c.want {
-			t.Errorf("isECRRef(%q, %q) = %v, want %v", c.ref, c.gatewayHost, got, c.want)
+	for ref, want := range cases {
+		if got := isECRRef(ref); got != want {
+			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
 		}
 	}
 }
-
-// TestECRResolver_Authorize_BareHostGateway asserts that a ref whose host
-// equals the agent's own gateway host authenticates via the co-located ECR
-// registry, even though it lacks a ".dkr.ecr." name.
-func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
-	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
-	}}
-	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
-
-	host := strings.TrimPrefix(srv.URL, "https://")
-	user, pass, _, err := r.Authorize(context.Background(), host+"/bunyip:latest")
-	if err != nil {
-		t.Fatalf("Authorize: %v", err)
-	}
-	if user != "AWS" || pass != "jwt" {
-		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
-	}
-}

cmd/ecs-agent/heartbeat.go

@@ -14,23 +14,18 @@ type heartbeater struct {
 	cp       controlPlane
 	id       identity
 	interval time.Duration
-	// runtimeReady reports whether a container runtime is resolved, so a beat
-	// advertises capacity only while there is something to run it with. nil
-	// means always ready.
-	runtimeReady func() bool
 }
 
-func newHeartbeater(cp controlPlane, id identity, interval time.Duration, runtimeReady func() bool) *heartbeater {
+func newHeartbeater(cp controlPlane, id identity, interval time.Duration) *heartbeater {
 	if interval <= 0 {
 		interval = defaultHeartbeat
 	}
-	return &heartbeater{cp: cp, id: id, interval: interval, runtimeReady: runtimeReady}
+	return &heartbeater{cp: cp, id: id, interval: interval}
 }
 
 // beat re-registers the instance once, refreshing its LastSeen.
 func (h *heartbeater) beat() error {
-	ready := h.runtimeReady == nil || h.runtimeReady()
-	return h.cp.Register(advertisedIdentity(h.id, ready))
+	return h.cp.Register(h.id)
 }
 
 // Run beats every interval until ctx is cancelled. A failed beat is logged and

cmd/ecs-agent/main.go

@@ -17,7 +17,7 @@ import (
 	"os/signal"
 	"syscall"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {

cmd/ecs-agent/metadata.go

@@ -13,10 +13,9 @@ import (
 
 // instanceMetadata is the IMDS-derived identity of the host the agent runs on.
 type instanceMetadata struct {
-	AccountID    string
-	InstanceID   string
-	AZ           string
-	InstanceType string
+	AccountID  string
+	InstanceID string
+	AZ         string
 }
 
 // fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
@@ -40,14 +39,10 @@ func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata,
 		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
 	}
 
-	// The instance type comes off the identity document already in hand rather
-	// than a further meta-data read; the scheduler sizes the host's awsvpc ENI
-	// capacity from it.
 	return instanceMetadata{
-		AccountID:    doc.AccountID,
-		InstanceID:   strings.TrimSpace(instanceID),
-		AZ:           strings.TrimSpace(az),
-		InstanceType: strings.TrimSpace(doc.InstanceType),
+		AccountID:  doc.AccountID,
+		InstanceID: strings.TrimSpace(instanceID),
+		AZ:         strings.TrimSpace(az),
 	}, nil
 }
 

cmd/ecs-agent/reconcile.go

@@ -24,11 +24,10 @@ import (
 // agent falls back to its pre-reconcile behaviour (poll + run).
 func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 	adopted := map[string]bool{}
-	runner := a.rt.get()
-	if runner == nil {
+	if a.runner == nil {
 		return adopted
 	}
-	containers, err := runner.List(ctx)
+	containers, err := a.runner.List(ctx)
 	if err != nil {
 		slog.Warn("ecs-agent: reconcile list failed", "err", err)
 		return adopted
@@ -97,13 +96,9 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
 // is STOPPED, so the scheduler stops counting a task this instance is not
 // running and its service can place a replacement.
 func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
-	runner := a.rt.get()
 	statuses := make([]bus.ContainerStatus, 0, len(dead))
 	for _, c := range dead {
-		if runner == nil {
-			break
-		}
-		if err := runner.Remove(ctx, c.ID); err != nil {
+		if err := a.runner.Remove(ctx, c.ID); err != nil {
 			slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
 		}
 		statuses = append(statuses, bus.ContainerStatus{

cmd/ecs-agent/reconcile_test.go

@@ -55,7 +55,7 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 			adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 
 	adopted := a.reconcile(context.Background())
@@ -84,7 +84,7 @@ func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
 			adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 
 	adopted := a.reconcile(context.Background())
@@ -116,7 +116,7 @@ func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
 			adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 
 	a.reconcile(context.Background())
@@ -145,7 +145,7 @@ func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
 		nil,
 	}}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{"t-001": true})
@@ -197,7 +197,7 @@ func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
 			adoptedContainer("t-ok", "web", "default", "", "", "", true),
 		},
 	}
-	a := newAgent(config{}, testIdentity(), cp, rt, nil)
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 
 	adopted := a.reconcile(context.Background())
 
@@ -213,13 +213,13 @@ func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
 
 // A nil runner or a List error degrades to an empty set without panicking.
 func TestReconcile_DegradesGracefully(t *testing.T) {
-	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil)
+	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil, nil)
 	if got := noRunner.reconcile(context.Background()); len(got) != 0 {
 		t.Errorf("nil runner: adopted = %v, want empty", got)
 	}
 
 	rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
-	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, nil)
+	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
 	if got := listErr.reconcile(context.Background()); len(got) != 0 {
 		t.Errorf("list error: adopted = %v, want empty", got)
 	}

cmd/ecs-agent/registration.go

@@ -3,9 +3,6 @@ package main
 import (
 	"fmt"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ecs"
-	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
@@ -17,61 +14,25 @@ type identity struct {
 	ClusterName  string
 	InstanceID   string
 	AZ           string
-	InstanceType string
 	Hostname     string
 	Capacity     bus.InstanceCapacity
 	AgentVersion string
 }
 
-// instanceAttributes builds the built-in container-instance attributes from the
-// host's identity. Only attributes whose value was actually discovered are
-// reported, so an empty one is absent rather than present and blank.
-func instanceAttributes(id identity) []*ecs.Attribute {
-	var attrs []*ecs.Attribute
-	add := func(name, value string) {
-		if value == "" {
-			return
-		}
-		attrs = append(attrs, &ecs.Attribute{Name: aws.String(name), Value: aws.String(value)})
-	}
-	add(handlers_ecs.AttrInstanceType, id.InstanceType)
-	add(handlers_ecs.AttrAvailabilityZone, id.AZ)
-	return attrs
-}
-
-// advertisedIdentity is the identity to register with, given whether a
-// container runtime is available. A host with no runtime can run nothing, so it
-// advertises no capacity: the scheduler places nothing on it, and the control
-// plane reads the zero as an involuntary drain. The identity is otherwise
-// unchanged, so the instance stays visible rather than disappearing.
-func advertisedIdentity(id identity, runtimeReady bool) identity {
-	if runtimeReady {
-		return id
-	}
-	id.Capacity = bus.InstanceCapacity{}
-	return id
-}
-
 // registrar registers the host as a container instance through the gateway.
 type registrar struct {
 	cp controlPlane
 	id identity
-	// runtimeReady reports whether a container runtime is resolved. nil means
-	// always ready, which is what a test with no gate wants.
-	runtimeReady func() bool
 }
 
-func newRegistrar(cp controlPlane, id identity, runtimeReady func() bool) *registrar {
-	return &registrar{cp: cp, id: id, runtimeReady: runtimeReady}
+func newRegistrar(cp controlPlane, id identity) *registrar {
+	return &registrar{cp: cp, id: id}
 }
 
-// ready reports the runtime state, defaulting to ready when no gate is wired.
-func (r *registrar) ready() bool { return r.runtimeReady == nil || r.runtimeReady() }
-
 // Register calls the gateway's RegisterContainerInstance. The scheduler records
 // (or refreshes) the container instance on receipt.
 func (r *registrar) Register() error {
-	if err := r.cp.Register(advertisedIdentity(r.id, r.ready())); err != nil {
+	if err := r.cp.Register(r.id); err != nil {
 		return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
 	}
 	return nil

cmd/ecs-agent/registration_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"errors"
-	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -18,16 +17,14 @@ import (
 type fakeCP struct {
 	mu sync.Mutex
 
-	registers      int
-	lastRegistered identity
-	states         []bus.TaskState
-	gpuReports     []gpuReport
-	pollAcks       [][]string
-	pollStopAcks   [][]string
-	pollReplies    [][]bus.Assign
-	stopReplies    [][]bus.StopDirective
-	stopPending    []bus.StopDirective
-	pollCalls      int
+	registers    int
+	states       []bus.TaskState
+	gpuReports   []gpuReport
+	pollAcks     [][]string
+	pollStopAcks [][]string
+	pollReplies  [][]bus.Assign
+	stopReplies  [][]bus.StopDirective
+	pollCalls    int
 
 	registerErr bool
 	submitErr   bool
@@ -42,24 +39,16 @@ type gpuReport struct {
 
 var _ controlPlane = (*fakeCP)(nil)
 
-func (f *fakeCP) Register(id identity) error {
+func (f *fakeCP) Register(_ identity) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	if f.registerErr {
 		return errors.New("register boom")
 	}
 	f.registers++
-	f.lastRegistered = id
 	return nil
 }
 
-// registered returns the identity of the most recent Register call.
-func (f *fakeCP) registered() identity {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return f.lastRegistered
-}
-
 func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -83,12 +72,6 @@ func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]
 	if f.pollCalls < len(f.stopReplies) {
 		stops = f.stopReplies[f.pollCalls]
 	}
-	// stopPending is the durable inbox the gateway keeps: an acked directive is
-	// deleted, and everything still in it is redelivered on every poll.
-	f.stopPending = slices.DeleteFunc(f.stopPending, func(sd bus.StopDirective) bool {
-		return slices.Contains(ackStops, sd.TaskID)
-	})
-	stops = slices.Concat(stops, f.stopPending)
 	f.pollCalls++
 	return out, stops, nil
 }
@@ -159,7 +142,7 @@ func testIdentityWithGPUs(uuids ...string) identity {
 
 func TestRegistrar_Register(t *testing.T) {
 	cp := &fakeCP{}
-	if err := newRegistrar(cp, testIdentity(), nil).Register(); err != nil {
+	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
 		t.Fatalf("Register: %v", err)
 	}
 	if cp.registerCount() != 1 {
@@ -169,14 +152,14 @@ func TestRegistrar_Register(t *testing.T) {
 
 func TestRegistrar_RegisterError(t *testing.T) {
 	cp := &fakeCP{registerErr: true}
-	if err := newRegistrar(cp, testIdentity(), nil).Register(); err == nil {
+	if err := newRegistrar(cp, testIdentity()).Register(); err == nil {
 		t.Fatal("expected register error")
 	}
 }
 
 func TestHeartbeater_Beat(t *testing.T) {
 	cp := &fakeCP{}
-	if err := newHeartbeater(cp, testIdentity(), time.Second, nil).beat(); err != nil {
+	if err := newHeartbeater(cp, testIdentity(), time.Second).beat(); err != nil {
 		t.Fatalf("beat: %v", err)
 	}
 	if cp.registerCount() != 1 {
@@ -185,7 +168,7 @@ func TestHeartbeater_Beat(t *testing.T) {
 }
 
 func TestHeartbeater_DefaultInterval(t *testing.T) {
-	h := newHeartbeater(&fakeCP{}, testIdentity(), 0, nil)
+	h := newHeartbeater(&fakeCP{}, testIdentity(), 0)
 	if h.interval != defaultHeartbeat {
 		t.Errorf("interval = %v, want default", h.interval)
 	}
@@ -193,7 +176,7 @@ func TestHeartbeater_DefaultInterval(t *testing.T) {
 
 func TestHeartbeater_RunStopsOnContext(t *testing.T) {
 	cp := &fakeCP{}
-	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond, nil)
+	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond)
 	ctx, cancel := context.WithCancel(context.Background())
 
 	done := make(chan struct{})
@@ -210,48 +193,3 @@ func TestHeartbeater_RunStopsOnContext(t *testing.T) {
 		t.Error("expected at least one re-register before cancel")
 	}
 }
-
-// TestRegistrar_NoRuntimeAdvertisesNoCapacity is the placement half of the boot
-// bug: an agent with no container runtime kept registering its full CPU and
-// memory, so the scheduler placed tasks it could only fail to start. Capacity is
-// advertised only while a runtime is there to use it.
-func TestRegistrar_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
-	cp := &fakeCP{}
-	ready := false
-	r := newRegistrar(cp, testIdentity(), func() bool { return ready })
-
-	if err := r.Register(); err != nil {
-		t.Fatalf("register: %v", err)
-	}
-	if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
-		t.Fatalf("advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
-	}
-
-	ready = true
-	if err := r.Register(); err != nil {
-		t.Fatalf("register: %v", err)
-	}
-	want := testIdentity().Capacity
-	if got := cp.registered().Capacity; got.CPU != want.CPU || got.MemoryMiB != want.MemoryMiB {
-		t.Fatalf("capacity not restored once the runtime resolved: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
-	}
-}
-
-// TestHeartbeater_NoRuntimeAdvertisesNoCapacity is the same rule on the beat,
-// which is how a runtime lost after boot reaches the control plane.
-func TestHeartbeater_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
-	cp := &fakeCP{}
-	h := newHeartbeater(cp, testIdentity(), time.Second, func() bool { return false })
-
-	if err := h.beat(); err != nil {
-		t.Fatalf("beat: %v", err)
-	}
-	if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
-		t.Fatalf("beat advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
-	}
-	// The instance still identifies itself, so it stays visible as drained rather
-	// than vanishing from the cluster.
-	if cp.registered().InstanceID != testIdentity().InstanceID {
-		t.Error("beat dropped the instance identity")
-	}
-}

cmd/ecs-agent/runtime/containerd.go

@@ -3,7 +3,6 @@ package runtime
 import (
 	"context"
 	"fmt"
-	"sync"
 	"time"
 
 	containerd "github.com/containerd/containerd/v2/client"
@@ -23,11 +22,6 @@ const servingProbeTimeout = 3 * time.Second
 // containerdPuller drives a real containerd daemon over its unix socket.
 type containerdPuller struct {
 	client *containerd.Client
-
-	// stdinDone holds one channel per interactive container, closed when that
-	// container is stopped or removed so its stdin reader returns EOF.
-	mu        sync.Mutex
-	stdinDone map[string]chan struct{}
 }
 
 var _ ImagePuller = (*containerdPuller)(nil)

cmd/ecs-agent/runtime/containerd_run.go

@@ -3,14 +3,9 @@ package runtime
 import (
 	"context"
 	"fmt"
-	"io"
 	"sort"
-	"strings"
-	"syscall"
-	"time"
 
 	containerd "github.com/containerd/containerd/v2/client"
-	"github.com/containerd/containerd/v2/core/containers"
 	"github.com/containerd/containerd/v2/pkg/cdi"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
@@ -46,106 +41,6 @@ func cdiSpecOpts(gpuIDs []string) []oci.SpecOpts {
 	return []oci.SpecOpts{cdi.WithCDIDevices(devices...)}
 }
 
-// ociCapName renders an ECS Linux capability (Docker's bare name, e.g.
-// "SYS_ADMIN") as the OCI runtime-spec name ("CAP_SYS_ADMIN") capabilities.Add/
-// Drop expect.
-func ociCapName(name string) string {
-	if strings.HasPrefix(name, "CAP_") {
-		return name
-	}
-	return "CAP_" + name
-}
-
-// ociCapNames maps a list of ECS capability names to their OCI form.
-func ociCapNames(names []string) []string {
-	if len(names) == 0 {
-		return nil
-	}
-	out := make([]string, len(names))
-	for i, n := range names {
-		out[i] = ociCapName(n)
-	}
-	return out
-}
-
-// withSysctls sets Linux.Sysctl on the OCI spec from systemControls namespace/
-// value pairs. containerd has no built-in SpecOpts for sysctls.
-func withSysctls(ctls []SystemControl) oci.SpecOpts {
-	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
-		if len(ctls) == 0 {
-			return nil
-		}
-		if s.Linux == nil {
-			s.Linux = &specs.Linux{}
-		}
-		if s.Linux.Sysctl == nil {
-			s.Linux.Sysctl = map[string]string{}
-		}
-		for _, c := range ctls {
-			s.Linux.Sysctl[c.Namespace] = c.Value
-		}
-		return nil
-	}
-}
-
-// heldStdin backs an interactive container's stdin: reads block instead of
-// hitting an immediate EOF, which is the "stdin held open" behaviour
-// interactive=true asks for, until done closes and the read reports EOF.
-type heldStdin struct{ done <-chan struct{} }
-
-func (h heldStdin) Read([]byte) (int, error) {
-	<-h.done
-	return 0, io.EOF
-}
-
-// ioStreamOpts describes the task IO for a container. A terminal container needs
-// a terminal FIFO set: oci.WithTTY alone puts Terminal in the OCI spec while the
-// shim creates no console socket, and runc refuses that pair outright. A
-// terminal carries no separate stderr, so that stream stays nil and the shim
-// merges it into the console.
-func ioStreamOpts(stdin io.Reader, terminal bool) []cio.Opt {
-	if terminal {
-		return []cio.Opt{cio.WithStreams(stdin, io.Discard, nil), cio.WithTerminal}
-	}
-	return []cio.Opt{cio.WithStreams(stdin, io.Discard, io.Discard)}
-}
-
-// containerIOCreator returns cio.NullIO for a container needing neither stdin
-// nor a terminal, else a creator carrying whichever it asked for. An interactive
-// container's stdin is held open until releaseStdin; containerd copies stdin on
-// a goroutine that exits only once the read returns, so a reader that never
-// returned would park it for the life of the agent. A terminal container that is
-// not interactive gets no stdin stream at all, which is what a nil reader means.
-func (p *containerdPuller) containerIOCreator(containerID string, interactive, terminal bool) cio.Creator {
-	if !interactive && !terminal {
-		return cio.NullIO
-	}
-	var stdin io.Reader
-	if interactive {
-		done := make(chan struct{})
-		p.mu.Lock()
-		if p.stdinDone == nil {
-			p.stdinDone = make(map[string]chan struct{})
-		}
-		p.stdinDone[containerID] = done
-		p.mu.Unlock()
-		stdin = heldStdin{done: done}
-	}
-	return cio.NewCreator(ioStreamOpts(stdin, terminal)...)
-}
-
-// releaseStdin closes a held stdin so containerd's copier goroutine exits. Safe
-// for a container that never held one, and safe to call more than once.
-func (p *containerdPuller) releaseStdin(containerID string) {
-	p.mu.Lock()
-	done, ok := p.stdinDone[containerID]
-	delete(p.stdinDone, containerID)
-	p.mu.Unlock()
-	if ok {
-		close(done)
-	}
-}
-
 var _ Runner = (*containerdPuller)(nil)
 
 // Run creates a container from an already-pulled image and starts its task. The
@@ -186,26 +81,6 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
 	}
 	specOpts = append(specOpts, cdiSpecOpts(spec.GPUIDs)...)
-	if spec.User != "" {
-		specOpts = append(specOpts, oci.WithUser(spec.User))
-	}
-	if spec.ReadonlyRootFilesystem != nil && *spec.ReadonlyRootFilesystem {
-		specOpts = append(specOpts, oci.WithRootFSReadonly())
-	}
-	if spec.Privileged != nil && *spec.Privileged {
-		specOpts = append(specOpts, oci.WithPrivileged)
-	}
-	terminal := spec.PseudoTerminal != nil && *spec.PseudoTerminal
-	if terminal {
-		specOpts = append(specOpts, oci.WithTTY)
-	}
-	specOpts = append(specOpts, withSysctls(spec.SystemControls))
-	if len(spec.CapAdd) > 0 {
-		specOpts = append(specOpts, oci.WithAddedCapabilities(ociCapNames(spec.CapAdd)))
-	}
-	if len(spec.CapDrop) > 0 {
-		specOpts = append(specOpts, oci.WithDroppedCapabilities(ociCapNames(spec.CapDrop)))
-	}
 
 	container, err := p.client.NewContainer(ctx, id,
 		containerd.WithNewSnapshot(id+"-snapshot", image),
@@ -216,15 +91,12 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 		return "", fmt.Errorf("create container %s: %w", id, err)
 	}
 
-	interactive := spec.Interactive != nil && *spec.Interactive
-	task, err := container.NewTask(ctx, p.containerIOCreator(id, interactive, terminal))
+	task, err := container.NewTask(ctx, cio.NullIO)
 	if err != nil {
-		p.releaseStdin(id)
 		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
 		return "", fmt.Errorf("create task %s: %w", id, err)
 	}
 	if err := task.Start(ctx); err != nil {
-		p.releaseStdin(id)
 		_, _ = task.Delete(ctx)
 		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
 		return "", fmt.Errorf("start task %s: %w", id, err)
@@ -276,35 +148,9 @@ func (p *containerdPuller) List(ctx context.Context) ([]Container, error) {
 	return out, nil
 }
 
-// Stop enforces a container's stopTimeout: SIGTERM the task, wait up to
-// timeout for it to exit on its own, then fall through to Remove's forced
-// kill+delete regardless of whether it exited in time. A container with no
-// live task (already gone) or that fails to accept the signal goes straight to
-// Remove, matching Remove's own already-gone tolerance.
-func (p *containerdPuller) Stop(ctx context.Context, containerID string, timeout time.Duration) error {
-	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
-	container, err := p.client.LoadContainer(ctx, containerID)
-	if err != nil {
-		return nil //nolint:nilerr // load failure means the container is already gone
-	}
-	task, terr := container.Task(ctx, nil)
-	if terr != nil {
-		return p.Remove(ctx, containerID)
-	}
-	statusC, werr := task.Wait(ctx)
-	if werr == nil && task.Kill(ctx, syscall.SIGTERM) == nil {
-		select {
-		case <-statusC:
-		case <-time.After(timeout):
-		}
-	}
-	return p.Remove(ctx, containerID)
-}
-
 // Remove kills and deletes the container's task, then the container + snapshot.
 func (p *containerdPuller) Remove(ctx context.Context, containerID string) error {
 	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
-	p.releaseStdin(containerID)
 	container, err := p.client.LoadContainer(ctx, containerID)
 	if err != nil {
 		return nil //nolint:nilerr // load failure means the container is already gone

cmd/ecs-agent/runtime/containerd_run_test.go

@@ -1,15 +1,8 @@
 package runtime
 
 import (
-	"context"
-	"errors"
-	"io"
 	"reflect"
 	"testing"
-	"time"
-
-	"github.com/containerd/containerd/v2/pkg/cio"
-	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
 // cdiDeviceNames maps a container's pinned GPU UUIDs to CDI device names
@@ -52,166 +45,3 @@ func TestCDISpecOpts_Wiring(t *testing.T) {
 		t.Error("GPU container: spec opt is nil")
 	}
 }
-
-// TestOCICapNames verifies ECS's bare capability names (Docker convention,
-// e.g. "SYS_PTRACE") are prefixed to the OCI runtime-spec form the process
-// capability sets expect, and a name already carrying the prefix passes
-// through unchanged.
-func TestOCICapNames(t *testing.T) {
-	cases := []struct {
-		name string
-		in   []string
-		want []string
-	}{
-		{name: "empty", in: nil, want: nil},
-		{name: "bare", in: []string{"SYS_PTRACE"}, want: []string{"CAP_SYS_PTRACE"}},
-		{name: "already prefixed", in: []string{"CAP_NET_ADMIN"}, want: []string{"CAP_NET_ADMIN"}},
-		{name: "mixed", in: []string{"SYS_PTRACE", "CAP_NET_ADMIN"}, want: []string{"CAP_SYS_PTRACE", "CAP_NET_ADMIN"}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			got := ociCapNames(tc.in)
-			if !reflect.DeepEqual(got, tc.want) {
-				t.Errorf("ociCapNames(%v) = %v, want %v", tc.in, got, tc.want)
-			}
-		})
-	}
-}
-
-// TestWithSysctls verifies systemControls namespace/value pairs land on
-// Linux.Sysctl, and an empty list leaves the spec's Linux section untouched
-// rather than allocating an empty map (the "requests nothing" case).
-func TestWithSysctls(t *testing.T) {
-	s := &specs.Spec{}
-	if err := withSysctls(nil)(context.Background(), nil, nil, s); err != nil {
-		t.Fatalf("empty ctls: %v", err)
-	}
-	if s.Linux != nil {
-		t.Errorf("empty ctls: want no Linux section, got %+v", s.Linux)
-	}
-
-	ctls := []SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
-	if err := withSysctls(ctls)(context.Background(), nil, nil, s); err != nil {
-		t.Fatalf("with ctls: %v", err)
-	}
-	if s.Linux == nil || s.Linux.Sysctl["net.core.somaxconn"] != "1024" {
-		t.Errorf("want sysctl net.core.somaxconn=1024, got %+v", s.Linux)
-	}
-}
-
-// An interactive container's stdin read blocks until the container is released,
-// then reports EOF so containerd's copier goroutine exits instead of parking
-// for the life of the agent.
-func TestHeldStdin_ReadsEOFOnceReleased(t *testing.T) {
-	p := &containerdPuller{}
-	p.containerIOCreator("c1", true, false)
-
-	p.mu.Lock()
-	done := p.stdinDone["c1"]
-	p.mu.Unlock()
-	if done == nil {
-		t.Fatal("interactive container registered no stdin channel")
-	}
-
-	read := make(chan error, 1)
-	go func() {
-		_, err := heldStdin{done: done}.Read(make([]byte, 1))
-		read <- err
-	}()
-
-	select {
-	case err := <-read:
-		t.Fatalf("read returned before release: %v", err)
-	case <-time.After(50 * time.Millisecond):
-	}
-
-	p.releaseStdin("c1")
-
-	select {
-	case err := <-read:
-		if !errors.Is(err, io.EOF) {
-			t.Fatalf("want io.EOF after release, got %v", err)
-		}
-	case <-time.After(time.Second):
-		t.Fatal("read did not return after release")
-	}
-}
-
-// A non-interactive container uses cio.NullIO and registers nothing to release.
-func TestContainerIOCreator_NonInteractiveRegistersNothing(t *testing.T) {
-	p := &containerdPuller{}
-	p.containerIOCreator("c1", false, false)
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	if len(p.stdinDone) != 0 {
-		t.Fatalf("want no registered stdin channels, got %d", len(p.stdinDone))
-	}
-}
-
-// applyIOOpts resolves the cio options onto a Streams the way cio.NewCreator
-// does, which is the only way to observe what the creator was built with.
-func applyIOOpts(opts []cio.Opt) cio.Streams {
-	var s cio.Streams
-	for _, o := range opts {
-		o(&s)
-	}
-	return s
-}
-
-// A terminal container needs the terminal flag set on its FIFO set, else the
-// shim creates no console socket and runc refuses to start the container at all.
-// The terminal carries no separate stderr stream.
-func TestIOStreamOpts_TerminalRequestsAConsole(t *testing.T) {
-	s := applyIOOpts(ioStreamOpts(nil, true))
-	if !s.Terminal {
-		t.Error("terminal container: want Terminal set")
-	}
-	if s.Stderr != nil {
-		t.Errorf("terminal container: want no stderr stream, got %#v", s.Stderr)
-	}
-	if s.Stdout == nil {
-		t.Error("terminal container: want a stdout stream")
-	}
-}
-
-// A container with stdin but no terminal keeps both output streams and asks for
-// no console.
-func TestIOStreamOpts_NonTerminalKeepsStderr(t *testing.T) {
-	s := applyIOOpts(ioStreamOpts(heldStdin{}, false))
-	if s.Terminal {
-		t.Error("non-terminal container: want Terminal unset")
-	}
-	if s.Stderr == nil {
-		t.Error("non-terminal container: want a stderr stream")
-	}
-	if s.Stdin == nil {
-		t.Error("non-terminal interactive container: want a stdin stream")
-	}
-}
-
-// A terminal container that is not interactive still gets a creator rather than
-// cio.NullIO, and registers no stdin to release.
-func TestContainerIOCreator_TerminalWithoutInteractive(t *testing.T) {
-	p := &containerdPuller{}
-	if c := p.containerIOCreator("c1", false, true); c == nil {
-		t.Fatal("terminal container: want a creator")
-	}
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	if len(p.stdinDone) != 0 {
-		t.Fatalf("want no registered stdin channels, got %d", len(p.stdinDone))
-	}
-}
-
-// Remove calls releaseStdin unconditionally and Stop falls through to Remove,
-// so an unknown container and a second release must both be no-ops.
-func TestReleaseStdin_UnknownAndRepeatedAreSafe(t *testing.T) {
-	p := &containerdPuller{}
-	p.releaseStdin("never-registered")
-
-	p.containerIOCreator("c1", true, false)
-	p.releaseStdin("c1")
-	p.releaseStdin("c1")
-}

cmd/ecs-agent/runtime/runtime.go

@@ -4,10 +4,7 @@
 // scheduler-only build run against a fake.
 package runtime
 
-import (
-	"context"
-	"time"
-)
+import "context"
 
 // PullSpec describes one image to pull.
 type PullSpec struct {
@@ -38,12 +35,6 @@ type ImagePuller interface {
 	Close() error
 }
 
-// SystemControl is a sysctl namespace/value pair applied to the OCI spec.
-type SystemControl struct {
-	Namespace string
-	Value     string
-}
-
 // RunSpec describes a single container the agent must create and start. v1 runs
 // the container in the host network namespace (bridge/CNI task networking lands
 // in a later sprint); Labels carry the mulga.ecs.* task identity for the reboot
@@ -63,22 +54,6 @@ type RunSpec struct {
 	// per GPU requested. Empty for non-GPU containers or if pinning fell short.
 	// The runner injects these as CDI devices into the OCI spec.
 	GPUIDs []string
-	// User is the OCI process user (oci.WithUser). Empty runs as the image
-	// default.
-	User string
-	// ReadonlyRootFilesystem, Privileged, PseudoTerminal and Interactive are
-	// pointers so the runner can tell "not requested" (nil) from an explicit
-	// false, which still needs echoing but applies nothing.
-	ReadonlyRootFilesystem *bool
-	Privileged             *bool
-	PseudoTerminal         *bool
-	Interactive            *bool
-	// SystemControls are sysctls set on the OCI spec's Linux.Sysctl map.
-	SystemControls []SystemControl
-	// CapAdd / CapDrop are Linux capabilities added/dropped from the OCI
-	// spec's process capability sets (linuxParameters.capabilities).
-	CapAdd  []string
-	CapDrop []string
 }
 
 // RunStatus is a finished container's outcome.
@@ -96,16 +71,12 @@ type Container struct {
 }
 
 // Runner creates and starts containers from already-pulled images. id is a
-// caller-unique container ID; Wait blocks until the container exits; Stop
-// requests a graceful exit (SIGTERM, then a forced kill+remove once timeout
-// elapses) — the agent's enforcement of a container's stopTimeout; Remove
-// tears down the container + its task immediately, with no grace period; List
-// enumerates known containers so the reboot reconciler can re-adopt the ones
-// still running.
+// caller-unique container ID; Wait blocks until the container exits; Remove
+// tears down the container + its task; List enumerates known containers so the
+// reboot reconciler can re-adopt the ones still running.
 type Runner interface {
 	Run(ctx context.Context, id string, spec RunSpec) (containerID string, err error)
 	Wait(ctx context.Context, containerID string) (RunStatus, error)
-	Stop(ctx context.Context, containerID string, timeout time.Duration) error
 	Remove(ctx context.Context, containerID string) error
 	List(ctx context.Context) ([]Container, error)
 }

cmd/ecs-agent/runtime/runtime_fake.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"sync"
-	"time"
 )
 
 // FakePuller is an in-memory ImagePuller for tests and the scheduler-only build.
@@ -27,21 +26,11 @@ type FakePuller struct {
 	Removed  []string
 	OnRun    func(string, RunSpec) (string, error)
 
-	// Stop bookkeeping: StopTimeouts pairs the container ID with the timeout it
-	// was stopped with, in call order, so a test can assert both the target and
-	// the enforced stopTimeout without a fake containerd.
-	Stopped      []string
-	StopTimeouts []time.Duration
-	StopErr      error
-
-	// List bookkeeping: Containers is replayed by List; ListErr forces a failure,
-	// limited to the first ListErrFor calls when that is non-zero, so a test can
-	// model a runtime that recovers. Listed and ListCalls record the calls.
+	// List bookkeeping: Containers is replayed by List; ListErr forces a failure;
+	// Listed records that List was called.
 	Containers []Container
 	ListErr    error
-	ListErrFor int
 	Listed     bool
-	ListCalls  int
 }
 
 var (
@@ -109,15 +98,6 @@ func (f *FakePuller) Wait(_ context.Context, containerID string) (RunStatus, err
 	return RunStatus{ExitCode: f.WaitCode}, f.WaitErr
 }
 
-// Stop records the container ID and the timeout it was stopped with.
-func (f *FakePuller) Stop(_ context.Context, containerID string, timeout time.Duration) error {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.Stopped = append(f.Stopped, containerID)
-	f.StopTimeouts = append(f.StopTimeouts, timeout)
-	return f.StopErr
-}
-
 // Remove records the container ID.
 func (f *FakePuller) Remove(_ context.Context, containerID string) error {
 	f.mu.Lock()
@@ -133,36 +113,13 @@ func (f *FakePuller) Waits() []string {
 	return append([]string(nil), f.Waited...)
 }
 
-// RunCalls returns a copy of the specs passed to Run, for assertions made while
-// the agent's goroutines may still be live.
-func (f *FakePuller) RunCalls() []RunSpec {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return append([]RunSpec(nil), f.Runs...)
-}
-
-// Reaped returns a copy of the container IDs passed to Stop.
-func (f *FakePuller) Reaped() []string {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return append([]string(nil), f.Stopped...)
-}
-
 // List replays the programmed container set (or error).
 func (f *FakePuller) List(_ context.Context) ([]Container, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.Listed = true
-	f.ListCalls++
-	if f.ListErr != nil && (f.ListErrFor == 0 || f.ListCalls <= f.ListErrFor) {
+	if f.ListErr != nil {
 		return nil, f.ListErr
 	}
 	return f.Containers, nil
 }
-
-// Lists reports how many times List was called.
-func (f *FakePuller) Lists() int {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return f.ListCalls
-}

cmd/ecs-agent/runtime/runtime_test.go

@@ -1,9 +1,71 @@
 package runtime
 
 import (
+	"context"
+	"errors"
 	"testing"
 )
 
+// staticResolver is a test Resolver returning fixed creds or an error.
+type staticResolver struct {
+	user, pass, endpoint string
+	err                  error
+}
+
+func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
+	return s.user, s.pass, s.endpoint, s.err
+}
+
+func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
+	f := &FakePuller{}
+	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
+	if err != nil {
+		t.Fatalf("Pull: %v", err)
+	}
+	if img.Ref != "repo/app:1" {
+		t.Errorf("Ref defaulted wrong: %q", img.Ref)
+	}
+	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
+		t.Errorf("Pulls = %+v", f.Pulls)
+	}
+	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
+		t.Errorf("Authzd = %+v", f.Authzd)
+	}
+}
+
+func TestFakePuller_ResolverError(t *testing.T) {
+	f := &FakePuller{}
+	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
+	if err == nil {
+		t.Fatal("expected resolver error to propagate")
+	}
+}
+
+func TestFakePuller_ProgrammedError(t *testing.T) {
+	f := &FakePuller{Err: errors.New("pull failed")}
+	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
+		t.Fatal("expected programmed error")
+	}
+}
+
+func TestFakePuller_OnPullOverrides(t *testing.T) {
+	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
+	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
+		return want, nil
+	}}
+	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
+	if err != nil || got != want {
+		t.Fatalf("got %+v, %v; want %+v", got, err, want)
+	}
+}
+
+func TestFakePuller_Close(t *testing.T) {
+	f := &FakePuller{}
+	if err := f.Close(); err != nil || !f.Closed {
+		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
+	}
+}
+
 func TestNormalizeRef(t *testing.T) {
 	cases := map[string]string{
 		"nginx:alpine":      "docker.io/library/nginx:alpine",

cmd/ecs-agent/runtimegate.go

@@ -1,116 +0,0 @@
-package main
-
-import (
-	"context"
-	"log/slog"
-	"sync"
-	"time"
-
-	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
-)
-
-// runtimeProbeTimeout bounds the liveness probe against a runtime the gate is
-// already holding, so a wedged socket cannot stall the watch loop.
-const runtimeProbeTimeout = 5 * time.Second
-
-// defaultRuntimeRetry is how often the gate dials a runtime it does not have,
-// and probes one it does. The node starts containerd and the agent back to
-// back, so the first few dials are expected to fail.
-const defaultRuntimeRetry = 10 * time.Second
-
-// runtimeGate holds the container runtime the agent has resolved, or nothing
-// when the socket has not answered. The runtime is resolved by a loop rather
-// than once at boot: the node's service manager reports the container daemon
-// started when the process spawned, not when its socket began accepting, so a
-// dial that fails in the first second of boot is a transient condition.
-//
-// Every read of the runtime goes through the gate, so a runtime that arrives
-// after boot is picked up on the next task without restarting the agent.
-type runtimeGate struct {
-	mu sync.RWMutex
-	rt ctrruntime.Runtime
-}
-
-// newRuntimeGate returns a gate holding rt, which may be nil.
-func newRuntimeGate(rt ctrruntime.Runtime) *runtimeGate {
-	return &runtimeGate{rt: rt}
-}
-
-// get returns the current runtime, or nil when none is resolved.
-func (g *runtimeGate) get() ctrruntime.Runtime {
-	g.mu.RLock()
-	defer g.mu.RUnlock()
-	return g.rt
-}
-
-// available reports whether a runtime is resolved. It is what registration
-// consults to decide whether the host has any capacity to advertise.
-func (g *runtimeGate) available() bool { return g.get() != nil }
-
-// set replaces the held runtime, closing the one it displaces.
-func (g *runtimeGate) set(rt ctrruntime.Runtime) {
-	g.mu.Lock()
-	old := g.rt
-	g.rt = rt
-	g.mu.Unlock()
-	if old != nil && old != rt {
-		_ = old.Close()
-	}
-}
-
-// clear drops and closes the held runtime, returning whether it held one.
-func (g *runtimeGate) clear() bool {
-	g.mu.Lock()
-	old := g.rt
-	g.rt = nil
-	g.mu.Unlock()
-	if old == nil {
-		return false
-	}
-	_ = old.Close()
-	return true
-}
-
-// watch keeps the gate current until ctx is cancelled: it dials while the gate
-// is empty, and probes while it is full, clearing the gate when the probe fails
-// so the next tick re-dials. A containerd stopped and restarted underneath a
-// running agent is therefore picked up the same way a slow boot is.
-func (g *runtimeGate) watch(ctx context.Context, dial func() (ctrruntime.Runtime, error), interval time.Duration) {
-	if interval <= 0 {
-		interval = defaultRuntimeRetry
-	}
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-ticker.C:
-			g.step(ctx, dial)
-		}
-	}
-}
-
-// step performs one pass of the watch loop: dial when empty, probe when full.
-func (g *runtimeGate) step(ctx context.Context, dial func() (ctrruntime.Runtime, error)) {
-	if rt := g.get(); rt != nil {
-		probeCtx, cancel := context.WithTimeout(ctx, runtimeProbeTimeout)
-		// A container list is what reconcile already asks the runtime at boot and
-		// the cheapest thing it answers, so it doubles as the liveness probe.
-		_, err := rt.List(probeCtx)
-		cancel()
-		if err != nil && ctx.Err() == nil {
-			slog.Warn("ecs-agent: container runtime stopped answering, will re-dial", "err", err)
-			g.clear()
-		}
-		return
-	}
-
-	rt, err := dial()
-	if err != nil {
-		slog.Debug("ecs-agent: container runtime still unavailable", "err", err)
-		return
-	}
-	slog.Info("ecs-agent: container runtime available")
-	g.set(rt)
-}

cmd/ecs-agent/runtimegate_test.go

@@ -1,122 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
-)
-
-// TestRuntimeGate_DialsUntilTheSocketAnswers is the boot case the agent used to
-// lose: the node starts containerd and the agent together, so the first dials
-// fail. Resolving once left the agent runtime-less for the life of the process.
-func TestRuntimeGate_DialsUntilTheSocketAnswers(t *testing.T) {
-	g := newRuntimeGate(nil)
-	if g.available() {
-		t.Fatal("empty gate reported a runtime")
-	}
-
-	rt := &ctrruntime.FakePuller{}
-	dials := 0
-	dial := func() (ctrruntime.Runtime, error) {
-		dials++
-		if dials < 3 {
-			return nil, errors.New("dial containerd: no such file or directory")
-		}
-		return rt, nil
-	}
-
-	for range 2 {
-		g.step(t.Context(), dial)
-		if g.available() {
-			t.Fatalf("gate resolved a runtime after %d failed dials", dials)
-		}
-	}
-
-	g.step(t.Context(), dial)
-	if g.get() != rt {
-		t.Fatalf("gate did not pick up the runtime once the socket answered (dials=%d)", dials)
-	}
-}
-
-// TestRuntimeGate_DropsARuntimeThatStopsAnswering covers a containerd restarted
-// underneath a live agent: the held runtime stops answering, so the gate clears
-// it and the next pass re-dials rather than handing out a dead client forever.
-func TestRuntimeGate_DropsARuntimeThatStopsAnswering(t *testing.T) {
-	dead := &ctrruntime.FakePuller{ListErr: errors.New("connection refused")}
-	fresh := &ctrruntime.FakePuller{}
-	g := newRuntimeGate(dead)
-
-	dial := func() (ctrruntime.Runtime, error) { return fresh, nil }
-
-	g.step(t.Context(), dial)
-	if g.available() {
-		t.Fatal("gate kept a runtime whose probe failed")
-	}
-	if !dead.Closed {
-		t.Error("displaced runtime was not closed")
-	}
-
-	g.step(t.Context(), dial)
-	if g.get() != fresh {
-		t.Fatal("gate did not re-dial after clearing")
-	}
-}
-
-// TestRuntimeGate_KeepsAHealthyRuntime guards the other direction: a probe that
-// succeeds must not churn the runtime, or every pass would drop a working client.
-func TestRuntimeGate_KeepsAHealthyRuntime(t *testing.T) {
-	rt := &ctrruntime.FakePuller{}
-	g := newRuntimeGate(rt)
-
-	dial := func() (ctrruntime.Runtime, error) {
-		t.Error("dialled while holding a healthy runtime")
-		return nil, errors.New("unexpected dial")
-	}
-	g.step(t.Context(), dial)
-
-	if g.get() != rt {
-		t.Fatal("gate dropped a runtime whose probe succeeded")
-	}
-	if rt.Closed {
-		t.Error("healthy runtime was closed")
-	}
-}
-
-// TestRuntimeGate_ClearIsIdempotent keeps Stop safe to call on an agent that
-// never resolved a runtime.
-func TestRuntimeGate_ClearIsIdempotent(t *testing.T) {
-	g := newRuntimeGate(nil)
-	if g.clear() {
-		t.Error("clear reported dropping a runtime the gate never held")
-	}
-
-	rt := &ctrruntime.FakePuller{}
-	g.set(rt)
-	if !g.clear() {
-		t.Error("clear did not report dropping the held runtime")
-	}
-	if !rt.Closed {
-		t.Error("cleared runtime was not closed")
-	}
-	if g.clear() {
-		t.Error("second clear reported dropping a runtime")
-	}
-}
-
-// TestRuntimeGate_WatchStopsWithContext keeps the watch loop from outliving the
-// agent it belongs to.
-func TestRuntimeGate_WatchStopsWithContext(t *testing.T) {
-	ctx, cancel := context.WithCancel(t.Context())
-	g := newRuntimeGate(nil)
-
-	done := make(chan struct{})
-	go func() {
-		g.watch(ctx, func() (ctrruntime.Runtime, error) { return nil, errors.New("no socket") }, 0)
-		close(done)
-	}()
-
-	cancel()
-	<-done
-}

cmd/ecs-agent/taskrun.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"log/slog"
 	"maps"
-	"strconv"
-	"sync"
 	"time"
 
 	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
@@ -14,10 +12,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
-// defaultStopTimeout is applied when a container carries no explicit
-// stopTimeout, matching the EC2 launch-type default ECS itself uses.
-const defaultStopTimeout = 30 * time.Second
-
 // pollAssignments drains the instance's assignment inbox on a ticker until ctx is
 // cancelled. Each assign is dispatched to runTask exactly once; the taskIDs seen
 // on a poll are acked on the next poll so the gateway can drop them — a crash
@@ -29,7 +23,7 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	ticker := time.NewTicker(a.cfg.PollInterval)
 	defer ticker.Stop()
 
-	reaps := newStopLedger()
+	stopping := map[string]bool{}
 	var ackAssigns, ackStops []string
 	for {
 		select {
@@ -46,18 +40,17 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 			ackStops = ackStops[:0]
 			for i := range stops {
 				sd := stops[i]
-				switch {
-				case reaps.done(sd.TaskID):
-					ackStops = append(ackStops, sd.TaskID)
-				case reaps.begin(sd.TaskID):
+				if !stopping[sd.TaskID] {
+					stopping[sd.TaskID] = true
 					slog.Info("ecs-agent: task stop requested", "task", sd.TaskID, "reason", sd.Reason)
-					go func() { reaps.end(sd.TaskID, a.stopTask(ctx, sd)) }()
+					go a.stopTask(ctx, sd)
 				}
+				ackStops = append(ackStops, sd.TaskID)
 			}
 			ackAssigns = ackAssigns[:0]
 			for i := range assigns {
 				as := assigns[i]
-				if !dispatched[as.TaskID] && !reaps.seen(as.TaskID) {
+				if !dispatched[as.TaskID] && !stopping[as.TaskID] {
 					dispatched[as.TaskID] = true
 					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
 					go a.runTask(ctx, &as)
@@ -68,83 +61,27 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	}
 }
 
-// stopLedger tracks reaps in flight and reaps that reported STOPPED. The two
-// are separate because only the second may be acked: acking the first deletes a
-// directive that was never performed, and nothing re-posts it.
-type stopLedger struct {
-	mu       sync.Mutex
-	inflight map[string]bool
-	stopped  map[string]bool
-}
-
-func newStopLedger() *stopLedger {
-	return &stopLedger{inflight: map[string]bool{}, stopped: map[string]bool{}}
-}
-
-// begin claims a task for a reap, reporting false when one is already running.
-func (l *stopLedger) begin(taskID string) bool {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	if l.inflight[taskID] {
-		return false
+// stopTask reaps a task's containers on a control-plane stop directive: it lists
+// the runtime's containers, removes the ones labeled with this task (kill +
+// delete, releasing host ports/netns), then reports the task STOPPED with the
+// directive's reason. Idempotent — a task with no live containers still reports
+// STOPPED so the scheduler releases its capacity.
+func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
+	if a.runner == nil {
+		return
 	}
-	l.inflight[taskID] = true
-	return true
-}
-
-// end releases the claim, recording the task as stopped only if the reap got
-// that far. A failed reap leaves nothing behind, so the next poll redelivers
-// the directive and dispatches it again.
-func (l *stopLedger) end(taskID string, reported bool) {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	delete(l.inflight, taskID)
-	if reported {
-		l.stopped[taskID] = true
-	}
-}
-
-// done reports whether the task's reap reported STOPPED, which is the only
-// condition under which its directive may be acked.
-func (l *stopLedger) done(taskID string) bool {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	return l.stopped[taskID]
-}
-
-// seen reports whether a stop for the task is in flight or complete, so an
-// assignment arriving alongside its own stop is never started.
-func (l *stopLedger) seen(taskID string) bool {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	return l.inflight[taskID] || l.stopped[taskID]
-}
-
-// stopTask reaps a task's containers on a control-plane stop directive: it
-// lists the runtime's containers, gracefully stops (SIGTERM, wait up to the
-// container's stopTimeout, then force kill + delete) the ones labeled with
-// this task, then reports the task STOPPED with the directive's reason.
-// Idempotent — a task with no live containers still reports STOPPED so the
-// scheduler releases its capacity. Reports whether it got as far as saying
-// STOPPED: a caller may only ack the directive when it did.
-func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) bool {
-	runner := a.rt.get()
-	if runner == nil {
-		slog.Warn("ecs-agent: stop deferred, no container runtime", "task", sd.TaskID)
-		return false
-	}
-	containers, err := runner.List(ctx)
+	containers, err := a.runner.List(ctx)
 	if err != nil {
 		slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
-		return false
+		return
 	}
 	statuses := make([]bus.ContainerStatus, 0)
 	for _, c := range containers {
 		if c.Labels[labelTaskID] != sd.TaskID {
 			continue
 		}
-		if rerr := runner.Stop(ctx, c.ID, containerStopTimeout(c.Labels)); rerr != nil {
-			slog.Warn("ecs-agent: stop failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
+		if rerr := a.runner.Remove(ctx, c.ID); rerr != nil {
+			slog.Warn("ecs-agent: stop remove failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
 		}
 		statuses = append(statuses, bus.ContainerStatus{
 			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
@@ -156,15 +93,13 @@ func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) bool {
 	}
 	a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
 	slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
-	return true
 }
 
 // runTask pulls each container image, starts the containers, and reports task
 // state on the bus. A missing runtime or any per-container failure reports the
 // task STOPPED with a reason; success reports RUNNING and waits for exit.
 func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
-	rt := a.rt.get()
-	if rt == nil {
+	if a.puller == nil || a.runner == nil {
 		a.reportTaskState(as, bus.TaskStatusStopped, "containerd unavailable on agent", nil)
 		return
 	}
@@ -182,12 +117,7 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 
 	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
 	for _, c := range as.Containers {
-		// startTimeout bounds pull-through-start as a unit: real ECS measures a
-		// container's time-to-RUNNING from the same starting point.
-		runCtx, cancel := containerStartCtx(ctx, c.StartTimeout)
-
-		if _, err := rt.Pull(runCtx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
-			cancel()
+		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
 			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
 			a.teardownTaskNetns(as)
 			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
@@ -196,29 +126,16 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 
 		cid := containerID(as.TaskID, c.Name)
 		gpuIDs := a.pinContainerGPUs(as.TaskID, c.Name, c.GPU)
-		labels := taskLabels(as, c.Name)
-		if c.StopTimeout != nil {
-			labels[labelStopTimeout] = strconv.FormatInt(*c.StopTimeout, 10)
-		}
 		spec := ctrruntime.RunSpec{
-			Image:                  c.Image,
-			Command:                c.Command,
-			Env:                    withCredEnv(c.Environment, credID),
-			Labels:                 labels,
-			NetnsPath:              netnsPath,
-			GPU:                    c.GPU,
-			GPUIDs:                 gpuIDs,
-			User:                   c.User,
-			ReadonlyRootFilesystem: c.ReadonlyRootFilesystem,
-			Privileged:             c.Privileged,
-			PseudoTerminal:         c.PseudoTerminal,
-			Interactive:            c.Interactive,
-			SystemControls:         toRuntimeSystemControls(c.SystemControls),
-			CapAdd:                 c.CapAdd,
-			CapDrop:                c.CapDrop,
+			Image:     c.Image,
+			Command:   c.Command,
+			Env:       withCredEnv(c.Environment, credID),
+			Labels:    taskLabels(as, c.Name),
+			NetnsPath: netnsPath,
+			GPU:       c.GPU,
+			GPUIDs:    gpuIDs,
 		}
-		id, err := rt.Run(runCtx, cid, spec)
-		cancel()
+		id, err := a.runner.Run(ctx, cid, spec)
 		if err != nil {
 			slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
 			a.teardownTaskNetns(as)
@@ -238,12 +155,7 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 // waitContainer blocks until a container exits, then reports the task STOPPED
 // with the exit code. v1 stops the whole task when its first container exits.
 func (a *Agent) waitContainer(ctx context.Context, as *bus.Assign, name, containerID string) {
-	runner := a.rt.get()
-	if runner == nil {
-		slog.Warn("ecs-agent: cannot wait on container, no container runtime", "task", as.TaskID, "container", name)
-		return
-	}
-	status, err := runner.Wait(ctx, containerID)
+	status, err := a.runner.Wait(ctx, containerID)
 	if err != nil {
 		slog.Warn("ecs-agent: wait container failed", "task", as.TaskID, "container", name, "err", err)
 		return
@@ -328,44 +240,6 @@ func withCredEnv(env map[string]string, credID string) map[string]string {
 	return out
 }
 
-// containerStartCtx bounds pull+run by a container's startTimeout, if set. A
-// nil startTimeout (the common case) returns ctx unchanged with a no-op
-// cancel, matching today's unbounded behaviour.
-func containerStartCtx(ctx context.Context, startTimeout *int64) (context.Context, context.CancelFunc) {
-	if startTimeout == nil {
-		return ctx, func() {}
-	}
-	return context.WithTimeout(ctx, time.Duration(*startTimeout)*time.Second)
-}
-
-// containerStopTimeout reads the stopTimeout stamped on a container's labels
-// by runTask, falling back to defaultStopTimeout when absent or unparseable.
-func containerStopTimeout(labels map[string]string) time.Duration {
-	v, ok := labels[labelStopTimeout]
-	if !ok {
-		return defaultStopTimeout
-	}
-	secs, err := strconv.ParseInt(v, 10, 64)
-	if err != nil {
-		return defaultStopTimeout
-	}
-	return time.Duration(secs) * time.Second
-}
-
-// toRuntimeSystemControls maps the assign's sysctl pairs to the runtime
-// package's own type, keeping cmd/ecs-agent/runtime decoupled from the ecs
-// handlers' bus package.
-func toRuntimeSystemControls(in []bus.SystemControl) []ctrruntime.SystemControl {
-	if len(in) == 0 {
-		return nil
-	}
-	out := make([]ctrruntime.SystemControl, len(in))
-	for i, sc := range in {
-		out[i] = ctrruntime.SystemControl{Namespace: sc.Namespace, Value: sc.Value}
-	}
-	return out
-}
-
 // pinContainerGPUs reserves n device UUIDs for a container from the local
 // ledger, before the container is started so the runner can inject them as
 // CDI devices. A short ledger (fewer free devices than requested, e.g.
@@ -443,9 +317,6 @@ const (
 	labelCredID        = "mulga.ecs.credID"
 	labelTaskRoleARN   = "mulga.ecs.taskRoleArn"
 	labelENIMac        = "mulga.ecs.eniMac"
-	// labelStopTimeout carries a container's stopTimeout (seconds) so stopTask
-	// can read it back from List's labels; it has no assign to consult there.
-	labelStopTimeout = "mulga.ecs.stopTimeout"
 )
 
 // taskLabels are the mulga.ecs.* labels stamped on a container. The cred/role/MAC

cmd/ecs-agent/taskrun_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"errors"
-	"slices"
 	"testing"
 	"time"
 
@@ -25,13 +24,13 @@ func testAssign() *bus.Assign {
 }
 
 func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
-	return newAgent(config{}, testIdentity(), cp, rt, nil)
+	return newAgent(config{}, testIdentity(), cp, rt, rt, nil)
 }
 
 // runTask with no runtime reports the task STOPPED instead of crashing.
 func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
 	cp := &fakeCP{}
-	a := newAgent(config{}, testIdentity(), cp, nil, nil)
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
 	a.runTask(context.Background(), testAssign())
 
 	st := cp.taskStates()
@@ -71,157 +70,6 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }
 
-// Each enforced field maps onto RunSpec exactly, asserted through the fake
-// runtime — the mapping this change exists to wire, without a real containerd.
-func TestRunTask_RuntimeFieldsCarriedToRunSpec(t *testing.T) {
-	cp := &fakeCP{}
-	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newRunAgent(cp, rt)
-	as := testAssign()
-	ro := true
-	priv := false
-	tty := true
-	interactive := true
-	as.Containers[0].User = "1000:1000"
-	as.Containers[0].ReadonlyRootFilesystem = &ro
-	as.Containers[0].Privileged = &priv
-	as.Containers[0].PseudoTerminal = &tty
-	as.Containers[0].Interactive = &interactive
-	as.Containers[0].SystemControls = []bus.SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
-	as.Containers[0].CapAdd = []string{"SYS_PTRACE"}
-	as.Containers[0].CapDrop = []string{"NET_RAW"}
-	a.runTask(context.Background(), as)
-
-	if len(rt.Runs) != 1 {
-		t.Fatalf("expected one run, got %+v", rt.Runs)
-	}
-	got := rt.Runs[0]
-	if got.User != "1000:1000" {
-		t.Errorf("User = %q, want 1000:1000", got.User)
-	}
-	if got.ReadonlyRootFilesystem == nil || !*got.ReadonlyRootFilesystem {
-		t.Errorf("ReadonlyRootFilesystem = %v, want true", got.ReadonlyRootFilesystem)
-	}
-	if got.Privileged == nil || *got.Privileged {
-		t.Errorf("Privileged = %v, want false (present, not applied)", got.Privileged)
-	}
-	if got.PseudoTerminal == nil || !*got.PseudoTerminal {
-		t.Errorf("PseudoTerminal = %v, want true", got.PseudoTerminal)
-	}
-	if got.Interactive == nil || !*got.Interactive {
-		t.Errorf("Interactive = %v, want true", got.Interactive)
-	}
-	if len(got.SystemControls) != 1 || got.SystemControls[0].Namespace != "net.core.somaxconn" {
-		t.Errorf("SystemControls = %+v, want one net.core.somaxconn entry", got.SystemControls)
-	}
-	if len(got.CapAdd) != 1 || got.CapAdd[0] != "SYS_PTRACE" {
-		t.Errorf("CapAdd = %+v, want [SYS_PTRACE]", got.CapAdd)
-	}
-	if len(got.CapDrop) != 1 || got.CapDrop[0] != "NET_RAW" {
-		t.Errorf("CapDrop = %+v, want [NET_RAW]", got.CapDrop)
-	}
-}
-
-// An assign carrying none of the new fields leaves RunSpec's pointer fields
-// nil rather than defaulting to a false indistinguishable from "explicitly
-// not requested".
-func TestRunTask_NilRuntimeFieldsLeaveRunSpecUnset(t *testing.T) {
-	cp := &fakeCP{}
-	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newRunAgent(cp, rt)
-	a.runTask(context.Background(), testAssign())
-
-	if len(rt.Runs) != 1 {
-		t.Fatalf("expected one run, got %+v", rt.Runs)
-	}
-	got := rt.Runs[0]
-	if got.ReadonlyRootFilesystem != nil || got.Privileged != nil || got.PseudoTerminal != nil || got.Interactive != nil {
-		t.Errorf("want all runtime-field pointers nil, got %+v", got)
-	}
-	if len(got.SystemControls) != 0 || len(got.CapAdd) != 0 || len(got.CapDrop) != 0 {
-		t.Errorf("want no sysctls/capabilities, got %+v", got)
-	}
-}
-
-// TestRunTask_StampsStopTimeoutLabel verifies runTask stamps a container's
-// stopTimeout as a label — the only way stopTask (which only has List's
-// labels, not the original assign) can recover it later.
-func TestRunTask_StampsStopTimeoutLabel(t *testing.T) {
-	cp := &fakeCP{}
-	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newRunAgent(cp, rt)
-	as := testAssign()
-	stop := int64(45)
-	as.Containers[0].StopTimeout = &stop
-	a.runTask(context.Background(), as)
-
-	if len(rt.Runs) != 1 {
-		t.Fatalf("expected one run, got %+v", rt.Runs)
-	}
-	if got := rt.Runs[0].Labels[labelStopTimeout]; got != "45" {
-		t.Errorf("stopTimeout label = %q, want 45", got)
-	}
-}
-
-// TestRunTask_NoStopTimeoutLabelWhenUnset verifies a container with no
-// stopTimeout carries no label, so stopTask falls back to the default rather
-// than reading a stray "0".
-func TestRunTask_NoStopTimeoutLabelWhenUnset(t *testing.T) {
-	cp := &fakeCP{}
-	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newRunAgent(cp, rt)
-	a.runTask(context.Background(), testAssign())
-
-	if len(rt.Runs) != 1 {
-		t.Fatalf("expected one run, got %+v", rt.Runs)
-	}
-	if _, ok := rt.Runs[0].Labels[labelStopTimeout]; ok {
-		t.Errorf("want no stopTimeout label, got %+v", rt.Runs[0].Labels)
-	}
-}
-
-// TestStopTask_UsesContainerStopTimeout verifies that stopTask reads a
-// container's stopTimeout label and calls the runner's graceful Stop (SIGTERM
-// + bounded wait) with it, instead of the hard kill+delete used before this
-// fix.
-func TestStopTask_UsesContainerStopTimeout(t *testing.T) {
-	cp := &fakeCP{}
-	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
-		ID: "t-001-web",
-		Labels: map[string]string{
-			labelTaskID: "t-001", labelContainerName: "web", labelStopTimeout: "45",
-		},
-	}}}
-	a := newRunAgent(cp, rt)
-
-	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
-
-	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
-		t.Fatalf("want t-001-web stopped, got %+v", rt.Stopped)
-	}
-	if len(rt.StopTimeouts) != 1 || rt.StopTimeouts[0] != 45*time.Second {
-		t.Fatalf("want stopTimeout 45s, got %+v", rt.StopTimeouts)
-	}
-}
-
-// TestStopTask_DefaultStopTimeoutWhenUnset verifies a container with no
-// stopTimeout label gets the EC2 launch-type default rather than an unbounded
-// or zero grace period.
-func TestStopTask_DefaultStopTimeoutWhenUnset(t *testing.T) {
-	cp := &fakeCP{}
-	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
-		ID:     "t-001-web",
-		Labels: map[string]string{labelTaskID: "t-001", labelContainerName: "web"},
-	}}}
-	a := newRunAgent(cp, rt)
-
-	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
-
-	if len(rt.StopTimeouts) != 1 || rt.StopTimeouts[0] != defaultStopTimeout {
-		t.Fatalf("want default stopTimeout %v, got %+v", defaultStopTimeout, rt.StopTimeouts)
-	}
-}
-
 // TestRunTask_GPUCarriedToRunSpec verifies that the AssignContainer's GPU count
 // reaches the runtime RunSpec unchanged; a
 // container with no GPU request keeps RunSpec.GPU at zero (regression).
@@ -252,7 +100,7 @@ func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
 func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
 	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
 	as := testAssign()
 	as.Containers = append(as.Containers, bus.AssignContainer{
 		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
@@ -296,7 +144,7 @@ func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
 func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
 	cp := &fakeCP{}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
 	as := testAssign()
 	as.Containers = append(as.Containers, bus.AssignContainer{
 		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
@@ -361,7 +209,7 @@ func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
 			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
 		}},
 	}}
-	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, nil)
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, rt, nil)
 	if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
 		t.Fatalf("seed pin: %v", err)
 	}
@@ -439,7 +287,7 @@ func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
 		nil,   // poll 3: gateway dropped it after the ack
 	}}
 	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{})
@@ -494,8 +342,8 @@ func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {
 
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
 
-	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
-		t.Fatalf("want only t-001-web stopped, got %+v", rt.Stopped)
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want only t-001-web removed, got %+v", rt.Removed)
 	}
 	st := cp.taskStates()
 	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
@@ -533,12 +381,12 @@ func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
 	as := testAssign()
 	cp := &fakeCP{
 		pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
-		stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
+		stopReplies: [][]bus.StopDirective{{{TaskID: as.TaskID, Reason: "bye"}}},
 	}
 	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
 		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
 	}}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go a.pollAssignments(ctx, map[string]bool{})
@@ -577,126 +425,10 @@ func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
 	}
 }
 
-// countStopAcks reports how many polls carried taskID in their stop-ack list.
-func countStopAcks(acks [][]string, taskID string) int {
-	n := 0
-	for _, ack := range acks {
-		if slices.Contains(ack, taskID) {
-			n++
-		}
-	}
-	return n
-}
-
-// A reap that cannot run is never acked, so the directive stays in the gateway's
-// inbox and is redelivered and retried until the runtime comes back.
-func TestPollAssignments_UnperformableStopIsNotAckedAndRetried(t *testing.T) {
-	cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
-	rt := &ctrruntime.FakePuller{ListErr: errors.New("containerd is down")}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go a.pollAssignments(ctx, map[string]bool{})
-
-	deadline := time.After(time.Second)
-	for rt.Lists() < 3 {
-		select {
-		case <-deadline:
-			t.Fatalf("failed reap was not retried; List calls=%d", rt.Lists())
-		case <-time.After(2 * time.Millisecond):
-		}
-	}
-	cancel()
-
-	if n := countStopAcks(cp.stopAcks(), "t-001"); n != 0 {
-		t.Errorf("a reap that never ran was acked %d times; stopAcks=%v", n, cp.stopAcks())
-	}
-	if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 0 {
-		t.Errorf("a reap that never ran reported STOPPED %d times", stopped)
-	}
-}
-
-// Once the runtime recovers, the redelivered directive is reaped, reported
-// STOPPED and acked exactly once, which deletes it from the inbox.
-func TestPollAssignments_RecoveredStopIsAckedExactlyOnce(t *testing.T) {
-	cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
-	rt := &ctrruntime.FakePuller{
-		ListErr:    errors.New("containerd is down"),
-		ListErrFor: 1,
-		Containers: []ctrruntime.Container{
-			{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
-		},
-	}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go a.pollAssignments(ctx, map[string]bool{})
-
-	deadline := time.After(2 * time.Second)
-	for countStopAcks(cp.stopAcks(), "t-001") == 0 {
-		select {
-		case <-deadline:
-			t.Fatalf("recovered reap was never acked; states=%+v stopAcks=%v", cp.taskStates(), cp.stopAcks())
-		case <-time.After(2 * time.Millisecond):
-		}
-	}
-	time.Sleep(30 * time.Millisecond)
-	cancel()
-
-	if n := countStopAcks(cp.stopAcks(), "t-001"); n != 1 {
-		t.Errorf("stop acked %d times, want exactly 1; stopAcks=%v", n, cp.stopAcks())
-	}
-	if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 1 {
-		t.Errorf("task reported STOPPED %d times, want exactly 1", stopped)
-	}
-	if reaped := rt.Reaped(); len(reaped) != 1 || reaped[0] != "t-001-web" {
-		t.Errorf("want t-001-web reaped once, got %+v", reaped)
-	}
-}
-
-// A task whose reap completed is never started again by an assign the gateway
-// is still redelivering.
-func TestPollAssignments_StoppedTaskIsNotRerunByRedeliveredAssign(t *testing.T) {
-	as := testAssign()
-	cp := &fakeCP{
-		pollReplies: [][]bus.Assign{{*as}, {*as}, {*as}, {*as}, {*as}, {*as}},
-		stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
-	}
-	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
-		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
-	}}
-	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go a.pollAssignments(ctx, map[string]bool{})
-
-	deadline := time.After(time.Second)
-	for countStopAcks(cp.stopAcks(), as.TaskID) == 0 {
-		select {
-		case <-deadline:
-			t.Fatalf("stop never completed; states=%+v", cp.taskStates())
-		case <-time.After(2 * time.Millisecond):
-		}
-	}
-	// Let the remaining scripted assigns arrive after the reap has finished.
-	time.Sleep(40 * time.Millisecond)
-	cancel()
-
-	if runs := rt.RunCalls(); len(runs) != 0 {
-		t.Errorf("stopped task was started by a redelivered assign: %+v", runs)
-	}
-	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
-		t.Errorf("stopped task reported RUNNING %d times", running)
-	}
-}
-
 // stopTask with no runtime is a no-op: nothing to reap, no state reported.
 func TestStopTask_NilRunnerNoOp(t *testing.T) {
 	cp := &fakeCP{}
-	a := newAgent(config{}, testIdentity(), cp, nil, nil)
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
 	if len(cp.taskStates()) != 0 {
 		t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
@@ -727,8 +459,8 @@ func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
 	a := newRunAgent(cp, rt)
 	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})
 
-	if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
-		t.Fatalf("want container t-001-web reaped, got %+v", rt.Stopped)
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want container t-001-web reaped, got %+v", rt.Removed)
 	}
 	st := cp.taskStates()
 	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
@@ -807,7 +539,7 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
 // exec-role resolver is built.
 func TestPullResolver(t *testing.T) {
 	fake := sentinelResolver{}
-	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, fake)
+	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, nil, fake)
 
 	if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
 		t.Fatalf("no execution role: want instance resolver fallback")

cmd/eks-gateway-fetch/main.go

@@ -40,7 +40,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 // stagedAddon mirrors the gateway's StagedAddonManifest wire shape; duplicated

cmd/eks-gateway-publish/main.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 const (

cmd/eks-konnectivity-cert/main.go

@@ -38,8 +38,6 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/eks-token-webhook/cert.go

@@ -15,7 +15,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )
 
 // serverTLSConfig is the webhook's loopback serving TLS config.
@@ -33,12 +33,12 @@ func serverTLSConfig(cert tls.Certificate) *tls.Config {
 // valid even if the webhook process is restarted. Returns the parsed cert and
 // the certificate PEM (for embedding as the kubeconfig CA bundle).
 func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
-	// Fall through to regenerate on a missing/corrupt/incompatible persisted
-	// pair. The cert PEM is re-read because tls.Certificate keeps only DER.
-	if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
-		if certPEM, err := os.ReadFile(certPath); err == nil {
+	if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
+		cert, err := tls.X509KeyPair(certPEM, keyPEM)
+		if err == nil {
 			return cert, certPEM, nil
 		}
+		// Fall through to regenerate on a corrupt/incompatible persisted pair.
 	}
 
 	certPEM, keyPEM, err := generateSelfSigned()
@@ -58,6 +58,18 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 	return cert, certPEM, nil
 }
 
+func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {
+	c, err := os.ReadFile(certPath)
+	if err != nil {
+		return nil, nil, false
+	}
+	k, err := os.ReadFile(keyPath)
+	if err != nil {
+		return nil, nil, false
+	}
+	return c, k, true
+}
+
 func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
 	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {