fix: write MariaDB's startup spelling for time_zone into the option file

mariadbd has no time_zone startup option — the name only exists for SET — so
the resolved parameter group aborted the server while it parsed defaults, before
it opened the datadir. No pidfile was ever written and the reconciler failed
every instance at the 20-minute engine-health deadline.

ParameterSpec gains an option-file spelling, used only when rendering the
include. time_zone is the one setting that needs it, as default_time_zone, which
resolves @@global.time_zone to the same value. The customer-facing name is
unchanged: the API still reports time_zone and a live apply still issues
SET GLOBAL time_zone. Names read back out of a generated file return to the
catalog's namespace first, so a startup spelling cannot read as an unknown name
and report a pending restart nothing could clear.

setup.sh now feeds the catalog's resolved defaults to the image's own mariadbd
at build time, so a name the server refuses fails the build rather than every
instance built from it.

.github/ISSUE_TEMPLATE/1-bug.yml

@@ -1,59 +0,0 @@
-name: Bug report
-description: Something is not working as it should.
-labels: ["bug"]
-body:
-  - type: dropdown
-    id: component
-    attributes:
-      label: Component
-      options:
-        - EC2 / instances
-        - EBS / volumes
-        - S3 / object store
-        - VPC / networking
-        - IAM / authentication
-        - EKS
-        - API gateway
-        - Web UI
-        - CLI / daemon
-        - Observability / telemetry
-        - Build / CI / packaging
-        - Documentation
-        - Other / not sure
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: Spinifex version
-      description: Output of `spinifex --version`, or the commit SHA you are on.
-      placeholder: v1.0.0, or commit abc1234
-    validations:
-      required: true
-
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: What happened
-      description: What you saw, what you expected instead, and how to reproduce it.
-    validations:
-      required: true
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs and output
-      description: Relevant log lines, error messages or command output.
-      render: shell
-
-  - type: textarea
-    id: platform
-    attributes:
-      label: Platform
-      description: Where you are running it, and how you deployed it.
-      placeholder: |
-        OS: Debian 13
-        Kernel: 6.12
-        Arch: amd64
-        Deployment: single node, systemd

.github/ISSUE_TEMPLATE/2-feature.yml

@@ -1,11 +0,0 @@
-name: Feature request
-description: Suggest something new.
-labels: ["enhancement"]
-body:
-  - type: textarea
-    id: proposal
-    attributes:
-      label: What would you like to see?
-      description: What you are trying to do, and what stops you today.
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/3-docs.yml

@@ -1,20 +0,0 @@
-name: Documentation
-description: Something in the docs is wrong, missing or unclear.
-labels: ["documentation"]
-body:
-  - type: input
-    id: location
-    attributes:
-      label: Where
-      description: Link to the page, or the path to the file in this repository.
-      placeholder: https://docs.mulgadc.com/..., or README.md
-    validations:
-      required: true
-
-  - type: textarea
-    id: problem
-    attributes:
-      label: What is wrong
-      description: What is inaccurate, missing or confusing, and what it should say instead.
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Documentation
-    url: https://docs.mulgadc.com
-    about: Guides, configuration reference and API compatibility notes.

.github/actions/e2e-analyze/analyze.go

@@ -91,12 +91,9 @@ type SuiteReport struct {
 	Label     string // short name, e.g. "single"
 	Total     int
 	FailCount int
-	// Unresolved counts testcases go-junit-report could not find an outcome
-	// for. They are not failures and the suite can be green with hundreds.
-	Unresolved int
-	Root       *Failure    // nil if no failures
-	Cascades   []Failure   // everything else that failed in this suite
-	Buckets    [][]Failure // failures grouped by signature (root's bucket first)
+	Root      *Failure    // nil if no failures
+	Cascades  []Failure   // everything else that failed in this suite
+	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
 	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
 	// the XML was produced — i.e. when the suite finished, not when it began.
 	EndedAt time.Time
@@ -369,7 +366,7 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 func computeRolledUpSet(cases []junitTC) map[string]bool {
 	rolledUp := make(map[string]bool, len(cases))
 	for _, tc := range cases {
-		if tc.Failure == nil && !isRealError(tc.Error) {
+		if tc.Failure == nil && tc.Error == nil {
 			continue
 		}
 		// Mark every ancestor: the failure belongs to the deepest case that
@@ -415,18 +412,6 @@ func parseStartTime(s string) time.Time {
 	return t
 }
 
-// noResultMessage is go-junit-report's marker for a testcase it saw start and
-// never saw finish. On cell-20 it accounted for 93 of the 120 testcases and 24
-// of the 25 reported failures, none of which had failed.
-const noResultMessage = "No test result found"
-
-// isRealError separates a genuine <error> from the parser's no-result marker.
-// A parent must not be treated as rolled up by a child whose outcome is merely
-// unknown, or the parent's own failure is dropped along with it.
-func isRealError(e *junitFailure) bool {
-	return e != nil && strings.TrimSpace(e.Message) != noResultMessage
-}
-
 // suiteLabel turns "junit-single.xml" into "single".
 func suiteLabel(path string) string {
 	base := filepath.Base(path)
@@ -472,17 +457,6 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 			case tc.Failure != nil:
 				body = tc.Failure.Body
 			case tc.Error != nil:
-				// go-junit-report emits this when a `=== RUN` line has no
-				// matching result line. It means the outcome is unknown, not
-				// that the test errored, and it attaches the test's own log
-				// output — which then reads as an assertion.
-				if !isRealError(tc.Error) {
-					rep.Unresolved++
-					if leaf {
-						cumul += tc.Time
-					}
-					continue
-				}
 				body = tc.Error.Body
 			default:
 				if leaf {
@@ -579,18 +553,6 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 }
 
 // Render writes the markdown report.
-// writeUnresolved names the testcases whose outcome the parser could not read.
-// Worth surfacing because a suite that leaves most of its subtests unresolved
-// is reporting on far less than its test count suggests.
-func writeUnresolved(b *strings.Builder, s SuiteReport) {
-	if s.Unresolved == 0 {
-		return
-	}
-	fmt.Fprintf(b, "ℹ️ %d of %d testcases have no result line in the log, so `go-junit-report` "+
-		"could not read their outcome. Not failures — but this suite reports on %d tests, not %d.\n\n",
-		s.Unresolved, s.Total, s.Total-s.Unresolved, s.Total)
-}
-
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
@@ -617,11 +579,9 @@ func Render(r Report) string {
 	for _, s := range r.Suites {
 		if s.FailCount == 0 {
 			fmt.Fprintf(&b, "### Suite `%s`: ✅ pass (%d tests)\n\n", s.Label, s.Total)
-			writeUnresolved(&b, s)
 			continue
 		}
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
-		writeUnresolved(&b, s)
 
 		if s.Root != nil {
 			// XML order is completion order, so "earliest" only means earliest

.github/actions/e2e-analyze/analyze_test.go

@@ -174,55 +174,6 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
-// go-junit-report emits <error message="No test result found"> for a `=== RUN`
-// with no matching result line, and attaches the test's own log output. On a
-// baremetal cell that was 93 of 120 testcases and 24 of 25 reported failures,
-// none of which had failed: "ALB internal: 20/20 successful" read as an error.
-func TestParseFile_NoResultFoundIsNotAFailure(t *testing.T) {
-	xml := []byte(`<?xml version="1.0"?>
-<testsuites><testsuite name="" tests="3" failures="1" errors="2">
-  <testcase name="TestLB/Internal_ALB" time="0.0"><error message="No test result found"><![CDATA[
-    lb_test.go:88: ALB internal: 20/20 successful, 2 unique
-]]></error></testcase>
-  <testcase name="TestLB/Internal_NLB" time="0.0"><error message="No test result found"></error></testcase>
-  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
-    foo_test.go:10:
-        Messages: real assertion message
-]]></failure></testcase>
-</testsuite></testsuites>`)
-	sr, err := ParseFile("junit-x.xml", xml)
-	if err != nil {
-		t.Fatalf("ParseFile: %v", err)
-	}
-	if sr.FailCount != 1 {
-		t.Fatalf("FailCount = %d, want 1 (no-result entries are not failures)", sr.FailCount)
-	}
-	if sr.Unresolved != 2 {
-		t.Fatalf("Unresolved = %d, want 2", sr.Unresolved)
-	}
-	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
-		t.Fatalf("expected the real failure as root, got %+v", sr.Root)
-	}
-}
-
-// An <error> that is not the parser's no-result marker is a genuine error and
-// must still be reported.
-func TestParseFile_KeepsARealError(t *testing.T) {
-	xml := []byte(`<?xml version="1.0"?>
-<testsuites><testsuite name="" tests="1" failures="0" errors="1">
-  <testcase name="TestX" time="0.5"><error message="panic"><![CDATA[
-    foo_test.go:10: panic: runtime error: index out of range
-]]></error></testcase>
-</testsuite></testsuites>`)
-	sr, err := ParseFile("junit-x.xml", xml)
-	if err != nil {
-		t.Fatalf("ParseFile: %v", err)
-	}
-	if sr.FailCount != 1 || sr.Unresolved != 0 {
-		t.Fatalf("FailCount = %d, Unresolved = %d, want 1 and 0", sr.FailCount, sr.Unresolved)
-	}
-}
-
 func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
 	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
 	// "DB instance … is gone" — which passing tests emit too — reads exactly

.github/actions/junit-to-es/action.yml

@@ -1,43 +0,0 @@
-name: JUnit to Elasticsearch
-description: >-
-  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
-  so per-test pass rates outlive the 90-day artifact retention. Call it once
-  per job right after the JUnit XML is produced (with `if: always()`), next to
-  the junit-report action that renders the same files into the job summary.
-
-inputs:
-  junit-glob:
-    description: Shell glob matching the JUnit XML files to ship.
-    required: true
-  cell:
-    description: >-
-      Cell label stored as labels.ci_cell — the same value the job puts in
-      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
-      Without it, results from different topologies collapse into one bucket.
-    required: true
-  es-sink:
-    description: host:port of the Elasticsearch sink.
-    required: false
-    default: "192.168.1.13:9200"
-  index:
-    description: Target index.
-    required: false
-    default: mulga-ci-test-results
-
-runs:
-  using: composite
-  steps:
-    - shell: bash
-      env:
-        JUNIT_GLOB: ${{ inputs.junit-glob }}
-        JUNIT_CELL: ${{ inputs.cell }}
-        MULGA_ES_SINK: ${{ inputs.es-sink }}
-        ES_INDEX: ${{ inputs.index }}
-        RUN_ID: ${{ github.run_id }}
-        RUN_ATTEMPT: ${{ github.run_attempt }}
-        WORKFLOW: ${{ github.workflow }}
-        BRANCH: ${{ github.head_ref || github.ref_name }}
-        SHA: ${{ github.sha }}
-        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-      run: |
-        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-"""Ship per-test JUnit results to the Elasticsearch sink.
-
-One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
-mulga-ci-runs summary says which suite failed; this says which test, with the
-failure text, so a per-test pass-rate trend outlives artifact retention.
-
-Inputs (env):
-    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
-                    a leg that never ran produces none.
-    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
-    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
-                    there, so no credentials are involved.
-    ES_INDEX        Target index; defaults to mulga-ci-test-results.
-    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
-"""
-
-from __future__ import annotations
-
-import glob
-import json
-import os
-import re
-import sys
-import urllib.error
-import urllib.request
-import xml.etree.ElementTree as ET
-from datetime import datetime, timezone
-
-# Go test output carries colour codes into the JUnit CDATA (harness.Phase
-# emits them), which are noise in a stored failure message.
-_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
-
-# Enough failure text to recognise a failure; the artifact log stays the full
-# record.
-_MAX_MESSAGE = 4000
-
-# go-junit-report emits this when a `=== RUN` line has no matching result line,
-# which means the log it was fed was truncated or filtered. It is a parse gap,
-# not a test outcome, so it is stored under its own status and left out of every
-# pass rate rather than counted as a failure.
-_NO_RESULT = "No test result found"
-
-
-def _strip_ansi(s: str) -> str:
-    return _ANSI_RE.sub("", s)
-
-
-def _truncate(s: str) -> str:
-    s = s.strip()
-    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
-
-
-def _suite_from_path(path: str) -> str:
-    base = os.path.basename(path)
-    if base.endswith(".xml"):
-        base = base[:-4]
-    if base.startswith("junit-"):
-        base = base[len("junit-") :]
-    return base
-
-
-def _cases(path: str) -> list[dict]:
-    """Parse one JUnit file into leaf test cases."""
-    try:
-        root = ET.parse(path).getroot()
-    except ET.ParseError as exc:
-        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
-        return []
-
-    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
-    # several. .iter() handles both.
-    out: list[dict] = []
-    for s in root.iter("testsuite"):
-        for tc in s.findall("testcase"):
-            status = "pass"
-            fail_type = ""
-            message = ""
-            for tag in ("failure", "error"):
-                node = tc.find(tag)
-                if node is not None:
-                    status = "fail" if tag == "failure" else "error"
-                    fail_type = tag
-                    message = _strip_ansi(
-                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
-                    )
-                    if (node.get("message") or "").strip() == _NO_RESULT:
-                        status = "unknown"
-                    break
-            if status == "pass" and tc.find("skipped") is not None:
-                status = "skip"
-            out.append(
-                {
-                    "name": tc.get("name", ""),
-                    "classname": tc.get("classname", ""),
-                    "time": float(tc.get("time", 0) or 0),
-                    "status": status,
-                    "fail_type": fail_type,
-                    "message": message,
-                }
-            )
-
-    # Drop container cases: Go reports a parent t.Run as a rollup of its
-    # children, so keeping both double-counts one outcome. Same rule the job
-    # summary renderer applies.
-    names = {c["name"] for c in out}
-    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
-    return [c for c in out if c["name"] not in containers]
-
-
-def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
-    full = case["name"]
-    top, _, sub = full.partition("/")
-    doc = {
-        "@timestamp": ts,
-        "suite": suite,
-        "status": case["status"],
-        "duration_seconds": case["time"],
-        "artifact_url": ci["artifact_url"],
-        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
-        "test": {
-            "name": top,
-            "subtest": sub,
-            "full_name": full,
-            "classname": case["classname"],
-        },
-        "ci": {
-            "workflow": ci["workflow"],
-            "branch": ci["branch"],
-            "sha": ci["sha"],
-            "run_attempt": ci["run_attempt"],
-        },
-    }
-    if case["fail_type"]:
-        doc["failure"] = {
-            "type": case["fail_type"],
-            "message": _truncate(case["message"]),
-        }
-    return doc
-
-
-def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
-    # Deterministic so a replayed post overwrites instead of double-counting.
-    # run_attempt is part of it because a re-run is a distinct result.
-    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
-
-
-def _bulk(sink: str, index: str, body: str) -> int:
-    req = urllib.request.Request(
-        f"http://{sink}/{index}/_bulk",
-        data=body.encode("utf-8"),
-        headers={"Content-Type": "application/x-ndjson"},
-        method="POST",
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=60) as resp:
-            payload = json.loads(resp.read().decode("utf-8"))
-    except (urllib.error.URLError, OSError) as exc:
-        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
-        return 1
-
-    if payload.get("errors"):
-        for item in payload.get("items", []):
-            err = next(iter(item.values())).get("error")
-            if err:
-                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
-                return 1
-    return 0
-
-
-def main() -> int:
-    pattern = os.environ.get("JUNIT_GLOB", "").strip()
-    if not pattern:
-        print("::error::junit-to-es: JUNIT_GLOB is empty")
-        return 1
-    cell = os.environ.get("JUNIT_CELL", "").strip()
-    if not cell:
-        print("::error::junit-to-es: JUNIT_CELL is empty")
-        return 1
-    sink = os.environ.get("MULGA_ES_SINK", "").strip()
-    if not sink:
-        print("::error::junit-to-es: MULGA_ES_SINK is empty")
-        return 1
-    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
-
-    ci = {
-        "run_id": os.environ.get("RUN_ID", ""),
-        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
-        "workflow": os.environ.get("WORKFLOW", ""),
-        "branch": os.environ.get("BRANCH", ""),
-        "sha": os.environ.get("SHA", ""),
-        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
-    }
-    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    lines: list[str] = []
-    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
-    for path in sorted(glob.glob(pattern)):
-        suite = _suite_from_path(path)
-        for case in _cases(path):
-            doc = _doc(cell, suite, case, ts, ci)
-            counts[case["status"]] = counts.get(case["status"], 0) + 1
-            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
-            lines.append(json.dumps(meta))
-            lines.append(json.dumps(doc))
-
-    if not lines:
-        # Not an error on its own: the leg may have been skipped, or died
-        # before any test ran. The run summary doc records that separately.
-        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
-        return 0
-
-    rc = _bulk(sink, index, "\n".join(lines) + "\n")
-    if rc == 0:
-        print(
-            f"junit-to-es: posted {sum(counts.values())} test docs "
-            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
-            f"{counts['fail']} fail, {counts['error']} error, "
-            f"{counts['skip']} skip, {counts['unknown']} unknown"
-        )
-    return rc
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/e2e-cells.json

@@ -1,147 +0,0 @@
-[
-  {
-    "cell": 1,
-    "install": "binary",
-    "topo": "single",
-    "net": "dhcp",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 2,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 3,
-    "install": "binary",
-    "topo": "single",
-    "net": "dhcp",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 8,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "dhcp",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 9,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "static",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 10,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "dhcp",
-    "host": "ubuntu-26.04",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": ""
-  },
-  {
-    "cell": 17,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-pseudo",
-    "extra": "tofu-examples",
-    "suite": ""
-  },
-  {
-    "cell": 18,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "reboot-single",
-    "suite": ""
-  },
-  {
-    "cell": 19,
-    "install": "binary",
-    "topo": "single",
-    "net": "nat",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "nat-single",
-    "suite": ""
-  },
-  {
-    "cell": 25,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single-nvme",
-    "extra": "eks",
-    "suite": "eks"
-  },
-  {
-    "cell": 26,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "ecs",
-    "suite": "ecs"
-  },
-  {
-    "cell": 27,
-    "install": "binary",
-    "topo": "single",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-single",
-    "extra": "rds",
-    "suite": "rds",
-    "timeout_minutes": 100
-  },
-  {
-    "cell": 29,
-    "install": "binary",
-    "topo": "real-multi",
-    "net": "static",
-    "host": "debian-13",
-    "arch": "x86_64",
-    "runner": "ci-multi",
-    "extra": "",
-    "suite": "storagefault",
-    "timeout_minutes": 100
-  }
-]

.github/resolve-sibling-refs.sh

@@ -1,21 +1,20 @@
 #!/usr/bin/env bash
-# Resolve which branch of each sibling repo (viperblock, predastore, northstar,
-# bluebottle) the CI workspace should build against. A cross-repo change is developed on a branch
+# Resolve which branch of each sibling repo (viperblock, predastore) the CI
+# workspace should build against. A cross-repo change is developed on a branch
 # of the SAME name in each repo; build against that branch when it exists so the
 # set can be tested together, and fall back to dev otherwise. This mirrors the
 # resolution the e2e path already does in
 # mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
 # agree on how sibling branches are selected.
 #
-# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref>,
-# predastore=<ref>, northstar=<ref> and bluebottle=<ref> to $GITHUB_OUTPUT for
-# the checkout steps to consume.
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
+# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
 set -euo pipefail
 
 : "${BRANCH:?BRANCH must be set to the triggering branch name}"
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
 
-for repo in viperblock predastore northstar bluebottle; do
+for repo in viperblock predastore; do
   if git ls-remote --exit-code --heads \
       "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
     ref="$BRANCH"

.github/scripts/ci-fmt.sh

@@ -61,37 +61,3 @@ quiet_run() {
     return "$rc"
   fi
 }
-
-# Connection-layer failures from the libvirt tofu provider. The socket drops
-# when a hypervisor is carrying several environments at once, which is the
-# steady state here. A resource that failed to build says something else.
-CI_LIBVIRT_TRANSIENT='Unable to Connect to Libvirt|error connecting to libvirt|procedure interrupted|Cannot recv data|Cannot write data|End of file while reading data'
-
-# retry_run LABEL LOG_FILE -- CMD ARGS …
-# quiet_run with a bounded retry, taken only when that attempt's own output
-# names a libvirt connection failure. Every other failure is reported on the
-# first attempt, so a real bug never becomes a slow flake.
-retry_run() {
-  local label="$1" log="$2"; shift 2
-  local _attempts="${CI_RETRY_ATTEMPTS:-3}" _delay="${CI_RETRY_DELAY:-30}" _i _rc=1 _mark
-  for ((_i = 1; _i <= _attempts; _i++)); do
-    _mark=$(wc -c < "$log" 2>/dev/null || echo 0)
-    # rc must be read inside the else: an `if` whose condition fails and has no
-    # else exits 0, so reading $? after `fi` reports success for a failed run.
-    if "$@" >> "$log" 2>&1; then
-      if [[ $_i -gt 1 ]]; then ok "$label (libvirt recovered on attempt $_i)"; else ok "$label"; fi
-      return 0
-    else
-      _rc=$?
-    fi
-    tail -c "+$((_mark + 1))" "$log" | grep -Eq "$CI_LIBVIRT_TRANSIENT" || break
-    [[ $_i -eq $_attempts ]] && break
-    warn "$label: libvirt connection dropped (attempt $_i/$_attempts), retrying in ${_delay}s"
-    sleep "$_delay"
-  done
-  bad "$label (rc=$_rc) — last 80 lines of $(basename "$log"):"
-  group_open "tail $(basename "$log")"
-  tail -n 80 "$log"
-  group_close
-  return "$_rc"
-}

.github/scripts/e2e-run-summary.py

@@ -1,220 +0,0 @@
-#!/usr/bin/env python3
-"""Render one run-level table for the e2e nightly into the job summary.
-
-Reading a nightly meant opening nineteen jobs, and the Actions UI does not
-distinguish a cell that failed from one that was cancelled out from under it —
-a distinction every investigation of this suite has needed.
-
-Inputs (env):
-    GITHUB_TOKEN         Token with actions:read.
-    GITHUB_REPOSITORY    owner/repo.
-    GITHUB_RUN_ID        Run to summarise.
-    JUNIT_ROOT           Directory holding downloaded junit-cell-* artifacts.
-    GITHUB_STEP_SUMMARY  Appended to.
-
-Test counts come from the junit artifacts; result, duration and runner come
-from the jobs API, so a cell that died before writing junit still gets a row.
-"""
-
-from __future__ import annotations
-
-import glob
-import json
-import os
-import sys
-import urllib.error
-import urllib.request
-import xml.etree.ElementTree as ET
-from datetime import datetime
-
-API = "https://api.github.com"
-
-# A cell whose job produced no verdict is not a pass and not a failure. Keeping
-# them apart is the point of the table.
-VERDICT_ICON = {
-    "success": "✅",
-    "failure": "❌",
-    "cancelled": "🚫",
-    "skipped": "⏭️",
-    "timed_out": "⏱️",
-}
-
-
-def fetch_jobs(repo: str, run_id: str, token: str) -> list[dict]:
-    jobs: list[dict] = []
-    page = 1
-    while True:
-        url = f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}"
-        req = urllib.request.Request(
-            url,
-            headers={
-                "Authorization": f"Bearer {token}",
-                "Accept": "application/vnd.github+json",
-                "X-GitHub-Api-Version": "2022-11-28",
-            },
-        )
-        try:
-            with urllib.request.urlopen(req, timeout=30) as resp:
-                payload = json.load(resp)
-        except (urllib.error.URLError, TimeoutError) as exc:
-            print(f"::warning::jobs API page {page} failed: {exc}", file=sys.stderr)
-            break
-        batch = payload.get("jobs", [])
-        jobs.extend(batch)
-        if len(batch) < 100:
-            break
-        page += 1
-    return jobs
-
-
-def duration_s(job: dict) -> int | None:
-    started, completed = job.get("started_at"), job.get("completed_at")
-    if not started or not completed:
-        return None
-    fmt = "%Y-%m-%dT%H:%M:%SZ"
-    try:
-        return int((datetime.strptime(completed, fmt) - datetime.strptime(started, fmt)).total_seconds())
-    except ValueError:
-        return None
-
-
-def human(seconds: int | None) -> str:
-    if seconds is None:
-        return "—"
-    if seconds < 0:
-        return "0s"
-    return f"{seconds // 60}m{seconds % 60:02d}s" if seconds >= 60 else f"{seconds}s"
-
-
-def junit_counts(root: str) -> dict[str, dict]:
-    """Map cell number to its aggregate junit counts and failing test names."""
-    per_cell: dict[str, dict] = {}
-    for path in glob.glob(os.path.join(root, "**", "junit-*.xml"), recursive=True):
-        cell = cell_from_artifact_path(path)
-        if cell is None:
-            continue
-        acc = per_cell.setdefault(cell, {"tests": 0, "failures": 0, "skipped": 0, "failing": []})
-        try:
-            root_el = ET.parse(path).getroot()
-        except ET.ParseError as exc:
-            print(f"::warning::unparseable junit {path}: {exc}", file=sys.stderr)
-            continue
-        suites = root_el.iter("testsuite") if root_el.tag != "testsuite" else [root_el]
-        for suite in suites:
-            acc["tests"] += int(suite.get("tests") or 0)
-            acc["failures"] += int(suite.get("failures") or 0) + int(suite.get("errors") or 0)
-            acc["skipped"] += int(suite.get("skipped") or 0)
-        for case in root_el.iter("testcase"):
-            if case.find("failure") is not None or case.find("error") is not None:
-                name = case.get("name") or "?"
-                if name not in acc["failing"]:
-                    acc["failing"].append(name)
-    return per_cell
-
-
-def cell_from_artifact_path(path: str) -> str | None:
-    """Recover the cell number from a junit-cell-<n>-<run_id> artifact directory."""
-    for part in path.split(os.sep):
-        if part.startswith("junit-cell-"):
-            rest = part[len("junit-cell-"):]
-            cell = rest.split("-", 1)[0]
-            return cell if cell.isdigit() else None
-    return None
-
-
-def cell_number(job_name: str) -> str | None:
-    """cell-19 (single/static/debian-13) -> 19."""
-    if not job_name.startswith("cell-"):
-        return None
-    cell = job_name[len("cell-"):].split(" ", 1)[0]
-    return cell if cell.isdigit() else None
-
-
-def main() -> int:
-    repo = os.environ.get("GITHUB_REPOSITORY", "")
-    run_id = os.environ.get("GITHUB_RUN_ID", "")
-    token = os.environ.get("GITHUB_TOKEN", "")
-    summary_path = os.environ.get("GITHUB_STEP_SUMMARY", "")
-    junit_root = os.environ.get("JUNIT_ROOT", "")
-
-    if not (repo and run_id and token and summary_path):
-        print("::warning::e2e-run-summary: missing required environment", file=sys.stderr)
-        return 0
-
-    jobs = fetch_jobs(repo, run_id, token)
-    counts = junit_counts(junit_root) if junit_root and os.path.isdir(junit_root) else {}
-
-    rows = []
-    for job in jobs:
-        cell = cell_number(job.get("name", ""))
-        if cell is None:
-            continue
-        rows.append((int(cell), job))
-    rows.sort(key=lambda r: r[0])
-
-    lines = [f"## e2e nightly — run summary ({len(rows)} cells)", ""]
-    if not rows:
-        lines.append("No cell jobs found for this run.")
-        write(summary_path, lines)
-        return 0
-
-    lines += [
-        "| Cell | Topology | Result | Duration | Runner | Tests | Failed | Skipped |",
-        "| --- | --- | --- | --- | --- | --- | --- | --- |",
-    ]
-
-    tally: dict[str, int] = {}
-    total_tests = total_failed = total_seconds = 0
-    failing_detail: list[tuple[int, list[str]]] = []
-
-    for cell, job in rows:
-        name = job.get("name", "")
-        # A cell dispatched to a reusable workflow reports as
-        # "cell-20 (baremetal/pxe/real-hw) / Bare-Metal Single-Node E2E".
-        topo = name.split("(", 1)[1].split(")", 1)[0] if "(" in name else "—"
-        verdict = job.get("conclusion") or job.get("status") or "unknown"
-        tally[verdict] = tally.get(verdict, 0) + 1
-        secs = duration_s(job)
-        total_seconds += secs or 0
-        c = counts.get(str(cell), {})
-        total_tests += c.get("tests", 0)
-        total_failed += c.get("failures", 0)
-        if c.get("failing"):
-            failing_detail.append((cell, c["failing"]))
-        lines.append(
-            f"| {cell} | {topo} | {VERDICT_ICON.get(verdict, '❔')} {verdict} | {human(secs)} "
-            f"| {job.get('runner_name') or '—'} | {c.get('tests', '—')} "
-            f"| {c.get('failures', '—')} | {c.get('skipped', '—')} |"
-        )
-
-    verdicts = ", ".join(f"{n} {v}" for v, n in sorted(tally.items(), key=lambda kv: -kv[1]))
-    lines += [
-        "",
-        f"**Cells:** {verdicts}",
-        f"**Tests:** {total_tests} run, {total_failed} failed",
-        f"**Cell wall clock:** {human(total_seconds)} across all cells",
-    ]
-
-    slowest = sorted(rows, key=lambda r: duration_s(r[1]) or 0, reverse=True)[:3]
-    if slowest:
-        lines.append(
-            "**Slowest:** " + ", ".join(f"cell {c} ({human(duration_s(j))})" for c, j in slowest)
-        )
-
-    if failing_detail:
-        lines += ["", "<details><summary>Failing tests</summary>", ""]
-        for cell, names in failing_detail:
-            lines.append(f"- **cell {cell}** — " + ", ".join(f"`{n}`" for n in names))
-        lines += ["", "</details>"]
-
-    write(summary_path, lines)
-    return 0
-
-
-def write(path: str, lines: list[str]) -> None:
-    with open(path, "a", encoding="utf-8") as fh:
-        fh.write("\n".join(lines) + "\n")
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/scripts/ensure-libguestfs.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-# Prepare a runner to build a guest image with build-system-image.sh.
-#
-# Installs libguestfs + qemu-utils if they are missing and makes a host kernel
-# readable, so the libguestfs appliance can boot. Nothing here touches a host
-# block device: virt-customize works inside an isolated userspace appliance, so
-# this is safe on a shared runner.
-#
-# Callers must still export LIBGUESTFS_BACKEND=direct themselves — it has to be
-# set in the environment the build runs in, which is not this subshell.
-set -euo pipefail
-
-# Three nightly service cells share one ci-single runner and start together, so
-# concurrent apt-get runs are the normal case rather than the exception. Two of
-# them lost the lock outright before this guard existed. Skipping the install
-# when the tools are already present avoids most of the contention; the lock
-# timeout covers the rest, including apt runs started by anything else on the
-# box.
-if command -v virt-customize >/dev/null && command -v qemu-img >/dev/null; then
-    echo "libguestfs-tools and qemu-utils already present"
-else
-    APT_OPTS=(-o DPkg::Lock::Timeout=600)
-    sudo apt-get "${APT_OPTS[@]}" update -qq
-    sudo apt-get "${APT_OPTS[@]}" install -y --no-install-recommends \
-        libguestfs-tools qemu-utils
-fi
-
-# Debian/Ubuntu ship /boot/vmlinuz-* mode 0600 and the appliance needs to read a
-# host kernel. Tolerated on failure: a runner whose kernel is already readable
-# has no vmlinuz to chmod under some images.
-sudo chmod 0644 /boot/vmlinuz-* || true

.github/scripts/suite-sets.sh

@@ -24,87 +24,23 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-# quota belongs to both topologies because the gate it asserts is gateway code
-# that runs identically on either, and because a quota left switched on would
-# cap every suite sharing the environment — so it must be proved to restore
-# itself wherever it runs, not only on one node.
-# storagefault runs last in both lists and belongs to both topologies, for
-# different reasons. On one node it is RS(1,0), so freezing the only predastore
-# is a total outage and the guest-survival assertion is unambiguous. On three it
-# is RS(2,1) with degraded writes on, so two must be frozen to break the shard
-# floor — and only there is a cross-node restart possible at all, which is the
-# case that takes a volume over from a node holding writes the backend never
-# received. It is last because it SIGSTOPs a cluster-wide
-# service: anything sharing the environment would see the same outage and report
-# a failure that says nothing about itself.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert quota storagefault"
+E2E_SUITES_MULTI="multinode lb cert"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
 E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 
-# What every e2e-nightly permutation cell must prove, deliberately narrower
-# than the two sets above: those cells have a 35-minute budget and exist to show
-# that every install / network / host-OS combination boots and serves. The
-# service suites are not deferred to the PR gate — eks, ecs and rds each run
-# nightly in their own dedicated cell, because each needs a guest AMI built
-# first and a service failure should not mask whether a host OS boots. These
-# are subsets, and e2e_suites_assert_subset below enforces that.
-#
-# storagefault has a dedicated cell for two reasons. It SIGSTOPs a cluster-wide
-# service, so in a permutation cell every suite sharing that environment would
-# report the outage as its own failure. And a full pass is ~63 minutes, nearly
-# all of it deliberate waiting, which is too long to add to the PR gate — where
-# it has never run, since the gate's default suite list does not name it.
-#
-# Its cell is three-node: the RS(2,1) shard floor and the cross-node takeover
-# after a failed seal do not exist on one node, and they are the assertions
-# worth a nightly.
-E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
+# e2e-nightly runs a narrower set than the two sets above, deliberately: its
+# ~19 cells each have a 35-minute budget and exist to prove that every
+# install / network / host-OS permutation boots and serves. Suite breadth is
+# the PR gate's job, and it already runs on every push. These are subsets, and
+# e2e_suites_assert_subset below enforces that.
+E2E_SUITES_NIGHTLY_SINGLE="single cert"
 E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
 
-# How long a suite gets, for the same reason the lists above live here: the
-# timeout was written out separately in each workflow and drifted, so rds ran
-# 50m on the PR gate and 30m nightly and panicked there. A suite needing more
-# than the default declares it once, here.
-#   e2e_suite_timeout <suite>
-e2e_suite_timeout() {
-  case "$1" in
-    # rds serialises its DB VMs behind a 3 GiB semaphore, so its wall clock is
-    # what the queue costs, not what the tests do: single tests logged 11m, 22m
-    # and 26m waiting for budget. 17 tests at that concurrency is over an hour
-    # before anything goes wrong, and the first pass to run deep into the suite
-    # still had three left at 60m.
-    rds) echo "90m" ;;
-    # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
-    # creates first. One pass is ~20 minutes on the bare-metal cell and a
-    # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
-    diskperf) echo "180m" ;;
-    # storagefault is mostly deliberate waiting: one test alone holds predastore
-    # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
-    # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
-    storagefault) echo "90m" ;;
-    *) echo "30m" ;;
-  esac
-}
-
-# One matrix leg per suite, carrying the job budget that suite needs. The outer
-# timeout must always be later than the `go test` one inside it: go test reports
-# where it stopped, a cancelled job reports only "The operation was canceled".
-#   e2e_suite_matrix <suite-list> <floor-minutes> [allowance-minutes]
-e2e_suite_matrix() {
-  local list="$1" floor="$2" allowance="${3:-15}" suite budget
-  for suite in ${list}; do
-    budget="$(e2e_suite_timeout "${suite}")"
-    budget=$((${budget%m} + allowance))
-    if [ "${budget}" -lt "${floor}" ]; then budget="${floor}"; fi
-    printf '%s\t%s\n' "${suite}" "${budget}"
-  done | jq -cRs 'split("\n")|map(select(length>0)|split("\t")|{suite:.[0],timeout:(.[1]|tonumber)})'
-}
-
 # Fail loudly when a narrowed list names a suite its topology cannot run. This
 # is the guard that stops the lists drifting again: a suite added to a nightly
 # subset without also being declared eligible above stops the job here rather

.github/workflows/apt-packages.yml

@@ -33,8 +33,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # The suites the nightly E2E matrix actually installs on. This matches
-        # what detect_os accepts: Debian 13+ and Ubuntu 24.04+.
+        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
+        # accepted by detect_os but has never been exercised and does not resolve.
         image:
           - ubuntu:24.04
           - ubuntu:26.04

.github/workflows/e2e-baremetal.yml

@@ -1,26 +1,7 @@
 name: E2E Bare-Metal
 
 on:
-  # Mirrors the suite selection of workflow_call so a baseline capture can be
-  # driven by hand. A dispatch run sees only the inputs declared here, so
-  # omitting these would silently fall back to the correctness set.
   workflow_dispatch:
-    inputs:
-      suites:
-        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance run names its own so its numbers come off an otherwise idle cell"
-        required: false
-        type: string
-        default: "single cert lb"
-      suite-env:
-        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
-        required: false
-        type: string
-        default: ""
-      cell:
-        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
-        required: false
-        type: string
-        default: "20"
   workflow_call:
     inputs:
       ref:
@@ -33,21 +14,6 @@ on:
         required: false
         type: string
         default: ""
-      suites:
-        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance caller names its own so its numbers come off an otherwise idle cell"
-        required: false
-        type: string
-        default: "single cert lb"
-      suite-env:
-        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
-        required: false
-        type: string
-        default: ""
-      cell:
-        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
-        required: false
-        type: string
-        default: "20"
 
 permissions:
   contents: read
@@ -64,18 +30,16 @@ jobs:
   e2e_baremetal:
     name: Bare-Metal Single-Node E2E
     runs-on: [ self-hosted, ci-baremetal ]
-    timeout-minutes: 240
+    timeout-minutes: 120
     concurrency:
       group: e2e-baremetal-cluster
       cancel-in-progress: false
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
-      BM_CELL: ${{ inputs.cell || '20' }}
-      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
-      # Caller-supplied on workflow_call. A dispatch run has no such input, and
-      # leaving it empty cost this cell its ci.run_id: its telemetry reached the
-      # sink but could not be found by run, only by guessing a time window.
-      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
+      # Caller-supplied (workflow_call input); empty for a manual
+      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
     steps:
       - name: Clean Workspace
         run: |
@@ -110,10 +74,6 @@ jobs:
           ssh-keygen -y -f "${RUNNER_TEMP}/bm-ci" > "${RUNNER_TEMP}/bm-ci.pub"
 
       - name: Build Artifacts
-        env:
-          # The node runs pre-built binaries, so a suite absent here cannot be
-          # selected later. Same list as the run step for that reason.
-          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
         run: |
           mulga/scripts/baremetal/bm-build-artifacts.sh \
             "$GITHUB_WORKSPACE" \
@@ -159,21 +119,6 @@ jobs:
       - name: Install go-junit-report on runner
         run: go install github.com/jstemmer/go-junit-report/v2@latest
 
-      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
-      # AMI. Built here because libguestfs needs source and network, which the
-      # runner has and the PXE-booted node does not; bm-bootstrap.sh imports it.
-      - name: Build eks-node AMI (libguestfs, runner)
-        if: contains(inputs.suites, 'eks')
-        run: |
-          set -euo pipefail
-          # direct backend = no libvirt. Set here rather than in the script:
-          # it has to be in the environment the build itself runs in.
-          export LIBGUESTFS_BACKEND=direct
-          # spinifex is the workspace root here, unlike the nightly where it is
-          # checked out into a subdirectory.
-          "${GITHUB_WORKSPACE}/.github/scripts/ensure-libguestfs.sh"
-          make build-eks-node-image
-
       - name: Run Bare-Metal E2E
         env:
           BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
@@ -181,27 +126,9 @@ jobs:
           BM_EPDU_ENV: ${{ runner.temp }}/epdu.env
           BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
           SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
-          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
-          BM_SUITE_ENV: ${{ inputs.suite-env }}
-          # Empty unless the eks build above ran, and bm-bootstrap.sh skips the
-          # import when unset, so the other suites are unaffected.
-          BM_EKS_IMAGE: ${{ contains(inputs.suites, 'eks') && '/tmp/eks-node-image-build/eks-node-alpine.raw' || '' }}
-          BM_EKS_MANIFEST: ${{ contains(inputs.suites, 'eks') && format('{0}/scripts/images/eks-node/manifest.conf', github.workspace) || '' }}
         run: |
           mkdir -p "${ARTIFACT_DIR}"
           set -o pipefail
-          # Per-suite budgets are declared once in suite-sets.sh. The script
-          # applies one timeout to every suite in the list, so the widest wins.
-          . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
-          BM_TEST_TIMEOUT=""
-          for suite in ${BM_SUITES}; do
-            tmo="$(e2e_suite_timeout "${suite}")"
-            if [ -z "${BM_TEST_TIMEOUT}" ] || [ "${tmo%m}" -gt "${BM_TEST_TIMEOUT%m}" ]; then
-              BM_TEST_TIMEOUT="${tmo}"
-            fi
-          done
-          export BM_TEST_TIMEOUT
-          echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
           sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
@@ -212,40 +139,22 @@ jobs:
       - name: Generate JUnit report
         if: always()
         run: |
-          # suite-output.log, not test-baremetal.log. The harness streams the
-          # full suite output to the first and a grep-filtered summary to its
-          # stdout, and that filter anchors --- PASS at column 0, so every
-          # indented subtest result is dropped. Parsed from the filtered log,
-          # go-junit-report saw a === RUN with no result and emitted
-          # <error message="No test result found"> — 17 of 39 testcases on a
-          # green cell-28, 93 of 120 on cell-20.
-          LOG="${ARTIFACT_DIR}/suite-output.log"
-          [ -s "$LOG" ] || LOG="${ARTIFACT_DIR}/test-baremetal.log"
+          LOG="${ARTIFACT_DIR}/test-baremetal.log"
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: ${{ env.BM_CELL }}
-
       - name: Analyze e2e failures
         if: always()
         uses: ./.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
-          title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
+          title: "E2E failure analysis (cell 20)"
 
       - uses: actions/upload-artifact@v7
         if: always()
         with:
-          name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
+          name: e2e-artifacts-cell-20-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14

.github/workflows/e2e-ddil.yml

@@ -10,14 +10,6 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
   contents: read
   # actions: read lets the otel-job-id action look up this job's numeric id via
@@ -111,7 +103,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VMs provisioned"
 
       - name: Bootstrap via Binary Install
@@ -188,16 +180,6 @@ jobs:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
-          cell: ddil
-
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster
@@ -236,7 +218,7 @@ jobs:
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
 

.github/workflows/e2e-suites.yml

@@ -43,14 +43,6 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
   contents: read
   # actions: read lets the otel-job-id action look up a provision job's numeric
@@ -151,9 +143,10 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
           e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # 45 is the floor this job has always had, so resolving each leg's
-          # budget can only raise it — storagefault and rds ask for more.
-          JSON=$(e2e_suite_matrix "${LIST}" 45)
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Reclaim docker build cache
@@ -189,7 +182,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -318,16 +311,14 @@ jobs:
           retention-days: 14
 
   single_suite:
-    name: Single env · ${{ matrix.suite }}
+    name: Single env · suite
     needs: single_provision
     runs-on: [self-hosted, ci-single]
-    # Per leg, from suite-sets.sh: one number for every suite would have to be
-    # storagefault's, and every other leg would then hang for 105 minutes.
-    timeout-minutes: ${{ matrix.timeout }}
+    timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
+        suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
     env:
       WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
       SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
@@ -352,8 +343,6 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
-          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
-          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${WAN_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -371,7 +360,7 @@ jobs:
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -405,16 +394,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: single
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -527,7 +506,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -638,7 +617,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -680,9 +659,10 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
           e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # 60 is the floor this job has always had; multi-node legs carry a
-          # heavier setup than single-node ones, which is why it is higher.
-          JSON=$(e2e_suite_matrix "${LIST}" 60)
+          # Unquoted on purpose: the space-separated list must word-split into
+          # one JSON element per suite.
+          # shellcheck disable=SC2086
+          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Build multinode binaries on runner
@@ -732,15 +712,14 @@ jobs:
           retention-days: 14
 
   multi_suite:
-    name: Multi env · ${{ matrix.suite }}
+    name: Multi env · suite
     needs: multi_provision
     runs-on: [self-hosted, ci-multi]
-    # See the note on single_suite: the budget is the suite's, not the job's.
-    timeout-minutes: ${{ matrix.timeout }}
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
+        suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
     env:
       NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
       SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
@@ -766,8 +745,6 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
-          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
-          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${NODE1_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -786,7 +763,7 @@ jobs:
                SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -819,16 +796,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: multi
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -912,7 +879,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/e2e.yml

@@ -7,7 +7,6 @@ on:
       - dev
     paths:
       - "**/*.go"
-      - "!**/*_test.go"
       - go.mod
       - go.sum
       - Makefile
@@ -24,14 +23,6 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
   contents: read
   statuses: write
@@ -49,57 +40,6 @@ env:
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # The inner timeout must fire before the outer one. `go test` reports where it
-  # stopped and the suite's own failure; a job cancelled on timeout reports only
-  # "The operation was canceled" and takes the artifacts with it.
-  #
-  # A job budget cannot be a literal here because the suite set is a dispatch
-  # input: storagefault asks for 90m and diskperf for 180m, both eligible on one
-  # node, so any fixed number is either wrong for them or wasteful for everyone
-  # else. Resolving it from suite-sets.sh keeps the one source of truth and
-  # leaves the default push set at exactly the 60m it has always had.
-  plan:
-    name: Resolve Job Budgets
-    runs-on: ubuntu-26.04
-    timeout-minutes: 5
-    outputs:
-      single_timeout: ${{ steps.budgets.outputs.single_timeout }}
-      multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          path: spinifex
-          sparse-checkout: .github/scripts
-
-      - name: Resolve Budgets
-        id: budgets
-        env:
-          # Provision, install and teardown around the suites, plus the headroom
-          # a multi-suite job needs for whatever ran before the one that hangs.
-          SETUP_ALLOWANCE_MIN: 15
-          # What the job has always had. Kept as a floor so resolving the budget
-          # can only ever raise it, never quietly shorten an existing run.
-          FLOOR_MIN: 60
-        run: |
-          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
-          # Only max() is a sound bound: the suites share one job, so a sum would
-          # demand hours for a set that has never taken them. Max is what makes
-          # the hanging suite panic on its own timeout first, which is the point.
-          budget() {
-            local want="${FLOOR_MIN}" s t
-            for s in $(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "$1"); do
-              t="$(e2e_suite_timeout "${s}")"
-              t=$(( ${t%m} + SETUP_ALLOWANCE_MIN ))
-              [ "${t}" -gt "${want}" ] && want="${t}"
-            done
-            echo "${want}"
-          }
-          single=$(budget "${E2E_SUITES_SINGLE_RE}")
-          multi=$(budget "${E2E_SUITES_MULTI_RE}")
-          echo "single_timeout=${single}" >> "$GITHUB_OUTPUT"
-          echo "multi_timeout=${multi}" >> "$GITHUB_OUTPUT"
-          echo "suites='${SUITES}' -> single ${single}m, multi ${multi}m"
-
   e2e_single:
     name: Single-Node E2E
     # The dispatch check gates on exactly the suites this job's Run step
@@ -111,9 +51,8 @@ jobs:
     # tear down. Push triggers keep running via the null-check on
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
-    needs: plan
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: ${{ fromJSON(needs.plan.outputs.single_timeout) }}
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -213,7 +152,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -418,10 +357,6 @@ jobs:
           # suite-sets.sh also carries the per-suite rationale.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_SINGLE_RE}" | tr '\n' ' ')
-          # Resolve each suite's timeout here, where suite-sets.sh is readable, and
-          # carry the pairs into the remote loop below — the VM has no checkout.
-          SUITE_TMOS=""
-          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -453,7 +388,7 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
-               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
+               case \$SUITE in rds) TMO=45m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
                  ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
@@ -529,16 +464,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: single
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -585,7 +510,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -614,9 +539,8 @@ jobs:
     # provisions the 3-node env when requested on its own, and the env stays
     # unprovisioned when none of them are asked for.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
-    needs: plan
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: ${{ fromJSON(needs.plan.outputs.multi_timeout) }}
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
@@ -712,7 +636,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -838,8 +762,6 @@ jobs:
           # Allow-list, not a deny-list — see the single-node job for why.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_MULTI_RE}" | tr '\n' ' ')
-          SUITE_TMOS=""
-          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -871,14 +793,13 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
-               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                  SPINIFEX_NODES='${NODES_CSV}' \
                  SPINIFEX_NODE_IPS='${NODES_CSV}' \
                  SPINIFEX_SSH_USER=tf-user \
                  SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -982,16 +903,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always()
-        continue-on-error: true
-        uses: ./spinifex/.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: multi
-
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -1041,7 +952,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/gpu-e2e.yml

@@ -287,16 +287,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
-      - name: Ship test results to Elasticsearch
-        if: always() && steps.gate.outputs.paused != 'true'
-        continue-on-error: true
-        uses: ./.github/actions/junit-to-es
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: gpu
-
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
         uses: ./.github/actions/e2e-analyze

.github/workflows/image-scripts.yml

@@ -37,39 +37,3 @@ jobs:
 
       - name: Run image script tests + shellcheck
         run: make test-images
-
-  # The mkosi EKS image is what customers run, and until now nothing built it:
-  # the composition existed in mkosi-build.sh but had no caller, so a change
-  # that broke it was only found by hand. Cell-25 still boots the Alpine image,
-  # so this is the only gate the Ubuntu one has.
-  #
-  # Self-hosted because the build needs Docker and several GB of disk, matching
-  # publish-system-ami.yml. Its own concurrency group, not e2e-hypervisors: the
-  # build provisions no libvirt guest, and sharing that group would queue image
-  # builds behind whole nightlies.
-  eks_mkosi_image:
-    name: Build mkosi EKS image
-    runs-on: [self-hosted, ci-single]
-    concurrency:
-      group: eks-mkosi-image-${{ github.ref }}
-      cancel-in-progress: true
-
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v7
-
-      # The profile binaries (eks-token-webhook, eks-gateway-publish, ...) are
-      # built on the host, not in the builder container, so Go is needed here
-      # even though the toolchain image carries mkosi itself.
-      - name: Set up Go
-        uses: actions/setup-go@v7
-        with:
-          go-version-file: "go.mod"
-          # Self-hosted: the module + build cache already persists on local
-          # disk, and cache-save would re-upload hundreds of MB for nothing.
-          cache: false
-
-      # stage_profile_binaries() sets GOWORK=off, so the sub-repo resolves its
-      # own go.mod and the sibling monorepo checkouts are not needed here.
-      - name: Build the image
-        run: make build-eks-node-mkosi-image

.github/workflows/publish-system-ami.yml

@@ -17,11 +17,10 @@ name: Publish System AMIs
 #   spx admin images import --name spinifex-rds-postgres
 #   spx admin images import --name spinifex-rds-mariadb
 #
-# Publishing overwrites the mirror artifact, so it is a deliberate action, not
-# something every push should trigger: it runs manually, or from release.yml
-# once a tag has passed its gates. The `variant` input scopes which images
-# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
-# DKMS build), so publishing only the CPU AMIs stays cheap.
+# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
+# action, not something every push should trigger. The `variant` input scopes
+# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
+# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
@@ -39,16 +38,6 @@ on:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
         default: false
-  workflow_call:
-    inputs:
-      variant:
-        description: "Which images to build (all|cpu|gpu|rds)"
-        type: string
-        default: all
-      dry_run:
-        description: "Build + convert only; skip the R2 upload"
-        type: boolean
-        default: false
 
 permissions:
   contents: read
@@ -112,20 +101,6 @@ jobs:
           ref: dev
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: dev
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: dev
-          path: bluebottle
-
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
@@ -143,7 +118,7 @@ jobs:
         # "go.work already exists".
         run: |
           rm -f go.work go.work.sum
-          go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+          go work init ./spinifex ./viperblock ./predastore
 
       - name: Install build + upload tooling
         run: |

.github/workflows/release.yml

@@ -28,25 +28,13 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          path: bluebottle
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.27.0"
+          go-version: "1.26.5"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -85,18 +73,6 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          path: bluebottle
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -148,13 +124,6 @@ jobs:
   verify:
     needs: release
     runs-on: [ self-hosted, ci-single ]
-    # Shared with the e2e workflows: this job provisions on the same runner
-    # pools and the same hypervisors, so two in flight compete for their disks
-    # and their libvirt guests. Job-level, so the build jobs are not queued
-    # behind a nightly that has nothing to do with them.
-    concurrency:
-      group: e2e-hypervisors
-      cancel-in-progress: false
     timeout-minutes: 25
     env:
       TOFU_LOG: /tmp/release-verify-${{ github.run_id }}.log
@@ -181,13 +150,6 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      # Only the shared shell helpers: this job builds nothing from source, but
-      # it drives the same libvirt provider the e2e cells retry around.
-      - uses: actions/checkout@v7
-        with:
-          sparse-checkout: .github/scripts
-          path: spinifex
-
       - name: Download Release Assets
         uses: actions/download-artifact@v8
         with:
@@ -197,12 +159,11 @@ jobs:
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
       - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
@@ -215,9 +176,8 @@ jobs:
         if: always()
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
-          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
   baremetal:
@@ -247,17 +207,6 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
-  # Refresh the system AMIs on the R2 mirror only once the release is out of
-  # draft. Shares the ci-single runner with `iso`, so it queues behind it.
-  system-images:
-    needs: promote
-    permissions:
-      contents: read
-    uses: ./.github/workflows/publish-system-ami.yml
-    with:
-      variant: all
-    secrets: inherit
-
   # Annotate the (still-draft) release if either gate failed.
   mark-failed:
     needs: [ verify, baremetal ]
@@ -319,18 +268,6 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          path: bluebottle
-
       - name: Checkout Mulga iso-builder
         uses: actions/checkout@v7
         with:
@@ -349,7 +286,7 @@ jobs:
           cache: false
 
       - name: Configure Go workspace
-        run: go work init ./spinifex ./predastore ./viperblock ./northstar ./bluebottle
+        run: go work init ./spinifex ./predastore ./viperblock
 
       - name: Install ISO builder dependencies
         run: |

.github/workflows/segscan-oracle.yml

@@ -51,20 +51,6 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
@@ -87,7 +73,7 @@ jobs:
           go-version-file: "spinifex/go.mod"
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Run Segscan Oracle Test
         env:

.github/workflows/spinifex.yml

@@ -53,68 +53,14 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
+          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      # Restore runs everywhere; the matching save is gated to dev and main, so
-      # a PR reads the base branch's entry without spending a minute uploading
-      # one only that PR would ever read.
-      - name: Restore Go build cache
-        uses: actions/cache/restore@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
-
-      # The EBS provider contract suite needs a real implementation on each
-      # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which
-      # shares no code with viperblock, and nbdkit plus viperblock's plugin
-      # back viperblockd, the provider production actually uses. Without them
-      # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries
-      # the headers and nbdkit.pc that cgo needs to build that plugin, and
-      # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that
-      # judges the published export by something outside this codebase.
-      - name: Install conformance tools
-        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin
-
-      - name: Build viperblock nbdkit plugin
-        run: make -C viperblock go_build_nbd
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Build Spinifex
         run: make -C spinifex build
@@ -123,37 +69,62 @@ jobs:
         run: make -C spinifex manifest-check manifest-lint
 
       - name: Run Tests with Coverage
-        env:
-          SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
-          VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so
-        run: make -C spinifex test-cover GOTESTFLAGS=-count=1
+        run: make -C spinifex test-cover
 
       - name: Run Action Tests
-        run: make -C spinifex test-actions GOTESTFLAGS=-count=1
+        run: make -C spinifex test-actions
 
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
-      # Compiled output stays valid whatever the tests said, and skipping the
-      # save on a red run would leave the next one rebuilding it, so this is
-      # gated on the branch rather than on success.
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
-      - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+  # In-process integration tier (gateway.SetupRoutes over embedded NATS
+  # JetStream). Its own job so it runs in parallel with the unit tiers rather
+  # than adding to their wall time, and it still blocks a PR — unlike e2e.yml's
+  # "Integration (front gate)" job, which is push-triggered against main/dev on
+  # a self-hosted runner and so never gates a PR before merge.
+  integration_tests:
+    name: Integration Tests
+    runs-on: ubuntu-26.04
 
-      - name: Save Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
         with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Integration Tests
+        run: make -C spinifex test-integration
 
   race_detection:
     name: Race Detection
@@ -186,78 +157,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
+          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      # Restore runs everywhere; the matching save is gated to dev and main, so
-      # a PR reads the base branch's entry without spending a minute uploading
-      # one only that PR would ever read.
-      - name: Restore Go build cache
-        uses: actions/cache/restore@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Run Tests with Race Detector
-        run: make -C spinifex test-race GOTESTFLAGS=-count=1
-
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
-      - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
-
-      - name: Save Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
+        run: make -C spinifex test-race
 
   lint_and_security:
-    name: Lint + Security + Integration
+    name: Lint and Security
     runs-on: ubuntu-26.04
 
     steps:
@@ -265,7 +178,6 @@ jobs:
         uses: actions/checkout@v7
         with:
           path: spinifex
-          fetch-depth: 2
 
       - name: Resolve sibling refs
         id: siblings
@@ -288,82 +200,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
+          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      # Restore runs everywhere; the matching save is gated to dev and main, so
-      # a PR reads the base branch's entry without spending a minute uploading
-      # one only that PR would ever read.
-      - name: Restore Go build cache
-        uses: actions/cache/restore@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+        run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.13.0
+          version: v2.12.2
           working-directory: spinifex
 
       - name: Govulncheck
         run: make -C spinifex govulncheck
-
-      - name: Run Integration Tests
-        if: ${{ !cancelled() }}
-        run: make -C spinifex test-integration
-
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
-      - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
-
-      - name: Save Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        uses: actions/cache/save@v6
-        with:
-          path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}

.gitignore

@@ -34,11 +34,9 @@ Thumbs.db
 # Environment
 .env
 .venv/
-.cache/
-
-# VScode
-.vscode/
 
 # Test reports
 *-report.txt
-/rds-agent
+
+# Per-worktree golangci-lint cache
+/.cache/

.golangci.yml

@@ -26,7 +26,7 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forbidigo # blocks forbidden identifiers
+    - forbidigo # blocks the legacy JetStream/KV symbols
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
@@ -69,18 +69,6 @@ linters:
             - pkg: math/rand$
               desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
 
-        # The EC2 control plane reaches block storage through ebsprovider, not
-        # by importing the engine. viperblock.New starts a chunk uploader, so
-        # every construction is a writer against a volume viperblockd serves.
-        # Nothing under handlers/ may import viperblock, in any form, tests
-        # included. A types-only import counts: one became an engine import.
-        ec2-control-plane:
-          files:
-            - '**/spinifex/handlers/**'
-          deny:
-            - pkg: github.com/mulgadc/viperblock
-              desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
-
     embeddedstructfieldcheck:
       forbid-mutex: true
 
@@ -92,8 +80,6 @@ linters:
     forbidigo:
       analyze-types: true
       forbid:
-        - pattern: '^panic$'
-          msg: panic terminates the process; return an error or explicitly justify the panic.
         - pattern: '\.JetStream$'
           pkg: '^github\.com/nats-io/nats\.go$'
           msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -107,7 +93,6 @@ linters:
         - G116 # bidi chars — bidichk is more thorough
         - G117 # use of net/http serve functions (we handle TLS separately)
         - G703 # path traversal taint analysis — false positives on config file loading
-        - G704 # SSRF taint analysis — false positives on resolver-derived endpoint URLs
         - G705 # XSS taint analysis — false positives on internal error responses
 
     govet:
@@ -119,7 +104,6 @@ linters:
 
     modernize:
       disable:
-        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
@@ -170,7 +154,6 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
-          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions

Makefile

@@ -1,10 +1,10 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
-# GOFIPS140 is a global build-config change: it alters every package's action ID,
-# stdlib included, so a target that omits it shares no build-cache entry with one
-# that sets it. Exported once here rather than per-recipe so the two can't drift.
-export GOFIPS140 := v1.0.0
+# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
+# path, not worktree path, so parallel worktrees of this module collide and
+# surface each other's issues. Scope it per worktree.
+export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
 
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
@@ -63,23 +63,19 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI
 
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
 
 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
 build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
 	@echo -e "\n....Building ecs-agent (static)"
-	CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
-
-build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
-	@echo -e "\n....Building spx-loadgen"
-	go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -103,12 +99,6 @@ import-eks-node-image: ## Build + register the eks-node AMI (requires a running
 publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
 	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
 
-# The Ubuntu/systemd EKS image, as opposed to build-eks-node-image's Alpine one.
-# Composition lives in mkosi-build.sh's image_profiles table, so the image name
-# is the whole interface here.
-build-eks-node-mkosi-image: ## Build the Ubuntu mkosi EKS node image (server+agent; role at first boot)
-	./scripts/mkosi-build.sh --image spinifex-eks-node
-
 build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
 	$(MAKE) build-system-image IMAGE=ecs-agent
 
@@ -150,45 +140,24 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
-# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
-# ship on every node (unlike scripts/images/, kept in preflight: a wrong
-# restart decision here is an outage, not asset churn).
-test-build-scripts:
-	@echo -e "\n....Running build/scripts/**/*_test.sh...."
-	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
-	done
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
-	@echo "  test-build-scripts ok"
-
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
-#
-# diskperf/ is here for the same reason: its baseline comparison and probe
-# accounting decide whether the disk-performance gate passes, and that
-# arithmetic has to be covered off-cluster or the gate is trusted untested.
 test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... ./tests/e2e/diskperf/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
 
 # In-process integration tier: the real gateway router against embedded NATS
 # JetStream, with only the daemon-side NATS subjects stubbed.
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
 AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
-# -count=1 is load-bearing: the conformance report is written by the test binary,
-# so a cached pass skips the run, leaves no report, and the cat below fails the
-# target. It also keeps the conformance gate honest — a cached result would mean
-# the check never ran against this commit.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
 	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
@@ -229,30 +198,25 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
-
-# Empty locally, where reusing a cached result between runs is the point.
-# CI passes -count=1 so a green run means the suite executed against that commit
-# rather than replaying results the persisted build cache carried over.
-GOTESTFLAGS ?=
+	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
+	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
 test-actions:
 	@echo -e "\n....Running action tests...."
-	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 60s ./.github/actions/...
+	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
 # Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
 # helpers baked into system images. Kept out of `preflight` (a dedicated CI
@@ -291,10 +255,6 @@ test-images:
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
-# Check that newly added test files use an external test package
-test-package-check:
-	@QUIET=$(QUIET) scripts/check-test-package.sh
-
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
@@ -349,9 +309,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -434,7 +394,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
@@ -12,11 +12,10 @@
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS Compatibility</a> ·
-  <a href="#live-demo">Live Demo</a> ·
+  <a href="#aws-compatibility">AWS services</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#self-hosted-installation">Installation</a> ·
+  <a href="#installation">Installation</a> ·
   <a href="https://docs.mulgadc.com">Docs</a>
 </p>
 
@@ -30,9 +29,11 @@ Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
 
 Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Built for teams that need:
+Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
 
-- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+It's built for teams that need:
+
+- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
 - Full control of the stack: your hardware, your network, your data, your keys
 - A cloud that keeps running offline, through disconnection, with no external control plane
 - An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
@@ -47,69 +48,51 @@ You change *where* your software runs, not *what* it is.
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
-## Three Ways To Deploy
+## Three ways to deploy
 
-The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
+One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
+Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
+Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
 
 ### Edge
 
 Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
-## AWS Compatibility
+## AWS compatibility
 
-Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
+Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
 | **EC2** | Compute | Available |
-| **EBS** | Block Storage | Available |
-| **S3** | Object Storage | Available |
+| **EBS** | Block storage | Available |
+| **S3** | Object storage | Available |
 | **VPC** | Networking | Available |
-| **IAM** | Identity & Auth | Available |
-| **ALB / NLB** | Load Balancers | Available |
+| **IAM** | Identity & auth | Available |
+| **ALB / NLB** | Load balancers | Available |
 | **EKS** | Kubernetes | Available |
-| **ECR** | Container Registry | Available |
-| **ECS** | Container Service | Available |
+| **ECR** | Container registry | Available |
+| **ECS** | Container service | Available |
 | **RDS** | Databases | Available |
 | **Bedrock** | AI Deployment | Q3 2026 |
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
-## Live Demo
-
-Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
-
-* Live in under a minute. No card, no install.
-* Yours for 72 hours to test and experiment.
-* Real AWS-compatible endpoint, access keys and console.
-* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
-* Point your existing AWS CLI or Terraform straight at it.
-
-Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
-
 ## Core Components
 
-### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
+### Spinifex (Compute Service – EC2 Alternative)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
 
-The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
-
-- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
-- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
-- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
-- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
-- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
-- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
-- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
+- EC2-like VM management on bare metal
+- Launches with cloud-init metadata support
+- Works with standard AWS tooling
 
 ### Viperblock (Block Storage – EBS Alternative)
 
@@ -118,25 +101,17 @@ The managed services build on that same instance layer. An EKS node group, an EC
 - Fast, durable virtual disks
 - Replication for resilience
 - Exposed over NBD or embedded in VMs
-- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
-- In-memory read/write block cache
+- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
+- In memory read/write block cache for blazing performance.
 
 ### Predastore (Object Storage – S3-Compatible)
 
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
+[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
 
 - S3-compatible API and auth
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
-### Northstar (Authoritative DNS)
-
-[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
-
-- Authoritative DNS over UDP, TCP, and DNS-over-TLS
-- Service discovery for cluster components via SRV records
-- Zone files stored locally or in S3-compatible storage
-
 ## Architecture at a Glance
 
 <p align="center">
@@ -145,9 +120,18 @@ The managed services build on that same instance layer. An EKS node group, an EC
 
 Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Self-hosted Installation
+## Key Features
 
-Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
+- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
+- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
+- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
+- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
+- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
+- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
+
+## Installation
+
+Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
 
 ### Bare Metal ISO
 
@@ -157,11 +141,16 @@ The recommended installation is a [bootable x86 installer](https://iso.mulgadc.c
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
+
 
 ### Single Node Install
 
->*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
+The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
+
+>*Prerequisite:* Linux bridge for networking.
+
+Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -179,23 +168,36 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
+
+### Component Repositories
+
+Spinifex coordinates these independent components:
+
+- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
+- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
+
+Each component can be developed independently. See component-specific documentation for focused development guides.
 
 ## Spinifex UI
 
-Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
 </p>
 
-Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
+The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
 
-- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
-- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
+```bash
+open https://YOUR_NODE_IP:3000
+```
 
-For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
+- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+
+For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
 
 ## Development Philosophy
 
@@ -203,9 +205,16 @@ For the full walkthrough, covering first-time TLS certificate trust, login, and
 
 Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
 
+**Real-World Experience:**
+
+- Production AWS service development and operations
+- Large-scale infrastructure deployment and management
+- Edge computing and resource-constrained environments
+- Enterprise security and compliance requirements
+
 ### AI-Assisted Development
 
-Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
+While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
 
 **How We Use Claude Code:**
 
@@ -221,10 +230,12 @@ Spinifex is architected and implemented by experienced engineers, with **Claude
 - **Performance Optimization**: Real-world performance tuning and benchmarking
 - **Production Operations**: Deployment strategies and operational procedures
 
+This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
+
 ## Trademarks
 
 "AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

SECURITY.md

@@ -1,24 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-Please do not open a public issue for a security vulnerability.
-
-Report it privately, either way works:
-
-- [Open a private security advisory](https://github.com/mulgadc/spinifex/security/advisories/new) on this repository.
-- Email <security@mulgadc.com>.
-
-Please include enough for us to reproduce it: the version or commit, the configuration involved, and the steps that trigger it. If you have a proof of concept, include that too.
-
-## What Happens Next
-
-We aim to acknowledge a report within three business days. We will confirm whether we can reproduce it, agree a fix and a disclosure timeline with you, and credit you in the advisory unless you would rather we did not.
-
-## Scope
-
-Spinifex is one part of the [Mulga](https://mulgadc.com) stack. If a vulnerability spans more than one component, report it once against whichever repository you found it in and we will handle the rest.
-
-## Supported Versions
-
-Fixes land on the latest release and on `main`. Older releases are not patched.

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.27-bookworm AS builder
+FROM golang:1.26-bookworm AS builder
 
 ARG VERSION
 
@@ -16,11 +16,9 @@ COPY predastore/go.mod predastore/go.sum ./predastore/
 COPY viperblock/go.mod viperblock/go.sum ./viperblock/
 COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
 COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
-COPY northstar/go.mod northstar/go.sum ./northstar/
-COPY bluebottle/go.mod bluebottle/go.sum ./bluebottle/
 
 # Generate go.work (gitignored, so not in checkout)
-RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit ../northstar ../bluebottle
+RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit
 
 # Download dependencies (cached layer)
 RUN cd spinifex && go mod download
@@ -29,8 +27,6 @@ RUN cd spinifex && go mod download
 COPY spinifex/ ./spinifex/
 COPY predastore/ ./predastore/
 COPY viperblock/ ./viperblock/
-COPY northstar/ ./northstar/
-COPY bluebottle/ ./bluebottle/
 
 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \

build/scripts/nats-js-watchdog.sh

@@ -18,97 +18,22 @@
 # So a health check alone misses case 2 entirely. Both probes run here, and a
 # sustained failure of either restarts the service. The multi-sample window
 # avoids restarting during transient startup/quorum 503s.
-#
-# Three more guards sit around that decision, because "restart when the
-# probes fail" is not safe on its own:
-#
-#   - Age gate: spinifex-nats.service is Type=simple, so it reports active the
-#     instant the process forks, long before JetStream finishes replaying its
-#     state. Acting before a recovery budget has elapsed restarts a server
-#     that is still coming up, not one that has failed.
-#   - Cooldown: a restart that did not fix things must not be retried by the
-#     very next run, seconds later.
-#   - Escalation: a restart that has not fixed things after several attempts
-#     is not going to fix it on the next one either, and an unrunnable spx
-#     must never be read as proof of a JetStream failure at all.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
 SPX="${SPX_BIN:-/usr/local/bin/spx}"
-SAMPLES="${SAMPLES:-3}"           # consecutive failing samples required before acting
-SAMPLE_GAP="${SAMPLE_GAP:-2}"     # seconds between samples
-PROBE_TIMEOUT="${PROBE_TIMEOUT:-5}"  # seconds for one canary round-trip
+SAMPLES=3        # consecutive failing samples required before acting
+SAMPLE_GAP=2     # seconds between samples
+PROBE_TIMEOUT=5  # seconds for one canary round-trip
 
-# Recovery budget: on casuarina NATS needed 3m49s between "Started" and
-# JetStream actually accepting writes, and the bead recorded ~6min elsewhere.
-# 600s clears both with margin.
-NATS_MIN_AGE="${NATS_MIN_AGE:-600}"
-
-# Minimum time between restarts this watchdog performs, independent of the
-# age gate: closes the case where a restart did not fix things and the very
-# next run pulls the trigger again before the new process could recover.
-RESTART_COOLDOWN="${RESTART_COOLDOWN:-600}"
-
-# A restart that has not fixed things after this many attempts is not going
-# to fix it on the next one either, and looping costs more than stopping.
-RESTART_ESCALATE_AT="${RESTART_ESCALATE_AT:-3}"
-
-STATE_DIR="${STATE_DIR:-/run/spinifex}"
-COOLDOWN_STAMP="$STATE_DIR/nats-js-watchdog.last-restart"
-RESTART_COUNT_FILE="$STATE_DIR/nats-js-watchdog.restart-count"
-# Marks that the escalation message has already been printed for the current
-# latch, so a node that stays broken says so once instead of once per tick.
-ESCALATED_STAMP="$STATE_DIR/nats-js-watchdog.escalated"
-
-# Monotonic clock source, so a wall-clock step during boot (NTP correcting an
-# unset RTC) cannot manufacture a false age or cooldown. Overridable so tests
-# can stub a clock without faking /proc.
-UPTIME_FILE="${UPTIME_FILE:-/proc/uptime}"
-
-uptime_seconds() {
-    awk '{printf "%d", $1}' "$UPTIME_FILE"
-}
-
-# nats_age_seconds prints how long $SERVICE has been running. It reads
-# ExecMainStartTimestampMonotonic rather than is-active: that property goes
-# true the instant Type=simple forks, long before JetStream has replayed its
-# state, so it cannot express the recovery window this gate exists to
-# protect. A unit systemd has never started prints 0, which also covers a
-# stopped/activating unit the way the old is-active guard did.
-nats_age_seconds() {
-    start_usec=$(systemctl show --property=ExecMainStartTimestampMonotonic --value "$SERVICE" 2>/dev/null) || start_usec=""
-    case "$start_usec" in
-        ''|*[!0-9]*) echo 0; return ;;
-    esac
-    if [ "$start_usec" -eq 0 ]; then
-        echo 0
-        return
-    fi
-    echo $(( $(uptime_seconds) - start_usec / 1000000 ))
-}
-
-age=$(nats_age_seconds)
-if [ "$age" -lt "$NATS_MIN_AGE" ]; then
+# Only act on a running service: a stopped/activating unit is systemd's to manage,
+# and try-restart on it would either no-op or fight an in-flight (re)start.
+if ! systemctl is-active --quiet "$SERVICE"; then
     exit 0
 fi
 
-if [ -f "$COOLDOWN_STAMP" ]; then
-    last=$(cat "$COOLDOWN_STAMP" 2>/dev/null) || last=""
-    case "$last" in ''|*[!0-9]*) last=0 ;; esac
-    if [ $(( $(uptime_seconds) - last )) -lt "$RESTART_COOLDOWN" ]; then
-        exit 0
-    fi
-fi
-
-restart_count=$(cat "$RESTART_COUNT_FILE" 2>/dev/null) || restart_count=""
-case "$restart_count" in ''|*[!0-9]*) restart_count=0 ;; esac
-
-# The escalation count is read here but acted on only after the probes, below.
-# Checking it up front would return before the one line that clears it, so a
-# latched watchdog could never observe recovery and never re-arm itself.
-
 # curl is the probe transport; if it is unavailable, do nothing rather than
 # false-restart a healthy server.
 if ! command -v curl >/dev/null 2>&1; then
@@ -120,22 +45,13 @@ js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
-# Round-trips a canary key through the cluster-state bucket. Exit 0 means the
-# write succeeded and exit 1 means JetStream refused it; anything else
-# (spx's own exit 2, a missing binary, or one that cannot run at all) means
-# the probe observed nothing about JetStream and must never be read as
-# evidence of a failure.
+# Round-trips a canary key through the cluster-state bucket. Absent spx this
+# degrades to the health check alone rather than restarting on a missing tool.
 js_writable() {
     if [ ! -x "$SPX" ]; then
-        return 2
+        return 0
     fi
     "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
-    rc=$?
-    case "$rc" in
-        0) return 0 ;;
-        1) return 1 ;;
-        *) return 2 ;;
-    esac
 }
 
 failure=""
@@ -143,18 +59,6 @@ i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
         if js_writable; then
-            wrc=0
-        else
-            wrc=$?
-        fi
-        if [ "$wrc" -eq 0 ]; then
-            # Recovery: drop the history so a later failure gets the full
-            # restart budget again, and re-arm the escalation message.
-            rm -f "$RESTART_COUNT_FILE" "$ESCALATED_STAMP"
-            exit 0
-        fi
-        if [ "$wrc" -eq 2 ]; then
-            echo "nats-js-watchdog: js-probe inconclusive (config, connect, bucket-open, or spx itself failed); not evidence of a JetStream failure, taking no action" >&2
             exit 0
         fi
         failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
@@ -167,24 +71,5 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
-# The failure is real and current, so the restart history now means what it
-# says. Above the threshold another restart will not help, so stop acting —
-# but keep probing on later runs, because that is what notices a recovery.
-if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
-    if [ ! -f "$ESCALATED_STAMP" ]; then
-        mkdir -p "$STATE_DIR"
-        : > "$ESCALATED_STAMP"
-        echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2
-    fi
-    exit 0
-fi
-
 echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
-mkdir -p "$STATE_DIR"
-uptime_seconds > "$COOLDOWN_STAMP"
-restart_count=$((restart_count + 1))
-echo "$restart_count" > "$RESTART_COUNT_FILE"
-if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
-    echo "nats-js-watchdog: restart $restart_count of $RESTART_ESCALATE_AT before escalation; if $SERVICE is still failing after this, the watchdog will stop acting until $RESTART_COUNT_FILE is cleared" >&2
-fi
 systemctl try-restart "$SERVICE"

build/scripts/nats-js-watchdog_test.sh

@@ -1,197 +0,0 @@
-#!/bin/sh
-# Self-contained POSIX test for nats-js-watchdog.sh's restart-decision logic:
-# the age gate, cooldown, escalation, and js-probe exit-2 handling. No real
-# NATS or spx: systemctl, curl, and spx are all stubbed on PATH ahead of the
-# real ones, driven entirely by env vars this test controls.
-#
-# Run: sh build/scripts/nats-js-watchdog_test.sh
-set -eu
-
-SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
-SCRIPT="${SCRIPT_DIR}/nats-js-watchdog.sh"
-WORK=$(mktemp -d)
-trap 'rm -rf "${WORK}"' EXIT
-
-FAKE_BIN="${WORK}/bin"
-mkdir -p "${FAKE_BIN}"
-
-FAILS=0
-fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
-pass() { echo "ok: $*"; }
-
-# Fake systemctl: `show` reports a controllable ExecMainStartTimestampMonotonic,
-# and `try-restart` just records that it was called rather than touching a
-# real unit.
-cat > "${FAKE_BIN}/systemctl" <<'EOF'
-#!/bin/sh
-case "$1" in
-    show)
-        echo "${FAKE_NATS_START_USEC:-0}"
-        ;;
-    try-restart)
-        echo "try-restart $2" >> "${FAKE_RESTART_LOG:-/dev/null}"
-        ;;
-esac
-EOF
-chmod +x "${FAKE_BIN}/systemctl"
-
-# Fake curl: js_healthy() only cares about the exit status.
-cat > "${FAKE_BIN}/curl" <<'EOF'
-#!/bin/sh
-[ "${FAKE_HEALTHY:-1}" = "1" ]
-EOF
-chmod +x "${FAKE_BIN}/curl"
-
-# Fake spx: js-probe's whole contract from the watchdog's side is its exit
-# code (0 ok, 1 JetStream refused, 2 inconclusive).
-cat > "${FAKE_BIN}/spx" <<'EOF'
-#!/bin/sh
-exit "${FAKE_SPX_EXIT:-0}"
-EOF
-chmod +x "${FAKE_BIN}/spx"
-
-PATH="${FAKE_BIN}:${PATH}"
-export PATH
-
-# reset_env <name>: fresh per-case dirs and a default env that already clears
-# the age, cooldown, and escalation gates (old enough NATS, no cooldown
-# stamp, no restart history) and a healthy, single-sample probe, so a case
-# only has to override what it is actually testing.
-reset_env() {
-    CASE="${WORK}/$1"
-    mkdir -p "${CASE}/state"
-    UPTIME_FILE="${CASE}/uptime"
-    printf '700.00 0.00\n' > "${UPTIME_FILE}"
-    STATE_DIR="${CASE}/state"
-    FAKE_RESTART_LOG="${CASE}/restart.log"
-    FAKE_NATS_START_USEC=1000000   # 1s: 700s uptime - 1s start = 699s age, clears the 600s default gate
-    FAKE_HEALTHY=1
-    FAKE_SPX_EXIT=0
-    SAMPLES=1
-    SAMPLE_GAP=0
-    NATS_MIN_AGE=600
-    RESTART_COOLDOWN=600
-    RESTART_ESCALATE_AT=3
-    SPX_BIN="${FAKE_BIN}/spx"
-    STDOUT="${CASE}/stdout"
-    STDERR="${CASE}/stderr"
-    export UPTIME_FILE STATE_DIR FAKE_RESTART_LOG FAKE_NATS_START_USEC \
-        FAKE_HEALTHY FAKE_SPX_EXIT SAMPLES SAMPLE_GAP NATS_MIN_AGE \
-        RESTART_COOLDOWN RESTART_ESCALATE_AT SPX_BIN
-}
-
-# Run the watchdog, capturing its exit code (without tripping set -e) + streams.
-invoke() { rc=0; sh "${SCRIPT}" >"${STDOUT}" 2>"${STDERR}" || rc=$?; }
-
-restarted() { [ -s "${FAKE_RESTART_LOG}" ]; }
-
-# --- Case 1: NATS younger than the age gate -> no restart ---
-reset_env young
-FAKE_NATS_START_USEC=690000000  # 690s: 700s uptime - 690s start = 10s age, well under 600s
-export FAKE_NATS_START_USEC
-invoke
-[ "${rc}" -eq 0 ] && pass "young: exit 0" || fail "young: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "young: must not restart while still in the recovery window" || pass "young: no restart"
-
-# --- Case 2: cooldown stamp fresh -> no restart ---
-reset_env cooldown
-echo 650 > "${STATE_DIR}/nats-js-watchdog.last-restart"
-invoke
-[ "${rc}" -eq 0 ] && pass "cooldown: exit 0" || fail "cooldown: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "cooldown: must not restart again inside the cooldown window" || pass "cooldown: no restart"
-
-# --- Case 3: escalation threshold reached while still failing -> no restart,
-# loud log. The probe must fail: a healthy probe at the threshold is case 3b,
-# and it re-arms rather than giving up. ---
-reset_env escalated
-FAKE_SPX_EXIT=1
-export FAKE_SPX_EXIT
-echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-[ "${rc}" -eq 0 ] && pass "escalated: exit 0" || fail "escalated: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "escalated: must not restart past the escalation threshold" || pass "escalated: no restart"
-grep -q 'giving up rather than looping' "${STDERR}" \
-    && pass "escalated: loud log emitted" || fail "escalated: escalation log missing: $(cat "${STDERR}")"
-
-# Second identical run: the node is still broken, so the decision stands, but
-# repeating the message every timer tick is what buried the real signal for 42
-# hours on node1.
-invoke
-grep -q 'giving up rather than looping' "${STDERR}" \
-    && fail "escalated: message repeated on every run instead of once per latch" \
-    || pass "escalated: message not repeated"
-
-# --- Case 3b: escalation threshold reached but JetStream has recovered ->
-# the watchdog must re-arm itself. This is the production bug: the escalation
-# check used to run before the probe, so the clear below was unreachable and
-# the node stayed disarmed until reboot. ---
-reset_env escalated_recovered
-echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-: > "${STATE_DIR}/nats-js-watchdog.escalated"
-invoke
-[ "${rc}" -eq 0 ] && pass "escalated-recovered: exit 0" || fail "escalated-recovered: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "escalated-recovered: must not restart a recovered server" || pass "escalated-recovered: no restart"
-[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
-    && fail "escalated-recovered: restart count must clear once JetStream is writable again" \
-    || pass "escalated-recovered: restart count cleared"
-[ -f "${STATE_DIR}/nats-js-watchdog.escalated" ] \
-    && fail "escalated-recovered: escalation marker must clear so a later failure is reported" \
-    || pass "escalated-recovered: escalation marker cleared"
-[ -s "${STDERR}" ] \
-    && fail "escalated-recovered: a recovered node must be silent: $(cat "${STDERR}")" \
-    || pass "escalated-recovered: silent"
-
-# --- Case 3c: re-armed watchdog acts again. One healthy run clears the latch,
-# and the next genuine failure must restart -- proving it is armed, not just
-# quiet. ---
-reset_env escalated_rearm
-echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-FAKE_SPX_EXIT=1
-export FAKE_SPX_EXIT
-invoke
-restarted && pass "rearm: restarts again after recovery cleared the latch" \
-    || fail "rearm: expected a restart once re-armed: $(cat "${STDERR}")"
-[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "1" ] \
-    && pass "rearm: restart count restarts from 1" \
-    || fail "rearm: expected count 1, got $(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)"
-
-# --- Case 4: js-probe exits 2 (inconclusive) -> no restart ---
-reset_env inconclusive
-FAKE_SPX_EXIT=2
-export FAKE_SPX_EXIT
-invoke
-[ "${rc}" -eq 0 ] && pass "inconclusive: exit 0" || fail "inconclusive: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "inconclusive: exit 2 must never be read as a JetStream failure" || pass "inconclusive: no restart"
-grep -q 'inconclusive' "${STDERR}" \
-    && pass "inconclusive: reason logged" || fail "inconclusive: log missing: $(cat "${STDERR}")"
-
-# --- Case 5: genuine JetStream write failure -> restart happens ---
-reset_env genuine_failure
-FAKE_SPX_EXIT=1
-export FAKE_SPX_EXIT
-echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-[ "${rc}" -eq 0 ] && pass "genuine-failure: exit 0" || fail "genuine-failure: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && pass "genuine-failure: restart triggered" || fail "genuine-failure: expected a restart: $(cat "${STDERR}")"
-[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "3" ] \
-    && pass "genuine-failure: restart count incremented" || fail "genuine-failure: restart count not incremented"
-[ -f "${STATE_DIR}/nats-js-watchdog.last-restart" ] \
-    && pass "genuine-failure: cooldown stamp written" || fail "genuine-failure: cooldown stamp missing"
-
-# --- Case 6: healthy NATS -> no restart, and prior restart history clears ---
-reset_env healthy
-echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
-invoke
-[ "${rc}" -eq 0 ] && pass "healthy: exit 0" || fail "healthy: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
-restarted && fail "healthy: must not restart a healthy server" || pass "healthy: no restart"
-[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
-    && fail "healthy: restart count must clear once JetStream is confirmed writable" \
-    || pass "healthy: restart count cleared"
-
-if [ "${FAILS}" -eq 0 ]; then
-    echo "PASS: all nats-js-watchdog cases"
-    exit 0
-fi
-echo "FAILED: ${FAILS} case(s)"
-exit 1

build/scripts/wait-for-nats.sh

@@ -26,7 +26,6 @@ fi
 # Final fallback
 NATS_HOST="${NATS_HOST:-127.0.0.1}"
 
-# shellcheck disable=SC2034  # loop count only, index unused
 for i in $(seq 1 "$TIMEOUT"); do
     if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
         exit 0

build/systemd/regenerate-ssh-host-keys.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Regenerate SSH host keys
 ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key

build/systemd/spinifex-awsgw.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target

build/systemd/spinifex-daemon.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -39,13 +38,12 @@ PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
-DeviceAllow=/dev/vhost-net rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS
-# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall
+# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
+# ProtectKernelTunables=yes would otherwise keep read-only.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf
 
 # RG-9: hardening baseline
 ProtectHome=yes

build/systemd/spinifex-firewall.service

@@ -1,25 +0,0 @@
-# spinifex-unit-version: 1
-[Unit]
-Description=Spinifex host firewall (nftables input policy)
-Documentation=file:/etc/spinifex/firewall/spinifex.nft
-Wants=network-pre.target
-Before=network-pre.target shutdown.target
-DefaultDependencies=no
-ConditionPathExists=/etc/spinifex/firewall/peers.nft
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-StandardInput=null
-ProtectSystem=full
-ProtectHome=true
-ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
-ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
-TimeoutStopSec=10s
-
-# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD
-# rules with it. Remove the policy deliberately with
-# `nft delete table inet spinifex_filter`.
-
-[Install]
-WantedBy=sysinit.target

build/systemd/spinifex-guests.slice

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex guest VM tier
 Before=slices.target

build/systemd/spinifex-nats-watchdog.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog
 PartOf=spinifex.target
@@ -16,10 +15,6 @@ ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
 # cutting it off. Only the failing path runs that long; a healthy node exits on
 # the first sample.
 TimeoutStopSec=45
-# Same 34s worst case bounds the start side: without this the unit inherits
-# oneshot's infinite default, so a genuinely hung run would sit in
-# activating(start) forever and could not be retriggered by its timer.
-TimeoutStartSec=120
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog timer
 PartOf=spinifex.target

build/systemd/spinifex-nats.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS Message Bus
 PartOf=spinifex.target

build/systemd/spinifex-northstar.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Northstar (Authoritative DNS)
 PartOf=spinifex.target

build/systemd/spinifex-predastore.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -7,9 +6,9 @@ Wants=spinifex-nats.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
-# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
-# instead, and it can see whether a trace errored. A head ratio decides before
-# the error exists, so it discarded failures at the same rate as successes.
+# Guest block I/O arrives with no caller trace; sample those roots at 5% so
+# they don't drown request traces. Request-linked spans are always kept.
+Environment=MULGA_ROOT_TRACE_RATIO=0.05
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -18,9 +17,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
-# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
-# predastore, but After= already stops viperblock first (within its own 150s
-# budget); this is the backstop if that ordering is ever bypassed.
+# Matches viperblock's drain budget: viperblock's WAL flush writes into
+# predastore, and After= already stops viperblock first, but this is the
+# backstop if that ordering is ever bypassed (direct stop, future edit).
 TimeoutStopSec=30
 OOMScoreAdjust=-500
 # Legitimately the most memory-hungry infra service (S3 buffering/caching);

build/systemd/spinifex-qmp-collector.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex QMP Telemetry Collector (guest metrics)
 PartOf=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex graceful guest drain
 PartOf=spinifex.target
@@ -12,11 +11,11 @@ RemainAfterExit=yes
 ExecStart=/bin/true
 # ExecStop fires on ANY target stop, including the stop half of `systemctl
 # restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
-# of whether the host is actually going down. --unless-restarting drains on a
-# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it
-# skips only when a restart of the stack is already queued, so QEMU/nbdkit
-# survive (KillMode=process) and the daemon reconnects on the way back up.
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting
+# of whether the host is actually going down. --only-if-host-stopping drains
+# only when systemd itself is unwinding into shutdown.target; a plain restart
+# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
+# (KillMode=process) and the daemon reconnects to them on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-system.slice

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
 Before=slices.target

build/systemd/spinifex-ui.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target
@@ -24,7 +23,6 @@ MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
 Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
-Environment=SPINIFEX_UI_OCHRE_ENABLED=false
 
 # Filesystem access
 NoNewPrivileges=yes

build/systemd/spinifex-viperblock.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
@@ -24,10 +23,8 @@ MemoryMax=25%
 # way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
 # that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
 KillMode=process
-# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)
-# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must
-# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.
-TimeoutStopSec=150
+# Bounds this process's own WAL drain, which the plugin caps at 20s.
+TimeoutStopSec=30
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
@@ -51,8 +48,7 @@ ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
-# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's
-# QEMU /proc entries (different uid) orphan nbdkit reaping must see.
+ProtectProc=invisible
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK

build/systemd/spinifex-vpcd.service

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target

build/systemd/spinifex.slice

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex resource-bounded slice (host ceiling)
 Before=slices.target

build/systemd/spinifex.target

@@ -1,4 +1,3 @@
-# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target

cmd/aws-model-coverage/main.go

@@ -6,8 +6,6 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/dhcptest/main.go

@@ -24,8 +24,6 @@ import (
 	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/ecr-credential-provider/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 const (

cmd/ecs-agent/credentials/imds.go

@@ -9,9 +9,9 @@ import (
 )
 
 // IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
-// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
-// which refetches once the real expiry passes. A failed refresh is an error
-// here, not an extended expiry. Safe for concurrent use.
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
+// (which refetches once the real expiry passes; ec2rolecreds also caps
+// Expires to at most an hour out). Safe for concurrent use.
 type IMDSProvider struct {
 	cache *aws.CredentialsCache
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -88,63 +88,6 @@ func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	}
 }
 
-// imdsStubFailAfterFirst serves creds on the first credential request and 500s
-// on every one after it, so a test can drive a refresh that fails against a
-// credential the cache has already stored.
-func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
-	t.Helper()
-	var hits atomic.Int32
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits.Add(1) > 1 {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-// A refresh that fails must surface as an error. Left to itself the SDK's
-// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
-// key with Expires pushed 5-15 minutes out, so the agent would sign requests
-// with a credential the control plane has already rejected — indefinitely,
-// since every subsequent refresh fails the same way.
-func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
-	srv := imdsStubFailAfterFirst(t, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		// Already expired, so the next Retrieve must go back to IMDS.
-		"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	first, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("first Retrieve: %v", err)
-	}
-
-	got, err := p.Retrieve(context.Background())
-	if err == nil {
-		t.Fatalf("refresh failure was reported as success: %+v", got)
-	}
-	if got.Expiration.After(first.Expiration) {
-		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
-	}
-}
-
 func TestRetrieve_CancelledContext(t *testing.T) {
 	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
 	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")

cmd/ecs-agent/ecrresolver.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"net/http"
-	"net/url"
 	"strings"
 	"sync"
 
@@ -61,25 +60,12 @@ func (e *ecrResolver) client() (*http.Client, error) {
 	return c, nil
 }
 
-// gatewayRegistryHost returns the host[:port] of the co-located ECR registry
-// from the gateway URL; empty/unparseable yields "" (matches no ref). The
-// registry shares the gateway listener, so its port is the gateway port.
-func gatewayRegistryHost(gatewayURL string) string {
-	u, err := url.Parse(gatewayURL)
-	if err != nil {
-		return ""
-	}
-	return u.Host
-}
-
-// isECRRef reports whether ref targets an ECR registry: an "<acct>.dkr.ecr."
-// host, or one equal to gatewayHost (the agent's own co-located registry
-// advertised as a bare host/IP). Other refs pull anonymously, no token.
-func isECRRef(ref, gatewayHost string) bool {
+// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
+// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
+// public registries (docker.io and friends) pull anonymously, so refs without
+// an ECR host must not drag in IMDS host credentials.
+func isECRRef(ref string) bool {
 	host, _, _ := strings.Cut(ref, "/")
-	if gatewayHost != "" && host == gatewayHost {
-		return true
-	}
 	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
 		return false // no registry host -> docker.io default
 	}
@@ -89,7 +75,7 @@ func isECRRef(ref, gatewayHost string) bool {
 // Authorize returns the ECR token's user/password and proxy endpoint for ref.
 // Non-ECR refs resolve to anonymous pull (empty credentials).
 func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
-	if !isECRRef(ref, gatewayRegistryHost(e.gatewayURL)) {
+	if !isECRRef(ref) {
 		return "", "", "", nil
 	}
 	c, err := e.creds.Retrieve(ctx)

cmd/ecs-agent/ecrresolver_test.go

@@ -115,44 +115,17 @@ func TestECRResolver_NonECRAnonymous(t *testing.T) {
 }
 
 func TestIsECRRef(t *testing.T) {
-	cases := []struct {
-		ref, gatewayHost string
-		want             bool
-	}{
-		{"nginx:alpine", "192.168.1.13:9999", false},
-		{"docker.io/library/nginx", "192.168.1.13:9999", false},
-		{"ghcr.io/org/app:1", "192.168.1.13:9999", false},
-		{"registry:5000/app:1", "192.168.1.13:9999", false},
-		{"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1", "192.168.1.13:9999", true},
-		{"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x", "192.168.1.13:9999", true},
-		// Co-located gateway registry advertised as a bare host/IP.
-		{"192.168.1.13:9999/bunyip:latest", "192.168.1.13:9999", true},
-		{"192.168.1.13:9999/bunyip:latest", "", false},
+	cases := map[string]bool{
+		"nginx:alpine":            false,
+		"docker.io/library/nginx": false,
+		"ghcr.io/org/app:1":       false,
+		"registry:5000/app:1":     false,
+		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
+		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
 	}
-	for _, c := range cases {
-		if got := isECRRef(c.ref, c.gatewayHost); got != c.want {
-			t.Errorf("isECRRef(%q, %q) = %v, want %v", c.ref, c.gatewayHost, got, c.want)
+	for ref, want := range cases {
+		if got := isECRRef(ref); got != want {
+			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
 		}
 	}
 }
-
-// TestECRResolver_Authorize_BareHostGateway asserts that a ref whose host
-// equals the agent's own gateway host authenticates via the co-located ECR
-// registry, even though it lacks a ".dkr.ecr." name.
-func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
-	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
-	}}
-	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
-
-	host := strings.TrimPrefix(srv.URL, "https://")
-	user, pass, _, err := r.Authorize(context.Background(), host+"/bunyip:latest")
-	if err != nil {
-		t.Fatalf("Authorize: %v", err)
-	}
-	if user != "AWS" || pass != "jwt" {
-		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
-	}
-}

cmd/ecs-agent/main.go

@@ -17,7 +17,7 @@ import (
 	"os/signal"
 	"syscall"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {

cmd/ecs-agent/runtime/runtime_test.go

@@ -1,9 +1,71 @@
 package runtime
 
 import (
+	"context"
+	"errors"
 	"testing"
 )
 
+// staticResolver is a test Resolver returning fixed creds or an error.
+type staticResolver struct {
+	user, pass, endpoint string
+	err                  error
+}
+
+func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
+	return s.user, s.pass, s.endpoint, s.err
+}
+
+func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
+	f := &FakePuller{}
+	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
+	if err != nil {
+		t.Fatalf("Pull: %v", err)
+	}
+	if img.Ref != "repo/app:1" {
+		t.Errorf("Ref defaulted wrong: %q", img.Ref)
+	}
+	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
+		t.Errorf("Pulls = %+v", f.Pulls)
+	}
+	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
+		t.Errorf("Authzd = %+v", f.Authzd)
+	}
+}
+
+func TestFakePuller_ResolverError(t *testing.T) {
+	f := &FakePuller{}
+	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
+	if err == nil {
+		t.Fatal("expected resolver error to propagate")
+	}
+}
+
+func TestFakePuller_ProgrammedError(t *testing.T) {
+	f := &FakePuller{Err: errors.New("pull failed")}
+	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
+		t.Fatal("expected programmed error")
+	}
+}
+
+func TestFakePuller_OnPullOverrides(t *testing.T) {
+	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
+	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
+		return want, nil
+	}}
+	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
+	if err != nil || got != want {
+		t.Fatalf("got %+v, %v; want %+v", got, err, want)
+	}
+}
+
+func TestFakePuller_Close(t *testing.T) {
+	f := &FakePuller{}
+	if err := f.Close(); err != nil || !f.Closed {
+		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
+	}
+}
+
 func TestNormalizeRef(t *testing.T) {
 	cases := map[string]string{
 		"nginx:alpine":      "docker.io/library/nginx:alpine",

cmd/eks-gateway-fetch/main.go

@@ -40,7 +40,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 // stagedAddon mirrors the gateway's StagedAddonManifest wire shape; duplicated

cmd/eks-gateway-publish/main.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 const (

cmd/eks-konnectivity-cert/main.go

@@ -38,8 +38,6 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/eks-token-webhook/cert.go

@@ -15,7 +15,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )
 
 // serverTLSConfig is the webhook's loopback serving TLS config.

cmd/eks-token-webhook/main.go

@@ -27,8 +27,6 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 // config is the webhook's runtime configuration, sourced from the first-boot

cmd/eks-webhook-cert/main.go

@@ -32,8 +32,6 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
-
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/installer/firstboot/firstboot.go

@@ -146,21 +146,6 @@ hostnamectl set-hostname %s
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
 resolvconf -u
 
-# The RTC on a fresh machine can be badly wrong — a flat CMOS battery is routine
-# on refurbished hardware. Credentials and TLS certificates are minted further
-# down, and a certificate issued against a wrong clock carries a NotBefore that
-# outlives the correction, so the clock has to be right before that happens.
-# chronyd is already running; makestep forces an immediate correction instead of
-# waiting out the slew. Bounded and non-fatal: an air-gapped install must still
-# complete, and rtcsync writes the corrected time back so later boots start close.
-echo "[firstboot] syncing clock before credentials are minted"
-chronyc -a makestep >/dev/null 2>&1 || true
-if chronyc waitsync 12 0.1 0 5 >/dev/null 2>&1; then
-    echo "[firstboot] clock synced: $(date -uIs)"
-else
-    echo "[firstboot] WARNING: clock not synced, continuing with $(date -uIs)"
-fi
-
 # Configure OVN networking.
 # br-wan (and br-lan if present) are Linux bridges managed by systemd-networkd
 # (declared in /etc/systemd/network/ by the installer). setup-ovn.sh
@@ -222,12 +207,6 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
-# setup.sh enables this on the package install path, but the ISO chroot returns
-# before that line (no systemd to talk to), so the ISO needs it here. An nft
-# ruleset is runtime state and does not survive a reboot; without this the unit
-# never reapplies the policy and the node comes back up open. Enabled, not
-# started: ConditionPathExists holds it inert until the daemon writes a peer file.
-systemctl enable spinifex-firewall.service
 # WantedBy=timers.target only self-activates once enabled — without this the
 # JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
 systemctl enable --now spinifex-nats-watchdog.timer

cmd/installer/install/blockdev.go

@@ -45,7 +45,6 @@ var (
 	sysClassBlockDir = "/sys/class/block"
 	devByIDDir       = "/dev/disk/by-id"
 	procMountsPath   = "/proc/mounts"
-	procSwapsPath    = "/proc/swaps"
 )
 
 // ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
@@ -133,144 +132,6 @@ func partitionPath(disk string, n int) string {
 	return fmt.Sprintf("%s%d", disk, n)
 }
 
-// readKernelPartitions returns the partition device names the kernel currently
-// publishes for a disk, sorted.
-//
-// It reads sysfs rather than /dev deliberately. A partition device node
-// outlives the table that created it, so a stale node is indistinguishable
-// from a fresh one by existence alone — which is how a disk whose old table
-// the kernel never dropped passes every check and then gets formatted at the
-// previous layout's offsets.
-func readKernelPartitions(disk string) ([]string, error) {
-	name := filepath.Base(disk)
-	entries, err := os.ReadDir(filepath.Join(sysBlockDir, name))
-	if err != nil {
-		return nil, fmt.Errorf("read the kernel's partition list for %s: %w", disk, err)
-	}
-	var out []string
-	for _, e := range entries {
-		// Partitions are the child directories carrying a "partition" attribute.
-		// "queue", "device" and the plain attribute files sit alongside them.
-		if _, err := os.Stat(filepath.Join(sysBlockDir, name, e.Name(), "partition")); err != nil {
-			continue
-		}
-		out = append(out, e.Name())
-	}
-	slices.Sort(out)
-	return out, nil
-}
-
-// kernelPartitions is indirected so tests can drive the partition-table
-// assertions without a real block device.
-var kernelPartitions = readKernelPartitions
-
-// partitionDevice maps a kernel partition name back to its device node,
-// resolved alongside the disk it belongs to rather than assuming /dev.
-func partitionDevice(disk, part string) string {
-	return filepath.Join(filepath.Dir(disk), part)
-}
-
-// diskHolders names the md, LVM or device-mapper devices claiming this disk or
-// one of its partitions. A holder is the usual reason the kernel refuses to
-// re-read a partition table, so it is what the operator needs told when the
-// installer gives up on a disk.
-func diskHolders(disk string) []string {
-	name := filepath.Base(disk)
-	dirs := []string{filepath.Join(sysBlockDir, name, "holders")}
-	parts, _ := kernelPartitions(disk)
-	for _, p := range parts {
-		dirs = append(dirs, filepath.Join(sysBlockDir, name, p, "holders"))
-	}
-
-	seen := map[string]bool{}
-	var out []string
-	for _, dir := range dirs {
-		entries, err := os.ReadDir(dir)
-		if err != nil {
-			continue
-		}
-		for _, e := range entries {
-			if seen[e.Name()] {
-				continue
-			}
-			seen[e.Name()] = true
-			out = append(out, e.Name())
-		}
-	}
-	slices.Sort(out)
-	return out
-}
-
-// diskMount is one filesystem mounted from a disk the installer is about to
-// erase.
-type diskMount struct {
-	Device     string
-	Mountpoint string
-}
-
-// devicesOnDisk is the set of device paths belonging to a disk: the whole disk
-// and every partition the kernel currently publishes. Matching /proc tables
-// against a set rather than a string prefix keeps /dev/nvme0n1 from claiming
-// /dev/nvme0n11.
-func devicesOnDisk(disk string) map[string]bool {
-	out := map[string]bool{disk: true}
-	parts, _ := kernelPartitions(disk)
-	for _, p := range parts {
-		out[partitionDevice(disk, p)] = true
-	}
-	return out
-}
-
-// mountsOnDisk returns the filesystems mounted from a disk, deepest mountpoint
-// first so a nested mount is released before the one it sits inside.
-func mountsOnDisk(disk string) []diskMount {
-	devs := devicesOnDisk(disk)
-	var out []diskMount
-	for _, fields := range procTable(procMountsPath, 0) {
-		if len(fields) >= 2 && devs[fields[0]] {
-			out = append(out, diskMount{Device: fields[0], Mountpoint: fields[1]})
-		}
-	}
-	slices.SortFunc(out, func(a, b diskMount) int {
-		return strings.Count(b.Mountpoint, "/") - strings.Count(a.Mountpoint, "/")
-	})
-	return out
-}
-
-// swapsOnDisk returns the active swap areas backed by a disk.
-func swapsOnDisk(disk string) []string {
-	devs := devicesOnDisk(disk)
-	var out []string
-	// The first line of /proc/swaps is a column header, not an entry.
-	for _, fields := range procTable(procSwapsPath, 1) {
-		if len(fields) >= 1 && devs[fields[0]] {
-			out = append(out, fields[0])
-		}
-	}
-	return out
-}
-
-// procTable splits a whitespace-delimited /proc table into fields per line,
-// dropping the first skip lines and any blank ones.
-func procTable(path string, skip int) [][]string {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return nil
-	}
-	var out [][]string
-	n := 0
-	for line := range strings.SplitSeq(string(data), "\n") {
-		n++
-		if n <= skip {
-			continue
-		}
-		if fields := strings.Fields(line); len(fields) > 0 {
-			out = append(out, fields)
-		}
-	}
-	return out
-}
-
 // byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
 // pool member by. wwn- ids are last: they are stable but opaque, so a degraded
 // pool reports a number nobody can map to a drive bay.

cmd/installer/install/blockdev_test.go

@@ -5,7 +5,6 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
-	"strings"
 	"testing"
 )
 
@@ -41,7 +40,6 @@ func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
 	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
 	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
 	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
-	swap(&procSwapsPath, filepath.Join(f.root, "proc/swaps"))
 
 	f.mkdirAll(sysBlockDir)
 	f.mkdirAll(sysClassBlockDir)
@@ -115,18 +113,6 @@ func (f *fakeSysfs) setMounts(content string) {
 	f.write(procMountsPath, content)
 }
 
-func (f *fakeSysfs) setSwaps(content string) {
-	f.t.Helper()
-	f.write(procSwapsPath, content)
-}
-
-// addHolder marks a partition as claimed by md, LVM or device-mapper — the
-// state that makes the kernel refuse to re-read the disk's partition table.
-func (f *fakeSysfs) addHolder(disk, part, holder string) {
-	f.t.Helper()
-	f.mkdirAll(filepath.Join(sysBlockDir, disk, part, "holders", holder))
-}
-
 func boolAttr(b bool) string {
 	if b {
 		return "1\n"
@@ -366,119 +352,3 @@ func TestHasAnyPrefix(t *testing.T) {
 		t.Error("sda should not match")
 	}
 }
-
-func TestReadKernelPartitionsListsOnlyPartitions(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{
-		name: "nvme0n1", sectors: 1 << 22,
-		partitions: []string{"nvme0n1p3", "nvme0n1p1", "nvme0n1p2"},
-	})
-
-	got, err := readKernelPartitions("/dev/nvme0n1")
-	if err != nil {
-		t.Fatalf("readKernelPartitions: %v", err)
-	}
-	// Sorted, and "queue" and "device" alongside them are not partitions.
-	want := []string{"nvme0n1p1", "nvme0n1p2", "nvme0n1p3"}
-	if !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestReadKernelPartitionsIsEmptyForAnUnpartitionedDisk(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{name: "nvme2n1", sectors: 1 << 22})
-
-	got, err := readKernelPartitions("/dev/nvme2n1")
-	if err != nil {
-		t.Fatalf("readKernelPartitions: %v", err)
-	}
-	if len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}
-
-func TestReadKernelPartitionsFailsOnAnUnknownDisk(t *testing.T) {
-	newFakeSysfs(t)
-
-	if _, err := readKernelPartitions("/dev/nvme9n1"); err == nil {
-		t.Fatal("expected an error for a disk with no sysfs entry")
-	}
-}
-
-func TestDiskHoldersNamesEveryClaimant(t *testing.T) {
-	f := newFakeSysfs(t, fakeDisk{
-		name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"},
-	})
-	f.addHolder("sda", "sda1", "md0")
-	f.addHolder("sda", "sda2", "dm-0")
-
-	if got, want := diskHolders("/dev/sda"), []string{"dm-0", "md0"}; !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestDiskHoldersIsEmptyWhenNothingClaimsTheDisk(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
-
-	if got := diskHolders("/dev/sda"); len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}
-
-func TestMountsOnDiskFindsOnlyThisDiskDeepestFirst(t *testing.T) {
-	f := newFakeSysfs(t,
-		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"}},
-		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
-	)
-	f.setMounts(strings.Join([]string{
-		"/dev/sda1 /oldroot ext4 rw 0 0",
-		"/dev/sda2 /oldroot/boot/efi vfat rw 0 0",
-		"/dev/sdb1 /other ext4 rw 0 0",
-		"proc /proc proc rw 0 0",
-	}, "\n") + "\n")
-
-	got := mountsOnDisk("/dev/sda")
-	want := []diskMount{
-		{Device: "/dev/sda2", Mountpoint: "/oldroot/boot/efi"},
-		{Device: "/dev/sda1", Mountpoint: "/oldroot"},
-	}
-	if !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-// /dev/nvme0n1 must not claim /dev/nvme0n11, which a string-prefix match would.
-func TestMountsOnDiskDoesNotMatchASiblingNamespace(t *testing.T) {
-	f := newFakeSysfs(t,
-		fakeDisk{name: "nvme0n1", sectors: 1 << 22, partitions: []string{"nvme0n1p1"}},
-		fakeDisk{name: "nvme0n11", sectors: 1 << 22, partitions: []string{"nvme0n11p1"}},
-	)
-	f.setMounts("/dev/nvme0n11p1 /elsewhere ext4 rw 0 0\n")
-
-	if got := mountsOnDisk("/dev/nvme0n1"); len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}
-
-func TestSwapsOnDiskSkipsTheHeaderAndOtherDisks(t *testing.T) {
-	f := newFakeSysfs(t,
-		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}},
-		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
-	)
-	f.setSwaps(strings.Join([]string{
-		"Filename\t\t\t\tType\t\tSize\tUsed\tPriority",
-		"/dev/sda1 partition 8388604 0 -2",
-		"/dev/sdb1 partition 8388604 0 -3",
-	}, "\n") + "\n")
-
-	if got, want := swapsOnDisk("/dev/sda"), []string{"/dev/sda1"}; !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestSwapsOnDiskIsEmptyWithoutASwapsTable(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
-
-	if got := swapsOnDisk("/dev/sda"); len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}

cmd/installer/install/disk.go

@@ -4,9 +4,6 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"path/filepath"
-	"slices"
-	"strings"
 	"time"
 )
 
@@ -50,95 +47,6 @@ const (
 	tailReserveMiB = 1024
 )
 
-// partitionSettleTimeout bounds the wait for the kernel and udev to publish a
-// disk's new partition table. A var so tests do not pay it.
-var partitionSettleTimeout = 15 * time.Second
-
-// releaseDisk detaches whatever the live environment has attached to a disk
-// before it is erased. A mounted filesystem or an active swap area makes
-// BLKRRPART return EBUSY, and the kernel then keeps serving the disk's old
-// partition table while sgdisk writes the new one and reports success.
-func releaseDisk(d Disk) {
-	for _, m := range mountsOnDisk(d.Path) {
-		err := runQuiet("umount", m.Device)
-		if err != nil {
-			// Lazy detach: a busy mount must not stall an install that is about to
-			// erase the filesystem underneath it anyway.
-			err = runQuiet("umount", "-l", m.Device)
-		}
-		if err != nil {
-			slog.Warn("could not unmount before erase", "device", m.Device, "mountpoint", m.Mountpoint, "err", err)
-			continue
-		}
-		slog.Info("released mount before erase", "device", m.Device, "mountpoint", m.Mountpoint)
-	}
-	for _, s := range swapsOnDisk(d.Path) {
-		if err := runQuiet("swapoff", s); err != nil {
-			slog.Warn("could not disable swap before erase", "device", s, "err", err)
-			continue
-		}
-		slog.Info("disabled swap before erase", "device", s)
-	}
-}
-
-// settlePartitions forces the kernel to re-read a disk's partition table and
-// blocks until its view matches want, naming the partitions by number.
-//
-// An empty want asserts the disk has no partitions at all. That is the only
-// state no leftover can satisfy: a disk whose previous table had the same
-// partition numbers as the one being written — an old OS install is p1, p2, p3,
-// exactly what the boot layout writes — passes an "are the expected partitions
-// there" check on stale nodes alone.
-func settlePartitions(d Disk, want ...int) error {
-	if err := runQuiet("partprobe", d.Path); err != nil {
-		// partprobe rescans the whole disk and fails if it cannot open any part of
-		// it; the ioctl on its own sometimes still gets through.
-		if err := runQuiet("blockdev", "--rereadpt", d.Path); err != nil {
-			slog.Warn("could not force a partition table re-read", "disk", d.Path, "err", err)
-		}
-	}
-	_ = runQuiet("udevadm", "settle", "--timeout=10")
-
-	wanted := make([]string, 0, len(want))
-	for _, n := range want {
-		wanted = append(wanted, filepath.Base(d.PartitionPath(n)))
-	}
-	slices.Sort(wanted)
-
-	deadline := time.Now().Add(partitionSettleTimeout)
-	for {
-		got, err := kernelPartitions(d.Path)
-		if err != nil {
-			return err
-		}
-		if slices.Equal(got, wanted) {
-			return nil
-		}
-		if time.Now().After(deadline) {
-			return staleTableError(d, wanted, got)
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-}
-
-// staleTableError explains a kernel partition view that never caught up. The
-// erase case gets its own wording because it is the dangerous one — proceeding
-// would format the previous layout — and because the holder is usually the
-// whole diagnosis.
-func staleTableError(d Disk, wanted, got []string) error {
-	if len(wanted) == 0 {
-		msg := fmt.Sprintf("%s still shows partitions [%s] after its table was erased — the kernel is using the old table, so formatting would write at the previous layout's offsets",
-			d.Path, strings.Join(got, ", "))
-		if h := diskHolders(d.Path); len(h) > 0 {
-			return fmt.Errorf("%s; it is held by %s, which must be stopped before this disk can be installed to",
-				msg, strings.Join(h, ", "))
-		}
-		return fmt.Errorf("%s; something in the live environment is holding it open", msg)
-	}
-	return fmt.Errorf("%s: the kernel shows partitions [%s] but the new table has [%s] — the table was written but never picked up",
-		d.Path, strings.Join(got, ", "), strings.Join(wanted, ", "))
-}
-
 // espSizeMiB picks the ESP size for a disk.
 func espSizeMiB(d Disk) int {
 	if d.Bytes >= espLargeDiskGiB<<30 {
@@ -190,7 +98,7 @@ func partitionDataDisk(rm RoleMount) error {
 	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
 	}
-	return settlePartitions(rm.Disk, dataPartNum)
+	return nil
 }
 
 // partitionOne writes the GPT for a single disk.
@@ -221,15 +129,25 @@ func partitionOne(d Disk, rootType string) error {
 	if err := run("sgdisk", "-G", d.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
 	}
-	return settlePartitions(d, biosPartNum, espPartNum, rootPartNum)
+	return nil
 }
 
 // waitForPartitions ensures every partition device node the installer is about
-// to use exists. The kernel's own view was already asserted per disk as each
-// table was written; udev publishes the /dev nodes and by-id symlinks after
-// that, and Trixie is slow enough about it that mkfs races them.
+// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
+// races it and fails with ENOENT on a device the kernel has already accepted.
 func waitForPartitions(cfg DiskConfig) error {
-	deadline := time.Now().Add(partitionSettleTimeout)
+	for _, d := range cfg.Disks {
+		// Best-effort: partprobe failing is not fatal, since udev may still act
+		// on the BLKRRPART ioctl sgdisk issued itself.
+		if err := run("partprobe", d.Path); err != nil {
+			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
+		}
+	}
+	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
+		slog.Warn("udevadm settle failed, continuing", "err", err)
+	}
+
+	deadline := time.Now().Add(15 * time.Second)
 	for _, d := range cfg.bootDisks() {
 		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
 		// The pool is built from by-id paths, so their symlinks have to exist

cmd/installer/install/steps_test.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"slices"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -18,69 +16,17 @@ import (
 type fakeExec struct {
 	calls []string
 	fail  func(name string, args []string) error
-
-	// parts is the kernel's partition view per disk. Seeded by the test and then
-	// moved by the sgdisk calls the installer makes, so the partition-table
-	// assertions can run without a block device.
-	parts map[string][]string
-
-	// stale marks disks whose view never moves however many tables are written
-	// to them: something holds the disk open, BLKRRPART returns EBUSY, and the
-	// kernel keeps serving the old table while sgdisk reports success.
-	stale map[string]bool
-}
-
-// holdOpen makes a disk keep its current partition view forever.
-func (f *fakeExec) holdOpen(disk string) { f.stale[disk] = true }
-
-// seedPartitions gives a disk a pre-existing partition table — the state a
-// drive is in when the node has been installed on before.
-func (f *fakeExec) seedPartitions(disk string, nums ...int) {
-	for _, n := range nums {
-		f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
-	}
-}
-
-// applySgdisk moves the fake kernel view in step with a table rewrite, the way
-// a kernel that honoured every BLKRRPART would: -Z empties it, each -n adds the
-// partition it creates.
-func (f *fakeExec) applySgdisk(args []string) {
-	if len(args) == 0 {
-		return
-	}
-	disk := args[len(args)-1]
-	if f.stale[disk] {
-		return
-	}
-	for i, a := range args {
-		switch {
-		case a == "-Z":
-			f.parts[disk] = nil
-		case a == "-n" && i+1 < len(args):
-			num, _, _ := strings.Cut(args[i+1], ":")
-			n, err := strconv.Atoi(num)
-			if err != nil {
-				continue
-			}
-			f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
-		}
-	}
 }
 
 func fakeCommands(t *testing.T) *fakeExec {
 	t.Helper()
-	f := &fakeExec{parts: map[string][]string{}, stale: map[string]bool{}}
-	prevRun, prevQuiet, prevEnv, prevParts := run, runQuiet, runEnv, kernelPartitions
+	f := &fakeExec{}
+	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
 
 	record := func(name string, args ...string) error {
 		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
 		if f.fail != nil {
-			if err := f.fail(name, args); err != nil {
-				return err
-			}
-		}
-		if name == "sgdisk" {
-			f.applySgdisk(args)
+			return f.fail(name, args)
 		}
 		return nil
 	}
@@ -89,21 +35,8 @@ func fakeCommands(t *testing.T) *fakeExec {
 	runEnv = func(env []string, name string, args ...string) error {
 		return record(name, append(env, args...)...)
 	}
-	kernelPartitions = func(disk string) ([]string, error) {
-		got := slices.Clone(f.parts[disk])
-		slices.Sort(got)
-		return got, nil
-	}
 
-	// A short settle so a test asserting the stale-table failure does not wait
-	// out the install's real budget.
-	prevTimeout := partitionSettleTimeout
-	partitionSettleTimeout = 10 * time.Millisecond
-
-	t.Cleanup(func() {
-		run, runQuiet, runEnv, kernelPartitions = prevRun, prevQuiet, prevEnv, prevParts
-		partitionSettleTimeout = prevTimeout
-	})
+	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
 	return f
 }
 
@@ -181,24 +114,6 @@ func targetSkeleton(t *testing.T, root string) {
 
 // tempDisks builds disks whose device nodes are real files, so the steps that
 // wait for partition nodes to appear can complete.
-// fakeProcTables points the mount and swap readers at fixture tables for the
-// duration of a test.
-func fakeProcTables(t *testing.T, mounts, swaps string) {
-	t.Helper()
-	dir := t.TempDir()
-	point := func(dst *string, name, content string) {
-		prev := *dst
-		p := filepath.Join(dir, name)
-		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
-			t.Fatalf("write %s: %v", p, err)
-		}
-		*dst = p
-		t.Cleanup(func() { *dst = prev })
-	}
-	point(&procMountsPath, "mounts", mounts)
-	point(&procSwapsPath, "swaps", swaps)
-}
-
 func tempDisks(t *testing.T, n int, gib int64) []Disk {
 	t.Helper()
 	dir := t.TempDir()
@@ -322,154 +237,21 @@ func TestEspSizeMiB(t *testing.T) {
 	}
 }
 
-// A drive arriving with a previous install's layout must have every partition
-// cleared, not just the one the root filesystem used to be on: a data drive's
-// partition is p1, so a p3-only labelclear leaves a former pool member's ZFS
-// label in place at both ends of the device.
-func TestClearDiskWipesEveryPartitionItFinds(t *testing.T) {
+func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
 	f := fakeCommands(t)
-	d := tempDisks(t, 1, 40)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-
-	if err := clearDisk(d); err != nil {
+	disks := tempDisks(t, 1, 40)
+	if err := clearDisk(disks[0]); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	for _, n := range []int{biosPartNum, espPartNum, rootPartNum} {
-		f.mustRun(t, "zpool labelclear -f "+d.PartitionPath(n))
-		f.mustRun(t, "wipefs -a "+d.PartitionPath(n))
-	}
-	f.mustRun(t, "wipefs -a "+d.Path)
-	f.mustRun(t, "sgdisk -Z "+d.Path)
-}
+	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
 
-func TestClearDiskProbesNoPartitionsOnABlankDrive(t *testing.T) {
-	f := fakeCommands(t)
+	f2 := fakeCommands(t)
 	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
-
 	if err := clearDisk(blank); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f.mustNotRun(t, "labelclear")
-	f.mustRun(t, "wipefs -a "+blank.Path)
-}
-
-// The reported field failure: a drive whose old table the kernel never dropped.
-// Proceeding would format the previous layout's offsets, so the install must
-// stop, and the message must say the table is stale rather than blaming the
-// device node that did not appear.
-func TestClearDiskFailsWhenTheKernelKeepsTheOldTable(t *testing.T) {
-	f := fakeCommands(t)
-	d := tempDisks(t, 1, 40)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-	// sgdisk writes to the platters and exits 0, but BLKRRPART returned EBUSY,
-	// so the kernel's view never changes.
-	f.holdOpen(d.Path)
-
-	err := clearDisk(d)
-	if err == nil {
-		t.Fatal("clearDisk succeeded against a stale partition table")
-	}
-	for _, want := range []string{d.Path, "still shows partitions", "previous layout"} {
-		if !strings.Contains(err.Error(), want) {
-			t.Errorf("err = %q, want it to mention %q", err, want)
-		}
-	}
-}
-
-func TestClearDiskReleasesMountsAndSwapFirst(t *testing.T) {
-	f := fakeCommands(t)
-	d := tempDisks(t, 1, 40)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-
-	root, esp := d.PartitionPath(rootPartNum), d.PartitionPath(espPartNum)
-	swapDev := d.PartitionPath(biosPartNum)
-	fakeProcTables(t,
-		fmt.Sprintf("%s /oldroot ext4 rw 0 0\n%s /oldroot/boot/efi vfat rw 0 0\n", root, esp),
-		fmt.Sprintf("Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n%s partition 1000 0 -2\n", swapDev),
-	)
-
-	if err := clearDisk(d); err != nil {
-		t.Fatalf("clearDisk: %v", err)
-	}
-	f.mustRun(t, "swapoff "+swapDev)
-
-	// Deepest mountpoint first: unmounting /oldroot while /oldroot/boot/efi is
-	// still attached fails with EBUSY and leaves the disk held.
-	var order []string
-	for _, c := range f.calls {
-		if c == "umount "+root || c == "umount "+esp {
-			order = append(order, c)
-		}
-	}
-	if want := []string{"umount " + esp, "umount " + root}; !slices.Equal(order, want) {
-		t.Errorf("unmount order:\n got %v\nwant %v", order, want)
-	}
-}
-
-// A drive held by md or LVM cannot be released — the ISO ships neither tool —
-// so the operator has to be told what is holding it.
-func TestStaleTableErrorNamesTheHolder(t *testing.T) {
-	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
-	f.addHolder("sda", "sda1", "md0")
-
-	prev := kernelPartitions
-	kernelPartitions = readKernelPartitions
-	t.Cleanup(func() { kernelPartitions = prev })
-
-	err := staleTableError(Disk{Path: "/dev/sda"}, nil, []string{"sda1"})
-	if err == nil || !strings.Contains(err.Error(), "held by md0") {
-		t.Fatalf("err = %v, want it to name the md0 holder", err)
-	}
-}
-
-// The reported field failure as a unit test: two drives arrive carrying a
-// previous install's partitions and the third has never been partitioned. All
-// three must be taken over, and the blank one must not be the only one to get
-// a new table.
-func TestPartitionDisksTakesOverDrivesThatAlreadyHavePartitions(t *testing.T) {
-	f := fakeCommands(t)
-	nodes := tempDisks(t, 3, 200)
-	f.seedPartitions(nodes[0].Path, biosPartNum, espPartNum, rootPartNum) // held an OS
-	f.seedPartitions(nodes[1].Path, dataPartNum)                          // held data
-	// nodes[2] arrives blank.
-
-	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
-		{Role: RoleOS, Disk: nodes[0]},
-		{Role: RoleSpinifex, Disk: nodes[1]},
-		{Role: RolePredastore, Disk: nodes[2]},
-	})
-	if err := partitionDisks(cfg); err != nil {
-		t.Fatalf("partitionDisks: %v", err)
-	}
-	for _, d := range nodes {
-		f.mustRun(t, "sgdisk -Z "+d.Path)
-	}
-	f.mustRun(t, "sgdisk", nodes[0].Path, "-n 3:0:-1024M")
-	for _, d := range nodes[1:] {
-		f.mustRun(t, "sgdisk", d.Path, "-n 1:1M:-1024M", "-t 1:8300")
-	}
-}
-
-// A previous OS install leaves p1, p2, p3 — exactly the numbering the boot
-// layout writes. Asserting the expected partitions exist would pass on those
-// stale nodes and the install would format the old offsets, so the erase-stage
-// assertion on an empty table is what has to catch it.
-func TestPartitionOneStopsWhenAnOldTableHasTheSameLayout(t *testing.T) {
-	f := fakeCommands(t)
-	d := tempDisks(t, 1, 200)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-	f.holdOpen(d.Path)
-
-	err := partitionOne(d, typeLinuxFS)
-	if err == nil {
-		t.Fatal("partitionOne succeeded against a stale partition table")
-	}
-	// Specifically the erase-stage message: reaching the post-write check would
-	// mean the stale nodes had already been accepted.
-	if !strings.Contains(err.Error(), "still shows partitions") {
-		t.Errorf("err = %q, want the erase-stage stale-table message", err)
-	}
-	f.mustNotRun(t, "sgdisk -n")
+	f2.mustNotRun(t, "labelclear")
+	f2.mustRun(t, "wipefs -a "+blank.Path)
 }
 
 func TestFormatESPsLabelsEveryMember(t *testing.T) {
@@ -887,6 +669,13 @@ func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
 	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
 }
 
+func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
+	// Log-only, but it is the operator's only record of the disk-replacement
+	// command, so both branches are walked.
+	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
+	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
+}
+
 func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	root := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
@@ -906,6 +695,14 @@ func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	}
 }
 
+func TestCopyAssetsAreBestEffort(t *testing.T) {
+	root := t.TempDir()
+	// Neither asset exists outside the live ISO; a missing one must not fail
+	// the install, only the branding.
+	copyGrubFont(root)
+	copySplashImage(root)
+}
+
 func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
 	root := withMountRoot(t)
 	targetSkeleton(t, root)

cmd/installer/install/zfs.go

@@ -182,7 +182,8 @@ func ImportablePools() ([]string, error) {
 		// zpool exits non-zero when there is nothing to import, which is the
 		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
 		// a machine with an existing pool looks empty to the caller.
-		if _, ok := errors.AsType[*exec.ExitError](err); ok {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("zpool import: %w", err)
@@ -339,35 +340,23 @@ func exportPool() {
 	}
 }
 
-// clearDisk takes a disk over unconditionally, whatever it holds: everything
-// attached to it is released, every signature on it is wiped, its partition
-// table is erased, and the kernel is made to agree the disk is now empty.
-//
-// The last part is the point. sgdisk writes to the platters and exits 0 even
-// when the kernel refuses to drop the old table, so without the assertion an
-// install proceeds against partition nodes describing the previous layout.
+// clearDisk removes any prior filesystem, partition table and ZFS label from a
+// disk so a retry after a partial install is not rejected for stale metadata —
+// a failure that otherwise reads as mysterious rather than as leftover debris.
 //
 // The probes are quiet because a blank disk makes all of them fail, and their
 // stderr on the install console reads like the install is going wrong.
 func clearDisk(d Disk) error {
-	releaseDisk(d)
-
-	// Every partition, not just the root slice. ZFS labels live at both ends of
-	// a device and wipefs does not know about them, so a data drive that was
-	// once a pool member keeps its label and fails a later zpool create.
-	parts, err := kernelPartitions(d.Path)
-	if err != nil {
-		return err
+	// ZFS labels live at both ends of the device and wipefs does not know about
+	// them, so an old pool member has to be cleared before its partition goes.
+	if old := d.PartitionPath(rootPartNum); exists(old) {
+		_ = runQuiet("zpool", "labelclear", "-f", old)
 	}
-	for _, p := range parts {
-		dev := partitionDevice(d.Path, p)
-		_ = runQuiet("zpool", "labelclear", "-f", dev)
-		_ = runQuiet("wipefs", "-a", dev)
-	}
-
 	_ = runQuiet("wipefs", "-a", d.Path)
-	if err := runQuiet("sgdisk", "-Z", d.Path); err != nil {
-		return fmt.Errorf("erase partition table: %w", err)
-	}
-	return settlePartitions(d)
+	return runQuiet("sgdisk", "-Z", d.Path)
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
 }

cmd/installer/main.go

@@ -5,10 +5,10 @@ import (
 	"os"
 	"strings"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 	"github.com/mulgadc/spinifex/cmd/installer/autoinstall"
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 	"github.com/mulgadc/spinifex/cmd/installer/ui"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {

cmd/installer/ui/wizard_test.go

@@ -1,896 +0,0 @@
-//test:in-package — the wizard model, its screen constants and the key handlers
-//under test are all unexported, and the tests read model fields directly to
-//assert which screen a key sequence landed on.
-
-package ui
-
-import (
-	"slices"
-	"strings"
-	"testing"
-
-	tea "github.com/charmbracelet/bubbletea"
-	"github.com/mulgadc/spinifex/cmd/installer/install"
-)
-
-// namedKeys maps the key names the handlers switch on to their tea.KeyType.
-// Anything not listed is sent as runes.
-var namedKeys = map[string]tea.KeyType{
-	"enter":     tea.KeyEnter,
-	"esc":       tea.KeyEsc,
-	"tab":       tea.KeyTab,
-	"shift+tab": tea.KeyShiftTab,
-	"up":        tea.KeyUp,
-	"down":      tea.KeyDown,
-	"left":      tea.KeyLeft,
-	"right":     tea.KeyRight,
-	" ":         tea.KeySpace,
-	"backspace": tea.KeyBackspace,
-	"ctrl+c":    tea.KeyCtrlC,
-}
-
-// keyMsg builds the tea.KeyMsg for a key name and checks it round-trips, so a
-// mis-built message fails here rather than silently exercising the wrong branch.
-func keyMsg(t *testing.T, key string) tea.KeyMsg {
-	t.Helper()
-	msg := tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(key)}
-	if kt, ok := namedKeys[key]; ok {
-		msg = tea.KeyMsg{Type: kt}
-	}
-	if got := msg.String(); got != key {
-		t.Fatalf("built key %q, which the handlers see as %q", key, got)
-	}
-	return msg
-}
-
-// press feeds keys through Update, which is the only entry point the bubbletea
-// runtime uses, and returns the resulting model.
-func press(t *testing.T, m model, keys ...string) model {
-	t.Helper()
-	for _, k := range keys {
-		next, _ := m.Update(keyMsg(t, k))
-		typed, ok := next.(model)
-		if !ok {
-			t.Fatalf("Update returned %T, not a model", next)
-		}
-		m = typed
-	}
-	return m
-}
-
-// typeText sends a string one rune at a time, the way a terminal does.
-func typeText(t *testing.T, m model, s string) model {
-	t.Helper()
-	for _, r := range s {
-		m = press(t, m, string(r))
-	}
-	return m
-}
-
-func wizardModel(t *testing.T, diskCount, nicCount int) model {
-	t.Helper()
-	disks := make([]install.Disk, 0, diskCount)
-	for i := range diskCount {
-		disks = append(disks, install.Disk{
-			Path:             "/dev/sd" + string(rune('a'+i)),
-			Bytes:            200 << 30,
-			LogicalBlockSize: 512,
-		})
-	}
-	m := newModel(disks, nics(nicCount))
-	m.width, m.height = 100, 40
-	return m
-}
-
-// The wizard has to be completable from the welcome screen using only the keys
-// the help lines advertise, and produce the config the operator entered.
-func TestWizardWalkthroughProducesTheEnteredConfig(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-
-	m = press(t, m, "enter") // welcome → disk
-	if m.screen != screenDisk {
-		t.Fatalf("screen = %v, want the disk screen", m.screen)
-	}
-
-	m = press(t, m, "enter") // preselected disk is valid → erase confirmation
-	if m.screen != screenDiskConfirm {
-		t.Fatalf("screen = %v, want the erase confirmation", m.screen)
-	}
-
-	m = typeText(t, m, "yes")
-	m = press(t, m, "enter") // → network roles
-	if m.screen != screenNetworkRoles {
-		t.Fatalf("screen = %v, want the network roles screen", m.screen)
-	}
-
-	m = press(t, m, "down", "down", "down", "enter") // Continue → identity
-	if m.screen != screenIdentity {
-		t.Fatalf("screen = %v, want the identity screen", m.screen)
-	}
-
-	m = typeText(t, m, "node1")
-	m = press(t, m, "enter") // → credentials
-	if m.screen != screenPassword {
-		t.Fatalf("screen = %v, want the credentials screen", m.screen)
-	}
-
-	m = typeText(t, m, "admin@example.com")
-	m = press(t, m, "enter") // email accepted → password
-	m = typeText(t, m, "hunter2hunter2")
-	m = press(t, m, "enter") // → confirm password
-	m = typeText(t, m, "hunter2hunter2")
-	m = press(t, m, "enter") // → final confirmation
-	if m.screen != screenConfirm {
-		t.Fatalf("screen = %v, want the final confirmation (err: %q)", m.screen, m.validationErr)
-	}
-
-	m = press(t, m, "y")
-	if m.screen != screenDone {
-		t.Fatalf("screen = %v, want done", m.screen)
-	}
-	if m.result == nil {
-		t.Fatal("result is nil after confirming")
-	}
-
-	cfg := m.result
-	if cfg.Hostname != "node1" {
-		t.Errorf("Hostname = %q, want node1", cfg.Hostname)
-	}
-	if cfg.Email != "admin@example.com" {
-		t.Errorf("Email = %q, want admin@example.com", cfg.Email)
-	}
-	if cfg.RootPassword != "hunter2hunter2" {
-		t.Errorf("RootPassword = %q, want what was typed", cfg.RootPassword)
-	}
-	if !slices.Equal(cfg.Storage.Paths(), []string{"/dev/sda"}) {
-		t.Errorf("Storage.Paths() = %v, want the preselected disk", cfg.Storage.Paths())
-	}
-	if cfg.WAN.Interface != "eno1" {
-		t.Errorf("WAN interface = %q, want the first NIC", cfg.WAN.Interface)
-	}
-	if err := cfg.Validate(); err != nil {
-		t.Errorf("the wizard produced a config it would refuse to install: %v", err)
-	}
-}
-
-// Esc has to walk back the way it came, or an operator who overshoots is stuck
-// with a wizard they can only cancel out of.
-func TestEscapeWalksBackThroughTheScreens(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m = press(t, m, "enter") // disk
-	m = press(t, m, "enter") // erase confirmation
-
-	for _, step := range []struct {
-		key  string
-		want screen
-	}{
-		{"esc", screenDisk},
-		{"esc", screenWelcome},
-	} {
-		m = press(t, m, step.key)
-		if m.screen != step.want {
-			t.Fatalf("after esc: screen = %v, want %v", m.screen, step.want)
-		}
-	}
-}
-
-// Ctrl+C is handled before any screen sees the key, so it works everywhere.
-func TestCtrlCCancelsWithAnError(t *testing.T) {
-	m := wizardModel(t, 1, 1)
-	next, cmd := m.Update(keyMsg(t, "ctrl+c"))
-	final, ok := next.(model)
-	if !ok {
-		t.Fatalf("Update returned %T", next)
-	}
-	if final.err == nil || !strings.Contains(final.err.Error(), "cancelled") {
-		t.Fatalf("err = %v, want a cancellation", final.err)
-	}
-	if cmd == nil {
-		t.Error("ctrl+c must return tea.Quit")
-	}
-}
-
-// A window resize has to reach the model, since every view lays out against it.
-func TestWindowResizeIsStored(t *testing.T) {
-	m := model{}
-	next, _ := m.Update(tea.WindowSizeMsg{Width: 120, Height: 50})
-	resized, ok := next.(model)
-	if !ok {
-		t.Fatalf("Update returned %T", next)
-	}
-	if resized.width != 120 || resized.height != 50 {
-		t.Errorf("size = %dx%d, want 120x50", resized.width, resized.height)
-	}
-}
-
-// The erase prompt is the last point of no return, so anything other than an
-// explicit "yes" has to keep the operator on it.
-func TestEraseConfirmationRequiresYes(t *testing.T) {
-	m := wizardModel(t, 1, 1)
-	m = press(t, m, "enter", "enter") // → erase confirmation
-
-	m = typeText(t, m, "no")
-	m = press(t, m, "enter")
-	if m.screen != screenDiskConfirm {
-		t.Fatalf("screen = %v, want to stay on the confirmation", m.screen)
-	}
-	if !strings.Contains(m.validationErr, "yes") {
-		t.Errorf("validationErr = %q, want it to say what to type", m.validationErr)
-	}
-	if !strings.Contains(m.viewDiskConfirm(100), m.validationErr) {
-		t.Error("the confirmation screen must show the error it set")
-	}
-}
-
-// The filesystem selector and the disk list share one screen, so the cursor has
-// to move between them and stop at both ends.
-func TestDiskScreenCursorStaysInRange(t *testing.T) {
-	m := wizardModel(t, 2, 1)
-	m.screen = screenDisk
-
-	m = press(t, m, "up", "up")
-	if m.storageCursor != fsRow {
-		t.Errorf("storageCursor = %d, want it clamped to the filesystem row", m.storageCursor)
-	}
-
-	m = press(t, m, "down", "down", "down", "down")
-	if m.storageCursor != len(m.disks) {
-		t.Errorf("storageCursor = %d, want it clamped to the last disk", m.storageCursor)
-	}
-}
-
-// Left/right only cycle the filesystem while the cursor is on that row, and
-// they wrap, so one key reaches every topology.
-func TestFilesystemCyclesOnlyOnItsOwnRow(t *testing.T) {
-	m := wizardModel(t, 2, 1)
-	m.screen = screenDisk
-
-	m = press(t, m, "left")
-	if got := m.fsType(); got != install.AllFSTypes[len(install.AllFSTypes)-1] {
-		t.Errorf("fsType = %v, want the cycle to wrap backwards to the last", got)
-	}
-	m = press(t, m, "right")
-	if got := m.fsType(); got != install.AllFSTypes[0] {
-		t.Errorf("fsType = %v, want the cycle to wrap forwards to the first", got)
-	}
-
-	m = press(t, m, "down") // onto the first disk
-	before := m.fsType()
-	m = press(t, m, "left", "right")
-	if m.fsType() != before {
-		t.Errorf("fsType changed to %v while the cursor was on a disk", m.fsType())
-	}
-}
-
-// Space toggles the disk under the cursor, and r cycles its role — the two keys
-// the storage screen exists for.
-func TestDiskScreenSelectionKeys(t *testing.T) {
-	m := wizardModel(t, 2, 1)
-	m.screen = screenDisk
-
-	// The second disk is unselected; the first arrives preselected as os.
-	m = press(t, m, "down", "down", " ")
-	if !slices.Contains(m.selected, 1) {
-		t.Fatalf("selected = %v, want the second disk added", m.selected)
-	}
-
-	roleBefore := m.roleOf[slices.Index(m.selected, 1)]
-	m = press(t, m, "r")
-	if got := m.roleOf[slices.Index(m.selected, 1)]; got == roleBefore {
-		t.Errorf("role stayed %q after r", got)
-	}
-
-	m = press(t, m, " ")
-	if slices.Contains(m.selected, 1) {
-		t.Fatalf("selected = %v, want the second disk removed again", m.selected)
-	}
-	if len(m.roleOf) != len(m.selected) {
-		t.Errorf("roleOf (%d) and selected (%d) drifted", len(m.roleOf), len(m.selected))
-	}
-}
-
-// A selection that cannot be installed has to be reported on the screen the
-// operator can still fix it on, not part-way through the install.
-func TestDiskScreenRefusesAnInvalidSelection(t *testing.T) {
-	m := wizardModel(t, 2, 1)
-	m.screen = screenDisk
-	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1) // needs 2 disks
-
-	m = press(t, m, "enter")
-	if m.screen != screenDisk {
-		t.Fatalf("screen = %v, want to stay on the disk screen", m.screen)
-	}
-	if m.validationErr == "" {
-		t.Fatal("an invalid selection must set a validation error")
-	}
-	if !strings.Contains(m.viewDisk(100), m.validationErr) {
-		t.Error("the disk screen must render the error it set")
-	}
-
-	// Selecting the second disk satisfies the mirror and clears the way.
-	m = press(t, m, "down", "down", " ", "enter")
-	if m.screen != screenDiskConfirm {
-		t.Fatalf("screen = %v, want to advance once the mirror has two disks", m.screen)
-	}
-}
-
-// The ZFS options screen is only reachable on a pool, since its settings have
-// no meaning on ext4.
-func TestZFSOptionsScreenIsZFSOnly(t *testing.T) {
-	m := wizardModel(t, 2, 1)
-	m.screen = screenDisk
-
-	m = press(t, m, "a")
-	if m.screen != screenDisk {
-		t.Fatalf("screen = %v, want a to do nothing on ext4", m.screen)
-	}
-
-	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
-	m = press(t, m, "a")
-	if m.screen != screenZFSOptions {
-		t.Fatalf("screen = %v, want the ZFS options screen", m.screen)
-	}
-}
-
-// Every option defaults to auto and cycles through its own choice list, with
-// the cursor clamped to the rows that exist.
-func TestZFSOptionsCycleTheirValues(t *testing.T) {
-	m := wizardModel(t, 2, 1)
-	m.screen = screenZFSOptions
-
-	if got := zfsOptions[0].value(&m.zfs); got != "auto" {
-		t.Errorf("ashift starts at %q, want auto", got)
-	}
-	m = press(t, m, "right")
-	if got := zfsOptions[0].value(&m.zfs); got == "auto" {
-		t.Error("right must move ashift off auto")
-	}
-	m = press(t, m, "left")
-	if got := zfsOptions[0].value(&m.zfs); got != "auto" {
-		t.Errorf("ashift = %q after cycling back, want auto", got)
-	}
-
-	m = press(t, m, "up", "up")
-	if m.zfsCursor != 0 {
-		t.Errorf("zfsCursor = %d, want it clamped to the first row", m.zfsCursor)
-	}
-	for range len(zfsOptions) + 2 {
-		m = press(t, m, "down")
-	}
-	if m.zfsCursor != len(zfsOptions)-1 {
-		t.Errorf("zfsCursor = %d, want it clamped to the last row", m.zfsCursor)
-	}
-
-	// The compression choices are strings rather than ints, so they exercise
-	// the other half of cycle.
-	m.zfsCursor = 1
-	m = press(t, m, "right")
-	if m.zfs.Compress == "" {
-		t.Error("right must set a compression algorithm")
-	}
-
-	m = press(t, m, "enter")
-	if m.screen != screenDisk {
-		t.Fatalf("screen = %v, want enter to return to the disk screen", m.screen)
-	}
-}
-
-// The screen has to show what each option is currently set to, since that is
-// the only place the pool's geometry is stated before it is created.
-func TestZFSOptionsViewShowsValuesAndTheFocusedNote(t *testing.T) {
-	m := wizardModel(t, 2, 1)
-	m.screen = screenZFSOptions
-	m.zfs.Compress = "zstd"
-
-	got := m.viewZFSOptions(100)
-	for _, want := range []string{"ashift", "auto", "compression", "zstd", zfsOptions[0].note} {
-		if !strings.Contains(got, want) {
-			t.Errorf("ZFS options screen is missing %q:\n%s", want, got)
-		}
-	}
-	if strings.Contains(got, zfsOptions[1].note) {
-		t.Error("only the focused row's note should be shown")
-	}
-}
-
-// Credentials are three fields on one screen, so focus has to cycle both ways
-// and land somewhere sensible after a validation failure.
-func TestCredentialsFocusCyclesBothWays(t *testing.T) {
-	m := wizardModel(t, 1, 1)
-	m.screen = screenPassword
-	m = m.setCredsFocus(0)
-
-	m = press(t, m, "tab")
-	if m.credsFocus != 1 || !m.passwordInput.Focused() {
-		t.Fatalf("credsFocus = %d, want the password field focused", m.credsFocus)
-	}
-	m = press(t, m, "tab", "tab")
-	if m.credsFocus != 0 || !m.emailInput.Focused() {
-		t.Fatalf("credsFocus = %d, want the cycle to wrap to email", m.credsFocus)
-	}
-	m = press(t, m, "shift+tab")
-	if m.credsFocus != 2 || !m.passwordConfirmInput.Focused() {
-		t.Fatalf("credsFocus = %d, want shift+tab to wrap backwards to confirm", m.credsFocus)
-	}
-}
-
-// Each credential rule has to stop the wizard on the screen and put the cursor
-// on the field at fault, not just refuse to advance.
-func TestCredentialValidation(t *testing.T) {
-	tests := []struct {
-		name       string
-		email      string
-		password   string
-		confirm    string
-		wantErr    string
-		wantFocus  int
-		wantScreen screen
-	}{
-		{
-			name: "rejects a malformed email", email: "not-an-email",
-			password: "hunter2hunter2", confirm: "hunter2hunter2",
-			wantErr: "email", wantFocus: 0, wantScreen: screenPassword,
-		},
-		{
-			name: "rejects an empty password", email: "admin@example.com",
-			wantErr: "Password is required", wantFocus: 1, wantScreen: screenPassword,
-		},
-		{
-			name: "rejects a mismatch", email: "admin@example.com",
-			password: "hunter2hunter2", confirm: "hunter2hunter3",
-			wantErr: "do not match", wantFocus: 2, wantScreen: screenPassword,
-		},
-		{
-			name: "accepts a matching pair", email: "admin@example.com",
-			password: "hunter2hunter2", confirm: "hunter2hunter2",
-			wantScreen: screenConfirm,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			m := wizardModel(t, 1, 1)
-			m.screen = screenPassword
-			m.emailInput.SetValue(tt.email)
-			m.passwordInput.SetValue(tt.password)
-			m.passwordConfirmInput.SetValue(tt.confirm)
-			m = m.setCredsFocus(2) // enter on confirm runs every rule
-
-			m = press(t, m, "enter")
-
-			if m.screen != tt.wantScreen {
-				t.Fatalf("screen = %v, want %v (err %q)", m.screen, tt.wantScreen, m.validationErr)
-			}
-			if tt.wantErr == "" {
-				if m.validationErr != "" {
-					t.Fatalf("validationErr = %q, want none", m.validationErr)
-				}
-				return
-			}
-			if !strings.Contains(strings.ToLower(m.validationErr), strings.ToLower(tt.wantErr)) {
-				t.Errorf("validationErr = %q, want it to mention %q", m.validationErr, tt.wantErr)
-			}
-			if m.credsFocus != tt.wantFocus {
-				t.Errorf("credsFocus = %d, want the cursor on field %d", m.credsFocus, tt.wantFocus)
-			}
-			if !strings.Contains(m.viewPassword(100), m.validationErr) {
-				t.Error("the credentials screen must render the error it set")
-			}
-		})
-	}
-}
-
-// The hostname ends up in the config and in DNS, so an empty one has to be
-// caught on the screen rather than installed.
-func TestHostnameIsRequired(t *testing.T) {
-	m := wizardModel(t, 1, 1)
-	m.screen = screenIdentity
-	m.hostnameInput.Focus()
-
-	m = press(t, m, "enter")
-	if m.screen != screenIdentity {
-		t.Fatalf("screen = %v, want to stay on identity", m.screen)
-	}
-	if !strings.Contains(m.validationErr, "Hostname") {
-		t.Errorf("validationErr = %q, want it to name the field", m.validationErr)
-	}
-	if !m.hostnameInput.Focused() {
-		t.Error("the hostname field must keep focus after a failed submit")
-	}
-	if !strings.Contains(m.viewIdentity(100), m.validationErr) {
-		t.Error("the identity screen must render the error it set")
-	}
-}
-
-// Tab order follows the visible fields, which change with the addressing mode,
-// so it has to be derived from them rather than fixed.
-func TestRoleEditorTabsThroughVisibleFieldsOnly(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m.screen = screenNetworkRole
-	m.roleCursor = 0 // wan, DHCP by default
-
-	m = press(t, m, "tab")
-	if m.roles[0].focus != roleFieldMethod {
-		t.Fatalf("focus = %v, want the IP method field", m.roles[0].focus)
-	}
-	m = press(t, m, "tab")
-	if m.roles[0].focus != roleFieldNIC {
-		t.Fatalf("focus = %v, want the DHCP form to wrap after two fields", m.roles[0].focus)
-	}
-
-	// Switching to static reveals the address fields, and tab must reach them.
-	m = press(t, m, "tab", "right")
-	if m.roles[0].dhcp {
-		t.Fatal("right on the method field must select static addressing")
-	}
-	m = press(t, m, "tab")
-	if m.roles[0].focus != roleFieldIP || !m.roles[0].ip.Focused() {
-		t.Fatalf("focus = %v, want the IP field focused", m.roles[0].focus)
-	}
-	m = press(t, m, "shift+tab")
-	if m.roles[0].focus != roleFieldMethod {
-		t.Fatalf("focus = %v, want shift+tab to go back", m.roles[0].focus)
-	}
-}
-
-// A focused text field has to receive typed runes, since that is the only way
-// an address is entered.
-func TestRoleEditorTypesIntoTheFocusedField(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m.screen = screenNetworkRole
-	m.roles[0].dhcp = false
-	m.roles[0].focus = roleFieldIP
-	m.roles[0].focusCurrent()
-
-	m = typeText(t, m, "192.168.1.10")
-	if got := m.roles[0].ip.Value(); got != "192.168.1.10" {
-		t.Fatalf("ip = %q, want what was typed", got)
-	}
-	if got := m.roles[0].mask.Value(); got != "" {
-		t.Errorf("mask = %q, want the keys to reach only the focused field", got)
-	}
-}
-
-// The interface selector wraps and offers the fold option on lan and vpc only,
-// so a single-NIC node stays configurable with one key.
-func TestNICSelectorWrapsAndFoldsWhereAllowed(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m.screen = screenNetworkRole
-
-	// wan cannot fold, so it wraps across the real NICs only.
-	m.roleCursor = 0
-	m = press(t, m, "left")
-	if m.roles[0].nic != len(m.nics)-1 {
-		t.Fatalf("wan nic = %d, want it to wrap to the last NIC", m.roles[0].nic)
-	}
-	m = press(t, m, "right")
-	if m.roles[0].nic != 0 {
-		t.Fatalf("wan nic = %d, want it to wrap back to the first", m.roles[0].nic)
-	}
-
-	// lan may fold, so the cycle includes the unbound position.
-	m.roleCursor = 1
-	m.roles[1].nic = 0
-	m = press(t, m, "left")
-	if m.roles[1].nic != foldedNIC {
-		t.Fatalf("lan nic = %d, want the fold position", m.roles[1].nic)
-	}
-	if !m.roles[1].folded() {
-		t.Error("the lan role should read as folded")
-	}
-}
-
-// Per-role validation runs on Enter and keeps the operator on the editor, since
-// an invalid address here fails the install after the disk is partitioned.
-func TestRoleEditorValidatesBeforeLeaving(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m.screen = screenNetworkRole
-	m.roleCursor = 0
-	m.roles[0].dhcp = false
-	m.roles[0].ip.SetValue("not-an-ip")
-
-	m = press(t, m, "enter")
-	if m.screen != screenNetworkRole {
-		t.Fatalf("screen = %v, want to stay on the role editor", m.screen)
-	}
-	if !strings.Contains(m.validationErr, "IP address") {
-		t.Errorf("validationErr = %q, want it to name the field", m.validationErr)
-	}
-	if !strings.Contains(m.viewNetworkRole(100), m.validationErr) {
-		t.Error("the role editor must render the error it set")
-	}
-
-	// A prefix length is rejected here because netgen parses the mask with
-	// net.ParseIP and would fail the install itself.
-	m.roles[0].ip.SetValue("192.168.1.10")
-	m.roles[0].mask.SetValue("24")
-	m = press(t, m, "enter")
-	if !strings.Contains(m.validationErr, "dotted-decimal") {
-		t.Errorf("validationErr = %q, want the mask rule", m.validationErr)
-	}
-
-	m.roles[0].mask.SetValue("255.255.255.0")
-	m.roles[0].gateway.SetValue("192.168.1.1")
-	m = press(t, m, "enter")
-	if m.screen != screenNetworkRoles {
-		t.Fatalf("screen = %v, want a valid role to return to the overview (err %q)", m.screen, m.validationErr)
-	}
-}
-
-// Esc abandons the editor without validating, so a half-typed address is not a
-// trap the operator has to finish before backing out.
-func TestRoleEditorEscapeSkipsValidation(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m.screen = screenNetworkRole
-	m.roles[0].dhcp = false
-	m.roles[0].ip.SetValue("192.")
-
-	m = press(t, m, "esc")
-	if m.screen != screenNetworkRoles {
-		t.Fatalf("screen = %v, want the overview", m.screen)
-	}
-	if m.validationErr != "" {
-		t.Errorf("validationErr = %q, want esc to clear it", m.validationErr)
-	}
-}
-
-// The editor shows only the fields that apply, so the DHCP form stays two lines
-// and Advanced is what reveals VLAN and MTU.
-func TestRoleEditorViewShowsTheApplicableFields(t *testing.T) {
-	m := wizardModel(t, 1, 3)
-	m.screen = screenNetworkRole
-	m.roleCursor = 0
-
-	dhcpView := m.viewNetworkRole(100)
-	if !strings.Contains(dhcpView, "DHCP") || !strings.Contains(dhcpView, "WAN") {
-		t.Errorf("DHCP view is missing the plane or the method:\n%s", dhcpView)
-	}
-	for _, hidden := range []string{"IP address", "Subnet mask", "VLAN id"} {
-		if strings.Contains(dhcpView, hidden) {
-			t.Errorf("DHCP view should not show %q:\n%s", hidden, dhcpView)
-		}
-	}
-
-	m.roles[0].dhcp = false
-	m.advanced = true
-	staticView := m.viewNetworkRole(100)
-	for _, want := range []string{"IP address", "Subnet mask", "Gateway", "DNS", "VLAN id", "MTU"} {
-		if !strings.Contains(staticView, want) {
-			t.Errorf("static advanced view is missing %q:\n%s", want, staticView)
-		}
-	}
-
-	// Gateway and DNS belong to the plane holding the default route, so a
-	// static lan must not offer them.
-	m.roleCursor = 1
-	m.roles[1].nic = 1
-	m.roles[1].dhcp = false
-	lanView := m.viewNetworkRole(100)
-	for _, hidden := range []string{"Gateway", "DNS"} {
-		if strings.Contains(lanView, hidden) {
-			t.Errorf("static lan view should not show %q:\n%s", hidden, lanView)
-		}
-	}
-}
-
-// A folded role names the plane it landed on, on both screens, so the operator
-// can see where the traffic actually goes before committing.
-func TestFoldedRolesAreNamedNotBlank(t *testing.T) {
-	m := wizardModel(t, 1, 1) // one NIC folds lan and vpc onto wan
-
-	overview := m.viewNetworkRoles(100)
-	if !strings.Contains(overview, "folds onto") {
-		t.Errorf("overview does not state where the folded roles land:\n%s", overview)
-	}
-
-	m.screen = screenNetworkRole
-	m.roleCursor = 2
-	if got := m.viewNetworkRole(100); !strings.Contains(got, "fold onto") {
-		t.Errorf("role editor does not offer the fold plainly:\n%s", got)
-	}
-
-	m.screen = screenConfirm
-	if got := m.viewConfirm(100); !strings.Contains(got, "folded onto") {
-		t.Errorf("confirmation does not state the folded planes:\n%s", got)
-	}
-}
-
-// Continue is refused while the config would fail, and the message is the one
-// install.Config.Validate produced.
-func TestRolesOverviewRefusesAnInvalidConfig(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m.screen = screenNetworkRoles
-	m.roles[1].nic = 0 // lan on the same NIC as wan, no VLAN to separate them
-	m.roleCursor = continueRow
-
-	m = press(t, m, "enter")
-	if m.screen != screenNetworkRoles {
-		t.Fatalf("screen = %v, want to stay on the overview", m.screen)
-	}
-	if !strings.Contains(m.validationErr, "VLAN") {
-		t.Errorf("validationErr = %q, want the shared-NIC rule", m.validationErr)
-	}
-	// The message is wrapped to the box, so match its opening rather than
-	// the whole line.
-	if !strings.Contains(m.viewNetworkRoles(100), "both bind eno1") {
-		t.Error("the overview must render the error it set")
-	}
-
-	// a toggles Advanced for the whole node rather than per role.
-	m = press(t, m, "a")
-	if !m.advanced {
-		t.Error("a must turn Advanced on")
-	}
-	if !strings.Contains(m.viewNetworkRoles(100), "Advanced: on") {
-		t.Error("the overview must show Advanced is on")
-	}
-}
-
-// The final screen is the last chance to catch a wrong disk or address, so the
-// summary has to carry what was entered.
-func TestConfirmScreenSummarisesTheEnteredConfig(t *testing.T) {
-	m := wizardModel(t, 1, 3)
-	m.screen = screenConfirm
-	m.hostnameInput.SetValue("node7")
-	m.roles[1].dhcp = false
-	m.roles[1].ip.SetValue("10.0.0.3")
-	m.roles[1].mask.SetValue("255.255.255.0")
-
-	got := m.viewConfirm(100)
-	for _, want := range []string{"node7", "/dev/sda", "ext4", "eno1", "10.0.0.3/255.255.255.0", "br-lan", "will erase"} {
-		if !strings.Contains(got, want) {
-			t.Errorf("confirmation is missing %q:\n%s", want, got)
-		}
-	}
-}
-
-// n on the confirmation cancels rather than installing.
-func TestConfirmScreenCancelsOnN(t *testing.T) {
-	m := wizardModel(t, 1, 1)
-	m.screen = screenConfirm
-
-	m = press(t, m, "n")
-	if m.err == nil {
-		t.Fatal("n must set the cancellation error")
-	}
-	if m.result != nil {
-		t.Error("a cancelled install must produce no config")
-	}
-}
-
-// View dispatches on the screen, so each one has to render its own content
-// rather than falling through to an empty frame.
-func TestViewRendersEveryScreen(t *testing.T) {
-	tests := []struct {
-		screen screen
-		want   string
-	}{
-		{screenWelcome, "Enter to begin"},
-		{screenDisk, "Select Installation Disk"},
-		{screenZFSOptions, "ZFS Options"},
-		{screenDiskConfirm, "Confirm Disk Erasure"},
-		{screenNetworkRoles, "Network Planes"},
-		{screenNetworkRole, "Configure WAN plane"},
-		{screenIdentity, "Node Identity"},
-		{screenPassword, "Administrator Credentials"},
-		{screenConfirm, "Confirm Installation"},
-		{screenDone, "Installation complete"},
-	}
-
-	for _, tt := range tests {
-		m := wizardModel(t, 1, 2)
-		m.screen = tt.screen
-		if got := m.View(); !strings.Contains(got, tt.want) {
-			t.Errorf("screen %v does not render %q:\n%s", tt.screen, tt.want, got)
-		}
-	}
-}
-
-// The disk screen has to state what the selection actually builds, since the
-// cost of the redundancy choice is otherwise invisible until after the install.
-func TestDiskViewStatesTheResultingGeometry(t *testing.T) {
-	m := wizardModel(t, 3, 1)
-	m.screen = screenDisk
-
-	// ext4 with a role per drive: the mountpoints are the summary that matters.
-	m = press(t, m, "down", "down", " ", "down", " ")
-	if got := m.viewDisk(100); !strings.Contains(got, "/var/lib/spinifex") {
-		t.Errorf("ext4 view does not state the role layout:\n%s", got)
-	}
-	// Matched spares are suggested, never applied.
-	if got := m.viewDisk(100); !strings.Contains(got, "survive a disk failure") {
-		t.Errorf("view does not suggest the pool the matched disks allow:\n%s", got)
-	}
-
-	// A mirror with too few disks quotes the requirement, not a capacity.
-	m = wizardModel(t, 3, 1)
-	m.screen = screenDisk
-	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
-	if got := m.viewDisk(100); !strings.Contains(got, "1 selected") {
-		t.Errorf("view does not state what the mirror is missing:\n%s", got)
-	}
-
-	// Once buildable it quotes usable capacity and the failures tolerated.
-	m = press(t, m, "down", "down", " ")
-	got := m.viewDisk(100)
-	if !strings.Contains(got, "usable") || !strings.Contains(got, "survives 1 disk failure") {
-		t.Errorf("view does not quote the pool geometry:\n%s", got)
-	}
-	if !strings.Contains(got, "A for ZFS options") {
-		t.Errorf("the ZFS help line should replace the role key:\n%s", got)
-	}
-}
-
-// The overview names the card behind each port, which is what distinguishes the
-// 25gbe uplink from the onboard 1gbe.
-func TestRolesOverviewNamesTheHardware(t *testing.T) {
-	m := wizardModel(t, 1, 2)
-	m.nics[0].AltName = "enp1s0f0"
-	m.advanced = true
-
-	got := m.viewNetworkRoles(100)
-	for _, want := range []string{"eno1", "25Gbps", "Mellanox", "enp1s0f0", "VLAN", "MTU"} {
-		if !strings.Contains(got, want) {
-			t.Errorf("overview is missing %q:\n%s", want, got)
-		}
-	}
-}
-
-func TestHumanBytes(t *testing.T) {
-	tests := []struct {
-		bytes int64
-		want  string
-	}{
-		{512, "512B"},
-		{2 << 20, "2.0M"},
-		{3 << 30, "3.0G"},
-		{2 << 40, "2.0T"},
-	}
-	for _, tt := range tests {
-		if got := humanBytes(tt.bytes); got != tt.want {
-			t.Errorf("humanBytes(%d) = %q, want %q", tt.bytes, got, tt.want)
-		}
-	}
-}
-
-// The overview writes an address the way an operator does, and falls back to
-// the raw mask rather than dropping it when it is not dotted-decimal.
-func TestFormatCIDR(t *testing.T) {
-	tests := []struct {
-		name       string
-		addr, mask string
-		want       string
-	}{
-		{"dotted mask becomes a prefix", "10.0.0.3", "255.255.255.0", "10.0.0.3/24"},
-		{"short mask keeps its prefix", "10.0.0.3", "255.255.0.0", "10.0.0.3/16"},
-		{"no mask yet", "10.0.0.3", "", "10.0.0.3"},
-		{"no address is blank", "", "255.255.255.0", ""},
-		{"unparseable mask is shown as typed", "10.0.0.3", "24", "10.0.0.3/24"},
-		{"whitespace is trimmed", " 10.0.0.3 ", " 255.255.255.0 ", "10.0.0.3/24"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := formatCIDR(tt.addr, tt.mask); got != tt.want {
-				t.Errorf("formatCIDR(%q, %q) = %q, want %q", tt.addr, tt.mask, got, tt.want)
-			}
-		})
-	}
-}
-
-// Messages that are not keys reach the focused input, which is what keeps the
-// cursor blinking on the two single-field screens.
-func TestNonKeyMessagesReachTheActiveInput(t *testing.T) {
-	m := wizardModel(t, 1, 1)
-	m.screen = screenIdentity
-	m.hostnameInput.Focus()
-
-	next, _ := m.Update(struct{ tea.Msg }{})
-	if _, ok := next.(model); !ok {
-		t.Fatalf("Update returned %T", next)
-	}
-
-	m.screen = screenWelcome
-	if _, cmd := m.Update(struct{ tea.Msg }{}); cmd != nil {
-		t.Error("a screen with no input must issue no command")
-	}
-}

cmd/lb-agent/main.go

@@ -18,17 +18,17 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
-	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {
 	// Without this the agent falls back to slog's default text handler, which folds
 	// attributes into the message so the console scraper recovers a level and nothing
 	// else. No OTLP export: the agent's only egress is the gateway it is watching.
-	otelsetup.SetDefaultJSONLogger("lb-agent", slog.LevelInfo)
+	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
 
 	var (
 		lbID       string

cmd/rds-agent/agent.go

@@ -15,7 +15,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
-	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 )
 
 // Overridable via -ldflags "-X main.version=...".
@@ -237,9 +236,10 @@ func New(cfg config) (*Agent, error) {
 }
 
 func (a *Agent) Run(ctx context.Context) error {
-	// Named before the first dial: when register cannot reach the gateway, the
-	// address it is dialing is what separates a mgmt NIC that came up without one
-	// from a control plane that is genuinely down.
+	// Named before the first dial. When register cannot reach the gateway the
+	// retry loop is all that survives, and the address it is dialing is what
+	// separates a mgmt NIC that came up without an address from a control plane
+	// that is genuinely down.
 	slog.Info("rds-agent: starting",
 		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
 		"agentVersion", a.id.AgentVersion)
@@ -407,10 +407,10 @@ func retryObserved(ctx context.Context, what string, fn func(context.Context) er
 		}
 		if attempt >= retryErrorAttempt {
 			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
-				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
+				"attempt", attempt, "retryIn", delay, "err", err)
 		} else {
 			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
-				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
+				"attempt", attempt, "retryIn", delay, "err", err)
 		}
 
 		select {

cmd/rds-agent/agent_test.go

@@ -192,7 +192,7 @@ func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *A
 
 // staticProbe is a probeRunner that always reports the same exit code.
 func staticProbe(code int) probeRunner {
-	return func(context.Context, string, ...string) (int, string, error) { return code, "", nil }
+	return func(context.Context, string, ...string) (int, error) { return code, nil }
 }
 
 func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
@@ -207,8 +207,6 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 		MasterUsername:       "master",
 		MasterUserPassword:   password,
 		Port:                 5432,
-		ListenAddress:        testListenAddress,
-		ClientCIDR:           testClientCIDR,
 		DataVolumeID:         "vol-data-01",
 		DataVolumeSerial:     "voldata01",
 		VMGeneration:         1,
@@ -219,13 +217,6 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 
 const testPayloadID = "bp-0123456789abcdef"
 
-// The customer ENI address the engine binds, and the VPC range its client
-// authentication admits. writeHandoff refuses a config without either.
-const (
-	testListenAddress = "10.0.1.20"
-	testClientCIDR    = "10.0.0.0/16"
-)
-
 // Writes the completion receipt rds-init leaves behind, so the acknowledgement
 // step finds what a finished initialization would have produced.
 func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {

cmd/rds-agent/bootstrap.go

@@ -32,9 +32,10 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 		if fetched == nil {
 			return fmt.Errorf("bootstrap fetch returned no config")
 		}
-		// A control plane predating the encrypted payload answers initialize with
-		// nothing to initialise with. Retrying re-dials the queue group until an
-		// upgraded node answers, so a mixed-version fleet heals within one boot.
+		// A control plane that predates the encrypted payload answers initialize
+		// with nothing to initialise with. Retrying re-dials the worker queue
+		// group until an upgraded node answers, so a mixed-version fleet heals
+		// within one boot rather than needing a reboot.
 		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
 			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
 			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
@@ -60,6 +61,7 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 	a.hb.clearBootstrapFailure()
 	if cfg.Port > 0 {
 		a.probe.setPort(int(cfg.Port))
+		a.engine.setPort(int(cfg.Port))
 	}
 	if cfg.BootstrapPending && cfg.PayloadID != "" {
 		a.pending = &pendingBootstrap{
@@ -96,15 +98,6 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 	if cfg.MasterUsername == "" {
 		return fmt.Errorf("bootstrap config carries no master username")
 	}
-	// Half a handoff would leave rds-init deciding what to bind and who to admit,
-	// and the only answers it could reach on its own are the wildcards this
-	// contract exists to remove.
-	if cfg.ListenAddress == "" {
-		return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
-	}
-	if cfg.ClientCIDR == "" {
-		return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
-	}
 	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
 		return fmt.Errorf("create handoff dir %s: %w", dir, err)
 	}
@@ -143,8 +136,6 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
 	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
 	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
-	writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
-	writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
 	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
@@ -191,8 +182,12 @@ func shellQuote(s string) string {
 }
 
 // postgresql.conf syntax, which MariaDB's option files also parse. Values are
-// quoted so a space or unit suffix survives. Names are the engine's startup
-// spellings, not the customer's: the wrong one leaves the engine unable to boot.
+// quoted so a setting with a space or unit suffix survives; the engine accepts
+// quoted numerics and booleans too.
+//
+// Names are the engine's rather than the customer's: a setting the server takes
+// only through SET has a different spelling at startup, and writing the wrong one
+// leaves the engine refusing to boot on a file already on the data volume.
 func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
 	engine, err := handlers_rds.LookupEngine(engineName)
 	if err != nil {

cmd/rds-agent/bootstrap_test.go

@@ -18,8 +18,6 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
-		ListenAddress:      "10.0.1.20",
-		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
 		Port:               5433,
@@ -87,8 +85,6 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
-		ListenAddress:  "10.0.1.20",
-		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
@@ -120,8 +116,6 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
-		ListenAddress:      "10.0.1.20",
-		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		Port:               5432,
 	}); err != nil {
@@ -143,8 +137,6 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeAttach,
 		Engine:             "postgres",
 		MasterUsername:     "master",
-		ListenAddress:      "10.0.1.20",
-		ClientCIDR:         "10.0.0.0/16",
 		Port:               5432,
 		ServingCertificate: "CERT",
 	}); err != nil {
@@ -185,8 +177,6 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
-		ListenAddress:  "10.0.1.20",
-		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)

cmd/rds-agent/command.go

@@ -50,13 +50,7 @@ func newCommandRegistry(engine engineOps, storage storageOps) commandRegistry {
 			if err != nil {
 				return "", err
 			}
-			if err := engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold); err != nil {
-				return "", err
-			}
-			// After the hold, so what is flushed is the engine's checkpointed
-			// state. Its own files are not the risk — those it repairs from the
-			// WAL — but the parameter file beside them it never opens.
-			return "", storage.SyncDataMount(ctx)
+			return "", engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold)
 		},
 		handlers_rds.CommandUnquiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
 			return "", engine.Unquiesce(ctx)

cmd/rds-agent/config.go

@@ -13,6 +13,7 @@ const (
 	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
+	defaultEngineHost = "127.0.0.1"
 	// Where setup.sh stamps the engine the image bakes. The agent builds its
 	// engine implementation from this file rather than from anything delivered.
 	defaultEngineFile         = "/etc/spinifex-rds/engine"
@@ -37,6 +38,7 @@ type config struct {
 	// version alone rather than clearing it.
 	EngineVersion string
 	HandoffDir    string
+	EngineHost    string
 	EnginePort    int
 	PollWait      time.Duration
 
@@ -61,10 +63,6 @@ type config struct {
 	// is not running at all.
 	EnginePidFile      string
 	EngineProbeTimeout time.Duration
-	// Where the engine records why it would not start, which the probe quotes
-	// when the server is up but not serving. Overridable so a test can point it
-	// at a fixture.
-	EngineErrorLog string
 
 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -87,6 +85,7 @@ func loadConfig(envFile string) config {
 		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
+		EngineHost:           get("RDS_ENGINE_HOST"),
 		EngineBinDir:         get("RDS_ENGINE_BIN"),
 		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
@@ -94,7 +93,6 @@ func loadConfig(envFile string) config {
 		RCService:            get("RDS_RC_SERVICE"),
 		EngineService:        get("RDS_ENGINE_SERVICE"),
 		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
-		EngineErrorLog:       get("RDS_ENGINE_LOG"),
 		DataMount:            get("RDS_DATA_MOUNT"),
 		PollWait:             defaultPollWait,
 		EngineProbeTimeout:   defaultEngineProbeTimeout,
@@ -105,6 +103,9 @@ func loadConfig(envFile string) config {
 	if cfg.HandoffDir == "" {
 		cfg.HandoffDir = defaultHandoffDir
 	}
+	if cfg.EngineHost == "" {
+		cfg.EngineHost = defaultEngineHost
+	}
 	if cfg.EngineFile == "" {
 		cfg.EngineFile = defaultEngineFile
 	}
@@ -163,9 +164,6 @@ func (c *config) applyLayout(layout engineLayout) {
 	if c.EnginePidFile == "" {
 		c.EnginePidFile = layout.pidFile
 	}
-	if c.EngineErrorLog == "" {
-		c.EngineErrorLog = layout.errorLog
-	}
 	if c.DataMount == "" {
 		c.DataMount = layout.dataMount
 	}

cmd/rds-agent/config_test.go

@@ -26,8 +26,8 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.PollWait != defaultPollWait {
-		t.Errorf("pollWait = %v, want the built-in default", cfg.PollWait)
+	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
+		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
 	}
 	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
 		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)

cmd/rds-agent/defaultparams_test.go

@@ -1,9 +1,5 @@
 package main
 
-//test:in-package — the agent is a main package, which has no external test
-// package to import it from, and this covers the unexported default parameter
-// set the image bakes.
-
 import (
 	"os"
 	"path/filepath"

cmd/rds-agent/engine.go

@@ -4,11 +4,9 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"os"
 	"os/exec"
 	"os/user"
 	"path/filepath"
-	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -42,9 +40,7 @@ type engineOps interface {
 // a control-plane directive: nothing issues it, the agent runs it when the
 // engine fails to come up after a parameter change.
 type parameterRecovery interface {
-	// Reports whether the installed set differed and was replaced. True with a
-	// non-nil error is a set already on disk whose re-derived TLS enforcement did
-	// not land: the caller still has to restart and report.
+	// Reports whether the installed set differed and was replaced.
 	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
 	Restart(ctx context.Context) error
 }
@@ -63,6 +59,9 @@ type engine interface {
 	engineOps
 	parameterRecovery
 	servingParameterRecorder
+	// Follows the port the control plane assigned, once the bootstrap fetch
+	// lands. Called from a different goroutine than the commands that read it.
+	setPort(port int)
 }
 
 // The guest layout an engine preset lays down, keyed by the engine the image
@@ -79,10 +78,6 @@ type engineLayout struct {
 	// Where the engine records the pid the probe checks for liveness. Empty for
 	// an engine whose probe answers without one.
 	pidFile string
-	// Where the engine records why it would not start. Empty for an engine whose
-	// image does not direct one, which leaves the probe reporting its own reason
-	// alone rather than guessing at a path.
-	errorLog string
 	// The engine's own liveness signal, which is the only part of engineProbe
 	// that is not shared.
 	newProbe func(cfg config, run probeRunner) *engineProbe
@@ -108,12 +103,10 @@ var engineLayouts = map[string]engineLayout{
 		service:   "postgresql",
 		dataMount: "/var/lib/postgresql",
 		port:      5432,
-		// logfile in the image's /etc/conf.d/postgresql. The postmaster states
-		// which setting it refused there and nowhere else — the console shows only
-		// that pg_ctl could not start it, on a guest with no SSH.
-		errorLog:  "/var/log/postgresql/postmaster.log",
 		newProbe:  newPostgresProbe,
-		newEngine: newPostgresEngineFromCatalog,
+		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+			return newPostgresEngine(cfg, run, startSess, probe), nil
+		},
 	},
 	engineMariaDB: {
 		binDir:    "/usr/bin",
@@ -123,74 +116,17 @@ var engineLayouts = map[string]engineLayout{
 		service:   "mariadb",
 		dataMount: "/var/lib/mysql",
 		port:      3306,
-		// Alpine's service passes --pid-file=/run/mysqld/$RC_SVCNAME.pid, and the
-		// command line beats the option file, so this is the path rather than
-		// anything the drop-in names. setup.sh asserts it at build time.
-		pidFile: "/run/mysqld/mariadb.pid",
-		// log_error in the platform drop-in rds-init writes. Without it
-		// mysqld_safe would send the server's errors to syslog, which nothing
-		// collects off a guest that has no SSH.
-		errorLog:  "/var/log/mysql/error.log",
+		// Alpine's packaged service starts mysqld_safe with
+		// --pid-file=/run/mysqld/$RC_SVCNAME.pid, and a command-line option beats
+		// the configuration file, so this is the path rather than anything the
+		// platform drop-in could name. setup.sh asserts it against the packaged
+		// init script at build time.
+		pidFile:   "/run/mysqld/mariadb.pid",
 		newProbe:  newMariaDBProbe,
 		newEngine: newMariaDBEngineFromCatalog,
 	},
 }
 
-// How much of the engine's error log a probe reason carries. The server names
-// what it refused on in its last few lines, and the reason reaches a customer
-// through StatusInfos, so this quotes the tail rather than the whole file.
-const (
-	engineErrorLogTailBytes = 4096
-	engineErrorLogTailLines = 4
-)
-
-// Appends the last few lines of the engine's error log to reason. An unset path,
-// an unreadable file or an empty one leaves reason alone: the probe's own
-// statement is still true, and a guessed cause would be worse than none.
-func withEngineLogTail(reason, path string) string {
-	tail := engineLogTail(path)
-	if tail == "" {
-		return reason
-	}
-	return reason + "; the engine's log ends: " + tail
-}
-
-func engineLogTail(path string) string {
-	if path == "" {
-		return ""
-	}
-	f, err := os.Open(path)
-	if err != nil {
-		return ""
-	}
-	defer func() { _ = f.Close() }()
-
-	info, err := f.Stat()
-	if err != nil {
-		return ""
-	}
-	offset := max(info.Size()-engineErrorLogTailBytes, 0)
-	buf := make([]byte, info.Size()-offset)
-	if _, err := f.ReadAt(buf, offset); err != nil {
-		return ""
-	}
-
-	lines := strings.Split(string(buf), "\n")
-	// A read that started mid-file almost certainly started mid-line, and half a
-	// line is more misleading than one fewer.
-	if offset > 0 && len(lines) > 0 {
-		lines = lines[1:]
-	}
-	kept := make([]string, 0, engineErrorLogTailLines)
-	for i := len(lines) - 1; i >= 0 && len(kept) < engineErrorLogTailLines; i-- {
-		if line := strings.TrimSpace(lines[i]); line != "" {
-			kept = append(kept, line)
-		}
-	}
-	slices.Reverse(kept)
-	return strings.Join(kept, " | ")
-}
-
 // Builds the health probe the image's engine supplies.
 func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
 	layout, err := layoutFor(cfg)
@@ -297,34 +233,6 @@ func serviceAction(ctx context.Context, run commandRunner, rcService, service, a
 	return nil
 }
 
-// Whether the installed set requires TLS of client connections. Read back from
-// the file rather than taken from the set being applied, because the restore and
-// repair paths put a file in place without holding one. Both engines' generated
-// parameter files are written in the same syntax, so one reader serves them.
-func installedTLSEnforcement(name, installedPath string) (bool, error) {
-	values, err := readOptionFile(installedPath)
-	if err != nil && !os.IsNotExist(err) {
-		return false, fmt.Errorf("read the installed parameters: %w", err)
-	}
-	value, ok := values[name]
-	if !ok {
-		// The ordinary state of an instance whose resolved set predates the
-		// parameter, and the whole of the migration story: it begins enforcing at
-		// its next boot with no control-plane work at all.
-		return true, nil
-	}
-	switch value {
-	case "1":
-		return true, nil
-	case "0":
-		return false, nil
-	}
-	// The resolver canonicalises every boolean, so anything else means the file
-	// was written by something other than the platform. The permissive reading is
-	// the one that must not be chosen for a setting that turns TLS off.
-	return false, fmt.Errorf("the installed parameters set %s to %q, which is neither 1 nor 0", name, value)
-}
-
 // A minimal environment for a child that is not carrying secrets.
 const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 

cmd/rds-agent/engine_contract_test.go

@@ -1,9 +1,5 @@
 package main
 
-//test:in-package — the agent is a main package, which has no external test
-// package to import it from, and these cases are built on the unexported
-// engine interface and each implementation's own constructor.
-
 import (
 	"context"
 	"errors"

cmd/rds-agent/engine_log_test.go

@@ -1,262 +0,0 @@
-package main
-
-//test:in-package — the agent is a main package, which has no external test
-// package to import it from, and this covers the unexported engineLogTail and
-// probe-reason helpers directly.
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-	"testing"
-)
-
-const mariadbRDSInitScript = "../../scripts/images/rds-mariadb/rds-init"
-
-func writeEngineLog(t *testing.T, body string) string {
-	t.Helper()
-	path := filepath.Join(t.TempDir(), "error.log")
-	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
-		t.Fatalf("write engine log: %v", err)
-	}
-	return path
-}
-
-// The whole point of the log: "not answering on its socket" is the same message
-// for a server still recovering and for one that refused to start, and only the
-// engine's own log separates them.
-func TestMariaDBProbe_QuotesTheEngineLogWhenTheSocketIsSilent(t *testing.T) {
-	cfg := mariadbTestProbeConfig(t)
-	cfg.EngineErrorLog = writeEngineLog(t, strings.Join([]string{
-		"2026-08-12  1:49:26 0 [Note] Starting MariaDB 11.8.0",
-		"2026-08-12  1:49:26 0 [ERROR] InnoDB: Cannot allocate memory for the buffer pool",
-		"2026-08-12  1:49:26 0 [ERROR] Plugin 'InnoDB' registration as a STORAGE ENGINE failed.",
-		"",
-	}, "\n"))
-	writePid(t, cfg.EnginePidFile, os.Getpid())
-
-	state, message := mariadbProbeState(cfg,
-		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
-		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
-
-	if state != engineRecovering {
-		t.Errorf("state = %v, want recovering", state)
-	}
-	if !strings.Contains(message, "Cannot allocate memory for the buffer pool") {
-		t.Errorf("message = %q, want the engine's own refusal quoted", message)
-	}
-}
-
-func TestMariaDBProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
-	cfg := mariadbTestProbeConfig(t)
-	cfg.EngineErrorLog = filepath.Join(t.TempDir(), "does-not-exist.log")
-	writePid(t, cfg.EnginePidFile, os.Getpid())
-
-	state, message := mariadbProbeState(cfg,
-		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
-		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
-
-	if state != engineRecovering {
-		t.Errorf("state = %v, want recovering", state)
-	}
-	if message != "engine is not answering on its socket yet (startup or crash recovery)" {
-		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
-	}
-}
-
-// A client refused during the handshake is closed by the server without the
-// server ever logging a cause, so the engine log alone leaves the failure
-// unexplained and only the client's own stderr names it.
-func TestMariaDBProbe_QuotesTheProbeClientWhenItIsRefused(t *testing.T) {
-	cfg := mariadbTestProbeConfig(t)
-	cfg.EngineErrorLog = writeEngineLog(t, "2026-08-12  4:32:39 148 [Warning] Aborted connection 148\n")
-	writePid(t, cfg.EnginePidFile, os.Getpid())
-
-	state, message := mariadbProbeState(cfg,
-		func(context.Context, string, ...string) (int, string, error) {
-			return 1, "ERROR 2026 (HY000): TLS/SSL error: self-signed certificate\n", nil
-		},
-		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
-
-	if state != engineRecovering {
-		t.Errorf("state = %v, want recovering", state)
-	}
-	if !strings.Contains(message, "self-signed certificate") {
-		t.Errorf("message = %q, want the probe client's own error quoted", message)
-	}
-	if !strings.Contains(message, "Aborted connection") {
-		t.Errorf("message = %q, want the engine log kept alongside the client's error", message)
-	}
-}
-
-func TestCollapseProbeStderr(t *testing.T) {
-	tests := []struct {
-		name string
-		in   string
-		want string
-	}{
-		{name: "folds to one line", in: "ERROR 2026:\n  TLS error\n", want: "ERROR 2026: TLS error"},
-		{name: "empty stays empty", in: "   \n", want: ""},
-		{
-			name: "bounded",
-			in:   strings.Repeat("y", mariadbProbeStderrMaxBytes+50),
-			want: strings.Repeat("y", mariadbProbeStderrMaxBytes) + "...",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := collapseProbeStderr(tt.in); got != tt.want {
-				t.Errorf("collapseProbeStderr() = %q, want %q", got, tt.want)
-			}
-		})
-	}
-}
-
-// A probe that says nothing must not append an empty clause: the reason reaches
-// a customer, and a dangling "reported:" would read as truncated output.
-func TestWithProbeStderrLeavesReasonAloneWhenTheClientIsSilent(t *testing.T) {
-	const reason = "engine is not answering on its socket yet"
-	if got := withProbeStderr(reason, "  \n "); got != reason {
-		t.Errorf("withProbeStderr() = %q, want the reason unchanged", got)
-	}
-}
-
-func TestEngineLogTail(t *testing.T) {
-	tests := []struct {
-		name        string
-		body        string
-		want        string
-		wantMissing string
-	}{
-		{
-			name: "keeps only the last lines",
-			body: "one\ntwo\nthree\nfour\nfive\nsix\n",
-			want: "three | four | five | six",
-		},
-		{
-			name: "skips blank lines",
-			body: "alpha\n\n\nbeta\n\n",
-			want: "alpha | beta",
-		},
-		{
-			name: "empty file yields nothing",
-			body: "",
-			want: "",
-		},
-		{
-			name:        "drops a line the byte cap cut in half",
-			body:        strings.Repeat("x", engineErrorLogTailBytes) + "\nlast\n",
-			want:        "last",
-			wantMissing: "x",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := engineLogTail(writeEngineLog(t, tt.body))
-			if got != tt.want {
-				t.Errorf("engineLogTail() = %q, want %q", got, tt.want)
-			}
-			if tt.wantMissing != "" && strings.Contains(got, tt.wantMissing) {
-				t.Errorf("engineLogTail() = %q, want no partial line", got)
-			}
-		})
-	}
-}
-
-func TestEngineLogTailUnsetPath(t *testing.T) {
-	if got := engineLogTail(""); got != "" {
-		t.Errorf("engineLogTail(\"\") = %q, want empty", got)
-	}
-}
-
-// The agent reads a path the image writes, and nothing at runtime would notice
-// the two drifting apart: the log would simply always be missing and every
-// refusal would go back unexplained again.
-func TestMariaDBErrorLogPathMatchesTheImage(t *testing.T) {
-	raw, err := os.ReadFile(mariadbRDSInitScript)
-	if err != nil {
-		t.Fatalf("read rds-init: %v", err)
-	}
-	script := string(raw)
-
-	logDir := shellDefault(t, script, "LOG_DIR", "RDS_LOG_DIR")
-	engineLog := shellDefault(t, script, "ENGINE_LOG", "RDS_ENGINE_LOG")
-	want := strings.Replace(engineLog, "${LOG_DIR}", logDir, 1)
-
-	if got := engineLayouts[engineMariaDB].errorLog; got != want {
-		t.Errorf("layout errorLog = %q, but rds-init writes log_error = %q", got, want)
-	}
-}
-
-// Pulls the fallback out of a `NAME="${OVERRIDE:-fallback}"` assignment.
-func shellDefault(t *testing.T, script, name, override string) string {
-	t.Helper()
-	re := regexp.MustCompile(`(?m)^` + name + `="\$\{` + override + `:-(.*)\}"$`)
-	m := re.FindStringSubmatch(script)
-	if m == nil {
-		t.Fatalf("rds-init has no %s assignment defaulting from %s", name, override)
-	}
-	return m[1]
-}
-
-func TestWithEngineLogTailLeavesReasonAloneWithoutALog(t *testing.T) {
-	const reason = "engine is not answering on its socket yet"
-	if got := withEngineLogTail(reason, ""); got != reason {
-		t.Errorf("withEngineLogTail() = %q, want the reason unchanged", got)
-	}
-}
-
-const postgresConfD = "../../scripts/images/rds-postgres/postgresql.confd"
-
-// "engine did not respond" is where a postmaster that refused a setting lands,
-// and it is the same message as an engine that never got as far as trying. Only
-// the server's own log separates them, and the guest has no other way out.
-func TestPostgresProbe_QuotesTheEngineLogWhenTheEngineIsAbsent(t *testing.T) {
-	log := writeEngineLog(t, strings.Join([]string{
-		`2026-09-06 10:20:33 UTC [712] LOG:  starting PostgreSQL 18.0`,
-		`2026-09-06 10:20:33 UTC [712] FATAL:  could not map anonymous shared memory`,
-		"",
-	}, "\n"))
-
-	state, message := postgresProbeState("/run/postgresql", log,
-		func(context.Context, string, ...string) (int, string, error) { return 2, "", nil })(t.Context(), 5432)
-
-	if state != engineAbsent {
-		t.Errorf("state = %v, want absent", state)
-	}
-	if !strings.Contains(message, "could not map anonymous shared memory") {
-		t.Errorf("message = %q, want the postmaster's own refusal quoted", message)
-	}
-}
-
-func TestPostgresProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
-	state, message := postgresProbeState("/run/postgresql", filepath.Join(t.TempDir(), "nope.log"),
-		func(context.Context, string, ...string) (int, string, error) { return 2, "", nil })(t.Context(), 5432)
-
-	if state != engineAbsent {
-		t.Errorf("state = %v, want absent", state)
-	}
-	if message != "engine did not respond on /run/postgresql:5432" {
-		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
-	}
-}
-
-// The layout names a path the image has to be directing the postmaster to, or
-// the tail above quotes nothing on the one failure it exists for.
-func TestPostgresErrorLogPathMatchesTheImage(t *testing.T) {
-	raw, err := os.ReadFile(postgresConfD)
-	if err != nil {
-		t.Fatalf("read postgresql.confd: %v", err)
-	}
-	m := regexp.MustCompile(`(?m)^logfile="(.*)"$`).FindStringSubmatch(string(raw))
-	if m == nil {
-		t.Fatal("postgresql.confd has no logfile assignment")
-	}
-	if got := engineLayouts[enginePostgres].errorLog; got != m[1] {
-		t.Errorf("layout errorLog = %q, but the image sets logfile = %q", got, m[1])
-	}
-}

cmd/rds-agent/engine_mariadb.go

@@ -10,6 +10,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -30,13 +31,6 @@ type controlPlaneRules struct {
 	// The catalog name behind a spelling read back out of an option file. Only a
 	// setting whose startup spelling differs from the customer's moves.
 	catalogName func(optionFileName string) string
-	// The catalog's data type for a setting, which decides how its value is
-	// rendered into SQL. Empty for a name the catalog does not carry.
-	dataType func(name string) string
-	// The engine's own name for the setting that requires TLS of a client
-	// connection. Held rather than spelled out here, so the guest derives
-	// enforcement from the same key the control plane resolves it under.
-	tlsEnforcementParameter string
 }
 
 func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
@@ -64,31 +58,32 @@ func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
 			}
 			return optionFileName
 		},
-		dataType: func(name string) string {
-			spec, ok := meta.LookupParameter(name)
-			if !ok {
-				return ""
-			}
-			return spec.DataType
-		},
-		tlsEnforcementParameter: meta.TLSEnforcementParameter(),
 	}
 }
 
-// Reached over its unix socket as root, which the datadir's unix_socket plugin
-// authenticates from the connecting process' own uid. That is why the agent
-// holds no password of its own.
+// The engine is reached over its unix socket as root, which the datadir's
+// unix_socket plugin authenticates from the connecting process' own uid — the
+// direct analogue of PostgreSQL's peer auth, and the reason the agent holds no
+// password of its own.
 type mariadbEngine struct {
 	quiesceState
-	parameterManager
 
 	rules     controlPlaneRules
 	run       commandRunner
 	startSess sessionRunner
 	client    string
+	admin     string
 	socket    string
 	rcService string
 	service   string
+	probe     *engineProbe
+
+	params parameterStore
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
 }
 
 var _ engine = (*mariadbEngine)(nil)
@@ -100,9 +95,6 @@ const (
 	// rds-init keeps for the platform and never hands to the customer.
 	mariadbSuperuser                  = "root"
 	mariadbProbeConnectTimeoutSeconds = 3
-	// How much of the probe client's stderr a reason carries, on the same grounds
-	// as the log tail above.
-	mariadbProbeStderrMaxBytes = 512
 	// Named by the platform drop-in rds-init writes, so both halves reach the
 	// same socket without either asserting it to the other.
 	mariadbSocketFile = "mysqld.sock"
@@ -140,25 +132,24 @@ func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, st
 		run:       run,
 		startSess: startSess,
 		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
+		admin:     filepath.Join(cfg.EngineBinDir, mariadbAdminBinary),
 		socket:    mariadbSocketPath(cfg),
 		rcService: cfg.RCService,
 		service:   cfg.EngineService,
-		parameterManager: parameterManager{
-			probe: probe,
-			params: parameterStore{
-				// The mount point rather than the datadir one level inside it: the
-				// include directory has to outlive the sweep a failed bootstrap runs.
-				dir:       filepath.Join(cfg.DataMount, "conf.d"),
-				installed: mariadbParametersFile,
-				lastGood:  mariadbLastGoodFile,
-				serving:   mariadbServingFile,
-				header:    mariadbParametersHeader,
-				osUser:    cfg.EngineUser,
-				engine:    engineMariaDB,
-			},
-			repairTimeout: parameterRepairTimeout,
-			repairPoll:    parameterRepairPoll,
+		probe:     probe,
+		params: parameterStore{
+			// The mount point rather than the datadir one level inside it: the
+			// include directory has to outlive the sweep a failed bootstrap runs.
+			dir:       filepath.Join(cfg.DataMount, "conf.d"),
+			installed: mariadbParametersFile,
+			lastGood:  mariadbLastGoodFile,
+			serving:   mariadbServingFile,
+			header:    mariadbParametersHeader,
+			osUser:    cfg.EngineUser,
+			engine:    engineMariaDB,
 		},
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
 	}
 }
 
@@ -166,15 +157,10 @@ func mariadbSocketPath(cfg config) string {
 	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
 }
 
-// Every local connection, from the probe's ping to a held quiesce session.
-// --no-defaults so no option file can move it; --skip-ssl because the serving
-// cert names the endpoint rather than this socket, so a verifying client fails.
-func mariadbSocketConnectArgs(socket string) []string {
-	return []string{
-		"--no-defaults", "--protocol=socket", "--socket=" + socket,
-		"--user=" + mariadbSuperuser, "--skip-ssl",
-	}
-}
+// Nothing this implementation runs needs the assigned port: every connection it
+// makes is over the engine's unix socket, and the platform drop-in rds-init
+// writes is what puts the server on the port. The probe follows it separately.
+func (e *mariadbEngine) setPort(int) {}
 
 // Reports whether a pid is a live process. EPERM is a process this agent may not
 // signal rather than one that is gone, which matters because a stale pidfile
@@ -193,21 +179,33 @@ func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
 	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
 }
 
-// Three stages: during InnoDB crash recovery mariadbd opens neither socket nor
-// port, so a ping cannot tell a recovering engine from an absent one — and
-// reading recovery as absent has the rollback guard restart a server mid-replay.
+// Three stages, because MariaDB has no single signal that separates an engine
+// still coming up from one that is not there at all: during InnoDB crash
+// recovery mariadbd opens neither its socket nor its port, so a ping fails
+// exactly as it would against nothing.
+//
+// Losing that distinction would break the rollback guard, which treats a
+// recovering engine as a reason to reset its deadline. An instance killed hard
+// mid-write can spend minutes replaying its redo log; read as absent, the guard
+// would roll the parameter file back and restart the server mid-recovery,
+// discarding the work — a rollback that exists to break a boot loop would create
+// one, on an instance whose parameters were never at fault.
+//
+// The port is not consulted. Both stages below reach the server over its unix
+// socket, whose path does not move with the port.
 func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
 	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
 	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
-	// The probe alone bounds the connect: it must come back and report, where a
-	// statement the agent issues is allowed to take as long as the engine needs.
-	connect := append(mariadbSocketConnectArgs(socket),
-		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds))
+	connect := []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser,
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds),
+	}
 	probeTimeout := cfg.EngineProbeTimeout
 	if probeTimeout <= 0 {
 		probeTimeout = defaultEngineProbeTimeout
 	}
-	runBounded := func(ctx context.Context, name string, args ...string) (int, string, error) {
+	runBounded := func(ctx context.Context, name string, args ...string) (int, error) {
 		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
 		defer cancel()
 		return run(probeCtx, name, args...)
@@ -225,56 +223,31 @@ func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) pro
 		}
 
 		// The process is up from here on, so every remaining failure is an engine
-		// not serving yet rather than one that is gone. Reporting absent against a
-		// live server would have the rollback guard restart one making progress.
-		switch code, stderr, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		// that is not serving yet rather than one that is gone — including a probe
+		// binary that will not run at all. Reporting absent against a live server
+		// would have the rollback guard restart one that is making progress.
+		switch code, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
 		case err != nil:
 			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
 		case code != 0:
-			// A server that is genuinely recovering and one that refused the client
-			// are indistinguishable from the socket. The server states its side in
-			// its own log, and the client states its side on stderr.
-			return engineRecovering, withProbeStderr(withEngineLogTail(
-				"engine is not answering on its socket yet (startup or crash recovery)",
-				cfg.EngineErrorLog), stderr)
+			return engineRecovering, "engine is not answering on its socket yet (startup or crash recovery)"
 		}
 
-		// ping answers even on ER_ACCESS_DENIED, so on its own it certifies a server
-		// that may be unable to execute anything. The statement rides argv rather
-		// than stdin: it carries nothing secret and a probe reads no result back.
+		// ping answers successfully even on ER_ACCESS_DENIED, so on its own it
+		// certifies a server that may be unable to execute anything at all. The
+		// statement is a literal in the argv rather than on stdin because it
+		// carries nothing secret and a probe reads no result back.
 		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
-		switch code, stderr, err := runBounded(ctx, client, query...); {
+		switch code, err := runBounded(ctx, client, query...); {
 		case err != nil:
 			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
 		case code != 0:
-			return engineRecovering, withProbeStderr(
-				"engine answered its socket but would not execute a statement", stderr)
+			return engineRecovering, "engine answered its socket but would not execute a statement"
 		}
 		return engineServing, ""
 	}
 }
 
-// Appends what the probe client said on stderr. A client that fails during the
-// handshake is refused by the server without the server ever logging why, so
-// this is the only account of that half of the exchange.
-func withProbeStderr(reason, stderr string) string {
-	stderr = strings.TrimSpace(stderr)
-	if stderr == "" {
-		return reason
-	}
-	return reason + "; the probe client reported: " + collapseProbeStderr(stderr)
-}
-
-// One line, bounded: the reason reaches a customer through StatusInfos, and a
-// client that repeats itself per attempt would otherwise crowd out the rest.
-func collapseProbeStderr(stderr string) string {
-	line := strings.Join(strings.Fields(strings.ReplaceAll(stderr, "\n", " ")), " ")
-	if len(line) > mariadbProbeStderrMaxBytes {
-		return line[:mariadbProbeStderrMaxBytes] + "..."
-	}
-	return line
-}
-
 func readPidFile(path string) (int, error) {
 	raw, err := os.ReadFile(path)
 	if err != nil {
@@ -292,8 +265,11 @@ func (e *mariadbEngine) Stop(ctx context.Context) error {
 }
 
 // Only the parameter rollback and a failed apply call this: a restart the
-// control plane wants goes through RebootDBInstance, which cycles the VM. The
-// serving copy is refreshed first, since the server starts on what is installed.
+// control plane wants goes through RebootDBInstance, which cycles the VM.
+//
+// The serving copy is refreshed first, because the server is about to start on
+// whatever is installed. That is exactly what rds-init does on every boot, and
+// it is what keeps the two equal by construction whenever the engine starts.
 func (e *mariadbEngine) Restart(ctx context.Context) error {
 	if err := e.recordServingCopy(); err != nil {
 		return err
@@ -322,39 +298,6 @@ func sqlLiteral(value string) string {
 	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
 }
 
-// The spellings MariaDB parses for a boolean system variable, and the only ones
-// the catalog's own enum narrows a boolean to.
-var mariadbBooleanValues = map[string]string{
-	"on": "ON", "off": "OFF", "true": "ON", "false": "OFF", "1": "ON", "0": "OFF",
-}
-
-// One SET GLOBAL right-hand side. MariaDB refuses a quoted literal for a numeric
-// or boolean with ER_WRONG_TYPE_FOR_VAR, so the catalog's data type picks the
-// form; re-rendering from the parsed value is what keeps it safe to interpolate.
-func mariadbSetValue(dataType, value string) (string, error) {
-	switch dataType {
-	case handlers_rds.ParamTypeInteger:
-		n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
-		if err != nil {
-			return "", fmt.Errorf("value %q is not an integer", value)
-		}
-		return strconv.FormatInt(n, 10), nil
-	case handlers_rds.ParamTypeReal:
-		f, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
-		if err != nil {
-			return "", fmt.Errorf("value %q is not a real number", value)
-		}
-		return strconv.FormatFloat(f, 'g', -1, 64), nil
-	case handlers_rds.ParamTypeBoolean:
-		if literal, ok := mariadbBooleanValues[strings.ToLower(strings.TrimSpace(value))]; ok {
-			return literal, nil
-		}
-		return "", fmt.Errorf("value %q is not a boolean", value)
-	default:
-		return "'" + sqlLiteral(value) + "'", nil
-	}
-}
-
 // Turns off the two logs that would otherwise copy a statement's text, and pins
 // the mode the escaping above assumes. All three are session-scoped: a rotation
 // must not leave the customer's own general log switched off behind it.
@@ -371,9 +314,10 @@ func (e *mariadbEngine) SetPassword(ctx context.Context, username, password stri
 		return fmt.Errorf("set-password requires both %s and %s",
 			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
 	}
-	// This runs as the server's own superuser, so a reserved name in the payload
-	// would rotate the platform's account. The whole rule matters here, not just
-	// the reserved set: the name is interpolated below rather than client-quoted.
+	// This runs as the server's own superuser over the socket, so a reserved name
+	// in the command payload would rotate the platform's account rather than the
+	// customer's. The rest of the rule matters just as much here, because the name
+	// is interpolated into the statement below rather than quoted by the client.
 	if err := e.rules.validateUsername(username); err != nil {
 		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
 	}
@@ -390,9 +334,14 @@ func (e *mariadbEngine) SetPassword(ctx context.Context, username, password stri
 	return nil
 }
 
-// Installs the resolved set and applies the half a running server can adopt.
-// MariaDB re-reads no file while up, so SET GLOBAL is the only immediate path.
-// It has no offline parser, so the safety net is the boot-time rollback.
+// Installs the resolved set and applies the half of it a running server can
+// adopt. MariaDB re-reads no configuration file while it is up, so SET GLOBAL is
+// the only way an immediate apply reaches it; the static half waits in the
+// installed file for the next start, and is what comes back as pending.
+//
+// There is no offline parser to check the set with first — MariaDB ships no
+// equivalent of postgres -C — so the safety net for a static value the server
+// will not start on is the boot-time rollback to the last accepted set.
 func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
 	e.paramMu.Lock()
 	defer e.paramMu.Unlock()
@@ -422,13 +371,6 @@ func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_r
 		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
 	}
 
-	// Before the set reaches the server: enforcement is one of the values below,
-	// and a server told to require a transport it cannot offer would refuse every
-	// client the statement after.
-	if err := e.requireServingTLSForEnforcement(ctx); err != nil {
-		return nil, errors.Join(err, restore())
-	}
-
 	if err := e.applyDynamicParameters(ctx, params); err != nil {
 		// The engine may have gone down after the first probe. In that case start it
 		// on the newly installed set rather than restoring one that already left it
@@ -438,37 +380,26 @@ func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_r
 		}
 		return nil, errors.Join(err, restore())
 	}
-	// Last, and on its own: the batch above is not a transaction, so moving the
-	// TLS posture inside it would leave enforcement changed on a set the server
-	// went on to refuse, with only the parameter file rolled back.
-	if err := e.applyTLSEnforcement(ctx); err != nil {
-		if state, _ := e.probe.state(ctx); state == engineAbsent {
-			return e.restartOnRepairSetLocked(ctx)
-		}
-		return nil, errors.Join(err, restore())
-	}
 	return e.pendingRestartParameters(ctx)
 }
 
-// The dynamic half, in one invocation that stops at the first refusal so the
-// server names the setting it would not take. Only catalog names are emitted,
-// which is what makes the identifier safe to build into the statement.
+// The dynamic half of the set, in one invocation that stops at the first refusal
+// so the server's own message names the setting it would not take.
 //
-// TLS enforcement is excluded: applyTLSEnforcement issues it after this batch
-// has succeeded.
+// Only names the catalog carries are emitted, which is also what keeps the
+// statement safe to build: a name is an identifier here, and the classification
+// that selects it comes from the same table the API validated the request
+// against. MariaDB cannot set several globals atomically, so a value set before
+// a refusal stays live until the next restart, when the restored file wins.
 func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
 	var b strings.Builder
 	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
 	applied := 0
 	for _, p := range params {
-		if p.Name == "" || p.Name == e.rules.tlsEnforcementParameter || e.rules.isStatic(p.Name) {
+		if p.Name == "" || e.rules.isStatic(p.Name) {
 			continue
 		}
-		value, err := mariadbSetValue(e.rules.dataType(p.Name), p.Value)
-		if err != nil {
-			return fmt.Errorf("apply the dynamic parameters: %s: %w", p.Name, err)
-		}
-		fmt.Fprintf(&b, "SET GLOBAL %s = %s;\n", p.Name, value)
+		fmt.Fprintf(&b, "SET GLOBAL %s = '%s';\n", p.Name, sqlLiteral(p.Value))
 		applied++
 	}
 	if applied == 0 {
@@ -480,106 +411,22 @@ func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []han
 	return nil
 }
 
-// Puts the installed set's enforcement on the running server. The value comes
-// from the installed file rather than the set in hand, so a set that predates
-// the parameter enforces here rather than leaving the server on mariadbd's own
-// default of off while the API reports enforcement.
-func (e *mariadbEngine) applyTLSEnforcement(ctx context.Context) error {
-	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
-	if err != nil {
-		return err
-	}
-	previous, err := e.readTLSEnforcement(ctx)
-	if err != nil {
-		return err
-	}
-	if previous == enforce {
-		return nil
-	}
-	applyErr := e.setTLSEnforcement(ctx, enforce)
-	if applyErr == nil {
-		return nil
-	}
-	// The client can fail without saying whether the server ran the statement, so
-	// the posture it left behind is read rather than assumed unchanged: the one
-	// outcome that must not survive a failed apply is plaintext nobody asked for.
-	live, readErr := e.readTLSEnforcement(ctx)
-	if readErr != nil {
-		return errors.Join(applyErr, readErr)
-	}
-	if live == previous {
-		return applyErr
-	}
-	return errors.Join(applyErr, e.setTLSEnforcement(ctx, previous))
-}
-
-func (e *mariadbEngine) readTLSEnforcement(ctx context.Context) (bool, error) {
-	name := e.rules.tlsEnforcementParameter
-	out, err := e.clientRun(ctx, "SELECT @@global."+name+";\n")
-	if err != nil {
-		return false, fmt.Errorf("read whether the engine requires TLS of clients: %w", err)
-	}
-	switch value := strings.TrimSpace(out); value {
-	case "1":
-		return true, nil
-	case "0":
-		return false, nil
-	default:
-		return false, fmt.Errorf("the engine reports %s as %q, which is neither 1 nor 0", name, value)
-	}
-}
-
-func (e *mariadbEngine) setTLSEnforcement(ctx context.Context, enforce bool) error {
-	name, value := e.rules.tlsEnforcementParameter, "0"
-	if enforce {
-		value = "1"
-	}
-	// Rendered the way the batch renders every other boolean, so the statement the
-	// server sees does not depend on which path issued it.
-	literal, err := mariadbSetValue(handlers_rds.ParamTypeBoolean, value)
-	if err != nil {
-		return fmt.Errorf("render %s: %w", name, err)
-	}
-	sql := fmt.Sprintf("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\nSET GLOBAL %s = %s;\n", name, literal)
-	if _, err := e.clientRun(ctx, sql); err != nil {
-		return fmt.Errorf("set %s to %s: %w", name, literal, err)
-	}
-	return nil
-}
-
-// Refuses to require TLS of clients from a server that is not serving it, which
-// would reject every TCP connection while the parameter path reported success.
-// Read back from the installed file rather than taken from the set in hand, so
-// what is checked is also what the server reads at its next start.
-func (e *mariadbEngine) requireServingTLSForEnforcement(ctx context.Context) error {
-	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
-	if err != nil {
-		return err
-	}
-	if !enforce {
-		return nil
-	}
-	// DISABLED is a server built with TLS and started without a certificate,
-	// which is what a boot that was handed none leaves behind. The agent's own
-	// connections are unaffected either way: MariaDB counts a unix socket as a
-	// secure transport.
-	out, err := e.clientRun(ctx, "SELECT @@global.have_ssl;\n")
-	if err != nil {
-		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
-	}
-	if serving := strings.TrimSpace(out); serving != "YES" {
-		return fmt.Errorf("the engine is serving without TLS (have_ssl = %q), so requiring it of clients would reject every connection", serving)
-	}
-	return nil
-}
-
 func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
 	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
 }
 
-// MariaDB has no pending_restart, so this is computed from the files: the
-// catalog-static keys whose installed value differs from the one the server
-// started on. Live values are not compared — mysqld silently rewrites them.
+// MariaDB has no pending_restart. Nothing in information_schema reports a
+// setting the server has stored but not adopted, so the answer is computed from
+// the files instead: the catalog-static keys whose value in the installed
+// drop-in differs from the one the server actually started on.
+//
+// Comparing live values against the desired set was rejected. mysqld silently
+// rewrites values at startup — rounding the buffer pool and log file sizes, and
+// downgrading max_connections when it cannot obtain enough file descriptors —
+// and each rewrite would read as a permanent pending restart: un-clearable
+// pending-reboot state, permanent drift, and a last-known-good copy frozen for
+// good, silently disabling a recovery mechanism only exercised during an
+// incident.
 func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
 	installed, err := readOptionFile(e.params.installedPath())
 	if err != nil {
@@ -682,14 +529,20 @@ func normaliseOptionName(name string) string {
 	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
 }
 
-// How long any one BACKUP STAGE waits for its metadata locks. Well under the
-// control plane's quiesce timeout, so a stage that cannot take its lock is
-// reported rather than left queued in front of live traffic.
+// How long any one BACKUP STAGE waits for the metadata locks it needs. Well
+// under the control plane's own quiesce timeout, so a stage that cannot take its
+// lock fails and is reported rather than being abandoned mid-wait with the
+// request still queued in front of live traffic.
 const mariadbQuiesceLockWait = 20 * time.Second
 
-// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK: FTWRL makes
-// the whole database read-only for the full hold, and its acquisition phase is
-// unbounded. BACKUP STAGE blocks commits instead, and covers Aria and MyISAM.
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK. FTWRL turns
+// a snapshot into a write outage: the whole database is read-only until it is
+// released, so a control plane that died after issuing quiesce would leave the
+// customer read-only for the full hold, on a schedule and unattended. Its
+// acquisition phase is unbounded too, with writes queueing behind a pending lock
+// request. BACKUP STAGE blocks commits rather than all writes and its flush
+// stage covers Aria and MyISAM, so it yields a better consistency point while
+// being far less disruptive — and it is what mariadb-backup itself uses.
 func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
 	if err := validateQuiesceRequest(label, hold); err != nil {
 		return err
@@ -715,9 +568,10 @@ func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Dur
 		return fmt.Errorf("open a backup session: %w", err)
 	}
 
-	// In order and on one connection: MariaDB releases the hold with the session
-	// that took it, which bounds a control plane that dies mid-snapshot. Fed one
-	// at a time so a stage that cannot take its locks is named.
+	// The stages run in order and on one connection: MariaDB releases the whole
+	// hold with the session that took it, which is what bounds a control plane
+	// that dies mid-snapshot. Fed one at a time so a stage that cannot take its
+	// locks is named rather than reported as a line number.
 	stages := []string{
 		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
 		"BACKUP STAGE START;\n",
@@ -738,8 +592,27 @@ func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Dur
 	return nil
 }
 
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held stage.
 func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
-	return e.releaseHold(ctx, "BACKUP STAGE END;\n")
+	held := e.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The release has to run on the session that took the stages, and the session
+	// is closed either way — the engine ends an unreleased backup with it.
+	execErr := held.session.Exec(ctx, "BACKUP STAGE END;\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
 }
 
 // Feeds sql on stdin rather than as an argument, so a statement is never visible
@@ -753,10 +626,13 @@ func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, erro
 	})
 }
 
+// --no-defaults so no option file can move the connection this agent makes, and
 // --unbuffered so a held session's output reaches the reader as each statement
-// finishes rather than when a pipe buffer fills. No User on the command: the
-// connection args carry it, and the agent runs as the account they name.
+// finishes rather than when a pipe buffer fills. No User: the agent is root, and
+// root is what the datadir's unix_socket plugin authenticates.
 func (e *mariadbEngine) clientArgs() []string {
-	return append(mariadbSocketConnectArgs(e.socket),
-		"--batch", "--skip-column-names", "--unbuffered")
+	return []string{
+		"--no-defaults", "--batch", "--skip-column-names", "--unbuffered",
+		"--protocol=socket", "--socket=" + e.socket, "--user=" + mariadbSuperuser,
+	}
 }