release: update go.mod for v1.19.0

All three need unexported access and cannot move to an external test
package: the daemon suite drives shared package fixtures and reads vmMgr,
resourceMgr and localStatePath, while the other two assert on buildArgs
and volumeVBConfig directly.

They were added on dev without the marker, so check-test-package.sh
failed for anyone running preflight on dev, where the base ref is
origin/main rather than origin/dev.

.github/ISSUE_TEMPLATE/1-bug.yml

@@ -0,0 +1,59 @@
+name: Bug report
+description: Something is not working as it should.
+labels: ["bug"]
+body:
+  - type: dropdown
+    id: component
+    attributes:
+      label: Component
+      options:
+        - EC2 / instances
+        - EBS / volumes
+        - S3 / object store
+        - VPC / networking
+        - IAM / authentication
+        - EKS
+        - API gateway
+        - Web UI
+        - CLI / daemon
+        - Observability / telemetry
+        - Build / CI / packaging
+        - Documentation
+        - Other / not sure
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: Spinifex version
+      description: Output of `spinifex --version`, or the commit SHA you are on.
+      placeholder: v1.0.0, or commit abc1234
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened
+      description: What you saw, what you expected instead, and how to reproduce it.
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs and output
+      description: Relevant log lines, error messages or command output.
+      render: shell
+
+  - type: textarea
+    id: platform
+    attributes:
+      label: Platform
+      description: Where you are running it, and how you deployed it.
+      placeholder: |
+        OS: Debian 13
+        Kernel: 6.12
+        Arch: amd64
+        Deployment: single node, systemd

.github/ISSUE_TEMPLATE/2-feature.yml

@@ -0,0 +1,11 @@
+name: Feature request
+description: Suggest something new.
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: proposal
+    attributes:
+      label: What would you like to see?
+      description: What you are trying to do, and what stops you today.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/3-docs.yml

@@ -0,0 +1,20 @@
+name: Documentation
+description: Something in the docs is wrong, missing or unclear.
+labels: ["documentation"]
+body:
+  - type: input
+    id: location
+    attributes:
+      label: Where
+      description: Link to the page, or the path to the file in this repository.
+      placeholder: https://docs.mulgadc.com/..., or README.md
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: What is wrong
+      description: What is inaccurate, missing or confusing, and what it should say instead.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Documentation
+    url: https://docs.mulgadc.com
+    about: Guides, configuration reference and API compatibility notes.

.github/actions/e2e-analyze/analyze.go

@@ -91,9 +91,12 @@ type SuiteReport struct {
 	Label     string // short name, e.g. "single"
 	Total     int
 	FailCount int
-	Root      *Failure    // nil if no failures
-	Cascades  []Failure   // everything else that failed in this suite
-	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// Unresolved counts testcases go-junit-report could not find an outcome
+	// for. They are not failures and the suite can be green with hundreds.
+	Unresolved int
+	Root       *Failure    // nil if no failures
+	Cascades   []Failure   // everything else that failed in this suite
+	Buckets    [][]Failure // failures grouped by signature (root's bucket first)
 	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
 	// the XML was produced — i.e. when the suite finished, not when it began.
 	EndedAt time.Time
@@ -366,7 +369,7 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 func computeRolledUpSet(cases []junitTC) map[string]bool {
 	rolledUp := make(map[string]bool, len(cases))
 	for _, tc := range cases {
-		if tc.Failure == nil && tc.Error == nil {
+		if tc.Failure == nil && !isRealError(tc.Error) {
 			continue
 		}
 		// Mark every ancestor: the failure belongs to the deepest case that
@@ -412,6 +415,18 @@ func parseStartTime(s string) time.Time {
 	return t
 }
 
+// noResultMessage is go-junit-report's marker for a testcase it saw start and
+// never saw finish. On cell-20 it accounted for 93 of the 120 testcases and 24
+// of the 25 reported failures, none of which had failed.
+const noResultMessage = "No test result found"
+
+// isRealError separates a genuine <error> from the parser's no-result marker.
+// A parent must not be treated as rolled up by a child whose outcome is merely
+// unknown, or the parent's own failure is dropped along with it.
+func isRealError(e *junitFailure) bool {
+	return e != nil && strings.TrimSpace(e.Message) != noResultMessage
+}
+
 // suiteLabel turns "junit-single.xml" into "single".
 func suiteLabel(path string) string {
 	base := filepath.Base(path)
@@ -457,6 +472,17 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 			case tc.Failure != nil:
 				body = tc.Failure.Body
 			case tc.Error != nil:
+				// go-junit-report emits this when a `=== RUN` line has no
+				// matching result line. It means the outcome is unknown, not
+				// that the test errored, and it attaches the test's own log
+				// output — which then reads as an assertion.
+				if !isRealError(tc.Error) {
+					rep.Unresolved++
+					if leaf {
+						cumul += tc.Time
+					}
+					continue
+				}
 				body = tc.Error.Body
 			default:
 				if leaf {
@@ -553,6 +579,18 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 }
 
 // Render writes the markdown report.
+// writeUnresolved names the testcases whose outcome the parser could not read.
+// Worth surfacing because a suite that leaves most of its subtests unresolved
+// is reporting on far less than its test count suggests.
+func writeUnresolved(b *strings.Builder, s SuiteReport) {
+	if s.Unresolved == 0 {
+		return
+	}
+	fmt.Fprintf(b, "ℹ️ %d of %d testcases have no result line in the log, so `go-junit-report` "+
+		"could not read their outcome. Not failures — but this suite reports on %d tests, not %d.\n\n",
+		s.Unresolved, s.Total, s.Total-s.Unresolved, s.Total)
+}
+
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
@@ -579,9 +617,11 @@ func Render(r Report) string {
 	for _, s := range r.Suites {
 		if s.FailCount == 0 {
 			fmt.Fprintf(&b, "### Suite `%s`: ✅ pass (%d tests)\n\n", s.Label, s.Total)
+			writeUnresolved(&b, s)
 			continue
 		}
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
+		writeUnresolved(&b, s)
 
 		if s.Root != nil {
 			// XML order is completion order, so "earliest" only means earliest

.github/actions/e2e-analyze/analyze_test.go

@@ -174,6 +174,55 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+// go-junit-report emits <error message="No test result found"> for a `=== RUN`
+// with no matching result line, and attaches the test's own log output. On a
+// baremetal cell that was 93 of 120 testcases and 24 of 25 reported failures,
+// none of which had failed: "ALB internal: 20/20 successful" read as an error.
+func TestParseFile_NoResultFoundIsNotAFailure(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="3" failures="1" errors="2">
+  <testcase name="TestLB/Internal_ALB" time="0.0"><error message="No test result found"><![CDATA[
+    lb_test.go:88: ALB internal: 20/20 successful, 2 unique
+]]></error></testcase>
+  <testcase name="TestLB/Internal_NLB" time="0.0"><error message="No test result found"></error></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (no-result entries are not failures)", sr.FailCount)
+	}
+	if sr.Unresolved != 2 {
+		t.Fatalf("Unresolved = %d, want 2", sr.Unresolved)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the real failure as root, got %+v", sr.Root)
+	}
+}
+
+// An <error> that is not the parser's no-result marker is a genuine error and
+// must still be reported.
+func TestParseFile_KeepsARealError(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="1" failures="0" errors="1">
+  <testcase name="TestX" time="0.5"><error message="panic"><![CDATA[
+    foo_test.go:10: panic: runtime error: index out of range
+]]></error></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 || sr.Unresolved != 0 {
+		t.Fatalf("FailCount = %d, Unresolved = %d, want 1 and 0", sr.FailCount, sr.Unresolved)
+	}
+}
+
 func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
 	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
 	// "DB instance … is gone" — which passing tests emit too — reads exactly

.github/e2e-cells.json

@@ -0,0 +1,147 @@
+[
+  {
+    "cell": 1,
+    "install": "binary",
+    "topo": "single",
+    "net": "dhcp",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 2,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 3,
+    "install": "binary",
+    "topo": "single",
+    "net": "dhcp",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 8,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "dhcp",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 9,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 10,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "dhcp",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 17,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-pseudo",
+    "extra": "tofu-examples",
+    "suite": ""
+  },
+  {
+    "cell": 18,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "reboot-single",
+    "suite": ""
+  },
+  {
+    "cell": 19,
+    "install": "binary",
+    "topo": "single",
+    "net": "nat",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "nat-single",
+    "suite": ""
+  },
+  {
+    "cell": 25,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single-nvme",
+    "extra": "eks",
+    "suite": "eks"
+  },
+  {
+    "cell": 26,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "ecs",
+    "suite": "ecs"
+  },
+  {
+    "cell": 27,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "rds",
+    "suite": "rds",
+    "timeout_minutes": 100
+  },
+  {
+    "cell": 29,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "storagefault",
+    "timeout_minutes": 100
+  }
+]

.github/scripts/ci-fmt.sh

@@ -61,3 +61,37 @@ quiet_run() {
     return "$rc"
   fi
 }
+
+# Connection-layer failures from the libvirt tofu provider. The socket drops
+# when a hypervisor is carrying several environments at once, which is the
+# steady state here. A resource that failed to build says something else.
+CI_LIBVIRT_TRANSIENT='Unable to Connect to Libvirt|error connecting to libvirt|procedure interrupted|Cannot recv data|Cannot write data|End of file while reading data'
+
+# retry_run LABEL LOG_FILE -- CMD ARGS …
+# quiet_run with a bounded retry, taken only when that attempt's own output
+# names a libvirt connection failure. Every other failure is reported on the
+# first attempt, so a real bug never becomes a slow flake.
+retry_run() {
+  local label="$1" log="$2"; shift 2
+  local _attempts="${CI_RETRY_ATTEMPTS:-3}" _delay="${CI_RETRY_DELAY:-30}" _i _rc=1 _mark
+  for ((_i = 1; _i <= _attempts; _i++)); do
+    _mark=$(wc -c < "$log" 2>/dev/null || echo 0)
+    # rc must be read inside the else: an `if` whose condition fails and has no
+    # else exits 0, so reading $? after `fi` reports success for a failed run.
+    if "$@" >> "$log" 2>&1; then
+      if [[ $_i -gt 1 ]]; then ok "$label (libvirt recovered on attempt $_i)"; else ok "$label"; fi
+      return 0
+    else
+      _rc=$?
+    fi
+    tail -c "+$((_mark + 1))" "$log" | grep -Eq "$CI_LIBVIRT_TRANSIENT" || break
+    [[ $_i -eq $_attempts ]] && break
+    warn "$label: libvirt connection dropped (attempt $_i/$_attempts), retrying in ${_delay}s"
+    sleep "$_delay"
+  done
+  bad "$label (rc=$_rc) — last 80 lines of $(basename "$log"):"
+  group_open "tail $(basename "$log")"
+  tail -n 80 "$log"
+  group_close
+  return "$_rc"
+}

.github/scripts/e2e-run-summary.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Render one run-level table for the e2e nightly into the job summary.
+
+Reading a nightly meant opening nineteen jobs, and the Actions UI does not
+distinguish a cell that failed from one that was cancelled out from under it —
+a distinction every investigation of this suite has needed.
+
+Inputs (env):
+    GITHUB_TOKEN         Token with actions:read.
+    GITHUB_REPOSITORY    owner/repo.
+    GITHUB_RUN_ID        Run to summarise.
+    JUNIT_ROOT           Directory holding downloaded junit-cell-* artifacts.
+    GITHUB_STEP_SUMMARY  Appended to.
+
+Test counts come from the junit artifacts; result, duration and runner come
+from the jobs API, so a cell that died before writing junit still gets a row.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime
+
+API = "https://api.github.com"
+
+# A cell whose job produced no verdict is not a pass and not a failure. Keeping
+# them apart is the point of the table.
+VERDICT_ICON = {
+    "success": "✅",
+    "failure": "❌",
+    "cancelled": "🚫",
+    "skipped": "⏭️",
+    "timed_out": "⏱️",
+}
+
+
+def fetch_jobs(repo: str, run_id: str, token: str) -> list[dict]:
+    jobs: list[dict] = []
+    page = 1
+    while True:
+        url = f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}"
+        req = urllib.request.Request(
+            url,
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/vnd.github+json",
+                "X-GitHub-Api-Version": "2022-11-28",
+            },
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                payload = json.load(resp)
+        except (urllib.error.URLError, TimeoutError) as exc:
+            print(f"::warning::jobs API page {page} failed: {exc}", file=sys.stderr)
+            break
+        batch = payload.get("jobs", [])
+        jobs.extend(batch)
+        if len(batch) < 100:
+            break
+        page += 1
+    return jobs
+
+
+def duration_s(job: dict) -> int | None:
+    started, completed = job.get("started_at"), job.get("completed_at")
+    if not started or not completed:
+        return None
+    fmt = "%Y-%m-%dT%H:%M:%SZ"
+    try:
+        return int((datetime.strptime(completed, fmt) - datetime.strptime(started, fmt)).total_seconds())
+    except ValueError:
+        return None
+
+
+def human(seconds: int | None) -> str:
+    if seconds is None:
+        return "—"
+    if seconds < 0:
+        return "0s"
+    return f"{seconds // 60}m{seconds % 60:02d}s" if seconds >= 60 else f"{seconds}s"
+
+
+def junit_counts(root: str) -> dict[str, dict]:
+    """Map cell number to its aggregate junit counts and failing test names."""
+    per_cell: dict[str, dict] = {}
+    for path in glob.glob(os.path.join(root, "**", "junit-*.xml"), recursive=True):
+        cell = cell_from_artifact_path(path)
+        if cell is None:
+            continue
+        acc = per_cell.setdefault(cell, {"tests": 0, "failures": 0, "skipped": 0, "failing": []})
+        try:
+            root_el = ET.parse(path).getroot()
+        except ET.ParseError as exc:
+            print(f"::warning::unparseable junit {path}: {exc}", file=sys.stderr)
+            continue
+        suites = root_el.iter("testsuite") if root_el.tag != "testsuite" else [root_el]
+        for suite in suites:
+            acc["tests"] += int(suite.get("tests") or 0)
+            acc["failures"] += int(suite.get("failures") or 0) + int(suite.get("errors") or 0)
+            acc["skipped"] += int(suite.get("skipped") or 0)
+        for case in root_el.iter("testcase"):
+            if case.find("failure") is not None or case.find("error") is not None:
+                name = case.get("name") or "?"
+                if name not in acc["failing"]:
+                    acc["failing"].append(name)
+    return per_cell
+
+
+def cell_from_artifact_path(path: str) -> str | None:
+    """Recover the cell number from a junit-cell-<n>-<run_id> artifact directory."""
+    for part in path.split(os.sep):
+        if part.startswith("junit-cell-"):
+            rest = part[len("junit-cell-"):]
+            cell = rest.split("-", 1)[0]
+            return cell if cell.isdigit() else None
+    return None
+
+
+def cell_number(job_name: str) -> str | None:
+    """cell-19 (single/static/debian-13) -> 19."""
+    if not job_name.startswith("cell-"):
+        return None
+    cell = job_name[len("cell-"):].split(" ", 1)[0]
+    return cell if cell.isdigit() else None
+
+
+def main() -> int:
+    repo = os.environ.get("GITHUB_REPOSITORY", "")
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    token = os.environ.get("GITHUB_TOKEN", "")
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY", "")
+    junit_root = os.environ.get("JUNIT_ROOT", "")
+
+    if not (repo and run_id and token and summary_path):
+        print("::warning::e2e-run-summary: missing required environment", file=sys.stderr)
+        return 0
+
+    jobs = fetch_jobs(repo, run_id, token)
+    counts = junit_counts(junit_root) if junit_root and os.path.isdir(junit_root) else {}
+
+    rows = []
+    for job in jobs:
+        cell = cell_number(job.get("name", ""))
+        if cell is None:
+            continue
+        rows.append((int(cell), job))
+    rows.sort(key=lambda r: r[0])
+
+    lines = [f"## e2e nightly — run summary ({len(rows)} cells)", ""]
+    if not rows:
+        lines.append("No cell jobs found for this run.")
+        write(summary_path, lines)
+        return 0
+
+    lines += [
+        "| Cell | Topology | Result | Duration | Runner | Tests | Failed | Skipped |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+    ]
+
+    tally: dict[str, int] = {}
+    total_tests = total_failed = total_seconds = 0
+    failing_detail: list[tuple[int, list[str]]] = []
+
+    for cell, job in rows:
+        name = job.get("name", "")
+        # A cell dispatched to a reusable workflow reports as
+        # "cell-20 (baremetal/pxe/real-hw) / Bare-Metal Single-Node E2E".
+        topo = name.split("(", 1)[1].split(")", 1)[0] if "(" in name else "—"
+        verdict = job.get("conclusion") or job.get("status") or "unknown"
+        tally[verdict] = tally.get(verdict, 0) + 1
+        secs = duration_s(job)
+        total_seconds += secs or 0
+        c = counts.get(str(cell), {})
+        total_tests += c.get("tests", 0)
+        total_failed += c.get("failures", 0)
+        if c.get("failing"):
+            failing_detail.append((cell, c["failing"]))
+        lines.append(
+            f"| {cell} | {topo} | {VERDICT_ICON.get(verdict, '❔')} {verdict} | {human(secs)} "
+            f"| {job.get('runner_name') or '—'} | {c.get('tests', '—')} "
+            f"| {c.get('failures', '—')} | {c.get('skipped', '—')} |"
+        )
+
+    verdicts = ", ".join(f"{n} {v}" for v, n in sorted(tally.items(), key=lambda kv: -kv[1]))
+    lines += [
+        "",
+        f"**Cells:** {verdicts}",
+        f"**Tests:** {total_tests} run, {total_failed} failed",
+        f"**Cell wall clock:** {human(total_seconds)} across all cells",
+    ]
+
+    slowest = sorted(rows, key=lambda r: duration_s(r[1]) or 0, reverse=True)[:3]
+    if slowest:
+        lines.append(
+            "**Slowest:** " + ", ".join(f"cell {c} ({human(duration_s(j))})" for c, j in slowest)
+        )
+
+    if failing_detail:
+        lines += ["", "<details><summary>Failing tests</summary>", ""]
+        for cell, names in failing_detail:
+            lines.append(f"- **cell {cell}** — " + ", ".join(f"`{n}`" for n in names))
+        lines += ["", "</details>"]
+
+    write(summary_path, lines)
+    return 0
+
+
+def write(path: str, lines: list[str]) -> None:
+    with open(path, "a", encoding="utf-8") as fh:
+        fh.write("\n".join(lines) + "\n")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/ensure-libguestfs.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Prepare a runner to build a guest image with build-system-image.sh.
+#
+# Installs libguestfs + qemu-utils if they are missing and makes a host kernel
+# readable, so the libguestfs appliance can boot. Nothing here touches a host
+# block device: virt-customize works inside an isolated userspace appliance, so
+# this is safe on a shared runner.
+#
+# Callers must still export LIBGUESTFS_BACKEND=direct themselves — it has to be
+# set in the environment the build runs in, which is not this subshell.
+set -euo pipefail
+
+# Three nightly service cells share one ci-single runner and start together, so
+# concurrent apt-get runs are the normal case rather than the exception. Two of
+# them lost the lock outright before this guard existed. Skipping the install
+# when the tools are already present avoids most of the contention; the lock
+# timeout covers the rest, including apt runs started by anything else on the
+# box.
+if command -v virt-customize >/dev/null && command -v qemu-img >/dev/null; then
+    echo "libguestfs-tools and qemu-utils already present"
+else
+    APT_OPTS=(-o DPkg::Lock::Timeout=600)
+    sudo apt-get "${APT_OPTS[@]}" update -qq
+    sudo apt-get "${APT_OPTS[@]}" install -y --no-install-recommends \
+        libguestfs-tools qemu-utils
+fi
+
+# Debian/Ubuntu ship /boot/vmlinuz-* mode 0600 and the appliance needs to read a
+# host kernel. Tolerated on failure: a runner whose kernel is already readable
+# has no vmlinuz to chmod under some images.
+sudo chmod 0644 /boot/vmlinuz-* || true

.github/scripts/suite-sets.sh

@@ -28,23 +28,83 @@
 # that runs identically on either, and because a quota left switched on would
 # cap every suite sharing the environment — so it must be proved to restore
 # itself wherever it runs, not only on one node.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota"
+# storagefault runs last in both lists and belongs to both topologies, for
+# different reasons. On one node it is RS(1,0), so freezing the only predastore
+# is a total outage and the guest-survival assertion is unambiguous. On three it
+# is RS(2,1) with degraded writes on, so two must be frozen to break the shard
+# floor — and only there is a cross-node restart possible at all, which is the
+# case that takes a volume over from a node holding writes the backend never
+# received. It is last because it SIGSTOPs a cluster-wide
+# service: anything sharing the environment would see the same outage and report
+# a failure that says nothing about itself.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert quota"
+E2E_SUITES_MULTI="multinode lb cert quota storagefault"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
 E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 
-# e2e-nightly runs a narrower set than the two sets above, deliberately: its
-# ~19 cells each have a 35-minute budget and exist to prove that every
-# install / network / host-OS permutation boots and serves. Suite breadth is
-# the PR gate's job, and it already runs on every push. These are subsets, and
-# e2e_suites_assert_subset below enforces that.
-E2E_SUITES_NIGHTLY_SINGLE="single cert"
+# What every e2e-nightly permutation cell must prove, deliberately narrower
+# than the two sets above: those cells have a 35-minute budget and exist to show
+# that every install / network / host-OS combination boots and serves. The
+# service suites are not deferred to the PR gate — eks, ecs and rds each run
+# nightly in their own dedicated cell, because each needs a guest AMI built
+# first and a service failure should not mask whether a host OS boots. These
+# are subsets, and e2e_suites_assert_subset below enforces that.
+#
+# storagefault has a dedicated cell for two reasons. It SIGSTOPs a cluster-wide
+# service, so in a permutation cell every suite sharing that environment would
+# report the outage as its own failure. And a full pass is ~63 minutes, nearly
+# all of it deliberate waiting, which is too long to add to the PR gate — where
+# it has never run, since the gate's default suite list does not name it.
+#
+# Its cell is three-node: the RS(2,1) shard floor and the cross-node takeover
+# after a failed seal do not exist on one node, and they are the assertions
+# worth a nightly.
+E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
 E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
 
+# How long a suite gets, for the same reason the lists above live here: the
+# timeout was written out separately in each workflow and drifted, so rds ran
+# 50m on the PR gate and 30m nightly and panicked there. A suite needing more
+# than the default declares it once, here.
+#   e2e_suite_timeout <suite>
+e2e_suite_timeout() {
+  case "$1" in
+    # rds serialises its DB VMs behind a 3 GiB semaphore, so its wall clock is
+    # what the queue costs, not what the tests do: single tests logged 11m, 22m
+    # and 26m waiting for budget. 17 tests at that concurrency is over an hour
+    # before anything goes wrong, and the first pass to run deep into the suite
+    # still had three left at 60m.
+    rds) echo "90m" ;;
+    # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
+    # creates first. One pass is ~20 minutes on the bare-metal cell and a
+    # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
+    diskperf) echo "180m" ;;
+    # storagefault is mostly deliberate waiting: one test alone holds predastore
+    # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
+    # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
+    storagefault) echo "90m" ;;
+    *) echo "30m" ;;
+  esac
+}
+
+# One matrix leg per suite, carrying the job budget that suite needs. The outer
+# timeout must always be later than the `go test` one inside it: go test reports
+# where it stopped, a cancelled job reports only "The operation was canceled".
+#   e2e_suite_matrix <suite-list> <floor-minutes> [allowance-minutes]
+e2e_suite_matrix() {
+  local list="$1" floor="$2" allowance="${3:-15}" suite budget
+  for suite in ${list}; do
+    budget="$(e2e_suite_timeout "${suite}")"
+    budget=$((${budget%m} + allowance))
+    if [ "${budget}" -lt "${floor}" ]; then budget="${floor}"; fi
+    printf '%s\t%s\n' "${suite}" "${budget}"
+  done | jq -cRs 'split("\n")|map(select(length>0)|split("\t")|{suite:.[0],timeout:(.[1]|tonumber)})'
+}
+
 # Fail loudly when a narrowed list names a suite its topology cannot run. This
 # is the guard that stops the lists drifting again: a suite added to a nightly
 # subset without also being declared eligible above stops the job here rather

.github/workflows/apt-packages.yml

@@ -33,8 +33,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
-        # accepted by detect_os but has never been exercised and does not resolve.
+        # The suites the nightly E2E matrix actually installs on. This matches
+        # what detect_os accepts: Debian 13+ and Ubuntu 24.04+.
         image:
           - ubuntu:24.04
           - ubuntu:26.04

.github/workflows/e2e-baremetal.yml

@@ -1,7 +1,26 @@
 name: E2E Bare-Metal
 
 on:
+  # Mirrors the suite selection of workflow_call so a baseline capture can be
+  # driven by hand. A dispatch run sees only the inputs declared here, so
+  # omitting these would silently fall back to the correctness set.
   workflow_dispatch:
+    inputs:
+      suites:
+        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance run names its own so its numbers come off an otherwise idle cell"
+        required: false
+        type: string
+        default: "single cert lb"
+      suite-env:
+        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
+        required: false
+        type: string
+        default: ""
+      cell:
+        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
+        required: false
+        type: string
+        default: "20"
   workflow_call:
     inputs:
       ref:
@@ -14,6 +33,21 @@ on:
         required: false
         type: string
         default: ""
+      suites:
+        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance caller names its own so its numbers come off an otherwise idle cell"
+        required: false
+        type: string
+        default: "single cert lb"
+      suite-env:
+        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
+        required: false
+        type: string
+        default: ""
+      cell:
+        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
+        required: false
+        type: string
+        default: "20"
 
 permissions:
   contents: read
@@ -30,16 +64,18 @@ jobs:
   e2e_baremetal:
     name: Bare-Metal Single-Node E2E
     runs-on: [ self-hosted, ci-baremetal ]
-    timeout-minutes: 120
+    timeout-minutes: 240
     concurrency:
       group: e2e-baremetal-cluster
       cancel-in-progress: false
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
-      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
-      # Caller-supplied (workflow_call input); empty for a manual
-      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
-      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
+      BM_CELL: ${{ inputs.cell || '20' }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
+      # Caller-supplied on workflow_call. A dispatch run has no such input, and
+      # leaving it empty cost this cell its ci.run_id: its telemetry reached the
+      # sink but could not be found by run, only by guessing a time window.
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
       - name: Clean Workspace
         run: |
@@ -74,6 +110,10 @@ jobs:
           ssh-keygen -y -f "${RUNNER_TEMP}/bm-ci" > "${RUNNER_TEMP}/bm-ci.pub"
 
       - name: Build Artifacts
+        env:
+          # The node runs pre-built binaries, so a suite absent here cannot be
+          # selected later. Same list as the run step for that reason.
+          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
         run: |
           mulga/scripts/baremetal/bm-build-artifacts.sh \
             "$GITHUB_WORKSPACE" \
@@ -119,6 +159,21 @@ jobs:
       - name: Install go-junit-report on runner
         run: go install github.com/jstemmer/go-junit-report/v2@latest
 
+      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
+      # AMI. Built here because libguestfs needs source and network, which the
+      # runner has and the PXE-booted node does not; bm-bootstrap.sh imports it.
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: contains(inputs.suites, 'eks')
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          # spinifex is the workspace root here, unlike the nightly where it is
+          # checked out into a subdirectory.
+          "${GITHUB_WORKSPACE}/.github/scripts/ensure-libguestfs.sh"
+          make build-eks-node-image
+
       - name: Run Bare-Metal E2E
         env:
           BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
@@ -126,9 +181,27 @@ jobs:
           BM_EPDU_ENV: ${{ runner.temp }}/epdu.env
           BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
           SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
+          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
+          BM_SUITE_ENV: ${{ inputs.suite-env }}
+          # Empty unless the eks build above ran, and bm-bootstrap.sh skips the
+          # import when unset, so the other suites are unaffected.
+          BM_EKS_IMAGE: ${{ contains(inputs.suites, 'eks') && '/tmp/eks-node-image-build/eks-node-alpine.raw' || '' }}
+          BM_EKS_MANIFEST: ${{ contains(inputs.suites, 'eks') && format('{0}/scripts/images/eks-node/manifest.conf', github.workspace) || '' }}
         run: |
           mkdir -p "${ARTIFACT_DIR}"
           set -o pipefail
+          # Per-suite budgets are declared once in suite-sets.sh. The script
+          # applies one timeout to every suite in the list, so the widest wins.
+          . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
+          BM_TEST_TIMEOUT=""
+          for suite in ${BM_SUITES}; do
+            tmo="$(e2e_suite_timeout "${suite}")"
+            if [ -z "${BM_TEST_TIMEOUT}" ] || [ "${tmo%m}" -gt "${BM_TEST_TIMEOUT%m}" ]; then
+              BM_TEST_TIMEOUT="${tmo}"
+            fi
+          done
+          export BM_TEST_TIMEOUT
+          echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
           sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
@@ -139,7 +212,15 @@ jobs:
       - name: Generate JUnit report
         if: always()
         run: |
-          LOG="${ARTIFACT_DIR}/test-baremetal.log"
+          # suite-output.log, not test-baremetal.log. The harness streams the
+          # full suite output to the first and a grep-filtered summary to its
+          # stdout, and that filter anchors --- PASS at column 0, so every
+          # indented subtest result is dropped. Parsed from the filtered log,
+          # go-junit-report saw a === RUN with no result and emitted
+          # <error message="No test result found"> — 17 of 39 testcases on a
+          # green cell-28, 93 of 120 on cell-20.
+          LOG="${ARTIFACT_DIR}/suite-output.log"
+          [ -s "$LOG" ] || LOG="${ARTIFACT_DIR}/test-baremetal.log"
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
@@ -151,7 +232,7 @@ jobs:
         uses: ./.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: "20"
+          cell: ${{ env.BM_CELL }}
 
       - name: Analyze e2e failures
         if: always()
@@ -159,12 +240,12 @@ jobs:
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
-          title: "E2E failure analysis (cell 20)"
+          title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
 
       - uses: actions/upload-artifact@v7
         if: always()
         with:
-          name: e2e-artifacts-cell-20-${{ github.run_id }}
+          name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14

.github/workflows/e2e-ddil.yml

@@ -10,6 +10,14 @@ on:
         required: false
         default: ""
 
+# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
+# them provisions on the same runner pools and the same hypervisors, so two in
+# flight compete for their disks and their libvirt guests. Queued rather than
+# cancelled — a run in progress is someone's result, not a stale one.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 permissions:
   contents: read
   # actions: read lets the otel-job-id action look up this job's numeric id via
@@ -103,7 +111,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VMs provisioned"
 
       - name: Bootstrap via Binary Install
@@ -228,7 +236,7 @@ jobs:
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
 

.github/workflows/e2e-nightly.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-19 tofu matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection"
         required: false
 
 permissions:
@@ -16,8 +16,22 @@ permissions:
   # context exposes the id itself.
   actions: read
 
+# Every cell builds its cluster on the same five hypervisors, so two runs in
+# flight at once compete for their disks and their libvirt guests. Queue the
+# second rather than cancelling it: a dispatch is usually a verification run
+# and discarding it loses the result someone is waiting on.
+#
+# The group is shared with e2e.yml, e2e-suites.yml and e2e-ddil.yml, which draw
+# on the same runner pools. A per-workflow group let a push-triggered E2E land
+# on top of a nightly cell and take libvirt down under both.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 # Scheduled runs are loaded from the default branch (main), so the schedule
-# trigger must stay in sync on main for cron to fire.
+# trigger must stay in sync on main for cron to fire — and so must the
+# concurrency block above. main currently has none, which is why the nightly
+# cron runs ungrouped and a dispatch on any branch can land on top of it.
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
   UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
@@ -27,61 +41,109 @@ env:
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
+  # The matrix is built here rather than filtered inside each cell. A job whose
+  # steps all skip still reports success, so gating steps made a one-cell
+  # dispatch look like a green nineteen-job suite. A cell that was not selected
+  # now produces no job at all.
+  select:
+    runs-on: ubuntu-26.04
+    outputs:
+      cells: ${{ steps.pick.outputs.cells }}
+      any: ${{ steps.pick.outputs.any }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: |
+            .github/e2e-cells.json
+            .github/scripts/suite-sets.sh
+          sparse-checkout-cone-mode: false
+      - id: pick
+        env:
+          CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
+          # Provision, install and teardown around the suite. Measured at ~6
+          # minutes single-node and ~10 multi-node, so this is the floor the
+          # job budget must carry on top of the suite's own.
+          SETUP_ALLOWANCE_MIN: 10
+        run: |
+          if [ -z "$CELLS" ]; then
+            sel=$(jq -c . .github/e2e-cells.json)
+          else
+            sel=$(jq -c --arg c "$CELLS" \
+              '[.[] | . as $e | select(($c | split(",")) | index($e.cell|tostring))]' \
+              .github/e2e-cells.json)
+          fi
+          echo "cells=$sel" >> "$GITHUB_OUTPUT"
+          [ "$(jq 'length' <<<"$sel")" -gt 0 ] && any=true || any=false
+          echo "any=$any" >> "$GITHUB_OUTPUT"
+          echo "selected: $(jq -c '[.[].cell]' <<<"$sel")"
+
+          # The inner timeout must always fire before the outer one. go test
+          # reports where it stopped and the suite's own failure; a job timeout
+          # reports only "The operation was canceled". cell-27 ran a 50m rds
+          # suite inside a 50m job and could never finish either way.
+          . .github/scripts/suite-sets.sh
+          # Cells with no suite are filtered in jq, not with an empty-field test:
+          # tab is IFS whitespace, so read collapses a run of them and an empty
+          # middle column silently shifts every later one left.
+          bad=0
+          while IFS=$'\t' read -r cell suite budget; do
+            want="$(e2e_suite_timeout "$suite")"; want="${want%m}"
+            if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
+              echo "::error::cell-$cell gives suite '$suite' ${want}m inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN)), or lower the suite budget in suite-sets.sh." >&2
+              bad=1
+            fi
+          done < <(jq -r '.[]|select((.suite//"")!="")|[.cell,.suite,(.timeout_minutes//50)]|@tsv' <<<"$sel")
+
+          # A permutation cell names no suite and runs the nightly subset for its
+          # topology, so its bound is the longest of those. max() and not sum():
+          # whichever suite hangs must panic on its own timeout first, and a sum
+          # would demand a budget the set has never come close to using.
+          while IFS=$'\t' read -r cell topo budget; do
+            case "$topo" in
+              *multi) set -- ${E2E_SUITES_NIGHTLY_MULTI} ;;
+              *)      set -- ${E2E_SUITES_NIGHTLY_SINGLE} ;;
+            esac
+            want=0
+            for suite in "$@"; do
+              t="$(e2e_suite_timeout "$suite")"; t="${t%m}"
+              [ "$t" -gt "$want" ] && want="$t"
+            done
+            if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
+              echo "::error::cell-$cell runs the $topo nightly subset, whose longest suite asks ${want}m, inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN))." >&2
+              bad=1
+            fi
+          done < <(jq -r '.[]|select((.suite//"")=="")|[.cell,.topo,(.timeout_minutes//50)]|@tsv' <<<"$sel")
+          [ "$bad" -eq 0 ]
+
   matrix:
     name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
       matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
+    needs: select
+    if: needs.select.outputs.any == 'true'
     strategy:
       fail-fast: false
       max-parallel: 3
       matrix:
-        include:
-          - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
-          - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
-          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
-
+        include: ${{ fromJSON(needs.select.outputs.cells) }}
     runs-on: [ self-hosted, "${{ matrix.runner }}" ]
-    timeout-minutes: ${{ matrix.extra == 'tofu-examples' && 45 || 35 }}
+    timeout-minutes: ${{ matrix.timeout_minutes || 50 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
       # Job-level env wins over the workflow-level default above, so telemetry
-      # from this cell carries ci.cell/ci.matrix. Nine cells run under one
+      # from this cell carries ci.cell/ci.matrix. Twelve cells run under one
       # run id and every cell names its hosts node1/node2/node3, so without
       # this every cell's telemetry collapses into one unattributable bucket.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
-      # blank or scheduled run = full matrix.
-      # Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2').
-      - name: Gate on Dispatch Input
-        id: gate
-        env:
-          CELL: ${{ matrix.cell }}
-          CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
-        run: |
-          if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then
-            echo "selected=true" >> "$GITHUB_OUTPUT"
-            echo "cell ${CELL} selected"
-          else
-            echo "selected=false" >> "$GITHUB_OUTPUT"
-            echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
-          fi
-
       - name: Clean Workspace
-        if: steps.gate.outputs.selected == 'true'
         run: |
           sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
           if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
-        if: steps.gate.outputs.selected == 'true'
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
@@ -89,7 +151,6 @@ jobs:
           repositories: mulga
 
       - uses: actions/checkout@v7
-        if: steps.gate.outputs.selected == 'true'
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -97,31 +158,47 @@ jobs:
           path: mulga
 
       - uses: actions/checkout@v7
-        if: steps.gate.outputs.selected == 'true'
         with:
           path: spinifex
 
-      # Nine cells share one run id and every cell names its hosts
+      # Twelve cells share one run id and every cell names its hosts
       # node1/node2/node3, so ci.cell above separates them and ci.job_id here
       # links each cell's telemetry to the job log that produced it. Runs
       # before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
       - name: Resolve CI job id
-        if: steps.gate.outputs.selected == 'true'
         uses: ./spinifex/.github/actions/otel-job-id
         with:
           job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
 
       - name: Build Install Artifacts
-        if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
+        env:
+          MATRIX_SUITE: ${{ matrix.suite }}
         run: |
+          # The script's default set is the nightly baseline. A service cell
+          # runs one suite outside it, and asks for that binary by name.
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --extra-suites "${MATRIX_SUITE}"
+
+      # Every runner agent runs on one host, while their nodes sit on several, so
+      # this samples the runner's host and not the node's hypervisor. Kept for
+      # runner-side contention; the node's own capacity is sampled after.
+      - name: Sample runner-host load (before)
+        if: always()
+        run: |
+          mkdir -p "${ARTIFACT_DIR}"
+          {
+            echo "=== before: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
+            echo "loadavg: $(cat /proc/loadavg)"
+            grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
+            echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
+            echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
+          } >> "${ARTIFACT_DIR}/hypervisor-load.txt"
 
       - name: Provision VMs
-        if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -136,14 +213,15 @@ jobs:
             ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
           esac
 
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          # shellcheck disable=SC2086
+          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false
 
       - name: Bootstrap via Binary Install
-        if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_NET: ${{ matrix.net }}
@@ -166,7 +244,7 @@ jobs:
             --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Run Single Node E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
+        if: matrix.topo == 'single' && matrix.extra == '' && matrix.suite == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
@@ -194,7 +272,7 @@ jobs:
              done; exit \$FAIL"
 
       - name: Run NAT Uplink E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
+        if: matrix.extra == 'nat-single'
         working-directory: mulga/scripts/tofu-cluster
         # Skips cleanly (exit 0) when the tested branch predates the natuplink
         # suite so scheduled runs against dev stay green until the merge lands.
@@ -216,7 +294,7 @@ jobs:
              if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
 
       - name: Run Multi Node E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
+        if: matrix.topo == 'real-multi' && matrix.suite == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
@@ -251,8 +329,221 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      # The dedicated suite cells only. Resolved once here rather than in each
+      # step below, which would repeat the same three lines five times.
+      - name: Resolve VM SSH details
+        id: vm
+        if: matrix.suite != '' && matrix.topo == 'single'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          echo "wan_ip=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY/#\~/$HOME}" >> "$GITHUB_OUTPUT"
+
+      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
+      # AMI, so that AMI must be in the VM's image store before the suite runs.
+      # build-system-image.sh customizes with libguestfs (an isolated userspace
+      # appliance touching no host block device), so it is safe on the shared
+      # runner. The build needs source + network (runner only) and the import
+      # needs predastore/NATS (VM only), hence the split.
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: matrix.suite == 'eks'
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
+          make build-eks-node-image
+
+      - name: Import eks-node AMI on VM
+        if: matrix.suite == 'eks'
+        run: |
+          set -euo pipefail
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/eks-node/manifest.conf)"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            /tmp/eks-node-image-build/eks-node-alpine.raw \
+            "$VM":/tmp/eks-node-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/eks-node-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
+
+      # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
+      # `aws eks get-token` -> kubectl -> apiserver -> token-webhook -> gateway
+      # broker -> STS verify path. The e2e binaries run ON the VM, so kubectl
+      # must be on the VM's PATH or harness.NewKubectl skips the subtest and the
+      # cell reports green having never exercised the data plane. Pinned to the
+      # k3s server version (setup.sh K3S_VERSION) and checksum-verified.
+      - name: Install kubectl on VM
+        if: matrix.suite == 'eks'
+        run: |
+          set -euo pipefail
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          ssh "${SSH_OPTS[@]}" tf-user@${{ steps.vm.outputs.wan_ip }} 'bash -s' <<'EOF'
+          set -euo pipefail
+          KVER=v1.32.5
+          case "$(uname -m)" in
+            x86_64)  ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+            *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
+          esac
+          cd /tmp
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
+          echo "$(cat kubectl.sha256)  kubectl" | sha256sum -c -
+          sudo install -m 0755 kubectl /usr/local/bin/kubectl
+          kubectl version --client
+          EOF
+
+      # RegisterContainerInstance boots the container instance from the ecs-node
+      # AMI, and resolveECSNodeAMI matches on the managed-by tag rather than the
+      # name, so the tag is not optional. The image directory is named for
+      # IMAGE_NAME in the manifest (ecs-agent), not for the make target.
+      - name: Build ecs-node AMI (libguestfs, runner)
+        if: matrix.suite == 'ecs'
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
+          make build-ecs-node-image
+
+      - name: Import ecs-node AMI on VM
+        if: matrix.suite == 'ecs'
+        run: |
+          set -euo pipefail
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/ecs-agent/manifest.conf)"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            /tmp/ecs-agent-image-build/ecs-agent-alpine.raw \
+            "$VM":/tmp/ecs-agent-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/ecs-agent-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
+
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
+      # must be in the store before the suite runs. The suite's psql and mariadb
+      # clients need no step here: they run inside a client VM that installs them
+      # from its own user-data, because a private-only endpoint is unreachable
+      # from the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
+        if: matrix.suite == 'rds'
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
+          for IMAGE in rds-postgres rds-mariadb; do
+            make "build-${IMAGE}-image"
+          done
+
+      - name: Import RDS AMIs on VM
+        if: matrix.suite == 'rds'
+        run: |
+          set -euo pipefail
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+          done
+
+      - name: Run Service Suite E2E
+        if: matrix.suite != '' && matrix.topo == 'single'
+        env:
+          SUITE: ${{ matrix.suite }}
+        run: |
+          set -euo pipefail
+          # Same guard as the permutation steps, against the full single-node
+          # set: a cell naming a suite this topology cannot fully exercise fails
+          # here rather than skipping half its assertions.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_SINGLE}"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
+
+          ssh -tt -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \
+            tf-user@${{ steps.vm.outputs.wan_ip }} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             echo \"::group::suite ${SUITE}\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
+             rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+             if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
+             exit \$rc"
+
+      # The multi-node twin of the step above. Separate rather than merged
+      # because the suite needs every node's address to reach the whole cluster,
+      # which the single-node path has no way to supply.
+      - name: Run Multi Suite E2E
+        if: matrix.suite != '' && matrix.topo == 'real-multi'
+        working-directory: mulga/scripts/tofu-cluster
+        env:
+          SUITE: ${{ matrix.suite }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_MULTI}"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
+
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+
+          NODES_CSV=""
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
+          done
+
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             echo \"::group::suite ${SUITE}\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
+             rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+             if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
+             exit \$rc"
+
       - name: Import RDS PostgreSQL AMI
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        if: matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
@@ -264,7 +555,7 @@ jobs:
             "spx admin images import --name spinifex-rds-postgres"
 
       - name: Run Tofu Workbooks
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        if: matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
         # The driver emits go test -v markers per workbook; teeing to
         # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
@@ -285,7 +576,7 @@ jobs:
           exit $rc
 
       - name: Run Reboot Resilience E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
+        if: matrix.extra == 'reboot-single'
         working-directory: mulga/scripts/tofu-cluster
         # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
         # systemctl-reboot-induced ssh disconnect mid-run.
@@ -320,8 +611,46 @@ jobs:
           if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
           exit $rc
 
+      - name: Sample runner-host load (after)
+        if: always()
+        run: |
+          mkdir -p "${ARTIFACT_DIR}"
+          {
+            echo "=== after: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
+            echo "loadavg: $(cat /proc/loadavg)"
+            grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
+            echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
+            echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
+          } >> "${ARTIFACT_DIR}/hypervisor-load.txt"
+
+      # The node, not the runner's host, is what the guests are scheduled on, so
+      # its load is the figure that explains a starved guest. Never fails the
+      # cell: a node already gone is the case this is trying to describe.
+      - name: Sample node load (after)
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          INVENTORY=$(tofu output -json inventory 2>/dev/null)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip' 2>/dev/null)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path' 2>/dev/null)
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          [ -n "$NODE1_IP" ] && [ "$NODE1_IP" != "null" ] || exit 0
+          [ -n "$SSH_KEY" ] && [ "$SSH_KEY" != "null" ] || exit 0
+          {
+            echo "=== node after: $(date -u +%Y-%m-%dT%H:%M:%SZ) ip=${NODE1_IP} ==="
+            ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 \
+              -i "$SSH_KEY" "tf-user@${NODE1_IP}" \
+              'echo "host: $(hostname)"; echo "loadavg: $(cat /proc/loadavg)";
+               grep -E "^(MemTotal|MemAvailable|SwapFree):" /proc/meminfo;
+               echo "cpu(steal is field 8): $(grep "^cpu " /proc/stat)";
+               echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"' 2>&1
+          } >> "${ARTIFACT_DIR}/node-load.txt"
+          exit 0
+
       - name: Collect e2e artifacts
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set +e
@@ -344,6 +673,11 @@ jobs:
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
             ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
               > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
+            # The unit filter above drops kernel and OOM messages, which are
+            # exactly what explains one node behaving unlike its siblings.
+            # Gzipped on the far side; the whole journal is far larger.
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl --no-pager --output=short-iso --since "90 min ago" | gzip -6' \
+              > "${ARTIFACT_DIR}/full-journal-node${idx}.log.gz" 2>/dev/null
             # ovn-controller and ovs-vswitchd log to files, never to journald:
             # without these a cross-chassis black-hole cannot be attributed to
             # IPsec versus SB port binding.
@@ -357,13 +691,13 @@ jobs:
           echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
 
       - name: Install go-junit-report
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         run: |
           go install github.com/jstemmer/go-junit-report/v2@latest
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Convert test logs to JUnit XML
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         run: |
           shopt -s nullglob
           for LOG in "${ARTIFACT_DIR}"/test-*.log; do
@@ -374,7 +708,7 @@ jobs:
       # Tolerated on failure: the sink being unreachable must not turn a
       # green test run red.
       - name: Ship test results to Elasticsearch
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         continue-on-error: true
         uses: ./spinifex/.github/actions/junit-to-es
         with:
@@ -382,7 +716,7 @@ jobs:
           cell: ${{ matrix.cell }}
 
       - name: Analyze e2e failures
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
@@ -390,18 +724,29 @@ jobs:
           title: "E2E failure analysis (cell ${{ matrix.cell }})"
 
       - uses: actions/upload-artifact@v7
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         with:
           name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14
 
+      # The XML on its own, so the roll-up can read every cell's counts without
+      # pulling nineteen journals it has no use for.
+      - name: Upload junit only
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          if-no-files-found: ignore
+          retention-days: 14
+
       # Gated like every other step, not just always(): a cell the dispatch
       # deselected never checked out or ran tofu init, so destroying would fail
       # on missing plugins and report the cell red for correctly doing nothing.
       - name: Teardown
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -412,13 +757,15 @@ jobs:
           case "$MATRIX_HOST" in
             ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
           esac
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          # shellcheck disable=SC2086
+          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
       - name: Upload tofu log
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         uses: actions/upload-artifact@v7
         with:
           name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -448,6 +795,34 @@ jobs:
       otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit
 
+  # The guest disk-performance gate. Same bare-metal node as cell 20 and
+  # therefore queued behind it by the shared concurrency group, which is the
+  # point: perf numbers off a cell still running other suites are noise, and
+  # the availability assertions measure host responsiveness under a single
+  # tenant's load, which a co-tenant correctness suite would confound.
+  #
+  # It reimages the cluster first, like every other caller of that workflow, so
+  # it is a fresh node rather than whatever cell-20 left behind.
+  diskperf:
+    name: cell-28 (baremetal/diskperf)
+    needs: baremetal
+    if: >-
+      !cancelled() && (
+        github.event_name == 'schedule' ||
+        inputs.cells == '' ||
+        contains(format(',{0},', inputs.cells), ',28,')
+      )
+    permissions:
+      contents: read
+      statuses: write
+    uses: ./.github/workflows/e2e-baremetal.yml
+    with:
+      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      cell: "28"
+      suites: diskperf
+      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=28,ci.matrix=baremetal/pxe/diskperf
+    secrets: inherit
+
   # ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
   # on the runner. The nodes are therefore L1, which keeps the instances they
   # launch at L2 and their benchmark numbers comparable between the two cells.
@@ -634,6 +1009,22 @@ jobs:
           EOF
           ./test/test-workload.sh --env /tmp/cell21.env --benchmark
 
+      # The node's own journal, which nothing else preserves: the serial log
+      # stops at login and the harness owns the only node there is. Pulled
+      # before teardown and never fatal, so a passing night keeps it too.
+      - name: Collect node journal
+        if: always() && steps.smoke.outputs.node_ip != ''
+        env:
+          NODE_IP: ${{ steps.smoke.outputs.node_ip }}
+        run: |
+          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
+          timeout 180 ssh -o BatchMode=yes -o StrictHostKeyChecking=no \
+            -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 \
+            -i "$HOME/.ssh/spinifex-key" "spinifex@${NODE_IP}" \
+            'sudo journalctl -b --no-pager --output=short-iso | gzip -6' \
+            > "/tmp/cell21-artifacts-$GITHUB_RUN_ID/node-journal.log.gz" ||
+            echo "::warning::could not pull the node journal from ${NODE_IP}"
+
       - name: Collect logs
         if: always()
         run: |
@@ -697,7 +1088,7 @@ jobs:
     # them together would need 32 GiB on one box.
     needs: [iso-single]
     if: >-
-      always() && (
+      !cancelled() && (
       github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',22,') )
@@ -852,13 +1243,14 @@ jobs:
   # bottlebrush, so "a different label" would not have separated them. needs: on
   # a matrix job waits for every leg, so this starts on a quiet machine.
   #
-  # always() because a failed cell must not silently skip the benchmark and
-  # leave a green-looking night with no number in it.
+  # !cancelled() rather than always(): a failed cell must not silently skip the
+  # benchmark and leave a green-looking night with no number in it, but always()
+  # also overrides cancellation and keeps the run starting jobs after a cancel.
   performance:
     name: cell-23 (predastore/throughput)
     needs: [matrix, baremetal, iso-single, iso-multi]
     if: >-
-      always() && (
+      !cancelled() && (
       github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',23,') )
@@ -970,12 +1362,15 @@ jobs:
     name: cell-24 (predastore/fault-injection)
     needs: [matrix, baremetal, iso-single, iso-multi, performance]
     if: >-
-      always() && (
+      !cancelled() && (
       github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',24,') )
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 30
+    # The thirteen scenarios run serially and are ~35 minutes of work on their
+    # own, measured from the per-scenario predastore matrix, before checkout
+    # and build. 30 could not have passed whatever the disk did.
+    timeout-minutes: 75
     env:
       # Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
       # the frozen node first in its meta read order and pays the full client
@@ -988,12 +1383,20 @@ jobs:
       - name: Clean Workspace
         run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
+      # STRESS_WORK_ROOT is set as well as TMPDIR because the harness ignores
+      # TMPDIR for the scenarios that need the most disk: large-object and
+      # multipart-upload default to $HOME/.cache/predastore-e2e, on the theory
+      # that TMPDIR is commonly tmpfs. Here it is a 938G nvme, so pointing them
+      # at it is both correct and the only path with room.
       - name: Configure work directories
         run: |
           E2E_ROOT=/mnt/nvme/e2e
-          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
-            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          for d in work stress-work; do
+            mkdir -p "$E2E_ROOT/$d" 2>/dev/null ||
+              { sudo mkdir -p "$E2E_ROOT/$d" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/$d"; }
+          done
           echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "STRESS_WORK_ROOT=$E2E_ROOT/stress-work" >> "$GITHUB_ENV"
           echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
 
       - name: Checkout Spinifex
@@ -1021,15 +1424,25 @@ jobs:
       - name: Preflight
         run: |
           set -euo pipefail
-          mkdir -p "$TMPDIR"
-          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-stress.*' \
-               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
-          df -h "$TMPDIR"
-          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
-          if [ "$avail_gb" -lt 10 ]; then
-            echo "::error::$TMPDIR has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark."
-            exit 1
-          fi
+          mkdir -p "$TMPDIR" "$STRESS_WORK_ROOT"
+          for d in "$TMPDIR" "$STRESS_WORK_ROOT" "$HOME/.cache/predastore-e2e"; do
+            [ -d "$d" ] || continue
+            find "$d" -maxdepth 1 -name 'predastore-e2e-stress.*' \
+                 -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          done
+          # Both paths, because they can be different filesystems and the one
+          # holding the large objects is not the one TMPDIR names. large-object
+          # needs 2GiB and 4GiB sharded with parity, multipart 1GiBx8.
+          rc=0
+          for d in "$TMPDIR" "$STRESS_WORK_ROOT"; do
+            df -h "$d"
+            avail_gb=$(df -BG --output=avail "$d" | tail -1 | tr -dc '0-9')
+            if [ "$avail_gb" -lt 40 ]; then
+              echo "::error::$d has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark and room for the 1GiBx8 multipart case."
+              rc=1
+            fi
+          done
+          exit "$rc"
 
       # Leftovers from a run killed before its cleanup trap: a surviving node
       # holds its ports and fails this run somewhere unrelated.
@@ -1059,3 +1472,30 @@ jobs:
         run: |
           pkill -KILL -f '/bin/s3d -config' || true
           rm -rf "$STRESS_RESULTS_ROOT"
+
+  # Reading a nightly meant opening every cell job, and the Actions UI does not
+  # separate a cell that failed from one cancelled out from under it. needs:
+  # every cell so this runs last; always() so a red run is the one it describes.
+  summary:
+    name: run summary
+    needs: [select, matrix, baremetal, diskperf, iso-single, iso-multi, performance, stress]
+    if: always()
+    runs-on: ubuntu-26.04
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: .github/scripts/e2e-run-summary.py
+          sparse-checkout-cone-mode: false
+
+      - name: Collect junit
+        uses: actions/download-artifact@v7
+        with:
+          pattern: junit-cell-*-${{ github.run_id }}
+          path: junit
+        continue-on-error: true
+
+      - name: Write run summary
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          JUNIT_ROOT: junit
+        run: python3 .github/scripts/e2e-run-summary.py

.github/workflows/e2e-suites.yml

@@ -43,6 +43,14 @@ on:
         required: false
         default: ""
 
+# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
+# them provisions on the same runner pools and the same hypervisors, so two in
+# flight compete for their disks and their libvirt guests. Queued rather than
+# cancelled — a run in progress is someone's result, not a stale one.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 permissions:
   contents: read
   # actions: read lets the otel-job-id action look up a provision job's numeric
@@ -143,10 +151,9 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
           e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # Unquoted on purpose: the space-separated list must word-split into
-          # one JSON element per suite.
-          # shellcheck disable=SC2086
-          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          # 45 is the floor this job has always had, so resolving each leg's
+          # budget can only raise it — storagefault and rds ask for more.
+          JSON=$(e2e_suite_matrix "${LIST}" 45)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Reclaim docker build cache
@@ -182,7 +189,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -311,14 +318,16 @@ jobs:
           retention-days: 14
 
   single_suite:
-    name: Single env · suite
+    name: Single env · ${{ matrix.suite }}
     needs: single_provision
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 45
+    # Per leg, from suite-sets.sh: one number for every suite would have to be
+    # storagefault's, and every other leg would then hang for 105 minutes.
+    timeout-minutes: ${{ matrix.timeout }}
     strategy:
       fail-fast: false
       matrix:
-        suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
+        include: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
     env:
       WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
       SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
@@ -343,6 +352,8 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${WAN_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -360,7 +371,7 @@ jobs:
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -516,7 +527,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -627,7 +638,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -669,10 +680,9 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
           e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # Unquoted on purpose: the space-separated list must word-split into
-          # one JSON element per suite.
-          # shellcheck disable=SC2086
-          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          # 60 is the floor this job has always had; multi-node legs carry a
+          # heavier setup than single-node ones, which is why it is higher.
+          JSON=$(e2e_suite_matrix "${LIST}" 60)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Build multinode binaries on runner
@@ -722,14 +732,15 @@ jobs:
           retention-days: 14
 
   multi_suite:
-    name: Multi env · suite
+    name: Multi env · ${{ matrix.suite }}
     needs: multi_provision
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: 60
+    # See the note on single_suite: the budget is the suite's, not the job's.
+    timeout-minutes: ${{ matrix.timeout }}
     strategy:
       fail-fast: false
       matrix:
-        suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
+        include: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
     env:
       NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
       SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
@@ -755,6 +766,8 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${NODE1_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -773,7 +786,7 @@ jobs:
                SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -899,7 +912,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/e2e.yml

@@ -7,6 +7,7 @@ on:
       - dev
     paths:
       - "**/*.go"
+      - "!**/*_test.go"
       - go.mod
       - go.sum
       - Makefile
@@ -23,6 +24,14 @@ on:
         required: false
         default: ""
 
+# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
+# them provisions on the same runner pools and the same hypervisors, so two in
+# flight compete for their disks and their libvirt guests. Queued rather than
+# cancelled — a run in progress is someone's result, not a stale one.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 permissions:
   contents: read
   statuses: write
@@ -40,6 +49,57 @@ env:
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
+  # The inner timeout must fire before the outer one. `go test` reports where it
+  # stopped and the suite's own failure; a job cancelled on timeout reports only
+  # "The operation was canceled" and takes the artifacts with it.
+  #
+  # A job budget cannot be a literal here because the suite set is a dispatch
+  # input: storagefault asks for 90m and diskperf for 180m, both eligible on one
+  # node, so any fixed number is either wrong for them or wasteful for everyone
+  # else. Resolving it from suite-sets.sh keeps the one source of truth and
+  # leaves the default push set at exactly the 60m it has always had.
+  plan:
+    name: Resolve Job Budgets
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    outputs:
+      single_timeout: ${{ steps.budgets.outputs.single_timeout }}
+      multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+          sparse-checkout: .github/scripts
+
+      - name: Resolve Budgets
+        id: budgets
+        env:
+          # Provision, install and teardown around the suites, plus the headroom
+          # a multi-suite job needs for whatever ran before the one that hangs.
+          SETUP_ALLOWANCE_MIN: 15
+          # What the job has always had. Kept as a floor so resolving the budget
+          # can only ever raise it, never quietly shorten an existing run.
+          FLOOR_MIN: 60
+        run: |
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          # Only max() is a sound bound: the suites share one job, so a sum would
+          # demand hours for a set that has never taken them. Max is what makes
+          # the hanging suite panic on its own timeout first, which is the point.
+          budget() {
+            local want="${FLOOR_MIN}" s t
+            for s in $(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "$1"); do
+              t="$(e2e_suite_timeout "${s}")"
+              t=$(( ${t%m} + SETUP_ALLOWANCE_MIN ))
+              [ "${t}" -gt "${want}" ] && want="${t}"
+            done
+            echo "${want}"
+          }
+          single=$(budget "${E2E_SUITES_SINGLE_RE}")
+          multi=$(budget "${E2E_SUITES_MULTI_RE}")
+          echo "single_timeout=${single}" >> "$GITHUB_OUTPUT"
+          echo "multi_timeout=${multi}" >> "$GITHUB_OUTPUT"
+          echo "suites='${SUITES}' -> single ${single}m, multi ${multi}m"
+
   e2e_single:
     name: Single-Node E2E
     # The dispatch check gates on exactly the suites this job's Run step
@@ -51,8 +111,9 @@ jobs:
     # tear down. Push triggers keep running via the null-check on
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
+    needs: plan
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 60
+    timeout-minutes: ${{ fromJSON(needs.plan.outputs.single_timeout) }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -152,7 +213,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -357,6 +418,10 @@ jobs:
           # suite-sets.sh also carries the per-suite rationale.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_SINGLE_RE}" | tr '\n' ' ')
+          # Resolve each suite's timeout here, where suite-sets.sh is readable, and
+          # carry the pairs into the remote loop below — the VM has no checkout.
+          SUITE_TMOS=""
+          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -388,7 +453,7 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
-               case \$SUITE in rds) TMO=50m ;; *) TMO=30m ;; esac; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
                  ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
@@ -520,7 +585,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -549,8 +614,9 @@ jobs:
     # provisions the 3-node env when requested on its own, and the env stays
     # unprovisioned when none of them are asked for.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
+    needs: plan
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: 60
+    timeout-minutes: ${{ fromJSON(needs.plan.outputs.multi_timeout) }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
@@ -646,7 +712,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -772,6 +838,8 @@ jobs:
           # Allow-list, not a deny-list — see the single-node job for why.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_MULTI_RE}" | tr '\n' ' ')
+          SUITE_TMOS=""
+          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -803,13 +871,14 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                  SPINIFEX_NODES='${NODES_CSV}' \
                  SPINIFEX_NODE_IPS='${NODES_CSV}' \
                  SPINIFEX_SSH_USER=tf-user \
                  SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -972,7 +1041,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/image-scripts.yml

@@ -37,3 +37,39 @@ jobs:
 
       - name: Run image script tests + shellcheck
         run: make test-images
+
+  # The mkosi EKS image is what customers run, and until now nothing built it:
+  # the composition existed in mkosi-build.sh but had no caller, so a change
+  # that broke it was only found by hand. Cell-25 still boots the Alpine image,
+  # so this is the only gate the Ubuntu one has.
+  #
+  # Self-hosted because the build needs Docker and several GB of disk, matching
+  # publish-system-ami.yml. Its own concurrency group, not e2e-hypervisors: the
+  # build provisions no libvirt guest, and sharing that group would queue image
+  # builds behind whole nightlies.
+  eks_mkosi_image:
+    name: Build mkosi EKS image
+    runs-on: [self-hosted, ci-single]
+    concurrency:
+      group: eks-mkosi-image-${{ github.ref }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      # The profile binaries (eks-token-webhook, eks-gateway-publish, ...) are
+      # built on the host, not in the builder container, so Go is needed here
+      # even though the toolchain image carries mkosi itself.
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: "go.mod"
+          # Self-hosted: the module + build cache already persists on local
+          # disk, and cache-save would re-upload hundreds of MB for nothing.
+          cache: false
+
+      # stage_profile_binaries() sets GOWORK=off, so the sub-repo resolves its
+      # own go.mod and the sibling monorepo checkouts are not needed here.
+      - name: Build the image
+        run: make build-eks-node-mkosi-image

.github/workflows/release.yml

@@ -148,6 +148,13 @@ jobs:
   verify:
     needs: release
     runs-on: [ self-hosted, ci-single ]
+    # Shared with the e2e workflows: this job provisions on the same runner
+    # pools and the same hypervisors, so two in flight compete for their disks
+    # and their libvirt guests. Job-level, so the build jobs are not queued
+    # behind a nightly that has nothing to do with them.
+    concurrency:
+      group: e2e-hypervisors
+      cancel-in-progress: false
     timeout-minutes: 25
     env:
       TOFU_LOG: /tmp/release-verify-${{ github.run_id }}.log
@@ -174,6 +181,13 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
+      # Only the shared shell helpers: this job builds nothing from source, but
+      # it drives the same libvirt provider the e2e cells retry around.
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: .github/scripts
+          path: spinifex
+
       - name: Download Release Assets
         uses: actions/download-artifact@v8
         with:
@@ -183,11 +197,12 @@ jobs:
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
 
       - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
@@ -200,8 +215,9 @@ jobs:
         if: always()
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
-          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
   baremetal:

.github/workflows/spinifex.yml

@@ -89,8 +89,11 @@ jobs:
       # Build artifacts follow source, so the key rolls per run and falls back to
       # this job's newest entry. Keyed per job because -race, -covermode=atomic
       # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -131,91 +134,26 @@ jobs:
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
-  # In-process integration tier (gateway.SetupRoutes over embedded NATS
-  # JetStream). Its own job so it runs in parallel with the unit tiers rather
-  # than adding to their wall time, and it still blocks a PR — unlike e2e.yml's
-  # "Integration (front gate)" job, which is push-triggered against main/dev on
-  # a self-hosted runner and so never gates a PR before merge.
-  integration_tests:
-    name: Integration Tests
-    runs-on: ubuntu-26.04
+      # Compiled output stays valid whatever the tests said, and skipping the
+      # save on a red run would leave the next one rebuilding it, so this is
+      # gated on the branch rather than on success.
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
 
-    steps:
-      - name: Checkout Spinifex
-        uses: actions/checkout@v7
-        with:
-          path: spinifex
-          fetch-depth: 2
-
-      - name: Resolve sibling refs
-        id: siblings
-        env:
-          BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
-        working-directory: spinifex
-
-      - name: Checkout Viperblock
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/viperblock
-          ref: ${{ steps.siblings.outputs.viperblock }}
-          path: viperblock
-
-      - name: Checkout Predastore
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/predastore
-          ref: ${{ steps.siblings.outputs.predastore }}
-          path: predastore
-
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
-      - name: Setup Go
-        uses: actions/setup-go@v7
-        with:
-          go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
-
-      - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
-
-      - name: Run Integration Tests
-        run: make -C spinifex test-integration
 
   race_detection:
     name: Race Detection
@@ -284,8 +222,11 @@ jobs:
       # Build artifacts follow source, so the key rolls per run and falls back to
       # this job's newest entry. Keyed per job because -race, -covermode=atomic
       # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -297,8 +238,26 @@ jobs:
       - name: Run Tests with Race Detector
         run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+
   lint_and_security:
-    name: Lint and Security
+    name: Lint + Security + Integration
     runs-on: ubuntu-26.04
 
     steps:
@@ -306,6 +265,7 @@ jobs:
         uses: actions/checkout@v7
         with:
           path: spinifex
+          fetch-depth: 2
 
       - name: Resolve sibling refs
         id: siblings
@@ -364,8 +324,11 @@ jobs:
       # Build artifacts follow source, so the key rolls per run and falls back to
       # this job's newest entry. Keyed per job because -race, -covermode=atomic
       # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -382,3 +345,25 @@ jobs:
 
       - name: Govulncheck
         run: make -C spinifex govulncheck
+
+      - name: Run Integration Tests
+        if: ${{ !cancelled() }}
+        run: make -C spinifex test-integration
+
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}

.gitignore

@@ -41,3 +41,4 @@ Thumbs.db
 
 # Test reports
 *-report.txt
+/rds-agent

.golangci.yml

@@ -107,6 +107,7 @@ linters:
         - G116 # bidi chars — bidichk is more thorough
         - G117 # use of net/http serve functions (we handle TLS separately)
         - G703 # path traversal taint analysis — false positives on config file loading
+        - G704 # SSRF taint analysis — false positives on resolver-derived endpoint URLs
         - G705 # XSS taint analysis — false positives on internal error responses
 
     govet:

Makefile

@@ -79,7 +79,7 @@ build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host
 
 build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
 	@echo -e "\n....Building spx-loadgen"
-	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
+	go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -103,6 +103,12 @@ import-eks-node-image: ## Build + register the eks-node AMI (requires a running
 publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
 	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
 
+# The Ubuntu/systemd EKS image, as opposed to build-eks-node-image's Alpine one.
+# Composition lives in mkosi-build.sh's image_profiles table, so the image name
+# is the whole interface here.
+build-eks-node-mkosi-image: ## Build the Ubuntu mkosi EKS node image (server+agent; role at first boot)
+	./scripts/mkosi-build.sh --image spinifex-eks-node
+
 build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
 	$(MAKE) build-system-image IMAGE=ecs-agent
 
@@ -163,9 +169,13 @@ test-build-scripts:
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
+#
+# diskperf/ is here for the same reason: its baseline comparison and probe
+# accounting decide whether the disk-performance gate passes, and that
+# arithmetic has to be covered off-cluster or the gate is trusted untested.
 test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... ./tests/e2e/diskperf/... $(_RACEQ)
 
 # In-process integration tier: the real gateway router against embedded NATS
 # JetStream, with only the daemon-side NATS subjects stubbed.

SECURITY.md

@@ -0,0 +1,24 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please do not open a public issue for a security vulnerability.
+
+Report it privately, either way works:
+
+- [Open a private security advisory](https://github.com/mulgadc/spinifex/security/advisories/new) on this repository.
+- Email <security@mulgadc.com>.
+
+Please include enough for us to reproduce it: the version or commit, the configuration involved, and the steps that trigger it. If you have a proof of concept, include that too.
+
+## What Happens Next
+
+We aim to acknowledge a report within three business days. We will confirm whether we can reproduce it, agree a fix and a disclosure timeline with you, and credit you in the advisory unless you would rather we did not.
+
+## Scope
+
+Spinifex is one part of the [Mulga](https://mulgadc.com) stack. If a vulnerability spans more than one component, report it once against whichever repository you found it in and we will handle the rest.
+
+## Supported Versions
+
+Fixes land on the latest release and on `main`. Older releases are not patched.

build/systemd/spinifex-ui.service

@@ -1,4 +1,4 @@
-# spinifex-unit-version: 1
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target
@@ -24,6 +24,7 @@ MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
 Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
+Environment=SPINIFEX_UI_OCHRE_ENABLED=false
 
 # Filesystem access
 NoNewPrivileges=yes

cmd/aws-model-coverage/main.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/dhcptest/main.go

@@ -24,6 +24,8 @@ import (
 	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/ecr-credential-provider/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 const (

cmd/ecs-agent/credentials/imds.go

@@ -9,9 +9,9 @@ import (
 )
 
 // IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
-// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
-// (which refetches once the real expiry passes; ec2rolecreds also caps
-// Expires to at most an hour out). Safe for concurrent use.
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
+// which refetches once the real expiry passes. A failed refresh is an error
+// here, not an extended expiry. Safe for concurrent use.
 type IMDSProvider struct {
 	cache *aws.CredentialsCache
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -88,6 +88,63 @@ func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	}
 }
 
+// imdsStubFailAfterFirst serves creds on the first credential request and 500s
+// on every one after it, so a test can drive a refresh that fails against a
+// credential the cache has already stored.
+func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
+	t.Helper()
+	var hits atomic.Int32
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits.Add(1) > 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(creds)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+// A refresh that fails must surface as an error. Left to itself the SDK's
+// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
+// key with Expires pushed 5-15 minutes out, so the agent would sign requests
+// with a credential the control plane has already rejected — indefinitely,
+// since every subsequent refresh fails the same way.
+func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
+	srv := imdsStubFailAfterFirst(t, map[string]string{
+		"Code":            "Success",
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Token":           "session",
+		// Already expired, so the next Retrieve must go back to IMDS.
+		"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+	})
+
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	first, err := p.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("first Retrieve: %v", err)
+	}
+
+	got, err := p.Retrieve(context.Background())
+	if err == nil {
+		t.Fatalf("refresh failure was reported as success: %+v", got)
+	}
+	if got.Expiration.After(first.Expiration) {
+		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
+	}
+}
+
 func TestRetrieve_CancelledContext(t *testing.T) {
 	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
 	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")

cmd/ecs-agent/main.go

@@ -17,7 +17,7 @@ import (
 	"os/signal"
 	"syscall"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/eks-gateway-fetch/main.go

@@ -40,7 +40,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 // stagedAddon mirrors the gateway's StagedAddonManifest wire shape; duplicated

cmd/eks-gateway-publish/main.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 const (

cmd/eks-konnectivity-cert/main.go

@@ -38,6 +38,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/eks-token-webhook/cert.go

@@ -15,7 +15,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
+	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
 )
 
 // serverTLSConfig is the webhook's loopback serving TLS config.

cmd/eks-token-webhook/main.go

@@ -27,6 +27,8 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 // config is the webhook's runtime configuration, sourced from the first-boot

cmd/eks-webhook-cert/main.go

@@ -32,6 +32,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/installer/main.go

@@ -5,10 +5,10 @@ import (
 	"os"
 	"strings"
 
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 	"github.com/mulgadc/spinifex/cmd/installer/autoinstall"
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 	"github.com/mulgadc/spinifex/cmd/installer/ui"
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {

cmd/installer/ui/wizard_test.go

@@ -0,0 +1,896 @@
+//test:in-package — the wizard model, its screen constants and the key handlers
+//under test are all unexported, and the tests read model fields directly to
+//assert which screen a key sequence landed on.
+
+package ui
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+)
+
+// namedKeys maps the key names the handlers switch on to their tea.KeyType.
+// Anything not listed is sent as runes.
+var namedKeys = map[string]tea.KeyType{
+	"enter":     tea.KeyEnter,
+	"esc":       tea.KeyEsc,
+	"tab":       tea.KeyTab,
+	"shift+tab": tea.KeyShiftTab,
+	"up":        tea.KeyUp,
+	"down":      tea.KeyDown,
+	"left":      tea.KeyLeft,
+	"right":     tea.KeyRight,
+	" ":         tea.KeySpace,
+	"backspace": tea.KeyBackspace,
+	"ctrl+c":    tea.KeyCtrlC,
+}
+
+// keyMsg builds the tea.KeyMsg for a key name and checks it round-trips, so a
+// mis-built message fails here rather than silently exercising the wrong branch.
+func keyMsg(t *testing.T, key string) tea.KeyMsg {
+	t.Helper()
+	msg := tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(key)}
+	if kt, ok := namedKeys[key]; ok {
+		msg = tea.KeyMsg{Type: kt}
+	}
+	if got := msg.String(); got != key {
+		t.Fatalf("built key %q, which the handlers see as %q", key, got)
+	}
+	return msg
+}
+
+// press feeds keys through Update, which is the only entry point the bubbletea
+// runtime uses, and returns the resulting model.
+func press(t *testing.T, m model, keys ...string) model {
+	t.Helper()
+	for _, k := range keys {
+		next, _ := m.Update(keyMsg(t, k))
+		typed, ok := next.(model)
+		if !ok {
+			t.Fatalf("Update returned %T, not a model", next)
+		}
+		m = typed
+	}
+	return m
+}
+
+// typeText sends a string one rune at a time, the way a terminal does.
+func typeText(t *testing.T, m model, s string) model {
+	t.Helper()
+	for _, r := range s {
+		m = press(t, m, string(r))
+	}
+	return m
+}
+
+func wizardModel(t *testing.T, diskCount, nicCount int) model {
+	t.Helper()
+	disks := make([]install.Disk, 0, diskCount)
+	for i := range diskCount {
+		disks = append(disks, install.Disk{
+			Path:             "/dev/sd" + string(rune('a'+i)),
+			Bytes:            200 << 30,
+			LogicalBlockSize: 512,
+		})
+	}
+	m := newModel(disks, nics(nicCount))
+	m.width, m.height = 100, 40
+	return m
+}
+
+// The wizard has to be completable from the welcome screen using only the keys
+// the help lines advertise, and produce the config the operator entered.
+func TestWizardWalkthroughProducesTheEnteredConfig(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+
+	m = press(t, m, "enter") // welcome → disk
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want the disk screen", m.screen)
+	}
+
+	m = press(t, m, "enter") // preselected disk is valid → erase confirmation
+	if m.screen != screenDiskConfirm {
+		t.Fatalf("screen = %v, want the erase confirmation", m.screen)
+	}
+
+	m = typeText(t, m, "yes")
+	m = press(t, m, "enter") // → network roles
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want the network roles screen", m.screen)
+	}
+
+	m = press(t, m, "down", "down", "down", "enter") // Continue → identity
+	if m.screen != screenIdentity {
+		t.Fatalf("screen = %v, want the identity screen", m.screen)
+	}
+
+	m = typeText(t, m, "node1")
+	m = press(t, m, "enter") // → credentials
+	if m.screen != screenPassword {
+		t.Fatalf("screen = %v, want the credentials screen", m.screen)
+	}
+
+	m = typeText(t, m, "admin@example.com")
+	m = press(t, m, "enter") // email accepted → password
+	m = typeText(t, m, "hunter2hunter2")
+	m = press(t, m, "enter") // → confirm password
+	m = typeText(t, m, "hunter2hunter2")
+	m = press(t, m, "enter") // → final confirmation
+	if m.screen != screenConfirm {
+		t.Fatalf("screen = %v, want the final confirmation (err: %q)", m.screen, m.validationErr)
+	}
+
+	m = press(t, m, "y")
+	if m.screen != screenDone {
+		t.Fatalf("screen = %v, want done", m.screen)
+	}
+	if m.result == nil {
+		t.Fatal("result is nil after confirming")
+	}
+
+	cfg := m.result
+	if cfg.Hostname != "node1" {
+		t.Errorf("Hostname = %q, want node1", cfg.Hostname)
+	}
+	if cfg.Email != "admin@example.com" {
+		t.Errorf("Email = %q, want admin@example.com", cfg.Email)
+	}
+	if cfg.RootPassword != "hunter2hunter2" {
+		t.Errorf("RootPassword = %q, want what was typed", cfg.RootPassword)
+	}
+	if !slices.Equal(cfg.Storage.Paths(), []string{"/dev/sda"}) {
+		t.Errorf("Storage.Paths() = %v, want the preselected disk", cfg.Storage.Paths())
+	}
+	if cfg.WAN.Interface != "eno1" {
+		t.Errorf("WAN interface = %q, want the first NIC", cfg.WAN.Interface)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("the wizard produced a config it would refuse to install: %v", err)
+	}
+}
+
+// Esc has to walk back the way it came, or an operator who overshoots is stuck
+// with a wizard they can only cancel out of.
+func TestEscapeWalksBackThroughTheScreens(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m = press(t, m, "enter") // disk
+	m = press(t, m, "enter") // erase confirmation
+
+	for _, step := range []struct {
+		key  string
+		want screen
+	}{
+		{"esc", screenDisk},
+		{"esc", screenWelcome},
+	} {
+		m = press(t, m, step.key)
+		if m.screen != step.want {
+			t.Fatalf("after esc: screen = %v, want %v", m.screen, step.want)
+		}
+	}
+}
+
+// Ctrl+C is handled before any screen sees the key, so it works everywhere.
+func TestCtrlCCancelsWithAnError(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	next, cmd := m.Update(keyMsg(t, "ctrl+c"))
+	final, ok := next.(model)
+	if !ok {
+		t.Fatalf("Update returned %T", next)
+	}
+	if final.err == nil || !strings.Contains(final.err.Error(), "cancelled") {
+		t.Fatalf("err = %v, want a cancellation", final.err)
+	}
+	if cmd == nil {
+		t.Error("ctrl+c must return tea.Quit")
+	}
+}
+
+// A window resize has to reach the model, since every view lays out against it.
+func TestWindowResizeIsStored(t *testing.T) {
+	m := model{}
+	next, _ := m.Update(tea.WindowSizeMsg{Width: 120, Height: 50})
+	resized, ok := next.(model)
+	if !ok {
+		t.Fatalf("Update returned %T", next)
+	}
+	if resized.width != 120 || resized.height != 50 {
+		t.Errorf("size = %dx%d, want 120x50", resized.width, resized.height)
+	}
+}
+
+// The erase prompt is the last point of no return, so anything other than an
+// explicit "yes" has to keep the operator on it.
+func TestEraseConfirmationRequiresYes(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m = press(t, m, "enter", "enter") // → erase confirmation
+
+	m = typeText(t, m, "no")
+	m = press(t, m, "enter")
+	if m.screen != screenDiskConfirm {
+		t.Fatalf("screen = %v, want to stay on the confirmation", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "yes") {
+		t.Errorf("validationErr = %q, want it to say what to type", m.validationErr)
+	}
+	if !strings.Contains(m.viewDiskConfirm(100), m.validationErr) {
+		t.Error("the confirmation screen must show the error it set")
+	}
+}
+
+// The filesystem selector and the disk list share one screen, so the cursor has
+// to move between them and stop at both ends.
+func TestDiskScreenCursorStaysInRange(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	m = press(t, m, "up", "up")
+	if m.storageCursor != fsRow {
+		t.Errorf("storageCursor = %d, want it clamped to the filesystem row", m.storageCursor)
+	}
+
+	m = press(t, m, "down", "down", "down", "down")
+	if m.storageCursor != len(m.disks) {
+		t.Errorf("storageCursor = %d, want it clamped to the last disk", m.storageCursor)
+	}
+}
+
+// Left/right only cycle the filesystem while the cursor is on that row, and
+// they wrap, so one key reaches every topology.
+func TestFilesystemCyclesOnlyOnItsOwnRow(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	m = press(t, m, "left")
+	if got := m.fsType(); got != install.AllFSTypes[len(install.AllFSTypes)-1] {
+		t.Errorf("fsType = %v, want the cycle to wrap backwards to the last", got)
+	}
+	m = press(t, m, "right")
+	if got := m.fsType(); got != install.AllFSTypes[0] {
+		t.Errorf("fsType = %v, want the cycle to wrap forwards to the first", got)
+	}
+
+	m = press(t, m, "down") // onto the first disk
+	before := m.fsType()
+	m = press(t, m, "left", "right")
+	if m.fsType() != before {
+		t.Errorf("fsType changed to %v while the cursor was on a disk", m.fsType())
+	}
+}
+
+// Space toggles the disk under the cursor, and r cycles its role — the two keys
+// the storage screen exists for.
+func TestDiskScreenSelectionKeys(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	// The second disk is unselected; the first arrives preselected as os.
+	m = press(t, m, "down", "down", " ")
+	if !slices.Contains(m.selected, 1) {
+		t.Fatalf("selected = %v, want the second disk added", m.selected)
+	}
+
+	roleBefore := m.roleOf[slices.Index(m.selected, 1)]
+	m = press(t, m, "r")
+	if got := m.roleOf[slices.Index(m.selected, 1)]; got == roleBefore {
+		t.Errorf("role stayed %q after r", got)
+	}
+
+	m = press(t, m, " ")
+	if slices.Contains(m.selected, 1) {
+		t.Fatalf("selected = %v, want the second disk removed again", m.selected)
+	}
+	if len(m.roleOf) != len(m.selected) {
+		t.Errorf("roleOf (%d) and selected (%d) drifted", len(m.roleOf), len(m.selected))
+	}
+}
+
+// A selection that cannot be installed has to be reported on the screen the
+// operator can still fix it on, not part-way through the install.
+func TestDiskScreenRefusesAnInvalidSelection(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1) // needs 2 disks
+
+	m = press(t, m, "enter")
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want to stay on the disk screen", m.screen)
+	}
+	if m.validationErr == "" {
+		t.Fatal("an invalid selection must set a validation error")
+	}
+	if !strings.Contains(m.viewDisk(100), m.validationErr) {
+		t.Error("the disk screen must render the error it set")
+	}
+
+	// Selecting the second disk satisfies the mirror and clears the way.
+	m = press(t, m, "down", "down", " ", "enter")
+	if m.screen != screenDiskConfirm {
+		t.Fatalf("screen = %v, want to advance once the mirror has two disks", m.screen)
+	}
+}
+
+// The ZFS options screen is only reachable on a pool, since its settings have
+// no meaning on ext4.
+func TestZFSOptionsScreenIsZFSOnly(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	m = press(t, m, "a")
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want a to do nothing on ext4", m.screen)
+	}
+
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	m = press(t, m, "a")
+	if m.screen != screenZFSOptions {
+		t.Fatalf("screen = %v, want the ZFS options screen", m.screen)
+	}
+}
+
+// Every option defaults to auto and cycles through its own choice list, with
+// the cursor clamped to the rows that exist.
+func TestZFSOptionsCycleTheirValues(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenZFSOptions
+
+	if got := zfsOptions[0].value(&m.zfs); got != "auto" {
+		t.Errorf("ashift starts at %q, want auto", got)
+	}
+	m = press(t, m, "right")
+	if got := zfsOptions[0].value(&m.zfs); got == "auto" {
+		t.Error("right must move ashift off auto")
+	}
+	m = press(t, m, "left")
+	if got := zfsOptions[0].value(&m.zfs); got != "auto" {
+		t.Errorf("ashift = %q after cycling back, want auto", got)
+	}
+
+	m = press(t, m, "up", "up")
+	if m.zfsCursor != 0 {
+		t.Errorf("zfsCursor = %d, want it clamped to the first row", m.zfsCursor)
+	}
+	for range len(zfsOptions) + 2 {
+		m = press(t, m, "down")
+	}
+	if m.zfsCursor != len(zfsOptions)-1 {
+		t.Errorf("zfsCursor = %d, want it clamped to the last row", m.zfsCursor)
+	}
+
+	// The compression choices are strings rather than ints, so they exercise
+	// the other half of cycle.
+	m.zfsCursor = 1
+	m = press(t, m, "right")
+	if m.zfs.Compress == "" {
+		t.Error("right must set a compression algorithm")
+	}
+
+	m = press(t, m, "enter")
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want enter to return to the disk screen", m.screen)
+	}
+}
+
+// The screen has to show what each option is currently set to, since that is
+// the only place the pool's geometry is stated before it is created.
+func TestZFSOptionsViewShowsValuesAndTheFocusedNote(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenZFSOptions
+	m.zfs.Compress = "zstd"
+
+	got := m.viewZFSOptions(100)
+	for _, want := range []string{"ashift", "auto", "compression", "zstd", zfsOptions[0].note} {
+		if !strings.Contains(got, want) {
+			t.Errorf("ZFS options screen is missing %q:\n%s", want, got)
+		}
+	}
+	if strings.Contains(got, zfsOptions[1].note) {
+		t.Error("only the focused row's note should be shown")
+	}
+}
+
+// Credentials are three fields on one screen, so focus has to cycle both ways
+// and land somewhere sensible after a validation failure.
+func TestCredentialsFocusCyclesBothWays(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenPassword
+	m = m.setCredsFocus(0)
+
+	m = press(t, m, "tab")
+	if m.credsFocus != 1 || !m.passwordInput.Focused() {
+		t.Fatalf("credsFocus = %d, want the password field focused", m.credsFocus)
+	}
+	m = press(t, m, "tab", "tab")
+	if m.credsFocus != 0 || !m.emailInput.Focused() {
+		t.Fatalf("credsFocus = %d, want the cycle to wrap to email", m.credsFocus)
+	}
+	m = press(t, m, "shift+tab")
+	if m.credsFocus != 2 || !m.passwordConfirmInput.Focused() {
+		t.Fatalf("credsFocus = %d, want shift+tab to wrap backwards to confirm", m.credsFocus)
+	}
+}
+
+// Each credential rule has to stop the wizard on the screen and put the cursor
+// on the field at fault, not just refuse to advance.
+func TestCredentialValidation(t *testing.T) {
+	tests := []struct {
+		name       string
+		email      string
+		password   string
+		confirm    string
+		wantErr    string
+		wantFocus  int
+		wantScreen screen
+	}{
+		{
+			name: "rejects a malformed email", email: "not-an-email",
+			password: "hunter2hunter2", confirm: "hunter2hunter2",
+			wantErr: "email", wantFocus: 0, wantScreen: screenPassword,
+		},
+		{
+			name: "rejects an empty password", email: "admin@example.com",
+			wantErr: "Password is required", wantFocus: 1, wantScreen: screenPassword,
+		},
+		{
+			name: "rejects a mismatch", email: "admin@example.com",
+			password: "hunter2hunter2", confirm: "hunter2hunter3",
+			wantErr: "do not match", wantFocus: 2, wantScreen: screenPassword,
+		},
+		{
+			name: "accepts a matching pair", email: "admin@example.com",
+			password: "hunter2hunter2", confirm: "hunter2hunter2",
+			wantScreen: screenConfirm,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := wizardModel(t, 1, 1)
+			m.screen = screenPassword
+			m.emailInput.SetValue(tt.email)
+			m.passwordInput.SetValue(tt.password)
+			m.passwordConfirmInput.SetValue(tt.confirm)
+			m = m.setCredsFocus(2) // enter on confirm runs every rule
+
+			m = press(t, m, "enter")
+
+			if m.screen != tt.wantScreen {
+				t.Fatalf("screen = %v, want %v (err %q)", m.screen, tt.wantScreen, m.validationErr)
+			}
+			if tt.wantErr == "" {
+				if m.validationErr != "" {
+					t.Fatalf("validationErr = %q, want none", m.validationErr)
+				}
+				return
+			}
+			if !strings.Contains(strings.ToLower(m.validationErr), strings.ToLower(tt.wantErr)) {
+				t.Errorf("validationErr = %q, want it to mention %q", m.validationErr, tt.wantErr)
+			}
+			if m.credsFocus != tt.wantFocus {
+				t.Errorf("credsFocus = %d, want the cursor on field %d", m.credsFocus, tt.wantFocus)
+			}
+			if !strings.Contains(m.viewPassword(100), m.validationErr) {
+				t.Error("the credentials screen must render the error it set")
+			}
+		})
+	}
+}
+
+// The hostname ends up in the config and in DNS, so an empty one has to be
+// caught on the screen rather than installed.
+func TestHostnameIsRequired(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenIdentity
+	m.hostnameInput.Focus()
+
+	m = press(t, m, "enter")
+	if m.screen != screenIdentity {
+		t.Fatalf("screen = %v, want to stay on identity", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "Hostname") {
+		t.Errorf("validationErr = %q, want it to name the field", m.validationErr)
+	}
+	if !m.hostnameInput.Focused() {
+		t.Error("the hostname field must keep focus after a failed submit")
+	}
+	if !strings.Contains(m.viewIdentity(100), m.validationErr) {
+		t.Error("the identity screen must render the error it set")
+	}
+}
+
+// Tab order follows the visible fields, which change with the addressing mode,
+// so it has to be derived from them rather than fixed.
+func TestRoleEditorTabsThroughVisibleFieldsOnly(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roleCursor = 0 // wan, DHCP by default
+
+	m = press(t, m, "tab")
+	if m.roles[0].focus != roleFieldMethod {
+		t.Fatalf("focus = %v, want the IP method field", m.roles[0].focus)
+	}
+	m = press(t, m, "tab")
+	if m.roles[0].focus != roleFieldNIC {
+		t.Fatalf("focus = %v, want the DHCP form to wrap after two fields", m.roles[0].focus)
+	}
+
+	// Switching to static reveals the address fields, and tab must reach them.
+	m = press(t, m, "tab", "right")
+	if m.roles[0].dhcp {
+		t.Fatal("right on the method field must select static addressing")
+	}
+	m = press(t, m, "tab")
+	if m.roles[0].focus != roleFieldIP || !m.roles[0].ip.Focused() {
+		t.Fatalf("focus = %v, want the IP field focused", m.roles[0].focus)
+	}
+	m = press(t, m, "shift+tab")
+	if m.roles[0].focus != roleFieldMethod {
+		t.Fatalf("focus = %v, want shift+tab to go back", m.roles[0].focus)
+	}
+}
+
+// A focused text field has to receive typed runes, since that is the only way
+// an address is entered.
+func TestRoleEditorTypesIntoTheFocusedField(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roles[0].dhcp = false
+	m.roles[0].focus = roleFieldIP
+	m.roles[0].focusCurrent()
+
+	m = typeText(t, m, "192.168.1.10")
+	if got := m.roles[0].ip.Value(); got != "192.168.1.10" {
+		t.Fatalf("ip = %q, want what was typed", got)
+	}
+	if got := m.roles[0].mask.Value(); got != "" {
+		t.Errorf("mask = %q, want the keys to reach only the focused field", got)
+	}
+}
+
+// The interface selector wraps and offers the fold option on lan and vpc only,
+// so a single-NIC node stays configurable with one key.
+func TestNICSelectorWrapsAndFoldsWhereAllowed(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+
+	// wan cannot fold, so it wraps across the real NICs only.
+	m.roleCursor = 0
+	m = press(t, m, "left")
+	if m.roles[0].nic != len(m.nics)-1 {
+		t.Fatalf("wan nic = %d, want it to wrap to the last NIC", m.roles[0].nic)
+	}
+	m = press(t, m, "right")
+	if m.roles[0].nic != 0 {
+		t.Fatalf("wan nic = %d, want it to wrap back to the first", m.roles[0].nic)
+	}
+
+	// lan may fold, so the cycle includes the unbound position.
+	m.roleCursor = 1
+	m.roles[1].nic = 0
+	m = press(t, m, "left")
+	if m.roles[1].nic != foldedNIC {
+		t.Fatalf("lan nic = %d, want the fold position", m.roles[1].nic)
+	}
+	if !m.roles[1].folded() {
+		t.Error("the lan role should read as folded")
+	}
+}
+
+// Per-role validation runs on Enter and keeps the operator on the editor, since
+// an invalid address here fails the install after the disk is partitioned.
+func TestRoleEditorValidatesBeforeLeaving(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roleCursor = 0
+	m.roles[0].dhcp = false
+	m.roles[0].ip.SetValue("not-an-ip")
+
+	m = press(t, m, "enter")
+	if m.screen != screenNetworkRole {
+		t.Fatalf("screen = %v, want to stay on the role editor", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "IP address") {
+		t.Errorf("validationErr = %q, want it to name the field", m.validationErr)
+	}
+	if !strings.Contains(m.viewNetworkRole(100), m.validationErr) {
+		t.Error("the role editor must render the error it set")
+	}
+
+	// A prefix length is rejected here because netgen parses the mask with
+	// net.ParseIP and would fail the install itself.
+	m.roles[0].ip.SetValue("192.168.1.10")
+	m.roles[0].mask.SetValue("24")
+	m = press(t, m, "enter")
+	if !strings.Contains(m.validationErr, "dotted-decimal") {
+		t.Errorf("validationErr = %q, want the mask rule", m.validationErr)
+	}
+
+	m.roles[0].mask.SetValue("255.255.255.0")
+	m.roles[0].gateway.SetValue("192.168.1.1")
+	m = press(t, m, "enter")
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want a valid role to return to the overview (err %q)", m.screen, m.validationErr)
+	}
+}
+
+// Esc abandons the editor without validating, so a half-typed address is not a
+// trap the operator has to finish before backing out.
+func TestRoleEditorEscapeSkipsValidation(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roles[0].dhcp = false
+	m.roles[0].ip.SetValue("192.")
+
+	m = press(t, m, "esc")
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want the overview", m.screen)
+	}
+	if m.validationErr != "" {
+		t.Errorf("validationErr = %q, want esc to clear it", m.validationErr)
+	}
+}
+
+// The editor shows only the fields that apply, so the DHCP form stays two lines
+// and Advanced is what reveals VLAN and MTU.
+func TestRoleEditorViewShowsTheApplicableFields(t *testing.T) {
+	m := wizardModel(t, 1, 3)
+	m.screen = screenNetworkRole
+	m.roleCursor = 0
+
+	dhcpView := m.viewNetworkRole(100)
+	if !strings.Contains(dhcpView, "DHCP") || !strings.Contains(dhcpView, "WAN") {
+		t.Errorf("DHCP view is missing the plane or the method:\n%s", dhcpView)
+	}
+	for _, hidden := range []string{"IP address", "Subnet mask", "VLAN id"} {
+		if strings.Contains(dhcpView, hidden) {
+			t.Errorf("DHCP view should not show %q:\n%s", hidden, dhcpView)
+		}
+	}
+
+	m.roles[0].dhcp = false
+	m.advanced = true
+	staticView := m.viewNetworkRole(100)
+	for _, want := range []string{"IP address", "Subnet mask", "Gateway", "DNS", "VLAN id", "MTU"} {
+		if !strings.Contains(staticView, want) {
+			t.Errorf("static advanced view is missing %q:\n%s", want, staticView)
+		}
+	}
+
+	// Gateway and DNS belong to the plane holding the default route, so a
+	// static lan must not offer them.
+	m.roleCursor = 1
+	m.roles[1].nic = 1
+	m.roles[1].dhcp = false
+	lanView := m.viewNetworkRole(100)
+	for _, hidden := range []string{"Gateway", "DNS"} {
+		if strings.Contains(lanView, hidden) {
+			t.Errorf("static lan view should not show %q:\n%s", hidden, lanView)
+		}
+	}
+}
+
+// A folded role names the plane it landed on, on both screens, so the operator
+// can see where the traffic actually goes before committing.
+func TestFoldedRolesAreNamedNotBlank(t *testing.T) {
+	m := wizardModel(t, 1, 1) // one NIC folds lan and vpc onto wan
+
+	overview := m.viewNetworkRoles(100)
+	if !strings.Contains(overview, "folds onto") {
+		t.Errorf("overview does not state where the folded roles land:\n%s", overview)
+	}
+
+	m.screen = screenNetworkRole
+	m.roleCursor = 2
+	if got := m.viewNetworkRole(100); !strings.Contains(got, "fold onto") {
+		t.Errorf("role editor does not offer the fold plainly:\n%s", got)
+	}
+
+	m.screen = screenConfirm
+	if got := m.viewConfirm(100); !strings.Contains(got, "folded onto") {
+		t.Errorf("confirmation does not state the folded planes:\n%s", got)
+	}
+}
+
+// Continue is refused while the config would fail, and the message is the one
+// install.Config.Validate produced.
+func TestRolesOverviewRefusesAnInvalidConfig(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRoles
+	m.roles[1].nic = 0 // lan on the same NIC as wan, no VLAN to separate them
+	m.roleCursor = continueRow
+
+	m = press(t, m, "enter")
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want to stay on the overview", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "VLAN") {
+		t.Errorf("validationErr = %q, want the shared-NIC rule", m.validationErr)
+	}
+	// The message is wrapped to the box, so match its opening rather than
+	// the whole line.
+	if !strings.Contains(m.viewNetworkRoles(100), "both bind eno1") {
+		t.Error("the overview must render the error it set")
+	}
+
+	// a toggles Advanced for the whole node rather than per role.
+	m = press(t, m, "a")
+	if !m.advanced {
+		t.Error("a must turn Advanced on")
+	}
+	if !strings.Contains(m.viewNetworkRoles(100), "Advanced: on") {
+		t.Error("the overview must show Advanced is on")
+	}
+}
+
+// The final screen is the last chance to catch a wrong disk or address, so the
+// summary has to carry what was entered.
+func TestConfirmScreenSummarisesTheEnteredConfig(t *testing.T) {
+	m := wizardModel(t, 1, 3)
+	m.screen = screenConfirm
+	m.hostnameInput.SetValue("node7")
+	m.roles[1].dhcp = false
+	m.roles[1].ip.SetValue("10.0.0.3")
+	m.roles[1].mask.SetValue("255.255.255.0")
+
+	got := m.viewConfirm(100)
+	for _, want := range []string{"node7", "/dev/sda", "ext4", "eno1", "10.0.0.3/255.255.255.0", "br-lan", "will erase"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("confirmation is missing %q:\n%s", want, got)
+		}
+	}
+}
+
+// n on the confirmation cancels rather than installing.
+func TestConfirmScreenCancelsOnN(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenConfirm
+
+	m = press(t, m, "n")
+	if m.err == nil {
+		t.Fatal("n must set the cancellation error")
+	}
+	if m.result != nil {
+		t.Error("a cancelled install must produce no config")
+	}
+}
+
+// View dispatches on the screen, so each one has to render its own content
+// rather than falling through to an empty frame.
+func TestViewRendersEveryScreen(t *testing.T) {
+	tests := []struct {
+		screen screen
+		want   string
+	}{
+		{screenWelcome, "Enter to begin"},
+		{screenDisk, "Select Installation Disk"},
+		{screenZFSOptions, "ZFS Options"},
+		{screenDiskConfirm, "Confirm Disk Erasure"},
+		{screenNetworkRoles, "Network Planes"},
+		{screenNetworkRole, "Configure WAN plane"},
+		{screenIdentity, "Node Identity"},
+		{screenPassword, "Administrator Credentials"},
+		{screenConfirm, "Confirm Installation"},
+		{screenDone, "Installation complete"},
+	}
+
+	for _, tt := range tests {
+		m := wizardModel(t, 1, 2)
+		m.screen = tt.screen
+		if got := m.View(); !strings.Contains(got, tt.want) {
+			t.Errorf("screen %v does not render %q:\n%s", tt.screen, tt.want, got)
+		}
+	}
+}
+
+// The disk screen has to state what the selection actually builds, since the
+// cost of the redundancy choice is otherwise invisible until after the install.
+func TestDiskViewStatesTheResultingGeometry(t *testing.T) {
+	m := wizardModel(t, 3, 1)
+	m.screen = screenDisk
+
+	// ext4 with a role per drive: the mountpoints are the summary that matters.
+	m = press(t, m, "down", "down", " ", "down", " ")
+	if got := m.viewDisk(100); !strings.Contains(got, "/var/lib/spinifex") {
+		t.Errorf("ext4 view does not state the role layout:\n%s", got)
+	}
+	// Matched spares are suggested, never applied.
+	if got := m.viewDisk(100); !strings.Contains(got, "survive a disk failure") {
+		t.Errorf("view does not suggest the pool the matched disks allow:\n%s", got)
+	}
+
+	// A mirror with too few disks quotes the requirement, not a capacity.
+	m = wizardModel(t, 3, 1)
+	m.screen = screenDisk
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	if got := m.viewDisk(100); !strings.Contains(got, "1 selected") {
+		t.Errorf("view does not state what the mirror is missing:\n%s", got)
+	}
+
+	// Once buildable it quotes usable capacity and the failures tolerated.
+	m = press(t, m, "down", "down", " ")
+	got := m.viewDisk(100)
+	if !strings.Contains(got, "usable") || !strings.Contains(got, "survives 1 disk failure") {
+		t.Errorf("view does not quote the pool geometry:\n%s", got)
+	}
+	if !strings.Contains(got, "A for ZFS options") {
+		t.Errorf("the ZFS help line should replace the role key:\n%s", got)
+	}
+}
+
+// The overview names the card behind each port, which is what distinguishes the
+// 25gbe uplink from the onboard 1gbe.
+func TestRolesOverviewNamesTheHardware(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.nics[0].AltName = "enp1s0f0"
+	m.advanced = true
+
+	got := m.viewNetworkRoles(100)
+	for _, want := range []string{"eno1", "25Gbps", "Mellanox", "enp1s0f0", "VLAN", "MTU"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("overview is missing %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestHumanBytes(t *testing.T) {
+	tests := []struct {
+		bytes int64
+		want  string
+	}{
+		{512, "512B"},
+		{2 << 20, "2.0M"},
+		{3 << 30, "3.0G"},
+		{2 << 40, "2.0T"},
+	}
+	for _, tt := range tests {
+		if got := humanBytes(tt.bytes); got != tt.want {
+			t.Errorf("humanBytes(%d) = %q, want %q", tt.bytes, got, tt.want)
+		}
+	}
+}
+
+// The overview writes an address the way an operator does, and falls back to
+// the raw mask rather than dropping it when it is not dotted-decimal.
+func TestFormatCIDR(t *testing.T) {
+	tests := []struct {
+		name       string
+		addr, mask string
+		want       string
+	}{
+		{"dotted mask becomes a prefix", "10.0.0.3", "255.255.255.0", "10.0.0.3/24"},
+		{"short mask keeps its prefix", "10.0.0.3", "255.255.0.0", "10.0.0.3/16"},
+		{"no mask yet", "10.0.0.3", "", "10.0.0.3"},
+		{"no address is blank", "", "255.255.255.0", ""},
+		{"unparseable mask is shown as typed", "10.0.0.3", "24", "10.0.0.3/24"},
+		{"whitespace is trimmed", " 10.0.0.3 ", " 255.255.255.0 ", "10.0.0.3/24"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := formatCIDR(tt.addr, tt.mask); got != tt.want {
+				t.Errorf("formatCIDR(%q, %q) = %q, want %q", tt.addr, tt.mask, got, tt.want)
+			}
+		})
+	}
+}
+
+// Messages that are not keys reach the focused input, which is what keeps the
+// cursor blinking on the two single-field screens.
+func TestNonKeyMessagesReachTheActiveInput(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenIdentity
+	m.hostnameInput.Focus()
+
+	next, _ := m.Update(struct{ tea.Msg }{})
+	if _, ok := next.(model); !ok {
+		t.Fatalf("Update returned %T", next)
+	}
+
+	m.screen = screenWelcome
+	if _, cmd := m.Update(struct{ tea.Msg }{}); cmd != nil {
+		t.Error("a screen with no input must issue no command")
+	}
+}

cmd/lb-agent/main.go

@@ -21,7 +21,7 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/rds-agent/command.go

@@ -50,7 +50,13 @@ func newCommandRegistry(engine engineOps, storage storageOps) commandRegistry {
 			if err != nil {
 				return "", err
 			}
-			return "", engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold)
+			if err := engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold); err != nil {
+				return "", err
+			}
+			// After the hold, so what is flushed is the engine's checkpointed
+			// state. Its own files are not the risk — those it repairs from the
+			// WAL — but the parameter file beside them it never opens.
+			return "", storage.SyncDataMount(ctx)
 		},
 		handlers_rds.CommandUnquiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
 			return "", engine.Unquiesce(ctx)

cmd/rds-agent/engine.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"os/user"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -107,6 +108,10 @@ var engineLayouts = map[string]engineLayout{
 		service:   "postgresql",
 		dataMount: "/var/lib/postgresql",
 		port:      5432,
+		// logfile in the image's /etc/conf.d/postgresql. The postmaster states
+		// which setting it refused there and nowhere else — the console shows only
+		// that pg_ctl could not start it, on a guest with no SSH.
+		errorLog:  "/var/log/postgresql/postmaster.log",
 		newProbe:  newPostgresProbe,
 		newEngine: newPostgresEngineFromCatalog,
 	},
@@ -131,6 +136,61 @@ var engineLayouts = map[string]engineLayout{
 	},
 }
 
+// How much of the engine's error log a probe reason carries. The server names
+// what it refused on in its last few lines, and the reason reaches a customer
+// through StatusInfos, so this quotes the tail rather than the whole file.
+const (
+	engineErrorLogTailBytes = 4096
+	engineErrorLogTailLines = 4
+)
+
+// Appends the last few lines of the engine's error log to reason. An unset path,
+// an unreadable file or an empty one leaves reason alone: the probe's own
+// statement is still true, and a guessed cause would be worse than none.
+func withEngineLogTail(reason, path string) string {
+	tail := engineLogTail(path)
+	if tail == "" {
+		return reason
+	}
+	return reason + "; the engine's log ends: " + tail
+}
+
+func engineLogTail(path string) string {
+	if path == "" {
+		return ""
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	offset := max(info.Size()-engineErrorLogTailBytes, 0)
+	buf := make([]byte, info.Size()-offset)
+	if _, err := f.ReadAt(buf, offset); err != nil {
+		return ""
+	}
+
+	lines := strings.Split(string(buf), "\n")
+	// A read that started mid-file almost certainly started mid-line, and half a
+	// line is more misleading than one fewer.
+	if offset > 0 && len(lines) > 0 {
+		lines = lines[1:]
+	}
+	kept := make([]string, 0, engineErrorLogTailLines)
+	for i := len(lines) - 1; i >= 0 && len(kept) < engineErrorLogTailLines; i-- {
+		if line := strings.TrimSpace(lines[i]); line != "" {
+			kept = append(kept, line)
+		}
+	}
+	slices.Reverse(kept)
+	return strings.Join(kept, " | ")
+}
+
 // Builds the health probe the image's engine supplies.
 func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
 	layout, err := layoutFor(cfg)

cmd/rds-agent/engine_log_test.go

@@ -148,7 +148,7 @@ func TestEngineLogTail(t *testing.T) {
 		},
 		{
 			name:        "drops a line the byte cap cut in half",
-			body:        strings.Repeat("x", mariadbErrorLogTailBytes) + "\nlast\n",
+			body:        strings.Repeat("x", engineErrorLogTailBytes) + "\nlast\n",
 			want:        "last",
 			wantMissing: "x",
 		},
@@ -209,3 +209,54 @@ func TestWithEngineLogTailLeavesReasonAloneWithoutALog(t *testing.T) {
 		t.Errorf("withEngineLogTail() = %q, want the reason unchanged", got)
 	}
 }
+
+const postgresConfD = "../../scripts/images/rds-postgres/postgresql.confd"
+
+// "engine did not respond" is where a postmaster that refused a setting lands,
+// and it is the same message as an engine that never got as far as trying. Only
+// the server's own log separates them, and the guest has no other way out.
+func TestPostgresProbe_QuotesTheEngineLogWhenTheEngineIsAbsent(t *testing.T) {
+	log := writeEngineLog(t, strings.Join([]string{
+		`2026-09-06 10:20:33 UTC [712] LOG:  starting PostgreSQL 18.0`,
+		`2026-09-06 10:20:33 UTC [712] FATAL:  could not map anonymous shared memory`,
+		"",
+	}, "\n"))
+
+	state, message := postgresProbeState("/run/postgresql", log,
+		func(context.Context, string, ...string) (int, string, error) { return 2, "", nil })(t.Context(), 5432)
+
+	if state != engineAbsent {
+		t.Errorf("state = %v, want absent", state)
+	}
+	if !strings.Contains(message, "could not map anonymous shared memory") {
+		t.Errorf("message = %q, want the postmaster's own refusal quoted", message)
+	}
+}
+
+func TestPostgresProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
+	state, message := postgresProbeState("/run/postgresql", filepath.Join(t.TempDir(), "nope.log"),
+		func(context.Context, string, ...string) (int, string, error) { return 2, "", nil })(t.Context(), 5432)
+
+	if state != engineAbsent {
+		t.Errorf("state = %v, want absent", state)
+	}
+	if message != "engine did not respond on /run/postgresql:5432" {
+		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
+	}
+}
+
+// The layout names a path the image has to be directing the postmaster to, or
+// the tail above quotes nothing on the one failure it exists for.
+func TestPostgresErrorLogPathMatchesTheImage(t *testing.T) {
+	raw, err := os.ReadFile(postgresConfD)
+	if err != nil {
+		t.Fatalf("read postgresql.confd: %v", err)
+	}
+	m := regexp.MustCompile(`(?m)^logfile="(.*)"$`).FindStringSubmatch(string(raw))
+	if m == nil {
+		t.Fatal("postgresql.confd has no logfile assignment")
+	}
+	if got := engineLayouts[enginePostgres].errorLog; got != m[1] {
+		t.Errorf("layout errorLog = %q, but the image sets logfile = %q", got, m[1])
+	}
+}

cmd/rds-agent/engine_mariadb.go

@@ -100,11 +100,6 @@ const (
 	// rds-init keeps for the platform and never hands to the customer.
 	mariadbSuperuser                  = "root"
 	mariadbProbeConnectTimeoutSeconds = 3
-	// How much of the engine's error log a probe reason carries. The server names
-	// what it refused on in its last few lines, and the reason reaches a customer
-	// through StatusInfos, so this quotes the tail rather than the whole file.
-	mariadbErrorLogTailBytes = 4096
-	mariadbErrorLogTailLines = 4
 	// How much of the probe client's stderr a reason carries, on the same grounds
 	// as the log tail above.
 	mariadbProbeStderrMaxBytes = 512
@@ -280,53 +275,6 @@ func collapseProbeStderr(stderr string) string {
 	return line
 }
 
-// Appends the last few lines of the engine's error log to reason. An unset path,
-// an unreadable file or an empty one leaves reason alone: the probe's own
-// statement is still true, and a guessed cause would be worse than none.
-func withEngineLogTail(reason, path string) string {
-	tail := engineLogTail(path)
-	if tail == "" {
-		return reason
-	}
-	return reason + "; the engine's log ends: " + tail
-}
-
-func engineLogTail(path string) string {
-	if path == "" {
-		return ""
-	}
-	f, err := os.Open(path)
-	if err != nil {
-		return ""
-	}
-	defer func() { _ = f.Close() }()
-
-	info, err := f.Stat()
-	if err != nil {
-		return ""
-	}
-	offset := max(info.Size()-mariadbErrorLogTailBytes, 0)
-	buf := make([]byte, info.Size()-offset)
-	if _, err := f.ReadAt(buf, offset); err != nil {
-		return ""
-	}
-
-	lines := strings.Split(string(buf), "\n")
-	// A read that started mid-file almost certainly started mid-line, and half a
-	// line is more misleading than one fewer.
-	if offset > 0 && len(lines) > 0 {
-		lines = lines[1:]
-	}
-	kept := make([]string, 0, mariadbErrorLogTailLines)
-	for i := len(lines) - 1; i >= 0 && len(kept) < mariadbErrorLogTailLines; i-- {
-		if line := strings.TrimSpace(lines[i]); line != "" {
-			kept = append(kept, line)
-		}
-	}
-	slices.Reverse(kept)
-	return strings.Join(kept, " | ")
-}
-
 func readPidFile(path string) (int, error) {
 	raw, err := os.ReadFile(path)
 	if err != nil {

cmd/rds-agent/engine_postgres.go

@@ -47,10 +47,10 @@ const postgresProbeBinary = "pg_isready"
 // enforcement rule — libpq reads a host beginning with / as a socket directory,
 // and the generated `local ... peer` line covers it.
 func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
-	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, run))
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, cfg.EngineErrorLog, run))
 }
 
-func postgresProbeState(host string, run probeRunner) probeStateFn {
+func postgresProbeState(host, errorLog string, run probeRunner) probeStateFn {
 	return func(ctx context.Context, port int64) (engineState, string) {
 		portArg := strconv.FormatInt(port, 10)
 		code, _, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
@@ -63,9 +63,14 @@ func postgresProbeState(host string, run probeRunner) probeStateFn {
 		case code == 0:
 			return engineServing, ""
 		case code == 1:
-			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+			return engineRecovering, withEngineLogTail(
+				"engine is rejecting connections (startup or recovery)", errorLog)
 		default:
-			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+			// A postmaster that refused a setting never opens the socket, so this is
+			// the state a bad parameter lands in. The reason for it is only in the
+			// server's own log, and it reaches the customer through StatusInfos.
+			return engineAbsent, withEngineLogTail(
+				fmt.Sprintf("engine did not respond on %s:%s", host, portArg), errorLog)
 		}
 	}
 }

cmd/rds-agent/heartbeat.go

@@ -90,10 +90,26 @@ func (h *heartbeater) beat(ctx context.Context) {
 	h.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
 }
 
+// How often a not-yet-serving engine is reported. The control plane owns the
+// steady-state cadence, but while the engine has never answered it is actively
+// waiting on this instance — a restart is not finished until a healthy beat
+// lands — and a full interval of silence there is pure added downtime.
+const startupHeartbeatInterval = 5 * time.Second
+
+func (h *heartbeater) nextInterval() time.Duration {
+	if h.probe != nil && !h.probe.seenHealthy && startupHeartbeatInterval < h.interval {
+		return startupHeartbeatInterval
+	}
+	return h.interval
+}
+
 // A timer rather than a ticker, so a cadence change takes effect on the next
-// beat, not at a restart.
+// beat, not at a restart. The first beat is not waited for: a just-registered
+// agent has told the control plane nothing about the engine yet, and holding
+// that back for an interval makes every restart look longer than it was.
 func (h *heartbeater) Run(ctx context.Context) {
-	timer := time.NewTimer(h.interval)
+	h.beat(ctx)
+	timer := time.NewTimer(h.nextInterval())
 	defer timer.Stop()
 	for {
 		select {
@@ -101,7 +117,7 @@ func (h *heartbeater) Run(ctx context.Context) {
 			return
 		case <-timer.C:
 			h.beat(ctx)
-			timer.Reset(h.interval)
+			timer.Reset(h.nextInterval())
 		}
 	}
 }

cmd/rds-agent/heartbeat_test.go

@@ -88,3 +88,34 @@ func (r *countingServingRecorder) RecordServingParameters(context.Context) error
 	r.calls++
 	return nil
 }
+
+// A restart is not finished until a healthy beat lands, so the two cadence
+// rules that shorten it are worth pinning: report at once rather than after a
+// full interval, and keep reporting quickly while the engine has never served.
+func TestHeartbeater_ReportsPromptlyUntilTheEngineHasServed(t *testing.T) {
+	code := 2
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
+	})
+	cp := newFakeControlPlane()
+	h := newHeartbeater(cp, probe, nil, 0)
+
+	if got := h.nextInterval(); got != startupHeartbeatInterval {
+		t.Errorf("interval before the engine has served = %s, want %s", got, startupHeartbeatInterval)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go h.Run(ctx)
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "the first heartbeat")
+	cancel()
+
+	if states := cp.snapshotStates(); states[0].health != handlers_rds.EngineHealthStarting {
+		t.Errorf("first beat health = %q, want %q", states[0].health, handlers_rds.EngineHealthStarting)
+	}
+
+	code = 0
+	h.beat(context.Background())
+	if got := h.nextInterval(); got != h.interval {
+		t.Errorf("interval after the engine served = %s, want the control plane's %s", got, h.interval)
+	}
+}

cmd/rds-agent/main.go

@@ -16,21 +16,32 @@ import (
 	"os/signal"
 	"syscall"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {
 	cfg := loadConfig(defaultEnvFile)
 
-	agent, err := New(cfg)
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	// Retried rather than fatal: construction resolves the IMDS credential chain,
+	// so a metadata service that is not answering yet would otherwise end the
+	// process with the engine still down and nothing left to bring it back.
+	var agent *Agent
+	err := retry(ctx, "startup", func(context.Context) error {
+		var err error
+		agent, err = New(cfg)
+		return err
+	})
 	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			return
+		}
 		slog.Error("rds-agent: startup failed", "err", err)
 		os.Exit(1)
 	}
 
-	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
-	defer stop()
-
 	// A shutdown signal cancels whatever boot retry loop was running; that is a
 	// clean stop, not a failure.
 	if err := agent.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {

cmd/rds-agent/quiesce_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"slices"
 	"strings"
 	"sync"
@@ -132,6 +133,50 @@ func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
 	}
 }
 
+// The engine's own hold settles only the files it owns. Everything else on that
+// volume — the parameter file the platform writes beside the datadir — is dirty
+// page cache until this, and a snapshot of it restores an engine that will not
+// start.
+func TestCommandRegistry_QuiesceFlushesTheDataMount(t *testing.T) {
+	storage := &fakeStorage{}
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, storage), 0).execute(context.Background(),
+		quiesceCommand("cmd-5a", "360"))
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if !storage.synced {
+		t.Error("the data mount was not flushed, so the snapshot may not hold what the guest wrote")
+	}
+}
+
+// A snapshot taken after a flush that did not happen is the failure this whole
+// path exists to prevent, so it is reported rather than swallowed. The hold is
+// left on the engine: it expires on its own deadline.
+func TestCommandRegistry_QuiesceFailsWhenTheDataMountCannotBeFlushed(t *testing.T) {
+	storage := &fakeStorage{syncErr: errors.New("input/output error")}
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, storage), 0).execute(context.Background(),
+		quiesceCommand("cmd-5b", "360"))
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Fatalf("reply = %+v, want failed", reply)
+	}
+	if !strings.Contains(reply.Message, "input/output error") {
+		t.Errorf("reply message = %q, want the flush error", reply.Message)
+	}
+}
+
+func quiesceCommand(id, deadline string) handlers_rds.Command {
+	return handlers_rds.Command{
+		CommandID: id,
+		Type:      handlers_rds.CommandQuiesce,
+		Parameters: []handlers_rds.Parameter{
+			{Name: handlers_rds.CommandParamQuiesceLabel, Value: "orders-db-pre-upgrade"},
+			{Name: handlers_rds.CommandParamQuiesceDeadlineSeconds, Value: deadline},
+		},
+	}
+}
+
 // A hold the guest chose the length of would not be the bound the control plane
 // is relying on, so a missing or unreadable deadline is refused.
 func TestCommandRegistry_QuiesceRejectsAnUnusableDeadline(t *testing.T) {

cmd/rds-agent/storage.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"golang.org/x/sys/unix"
 )
 
 // The in-guest half of a storage grow: the control plane has already grown the
@@ -17,6 +19,9 @@ type storageOps interface {
 	// Extends the filesystem at the data mount onto its whole device, and
 	// reports what it did for the command reply.
 	GrowFilesystem(ctx context.Context) (string, error)
+	// Flushes every dirty page on the data mount to its device. A backup hold
+	// settles the engine's own files and nothing else on that filesystem.
+	SyncDataMount(ctx context.Context) error
 }
 
 // Tool names rather than paths: the image installs them from
@@ -50,6 +55,23 @@ func newGuestStorage(cfg config, run commandRunner) *guestStorage {
 	}
 }
 
+// syncfs rather than fsync: the whole filesystem, not one file. Config the
+// engine never opens lives on this volume too, and a snapshot that catches it
+// half-written leaves a restored instance the engine cannot start from.
+func (g *guestStorage) SyncDataMount(ctx context.Context) error {
+	f, err := os.Open(g.dataMount)
+	if err != nil {
+		return fmt.Errorf("open the data mount %s to flush it: %w", g.dataMount, err)
+	}
+	defer func() { _ = f.Close() }()
+
+	if err := unix.Syncfs(int(f.Fd())); err != nil {
+		return fmt.Errorf("flush the data mount %s: %w", g.dataMount, err)
+	}
+	slog.InfoContext(ctx, "rds-agent: data mount flushed for backup", "dataMount", g.dataMount)
+	return nil
+}
+
 // The device and filesystem behind the data mount, resolved from the kernel's
 // own mount table rather than from anything the control plane asserted.
 type dataMount struct {

cmd/rds-agent/storage_test.go

@@ -15,8 +15,10 @@ import (
 // is that the command reaches it rather than what it does to a filesystem.
 type fakeStorage struct {
 	grown   bool
+	synced  bool
 	message string
 	err     error
+	syncErr error
 }
 
 var _ storageOps = (*fakeStorage)(nil)
@@ -26,6 +28,11 @@ func (f *fakeStorage) GrowFilesystem(context.Context) (string, error) {
 	return f.message, f.err
 }
 
+func (f *fakeStorage) SyncDataMount(context.Context) error {
+	f.synced = true
+	return f.syncErr
+}
+
 // runLog records what a grow shelled out to, in order, so the test asserts on
 // the tools invoked rather than on a filesystem it would have to create.
 type runLog struct {
@@ -227,3 +234,25 @@ func TestCommandRegistry_GrowFilesystemReachesTheGuestStorage(t *testing.T) {
 		t.Errorf("reply = %+v, want succeeded carrying what the grow reported", reply)
 	}
 }
+
+func TestSyncDataMount_FlushesTheMountItWasGiven(t *testing.T) {
+	g := &guestStorage{dataMount: t.TempDir()}
+
+	if err := g.SyncDataMount(context.Background()); err != nil {
+		t.Fatalf("SyncDataMount: %v", err)
+	}
+}
+
+// The control plane snapshots on the strength of this reply, so a mount that is
+// not there has to be an error rather than a flush of nothing.
+func TestSyncDataMount_FailsWhenTheMountIsNotThere(t *testing.T) {
+	g := &guestStorage{dataMount: filepath.Join(t.TempDir(), "absent")}
+
+	err := g.SyncDataMount(context.Background())
+	if err == nil {
+		t.Fatal("SyncDataMount succeeded on a mount that does not exist")
+	}
+	if !strings.Contains(err.Error(), "absent") {
+		t.Errorf("error = %v, want the mount it could not open", err)
+	}
+}

cmd/spinifex/cmd/admin.go

@@ -27,18 +27,20 @@ import (
 	"time"
 	"uuid"
 
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
+	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/hostdns"
 	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/services/viperblockd/vbwire"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -217,6 +219,18 @@ decision made per volume.`,
 	Run: runVolumesOrphansCmd,
 }
 
+var volumesUnsealedCmd = &cobra.Command{
+	Use:   "unsealed",
+	Short: "Report volumes whose writes have not been confirmed to the backend",
+	Long: `List volumes a node has open, or had open, without a confirmed seal to the
+backend. The node named holds writes that predastore may not have.
+
+Starting such a volume elsewhere is allowed and is preferred over leaving the
+instance unable to run, but it opens from the last checkpoint that did reach
+the backend. This is the list of volumes where that would cost something.`,
+	Run: runVolumesUnsealedCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -317,6 +331,7 @@ func init() {
 
 	adminCmd.AddCommand(volumesCmd)
 	volumesCmd.AddCommand(volumesOrphansCmd)
+	volumesCmd.AddCommand(volumesUnsealedCmd)
 
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
@@ -445,7 +460,7 @@ func amiVolumeSizeGiB(sizeBytes int64) uint64 {
 	if sizeBytes <= 0 {
 		return 0
 	}
-	return utils.SafeInt64ToUint64((sizeBytes + bytesPerGiB - 1) / bytesPerGiB)
+	return safecast.Int64ToUint64((sizeBytes + bytesPerGiB - 1) / bytesPerGiB)
 }
 
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
@@ -693,7 +708,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	err = admin.ImportImage(context.Background(), provider, admin.ImportOpts{
 		VolumeID:         volumeId,
 		NodeID:           appConfig.Node,
-		SizeBytes:        utils.SafeUint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
+		SizeBytes:        safecast.Uint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
 		AvailabilityZone: node.AZ,
 		SourcePath:       extractedImagePath,
 		Snapshot:         true,
@@ -731,10 +746,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 // DescribeSnapshots call cannot resolve it. The volume ID matches ami.ImageID:
 // admin.ImportImage creates the volume under the AMI's own ID.
 func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, ami ebsmetadata.AMI, az string, encrypted bool) error {
-	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+	cfg := ebsmetadata.Snapshot{
 		SnapshotID:       ami.SnapshotID,
 		VolumeID:         ami.ImageID,
-		VolumeSize:       utils.SafeUint64ToInt64(ami.VolumeSizeGiB),
+		VolumeSize:       safecast.Uint64ToInt64(ami.VolumeSizeGiB),
 		State:            "completed",
 		Progress:         "100%",
 		StartTime:        time.Now(),
@@ -743,7 +758,7 @@ func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, a
 		OwnerID:          utils.GlobalAccountID,
 		AvailabilityZone: az,
 	}
-	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, ami.SnapshotID, cfg); err != nil {
+	if err := ebsmetadata.NewStore(store, bucket).PutSnapshot(context.Background(), cfg); err != nil {
 		return fmt.Errorf("register snapshot metadata: %w", err)
 	}
 	return nil
@@ -842,9 +857,9 @@ func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 		}
 	}
 	fmt.Printf("  Backing storage: %s/      (%d objects, %s)\n",
-		preview.ImageID, preview.AMIObjectCount, utils.HumanBytes(utils.SafeInt64ToUint64(preview.AMIBytesTotal)))
+		preview.ImageID, preview.AMIObjectCount, utils.HumanBytes(safecast.Int64ToUint64(preview.AMIBytesTotal)))
 	fmt.Printf("                   %s/ (%d objects, %s)\n",
-		admin.SnapPrefix(preview.ImageID), preview.SnapObjectCount, utils.HumanBytes(utils.SafeInt64ToUint64(preview.SnapBytesTotal)))
+		admin.SnapPrefix(preview.ImageID), preview.SnapObjectCount, utils.HumanBytes(safecast.Int64ToUint64(preview.SnapBytesTotal)))
 	fmt.Println()
 
 	// Account-owned guard before salvage / dependents — the AWS-flow hint is
@@ -900,7 +915,7 @@ func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 	// BytesDeleted is logical: predastore reclaims the underlying disk space
 	// asynchronously via background compaction, not at delete time.
 	fmt.Printf("✅ Removed AMI %s (%d objects, %s marked for deletion; disk space is reclaimed by background compaction).\n",
-		imageID, res.ObjectsDeleted, utils.HumanBytes(utils.SafeInt64ToUint64(res.BytesDeleted)))
+		imageID, res.ObjectsDeleted, utils.HumanBytes(safecast.Int64ToUint64(res.BytesDeleted)))
 }
 
 func printDependents(w io.Writer, d admin.Dependents) {
@@ -1599,6 +1614,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ClusterName:   clusterName,
 
 		PredastoreHostID:          predastoreHostID,
+		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 
@@ -1900,6 +1916,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterName:   clusterName,
 
 		PredastoreHostID:          predastoreHostID,
+		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(allNodes, node, northstarConfigPath),
@@ -2459,6 +2476,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterName:   creds.ClusterName,
 
 		PredastoreHostID:          predastoreHostID,
+		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(statusResp.Nodes, node, northstarConfigPath),
@@ -2578,7 +2596,7 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
 	}
 
 	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
-	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	masterKey, err := masterkey.ReadShared(masterKeyPath)
 	if err != nil {
 		nc.Close()
 		return nil, nil, nil, nil, fmt.Errorf("load master key: %w", err)
@@ -2840,9 +2858,44 @@ func northstarFromFormation(creds *formation.SharedCredentials, dirs configDirs)
 	}, filepath.Join(dirs.Northstar, "northstar.toml")
 }
 
+// resolverAllowFrom returns the addresses permitted to recurse against this
+// node's northstar: every cluster node, on whichever address its peers dial.
+//
+// Guests are covered without appearing here. vpcd's per-tap DNS shim relays
+// guest queries to a node's :5300 from a node address, so trusting the nodes
+// trusts every guest behind them, and no guest CIDR is needed.
+func resolverAllowFrom(settings admin.ConfigSettings) []string {
+	seen := make(map[string]bool)
+	var addrs []string
+	add := func(ip string) {
+		ip = strings.TrimSpace(ip)
+		if ip == "" || ip == "0.0.0.0" || seen[ip] {
+			return
+		}
+		seen[ip] = true
+		addrs = append(addrs, ip)
+	}
+
+	// Both local addresses: a node resolving through its own advertised address
+	// is seen arriving from that address over lo, not from 127.0.0.1.
+	add(settings.AdvertiseIP)
+	add(settings.BindIP)
+	add(settings.ClusterBindIP)
+	for _, peer := range settings.RemoteNodes {
+		host := peer.Host
+		if h, _, err := net.SplitHostPort(host); err == nil {
+			host = h
+		}
+		add(host)
+	}
+	return addrs
+}
+
 // generateAndWriteConfigs renders the standard config files (spinifex.toml,
 // awsgw.toml, nats.conf, and optionally predastore.toml) from templates.
 func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings admin.ConfigSettings, skipPredastore bool) error {
+	settings.ResolverAllowFrom = resolverAllowFrom(settings)
+
 	// A one-sided pair must disable Northstar wholesale. Rendering only the
 	// public stanza or only the secret file advertises a resolver that cannot run.
 	northstarEnabled := settings.NorthstarAccessKey != "" && settings.NorthstarSecretKey != ""
@@ -2984,7 +3037,7 @@ type writeBootstrapResult struct {
 func ensureMasterKey(configDir string) (key []byte, existed bool, err error) {
 	keyPath := filepath.Join(configDir, "master.key")
 	if admin.FileExists(keyPath) {
-		key, err = handlers_iam.LoadMasterKey(keyPath)
+		key, err = masterkey.ReadShared(keyPath)
 		if err != nil {
 			return nil, false, fmt.Errorf("load master key: %w", err)
 		}
@@ -3077,7 +3130,7 @@ func ensureViperblockEncryptionKey(configDir string) ([]byte, string, error) {
 	}
 	keyPath := filepath.Join(viperblockDir, "encryption.key")
 	if admin.FileExists(keyPath) {
-		key, err := handlers_iam.LoadMasterKey(keyPath)
+		key, err := masterkey.ReadShared(keyPath)
 		if err != nil {
 			return nil, "", fmt.Errorf("load viperblock encryption key: %w", err)
 		}
@@ -3658,3 +3711,37 @@ func printChecksumError(w io.Writer, imageFile, imageName string, image utils.Im
 	fmt.Fprintln(w, "The cached file was left in place. To re-download and retry:")
 	fmt.Fprintf(w, "  spx admin images import --name %s --force\n", imageName)
 }
+
+func runVolumesUnsealedCmd(_ *cobra.Command, _ []string) {
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	unsealed, err := vbwire.ListUnsealedVolumes(context.Background(), nc)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Could not read the unsealed volumes:", err)
+		os.Exit(1)
+	}
+	if len(unsealed) == 0 {
+		fmt.Println("No unsealed volumes: every volume's writes are confirmed to the backend.")
+		return
+	}
+
+	sort.Slice(unsealed, func(i, j int) bool { return unsealed[i].VolumeID < unsealed[j].VolumeID })
+	fmt.Printf("%d volume(s) with writes the backend may not have:\n\n", len(unsealed))
+	for _, volume := range unsealed {
+		fmt.Printf("  %s\n", volume.VolumeID)
+		fmt.Printf("    held by: %s\n", volume.Owner)
+		if !volume.Since.IsZero() {
+			fmt.Printf("    since:   %s\n", volume.Since.Format(time.RFC3339))
+		}
+		fmt.Printf("    reason:  %s\n", volume.Reason)
+	}
+	fmt.Println()
+	fmt.Println("A volume open right now is listed here and that is normal — the entry")
+	fmt.Println("clears on its next successful seal. One naming a node that is gone is")
+	fmt.Println("the case worth acting on: starting it elsewhere loses writes made since.")
+}

cmd/spinifex/cmd/admin_ochre.go

@@ -16,13 +16,14 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
+	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
 	"github.com/mulgadc/spinifex/spinifex/gateway/bedrock/hfhub"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
-	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
@@ -350,6 +351,17 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.
 	return uploaded, nil
 }
 
+// weightsStagingRefusal builds the model-ID refusal error weights staging and
+// pulling share. Unknown IDs and provider-served IDs get distinct wording so
+// a caller can tell which one it hit; taking found/selfHost as plain bools
+// keeps this testable without a real catalog entry for the provider case.
+func weightsStagingRefusal(modelID, operation string, found, _ bool) error {
+	if !found {
+		return fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
+	}
+	return fmt.Errorf("%q is a provider-served model, not self-host; weights %s does not apply", modelID, operation)
+}
+
 // runPullWeights holds 'ochre weights pull' decision and side-effect logic:
 // catalog validation, ref resolution to an immutable commit SHA (D3), tree
 // listing and safetensors-only filtering (D4), streaming each selected file
@@ -359,10 +371,7 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.
 func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision string, revisionExplicit bool, s3URIFlag string) (string, error) {
 	spec, found, selfHost := gateway_bedrock.LookupServingSpec(modelID)
 	if !found || !selfHost {
-		if !found {
-			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
-		}
-		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights pull does not apply", modelID)
+		return "", weightsStagingRefusal(modelID, "pull", found, selfHost)
 	}
 
 	// The catalog carries the canonical repo and a pinned revision so a bare
@@ -624,12 +633,12 @@ func buildWeightsImage(srcDir, imagePath string, contentBytes int64) error {
 // metadata.json sitting alongside them, so without this the endpoint launcher
 // cannot see the snapshot at all and fails with InvalidSnapshot.NotFound.
 func registerWeightsSnapshot(store objectstore.ObjectStore, bucket, snapshotID, volumeID string, volumeSize uint64, az string, encrypted bool) error {
-	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+	cfg := ebsmetadata.Snapshot{
 		SnapshotID: snapshotID,
 		VolumeID:   volumeID,
 		// GiB, not bytes: CreateVolume compares this against a requested Size
 		// already in GiB, and would reject every clone if handed raw bytes.
-		VolumeSize:       utils.SafeUint64ToInt64(volumeSize / bytesPerGiB),
+		VolumeSize:       safecast.Uint64ToInt64(volumeSize / bytesPerGiB),
 		State:            "completed",
 		Progress:         "100%",
 		StartTime:        time.Now(),
@@ -638,7 +647,7 @@ func registerWeightsSnapshot(store objectstore.ObjectStore, bucket, snapshotID,
 		OwnerID:          utils.GlobalAccountID,
 		AvailabilityZone: az,
 	}
-	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, snapshotID, cfg); err != nil {
+	if err := ebsmetadata.NewStore(store, bucket).PutSnapshot(context.Background(), cfg); err != nil {
 		return fmt.Errorf("register snapshot metadata: %w", err)
 	}
 	return nil
@@ -666,10 +675,7 @@ type weightsSnapshotChecker func(ctx context.Context, snapshotID string) (bool,
 // fake materializer.
 func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weightsStore *gateway_bedrock.WeightsStore, tmpDirFlag, modelID, s3URI string, materialize weightsMaterializer, checkSnapshotLive weightsSnapshotChecker) (string, error) {
 	if _, found, selfHost := gateway_bedrock.LookupServingSpec(modelID); !found || !selfHost {
-		if !found {
-			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
-		}
-		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights staging does not apply", modelID)
+		return "", weightsStagingRefusal(modelID, "staging", found, selfHost)
 	}
 
 	bucket, prefix, err := parseWeightsS3URI(s3URI)
@@ -779,7 +785,7 @@ func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvi
 	if err := admin.ImportImage(ctx, provider, admin.ImportOpts{
 		VolumeID:         volumeId,
 		NodeID:           nodeID,
-		SizeBytes:        utils.SafeUint64ToInt64(volumeBytes),
+		SizeBytes:        safecast.Uint64ToInt64(volumeBytes),
 		AvailabilityZone: node.AZ,
 		SourcePath:       imagePath,
 		Snapshot:         true,
@@ -799,13 +805,14 @@ func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvi
 }
 
 // newWeightsSnapshotChecker is the real weightsSnapshotChecker: it reads the
-// EC2 control plane's metadata.json for snapshotID from the node's
-// predastore bucket -- the same record registerWeightsSnapshot writes at
-// stage time -- and treats a missing object as gone. Any other read failure
-// is a real error, not a signal to guess the snapshot's existence.
+// EC2 control plane's snapshot document for snapshotID under the global
+// account -- the same record registerWeightsSnapshot writes at stage time --
+// and treats a missing object as gone. Any other read failure is a real
+// error, not a signal to guess the snapshot's existence.
 func newWeightsSnapshotChecker(store objectstore.ObjectStore, bucket string) weightsSnapshotChecker {
+	metadata := ebsmetadata.NewStore(store, bucket)
 	return func(ctx context.Context, snapshotID string) (bool, error) {
-		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, store, bucket, snapshotID); err != nil {
+		if _, err := metadata.GetSnapshot(ctx, utils.GlobalAccountID, snapshotID); err != nil {
 			if objectstore.IsNoSuchKeyError(err) {
 				return false, nil
 			}
@@ -999,7 +1006,7 @@ func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.Cluster
 	}
 
 	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
-	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	masterKey, err := masterkey.ReadShared(masterKeyPath)
 	if err != nil {
 		return ""
 	}
@@ -1071,7 +1078,7 @@ func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
 		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
 	}
 	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
-	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	masterKey, err := masterkey.ReadShared(masterKeyPath)
 	if err != nil {
 		nc.Close()
 		return nil, nil, fmt.Errorf("load master key: %w", err)

cmd/spinifex/cmd/admin_ochre_pull_test.go

@@ -235,10 +235,12 @@ func TestRunPullWeights_UnknownModelRefused(t *testing.T) {
 	assert.Contains(t, err.Error(), "not present in the Ochre catalog")
 }
 
-// TestRunPullWeights_ProviderModelRefused covers the self-host gate: a
-// provider-served model must refuse before any network work, same as stage.
-func TestRunPullWeights_ProviderModelRefused(t *testing.T) {
-	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, providerModelID, testHFRepo, "main", true, "")
+// TestWeightsStagingRefusal_PullOperationWording covers the self-host gate's
+// message for the 'pull' operation. v1 ships no provider-tier catalog entry,
+// so this drives weightsStagingRefusal directly rather than through a real
+// LookupServingSpec lookup.
+func TestWeightsStagingRefusal_PullOperationWording(t *testing.T) {
+	err := weightsStagingRefusal(providerModelID, "pull", true, false)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "not self-host")
 }

cmd/spinifex/cmd/admin_ochre_test.go

@@ -15,20 +15,22 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// selfHostModelID and providerModelID are real catalog entries: the former
-// is self-host (weights staging applies), the latter is provider-served
-// (weights staging must refuse it), so tests exercise LookupServingSpec's
-// found/selfHost distinction against the real catalog rather than a stub.
+// selfHostModelID is a real catalog entry (weights staging applies).
+// providerModelID is an arbitrary ID not present in the catalog at all: v1
+// ships no provider-tier entry, so the found-but-not-selfHost case is
+// exercised directly against weightsStagingRefusal instead, not through a
+// real LookupServingSpec lookup.
 const (
 	selfHostModelID = "meta.llama3-2-1b-instruct-v1:0"
 	providerModelID = "anthropic.claude-3-5-sonnet-20240620-v1:0"
@@ -404,24 +406,17 @@ func TestRunStageWeights_UnknownModelIDRefused(t *testing.T) {
 	assert.Empty(t, entries, "refusal must not write a KV entry")
 }
 
-// TestRunStageWeights_ProviderModelIDRefused covers refusal on a model ID
-// that exists in the catalog but is provider-served, not self-host. The
-// error must be distinguishable from the unknown-model-ID case, since
-// LookupServingSpec returns found and selfHost separately for exactly this
-// reason.
-func TestRunStageWeights_ProviderModelIDRefused(t *testing.T) {
-	weightsStore := newWeightsStoreForTest(t)
-
-	_, err := runStageWeights(context.Background(), explodingObjectStore{t: t}, weightsStore, t.TempDir(),
-		providerModelID, "s3://models/claude/", explodingMaterializer(t), explodingSnapshotChecker(t))
+// TestWeightsStagingRefusal_ProviderServedDistinguishesFromUnknown covers
+// refusal for a model ID that is found in the catalog but is provider-served,
+// not self-host. The error must be distinguishable from the unknown-model-ID
+// case; v1 ships no provider-tier entry, so this drives weightsStagingRefusal
+// directly with explicit found/selfHost values rather than a real lookup.
+func TestWeightsStagingRefusal_ProviderServedDistinguishesFromUnknown(t *testing.T) {
+	err := weightsStagingRefusal(providerModelID, "staging", true, false)
 
 	require.Error(t, err)
 	assert.ErrorContains(t, err, "provider-served")
 	assert.NotContains(t, err.Error(), "unknown model ID", "must be distinguishable from the unknown-model-ID refusal")
-
-	entries, listErr := weightsStore.ListWeights(context.Background())
-	require.NoError(t, listErr)
-	assert.Empty(t, entries, "refusal must not write a KV entry")
 }
 
 // TestRunStageWeights_MissingRequiredFilesRefusedBeforeMaterializing covers
@@ -1100,7 +1095,7 @@ func TestRegisterWeightsSnapshot_WritesEC2ReadableMetadata(t *testing.T) {
 	require.NoError(t, registerWeightsSnapshot(store, bucket,
 		"snap-vol-abc", "vol-abc", 12*bytesPerGiB, "ap-southeast-2a", true))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, "snap-vol-abc")
+	cfg, err := ebsmetadata.NewStore(store, bucket).GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-vol-abc")
 	require.NoError(t, err)
 	assert.Equal(t, "snap-vol-abc", cfg.SnapshotID)
 	assert.Equal(t, "vol-abc", cfg.VolumeID)
@@ -1117,7 +1112,7 @@ func TestRegisterWeightsSnapshot_UnencryptedVolume(t *testing.T) {
 	require.NoError(t, registerWeightsSnapshot(store, "predastore",
 		"snap-vol-plain", "vol-plain", bytesPerGiB, "ap-southeast-2a", false))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, "predastore", "snap-vol-plain")
+	cfg, err := ebsmetadata.NewStore(store, "predastore").GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-vol-plain")
 	require.NoError(t, err)
 	assert.False(t, cfg.Encrypted)
 }

cmd/spinifex/cmd/admin_ochre_vector_test.go

@@ -30,6 +30,8 @@ type fakeVectorService struct {
 	queryErr       error
 	listJobs       []handlers_ochrevector.JobRecord
 	listJobsErr    error
+	stopJob        handlers_ochrevector.JobRecord
+	stopJobErr     error
 
 	createIndexInputs []*handlers_ochrevector.CreateIndexRequest
 	deleteIndexInputs []*handlers_ochrevector.DeleteIndexRequest
@@ -94,6 +96,17 @@ func (f *fakeVectorService) ListJobs(_ context.Context, _ *handlers_ochrevector.
 	return &handlers_ochrevector.ListJobsResponse{Jobs: f.listJobs}, nil
 }
 
+func (f *fakeVectorService) StopJob(_ context.Context, req *handlers_ochrevector.StopJobRequest, _ string) (*handlers_ochrevector.StopJobResponse, error) {
+	if f.stopJobErr != nil {
+		return nil, f.stopJobErr
+	}
+	job := f.stopJob
+	if job.ID == "" {
+		job.ID = req.JobID
+	}
+	return &handlers_ochrevector.StopJobResponse{Job: job}, nil
+}
+
 var _ handlers_ochrevector.VectorService = (*fakeVectorService)(nil)
 
 const testIndexID = "idx-0123456789abcdef0"

cmd/spinifex/cmd/admin_test.go

@@ -4,11 +4,14 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/mulgadc/northstar/pkg/backend"
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/formation"
@@ -1358,3 +1361,58 @@ func TestJoinRetryable(t *testing.T) {
 		})
 	}
 }
+
+// The rendered resolver must refuse the internet and still serve the cluster.
+// Both halves matter: an open resolver is the bug, and a resolver that refuses
+// its own nodes silently stops internal names resolving.
+func TestGenerateAndWriteConfigs_NorthstarRecursionIsClosedButServesTheCluster(t *testing.T) {
+	dirs, err := createConfigSubdirs(t.TempDir())
+	require.NoError(t, err)
+	nsPath := filepath.Join(dirs.Northstar, "northstar.toml")
+
+	settings := multinodeNorthstarSettings(nsPath)
+	settings.BindIP = "10.2.0.2"
+	settings.AdvertiseIP = "72.52.77.228"
+	settings.RemoteNodes = []admin.RemoteNode{
+		{Name: "node2", Host: "72.52.77.229", NorthstarConfigPath: nsPath},
+		{Name: "node3", Host: "72.52.77.230:4432", NorthstarConfigPath: nsPath},
+	}
+
+	require.NoError(t, generateAndWriteConfigs(dirs, filepath.Join(t.TempDir(), "spinifex.toml"), settings, true))
+
+	raw, err := os.ReadFile(nsPath)
+	require.NoError(t, err)
+	rendered := string(raw)
+
+	assert.Contains(t, rendered, "allow_recursion = false",
+		"a node bound to a public address must not be an open resolver")
+
+	// The node's own advertised address: resolv.conf points there, so the query
+	// arrives from it over lo rather than from 127.0.0.1.
+	assert.Contains(t, rendered, `"72.52.77.228"`)
+	assert.Contains(t, rendered, `"10.2.0.2"`)
+	// Peers, with any port stripped.
+	assert.Contains(t, rendered, `"72.52.77.229"`)
+	assert.Contains(t, rendered, `"72.52.77.230"`)
+	assert.NotContains(t, rendered, "72.52.77.230:4432")
+
+	// It must parse as the config northstar will actually load.
+	cfg, err := nsconfig.LoadServerConfig(nsPath)
+	require.NoError(t, err)
+	assert.False(t, cfg.Upstream.AllowRecursion)
+
+	prefixes, err := cfg.Upstream.ParseAllowRecursionFrom()
+	require.NoError(t, err)
+	policy := backend.NewRecursionPolicy(cfg.Upstream.AllowRecursion, prefixes)
+
+	// The cluster resolves.
+	assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.228")), "node must resolve through itself")
+	assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.229")), "peer node must resolve")
+	assert.True(t, policy.Allows(netip.MustParseAddr("127.0.0.1")), "loopback is always allowed")
+	// Guests reach northstar via vpcd's shim, which relays from a node address,
+	// so a guest query is seen as one of the addresses asserted above.
+
+	// The internet does not.
+	assert.False(t, policy.Allows(netip.MustParseAddr("198.51.100.7")))
+	assert.False(t, policy.Allows(netip.MustParseAddr("8.8.8.8")))
+}

cmd/spinifex/cmd/register_imported_ami_snapshot_test.go

@@ -5,7 +5,6 @@ import (
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/stretchr/testify/assert"
@@ -29,7 +28,7 @@ func TestRegisterImportedAMISnapshot_WritesEC2ReadableMetadata(t *testing.T) {
 
 	require.NoError(t, registerImportedAMISnapshot(store, bucket, ami, "ap-southeast-2a", true))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, "snap-ami-import01")
+	cfg, err := ebsmetadata.NewStore(store, bucket).GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-ami-import01")
 	require.NoError(t, err)
 	assert.Equal(t, "snap-ami-import01", cfg.SnapshotID)
 	assert.Equal(t, "ami-import01", cfg.VolumeID, "the import path creates the volume under the AMI's own ID")
@@ -50,7 +49,7 @@ func TestRegisterImportedAMISnapshot_UnencryptedVolume(t *testing.T) {
 
 	require.NoError(t, registerImportedAMISnapshot(store, "predastore", ami, "ap-southeast-2a", false))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, "predastore", "snap-ami-plain01")
+	cfg, err := ebsmetadata.NewStore(store, "predastore").GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-ami-plain01")
 	require.NoError(t, err)
 	assert.False(t, cfg.Encrypted)
 }

cmd/spinifex/cmd/service.go

@@ -386,6 +386,7 @@ var viperblockStartCmd = &cobra.Command{
 			NodeName:          clusterConfig.Node,
 			ShardWAL:          shardWAL,
 			GCEnabled:         gcEnabled,
+			WALBaseDir:        nodeConfig.Viperblock.WALBaseDir,
 			EncryptionKeyFile: encryptionKeyFile,
 			Threads:           nodeConfig.EBS.ResolvedThreads(),
 			CacheSizeMB:       nodeConfig.EBS.ResolvedCacheSizeMB(),
@@ -840,6 +841,7 @@ var spinifexUIStartCmd = &cobra.Command{
 		tlsCert := viper.GetString("spinifex-ui-tls-cert")
 		tlsKey := viper.GetString("spinifex-ui-tls-key")
 		baseDir := viper.GetString("spinifex-ui-base-dir")
+		ochreEnabled := viper.GetBool("spinifex-ui-ochre-enabled")
 
 		cfg, err := loadLocalConfig()
 		if err != nil {
@@ -853,12 +855,13 @@ var spinifexUIStartCmd = &cobra.Command{
 		defer initTelemetry("spinifex-ui", false)()
 
 		svc, err := service.New("spinifex-ui", &spinifexui.Config{
-			Port:    port,
-			Host:    host,
-			TLSCert: tlsCert,
-			TLSKey:  tlsKey,
-			BaseDir: baseDir,
-			Region:  region,
+			Port:         port,
+			Host:         host,
+			TLSCert:      tlsCert,
+			TLSKey:       tlsKey,
+			BaseDir:      baseDir,
+			Region:       region,
+			OchreEnabled: ochreEnabled,
 		})
 
 		if err != nil {
@@ -1012,6 +1015,7 @@ var vpcdStartCmd = &cobra.Command{
 			OVNNBAddr:               nodeConfig.VPCD.OVNNBAddr,
 			OVNSBAddr:               nodeConfig.VPCD.OVNSBAddr,
 			BaseDir:                 nodeConfig.BaseDir,
+			DataDir:                 nodeConfig.DataDir,
 			Debug:                   false,
 			ExternalMode:            clusterConfig.Network.ExternalMode,
 			ExternalPools:           extPools,
@@ -1025,6 +1029,8 @@ var vpcdStartCmd = &cobra.Command{
 			NATExemptCIDRs:          clusterConfig.Network.NATExemptCIDRs,
 			IPSecEnabled:            clusterConfig.Network.IPSecEnabled,
 			UnderlayMTU:             clusterConfig.Network.UnderlayMTU,
+			BlockedWANPorts:         clusterConfig.Network.ResolvedBlockedWANPorts(),
+			EgressBlockExemptVPCs:   clusterConfig.Network.EgressBlockExemptVPCs,
 		})
 		if err != nil {
 			return fmt.Errorf("create vpcd service: %w", err)
@@ -1535,6 +1541,10 @@ func init() {
 	viper.BindEnv("spinifex-ui-base-dir", "SPINIFEX_UI_BASE_DIR")
 	viper.BindPFlag("spinifex-ui-base-dir", spinifexUICmd.PersistentFlags().Lookup("base-dir"))
 
+	spinifexUICmd.PersistentFlags().Bool("ochre-enabled", false, "enable the Ochre console")
+	viper.BindEnv("spinifex-ui-ochre-enabled", "SPINIFEX_UI_OCHRE_ENABLED")
+	viper.BindPFlag("spinifex-ui-ochre-enabled", spinifexUICmd.PersistentFlags().Lookup("ochre-enabled"))
+
 	spinifexUICmd.AddCommand(spinifexUIStartCmd)
 	spinifexUICmd.AddCommand(spinifexUIStopCmd)
 	spinifexUICmd.AddCommand(spinifexUIStatusCmd)

cmd/spinifex/cmd/templates/northstar.toml

@@ -38,3 +38,25 @@ secret_key = "{{.NorthstarSecretKey}}"
 # Forwarders for non-authoritative (recursive) queries. Empty => REFUSED.
 [upstream]
 nameservers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}:53"{{end}}]
+
+# Who may have a query forwarded to those nameservers. Authoritative answers are
+# never gated by this — a public authoritative server must answer everyone.
+#
+# allow_recursion = true serves recursion to ANY client, including the whole
+# internet, because :53 above is bound to the node's public address. Do not set
+# it on a node reachable from the internet. An open resolver is found by scanners
+# within hours and is then used as a DDoS reflector, which gets the node's whole
+# address range blocklisted — an outage for every tenant on it, and slow to undo.
+# It is here for an isolated or lab deployment where the DNS port is not exposed.
+#
+# Trusted clients do not need it. List them in allow_recursion_from instead,
+# which is unaffected by allow_recursion and is how the cluster resolves:
+#   - Cluster nodes resolve through northstar (resolv.conf points at the node's
+#     own advertised address, so the query arrives from that public IP over lo,
+#     not from 127.0.0.1).
+#   - EC2 instances reach vpcd's per-tap DNS shim on 169.254.169.253, which
+#     relays to northstar's :5300 from a node address — so listing the nodes
+#     covers every guest, and guest addresses never appear here.
+# Loopback is always allowed and does not need listing.
+allow_recursion = false
+allow_recursion_from = [{{range $i, $s := .ResolverAllowFrom}}{{if $i}}, {{end}}"{{$s}}"{{end}}]

cmd/spinifex/cmd/templates/predastore-multinode.toml

@@ -36,6 +36,7 @@ data_dir = "{{$.PredastoreDataDir}}"
 tls_cert = "{{$.ConfigDir}}/server.pem"
 tls_key = "{{$.ConfigDir}}/server.key"
 encryption_key = "{{$.ConfigDir}}/predastore/encryption.key"
+admin_port = {{$.AdminPort}}
 {{- range $.ClusterNodes}}
 {{- if eq .HostID $host.ID}}
 

cmd/spinifex/cmd/templates/predastore.toml

@@ -32,6 +32,7 @@ data_dir = "{{.PredastoreDataDir}}"
 tls_cert = "{{.ConfigDir}}/server.pem"
 tls_key = "{{.ConfigDir}}/server.key"
 encryption_key = "{{.ConfigDir}}/predastore/encryption.key"
+admin_port = {{.PredastoreAdminPort}}
 
 # The gate serves the S3 API. Without one the host answers no S3 request.
 # S3 is the public plane, so it binds every interface while the host's

cmd/spinifex/cmd/templates/spinifex.toml

@@ -14,6 +14,13 @@ internal_suffix = "spinifex.internal"
 
 [network]
 ipsec_enabled = {{.IPSecEnabled}}
+# Outbound TCP ports blocked for guest egress to public destinations, like AWS's
+# default outbound-SMTP block. Omit the key for the default [25, 465, 587]; set
+# [] to disable. Private destinations are always exempt.
+# blocked_ports_wan = [25, 465, 587]
+# VPC IDs exempted from the block — the operator workaround for a tenant that
+# legitimately needs to send mail, until per-account exceptions exist.
+# egress_block_exempt_vpcs = ["vpc-..."]
 {{- if .ExternalMode}}
 external_mode = "{{.ExternalMode}}"
 {{- range .Pools}}
@@ -111,6 +118,14 @@ token = "{{.NatsToken}}"
 [nodes.{{.Node}}.viperblock]
 shardwal = false
 gc_enabled = true
+#
+# wal_base_dir: put every volume's WAL under this directory instead of the
+# node's base_dir. A WAL fsync shares the device queue with predastore, the
+# chunk cache and every other volume on the node, so a busy node lengthens the
+# fsync tail that a datastore guest measures as its commit latency. Pointing
+# this at its own device keeps that traffic out of the WAL's queue. Unset
+# keeps the WAL under base_dir.
+#wal_base_dir = "/var/lib/spinifex-wal"
 {{- if .EncryptionKeyFile}}
 encryption_key_file = "{{.EncryptionKeyFile}}"
 {{- end}}

cmd/spinifex/main.go

@@ -1,8 +1,8 @@
 package main
 
 import (
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {

cmd/spx-loadgen/main.go

@@ -38,6 +38,8 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/loadgen"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

docs/COMMANDS.md

@@ -6,37 +6,74 @@ Platform management commands not exposed via the AWS gateway API. These are CLI-
 
 ### Service Management
 
-Service lifecycle commands for starting, stopping, and checking status of all Spinifex cluster services. Each service subcommand supports `start`, `stop`, and `status` operations.
+Services run under systemd. Units live in `spinifex/build/systemd/`, installed by `scripts/setup.sh` on install and reconciled by `spx admin upgrade`.
+
+#### Units
+
+Every service is `PartOf=spinifex.target` and ordered by its own `After=`/`Wants=`.
+
+| Unit | ExecStart | Description |
+|------|-----------|-------------|
+| `spinifex.target` | — | The whole stack: every service below plus the watchdog timer, openvswitch and ovn-controller |
+| `spinifex-nats.service` | `spx service nats start` | NATS message bus with JetStream. Config at `/etc/spinifex/nats/nats.conf`; dependants block on `wait-for-nats.sh` as an `ExecStartPre` |
+| `spinifex-predastore.service` | `spx service predastore start` | S3-compatible object storage, serving that host's nodes and the S3 gate among them |
+| `spinifex-northstar.service` | `spx service northstar start` | Authoritative DNS for `*.spx3.net`, recursive via upstream forwarders, syncing zones from its S3 bucket |
+| `spinifex-viperblock.service` | `spx service viperblock start` | Block storage with the NBD plugin. `KillMode=process`: nbdkit serving a guest survives a restart of this unit |
+| `spinifex-daemon.service` | `spx service spinifex start` | VM orchestration, NATS subscriptions, health endpoint. `KillMode=process`: guest QEMU survives a restart and the new daemon reattaches. `SIGHUP` reloads config |
+| `spinifex-awsgw.service` | `spx service awsgw start` | AWS-compatible gateway: SigV4 auth, IAM policy enforcement, TLS. Also serves the Private Admin API |
+| `spinifex-vpcd.service` | `spx service vpcd start` | VPC daemon: translates `vpc.*` NATS events to OVN logical switches, ports and routers |
+| `spinifex-ui.service` | `spx service spinifex-ui start` | Web UI serving the React frontend |
+| `spinifex-qmp-collector.service` | `spx service qmp-collector start` | Polls per-VM telemetry QMP sockets and tap counters, publishes to NATS `metrics.ec2.*` |
+| `spinifex-shutdown.service` | `/bin/true`; the work is its `ExecStop` | Runs `spx admin node drain --local --unless-restarting`. `After=` every service, so it stops first and drains guests while the stack is still up |
+| `spinifex-nats-watchdog.service` / `.timer` | `nats-js-watchdog.sh` | JetStream liveness probe every 30s. `try-restart`s NATS when JetStream stops accepting writes while the process stays alive — what `Restart=on-failure` cannot catch |
+| `spinifex-firewall.service` | `spinifex-firewall-apply` | Oneshot nftables input policy, `WantedBy=sysinit.target` rather than the target. No `ExecStop`: remove it with `nft delete table inet spinifex_filter` |
+| `regenerate-ssh-host-keys.service` | `ssh-keygen -A` | Oneshot for AMI-built nodes, conditional on the host key being absent |
+
+#### Operations
+
+| Command | Description |
+|---------|-------------|
+| `sudo systemctl start spinifex.target` | Starts the stack in dependency order |
+| `sudo systemctl stop spinifex.target` | Stops the stack, draining guests first via `spinifex-shutdown.service` |
+| `sudo systemctl restart spinifex.target` | Restarts the stack. Does **not** drain: QEMU and nbdkit survive on `KillMode=process` and the daemon reattaches |
+| `sudo systemctl restart spinifex-daemon` | Restarts one service. Required after a binary swap — services keep executing the replaced inode until they restart (`spx admin preflight` reports this) |
+| `systemctl status 'spinifex-*'` | Per-service state, PID, memory and recent log lines |
+| `journalctl -u spinifex-daemon -f` | Follows one service; `-u 'spinifex-*'` follows all |
+| `sudo systemctl kill -s HUP spinifex-daemon` | Reloads daemon config without a restart |
+| `sudo systemctl edit spinifex-daemon` | Drop-in override. Preferred over editing the installed unit, which `spx admin upgrade` then reports as operator-modified and refuses to touch |
+
+A cluster-wide stop is `spx admin cluster shutdown`, not a `systemctl` loop across nodes — it sequences GATE → DRAIN → STORAGE → PERSIST → INFRA with a per-phase ACK from every node.
+
+#### Service configuration
+
+Under systemd the services take no flags: each unit supplies `Environment=` entries, and the flags below are reachable as their `SPINIFEX_*` equivalents. Precedence is flag, then env var, then the node's `spinifex.toml` section, then the default. Environment files are `/etc/spinifex/telemetry.env` (all services), `/etc/spinifex/systemd.env` (viperblock's plugin path) and `/etc/spinifex/host.env` (the daemon's `SPINIFEX_RESERVED_MEM_GB`).
+
+| Service | Flag | Environment variable | Default / notes |
+|---------|------|----------------------|-----------------|
+| predastore | `--port` | `SPINIFEX_PREDASTORE_PORT` | 8443, the host's S3 gate port |
+| predastore | `--host` | `SPINIFEX_PREDASTORE_HOST` | 0.0.0.0, the address the host's sockets bind |
+| predastore | `--base-path` | `SPINIFEX_PREDASTORE_BASE_PATH` | Directory holding the service pid file; the unit sets `SPINIFEX_BASE_DIR` |
+| predastore | `--config-path` | `SPINIFEX_PREDASTORE_CONFIG_PATH` | Required |
+| predastore | `--tls-cert` / `--tls-key` | `SPINIFEX_PREDASTORE_TLS_CERT` / `_TLS_KEY` | Required |
+| predastore | `--encryption-key-file` | `SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE` | Required |
+| predastore | `--host-id` | `SPINIFEX_PREDASTORE_HOST_ID` | Required; names a `[[host]]` in the predastore config. When `SPINIFEX_CONFIG_PATH` points at a cluster spinifex.toml, host/port/host-id default to that node's `[nodes.<node>.predastore]` section rather than the defaults above. `host_id` absent from spinifex.toml resolves to 0, which names no host and is rejected |
+| viperblock | `--s3-host` | `SPINIFEX_VIPERBLOCK_S3_HOST` | 0.0.0.0:8443 |
+| viperblock | `--s3-bucket` | `SPINIFEX_VIPERBLOCK_S3_BUCKET` | predastore |
+| viperblock | `--s3-region` | `SPINIFEX_VIPERBLOCK_S3_REGION` | ap-southeast-2 |
+| viperblock | `--plugin-path` | `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` | `/opt/spinifex/lib/nbdkit-viperblock-plugin.so`; set in `/etc/spinifex/systemd.env` |
+| viperblock | `--encryption-key-file` | `SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE` | 32-byte AES-256 master key for at-rest encryption; empty disables it |
+| viperblock | `--debug` | `SPINIFEX_VIPERBLOCK_DEBUG` | |
+| nats | `--port` / `--host` / `--debug` | `SPINIFEX_NATS_PORT` / `SPINIFEX_NATS_HOST` / `SPINIFEX_NATS_DEBUG` | 4222 / 0.0.0.0 |
+| nats | `--data-dir` / `--jetstream` | `SPINIFEX_NATS_DATA_DIR` / `SPINIFEX_NATS_JETSTREAM` | The unit sets neither: it points `SPINIFEX_CONFIG_PATH` at `nats.conf`, which carries both |
+| spinifex (daemon) | `--wal-dir` | `SPINIFEX_WAL_DIR` | The unit sets it alongside `SPINIFEX_BASE_DIR` |
+| awsgw | `--host` | `SPINIFEX_AWSGW_HOST` | 0.0.0.0:9999 |
+| awsgw | `--tls-cert` / `--tls-key` | `SPINIFEX_AWSGW_TLS_CERT` / `_TLS_KEY` | `/etc/spinifex/server.pem` and `server.key` |
+| awsgw | `--debug` | `SPINIFEX_AWSGW_DEBUG` | |
+| spinifex-ui | `--port` / `--host` | `SPINIFEX_UI_PORT` / `SPINIFEX_UI_HOST` | 3000 / 0.0.0.0. Command aliases: `ui`, `spinifexui` |
+| spinifex-ui | `--tls-cert` / `--tls-key` | `SPINIFEX_UI_TLS_CERT` / `SPINIFEX_UI_TLS_KEY` | `/etc/spinifex/server.pem` and `server.key` |
+| spinifex-ui | `--base-dir` | `SPINIFEX_UI_BASE_DIR` | `/var/lib/spinifex/spinifex-ui/` |
+| northstar | `--northstar-config` | `SPINIFEX_NORTHSTAR_CONFIG` | Overrides `nodes.<node>.northstar.config_path` |
 
-| Command | Flags | Description |
-|---------|-------|-------------|
-| `spx service predastore start` | `--port` (default: 8443, the host's S3 gate port), `--host` (default: 0.0.0.0, the address the host's sockets bind), `--base-path` (directory holding the service pid file), `--config-path` (required), `--tls-cert` (required), `--tls-key` (required), `--encryption-key-file` (required), `--host-id` (required; names a `[[host]]` in the predastore config) | Creates predastore service instance with S3-compatible storage backend → starts service, serving that host's nodes and the S3 gate among them until the process is signalled. When `--config`/`SPINIFEX_CONFIG_PATH` points at a cluster spinifex.toml, `--host`/`--port`/`--host-id` default to that node's `[nodes.<node>.predastore]` section instead of the flag defaults above; an explicit flag or `SPINIFEX_PREDASTORE_*` env var still overrides it. `host_id` absent from spinifex.toml resolves to 0, which names no host and is rejected. |
-| `spx service predastore stop` | — | Stops the predastore service |
-| `spx service predastore status` | — | Reports predastore service status |
-| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin |
-| `spx service viperblock stop` | — | Stops the viperblock service |
-| `spx service viperblock status` | — | Reports viperblock service status |
-| `spx service nats start` | `--port` (default: 4222), `--host` (default: 0.0.0.0), `--debug`, `--data-dir`, `--jetstream` | Starts embedded NATS server with optional JetStream |
-| `spx service nats stop` | — | Stops the NATS service |
-| `spx service nats status` | — | Reports NATS service status |
-| `spx service spinifex start` | `--wal-dir` | Loads cluster config → starts spinifex daemon (VM orchestration, NATS subscriptions, health endpoint) |
-| `spx service spinifex stop` | — | Stops the spinifex daemon service |
-| `spx service spinifex status` | — | Reports spinifex daemon service status |
-| `spx service awsgw start` | `--host` (default: 0.0.0.0:9999), `--tls-cert`, `--tls-key`, `--debug` | Loads cluster config → starts AWS-compatible gateway with SigV4 auth, IAM policy enforcement, TLS |
-| `spx service awsgw stop` | — | Stops the AWS gateway service |
-| `spx service awsgw status` | — | Reports AWS gateway service status |
-| `spx service spinifex-ui start` | `--port` (default: 3000), `--host` (default: 0.0.0.0), `--tls-cert`, `--tls-key` | Starts embedded web UI server serving the React frontend. Aliases: `ui`, `spinifexui` |
-| `spx service spinifex-ui stop` | — | Stops the spinifex-ui service |
-| `spx service spinifex-ui status` | — | Reports spinifex-ui service status |
-| `spx service vpcd start` | — | Loads cluster config → starts VPC daemon (subscribes to `vpc.*` NATS events, translates to OVN logical switches/ports/routers) |
-| `spx service vpcd stop` | — | Stops the vpcd service |
-| `spx service vpcd status` | — | Reports vpcd service status |
-| `spx service northstar start` | `--northstar-config` (overrides `nodes.<node>.northstar.config_path`) | Loads cluster config → reads `northstar.toml` → starts the northstar DNS server (authoritative for internal `*.spx3.net`, recursive via upstream forwarders), syncing zones from its S3 bucket. Guests resolve via `169.254.169.253`, served by vpcd's per-tap DNS shim which forwards to northstar |
-| `spx service northstar stop` | — | Stops the northstar service |
-| `spx service northstar status` | — | Reports northstar service status |
-| `spx service qmp-collector start` | — | Starts the guest-metrics collector (polls per-VM telemetry QMP sockets + tap counters, publishes CloudWatch-shaped series to NATS `metrics.ec2.*`) |
-| `spx service qmp-collector stop` | — | Stops the qmp-collector service |
-| `spx service qmp-collector status` | — | Reports qmp-collector service status |
 
 ### Cluster Inspection
 
@@ -46,6 +83,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx get nodes` | `--timeout` (default: 3s) | Loads config → publishes to `spinifex.node.status` fan-out topic → collects responses within timeout → merges config-known nodes with NATS responders → nodes that don't respond shown as `NotReady` |
 | `spx get vms` | `--timeout` (default: 3s) | Publishes to `spinifex.node.vms` fan-out topic → collects VM info from all nodes → sorts by node then instance ID → prints table. Alias: `spx get instances` |
+| `spx get dhcp-leases` | `--timeout` (default: 3s), `--orphans` (show only leases whose owning resource is gone), `--skip-owner-check` (skip the owner lookup; faster, OWNER reads as unchecked) | Lists the upstream DHCP leases vpcd holds for `source="dhcp"` external pools, with the resource each was taken for and whether that resource still exists. An OWNER of `gone` is a leaked address — nothing renews it on purpose and nothing will release it, so it stays out of the upstream scope until its TTL expires. `unknown` means the owning service could not be reached, which is not the same finding. Alias: `spx get leases` |
 
 ### Resource Monitoring
 
@@ -57,8 +95,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--skip-host-dns`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway <ip>` or `--external-source=dhcp [--external-bind-bridge <iface>]` for a public pool with full EIP support; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
-| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services`, `--skip-host-dns` | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations → points the host resolver at this node's local northstar so the Spinifex zones resolve from the node itself (opt out with `--skip-host-dns`) |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--advertise` (external IP off-host clients dial; auto-detected from the default route), `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--predastore-compaction-interval` (seconds; test clusters set a short one), `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--skip-host-dns`, `--email` (operator address for update and security notices), `--no-telemetry`, `--gpu-passthrough`, `--ipsec` (default true; encrypt intra-AZ Geneve via OVN native IPsec cluster-wide — disable only for a trusted single-rack lab), `--host` (leader to join; falls back to multicast discovery), `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--external-iface` (WAN NIC for br-external, auto-detected), `--external-source` (`dhcp` \| `static`), `--external-pool` (`start-end`), `--external-prefix-len` (default 24), `--external-gateway` (auto-detected), `--external-bind-bridge` (default `br-wan` when `--external-source=dhcp`), `--gateway-ip` (OVN gateway router's external IP for SNAT; defaults to the pool's range start, required for `nat` mode without DHCP) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
+| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--advertise` (defaults to `--bind`, or the auto-detected WAN IP when `--bind` is 0.0.0.0), `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services`, `--skip-host-dns`, `--email`, `--no-telemetry`, `--predastore-compaction-interval`, `--join-timeout` (how long to keep retrying while the formation server is unreachable, default 20m), `--force` (join even though this node is already initialized, **discarding its own CA and master key**) | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations → points the host resolver at this node's local northstar so the Spinifex zones resolve from the node itself (opt out with `--skip-host-dns`) |
 
 ### Version
 
@@ -72,7 +110,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
 | `spx admin cluster drain-dhcp` | `--timeout` (reply-collection window, default 30s) | Asks each vpcd to DHCPRELEASE every external-pool DHCP lease it currently holds, returning them to the upstream DHCP server. Run on teardown before stopping services — an env reset otherwise strands held leases upstream until their TTL expires, eventually exhausting the upstream scope. Best-effort: warns and exits 0 if the cluster is already down. |
-| `spx admin node drain --local` | `--local` (drain the local node only, required), `--timeout` (max wait per phase, default 120s) | Runs the GATE and DRAIN phases against the local node only: powers down its guests via QMP and unmounts their volumes (flushing the viperblock WAL) while every service is still up. STORAGE/PERSIST/INFRA are left to systemd's ordered teardown. Wired as the `ExecStop` of `spinifex-shutdown.service`, so `systemctl stop spinifex.target` and host reboot/poweroff drain guests before any storage service stops. |
+| `spx admin node drain --local` | `--local` (drain the local node only, required), `--timeout` (max wait per phase, default 120s), `--unless-restarting` (drain on a real host shutdown or a plain target stop, and skip only when a restart of the stack is already queued; unset, it always drains) | Runs the GATE and DRAIN phases against the local node only: powers down its guests via QMP and unmounts their volumes (flushing the viperblock WAL) while every service is still up. STORAGE/PERSIST/INFRA are left to systemd's ordered teardown. Wired as the `ExecStop` of `spinifex-shutdown.service`, so `systemctl stop spinifex.target` and host reboot/poweroff drain guests before any storage service stops. |
 
 ### Certificate Management
 
@@ -80,6 +118,22 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. |
 | `spx admin cert create-tenant-ca` | `--domain` (required, repeatable — permitted domains baked into the CA's name constraints), `--regenerate` (replace an existing tenant CA, requires confirmation), `--yes` (skip the `--regenerate` confirmation prompt) | Creates the tenant root CA that ACM's `PRIVATE_CA` validation mode signs leaf certificates from — a separate, independent root from the platform CA. Idempotent: running it again against an existing tenant CA reports the current state rather than regenerating; `--regenerate` is required to replace the root, which invalidates every device's existing trust. |
+| `spx admin cert force-renew` | `--arn` (required, certificate ARN), `--account` (defaults to the bootstrap account) | Reissues an ACM certificate under its existing ARN immediately, instead of waiting for the renewal worker's proportional window. Only `PRIVATE_CA` (tenant-CA-issued) certificates can be force-renewed; imported and public ones are refused. Talks to the running daemon over NATS and takes the same per-certificate lease the scheduled worker uses, so it is safe against a live deployment. It exists mainly to verify the ELBv2 fan-out — that every load balancer referencing the certificate re-renders with the new leaf — without waiting out the real window. |
+
+### Node Diagnostics
+
+| Command | Flags | Description |
+|---------|-------|-------------|
+| `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. Exits non-zero if any asset is Missing, Stale or Ungranted. |
+| `spx admin node js-probe` | `--timeout` (default: 10s) | Round-trips a canary key through the cluster-state KV bucket and exits non-zero if the write or the read-back fails. JetStream can latch a write error per message block and keep serving reads — nats-server's filestore records the failure on the block and returns it on every later write, and nothing clears it short of a restart. The HTTP health endpoint reports the server-wide JetStream state, which that latch never touches, so only an actual write observes it. |
+| `spx admin banner` | `--boot-check` | Writes the node information banner to `/etc/issue` (shown before login on the physical/serial console) and `/etc/motd` (shown after SSH login). With `--boot-check` it also detects a management IP that has changed since last boot and updates `/etc/spinifex/node.conf` accordingly. |
+
+### Storage Inspection
+
+| Command | Flags | Description |
+|---------|-------|-------------|
+| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. |
+| `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is allowed, and is preferred over leaving the instance unable to run, but it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
 
 ### Upgrade Management
 
@@ -97,10 +151,10 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
 | `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
 | `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
-| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Bedrock model endpoints are reaped ahead of instances, because only their own delete releases the ENI and weights volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
+| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
 | `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
 | `spx admin account delete --remote` | as above, plus `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Same teardown driven through `POST /admin/DeleteAccount` over SigV4 instead of NATS, then follows `DescribeAccountDeletion` to completion, printing each stage as it lands. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. |
-| `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
+| `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method — `CreateAccount`, `DeleteAccount`, `DescribeAccountDeletion`, `GetAccountQuota`, `ListAccounts`, `PutAccountQuota`) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
 | `spx admin principal list` | — | Lists every principal in the super-admin account with its access-key count and what it may call. An attached policy is reported by name (`AdministratorAccess (attached)`) rather than expanded, which is how an unscoped principal is spotted. |
 | `spx admin principal revoke <name>` | — | Deletes every access key the named principal holds, leaving the user and its policy in place. Immediate and cluster-wide; re-issue with `principal create`. |
 | `spx admin principal delete <name>` | — | Removes the principal entirely: its access keys, its inline policies and the user. `admin` is refused, as it is for `create`. Use `revoke` instead when the principal should stay and only its credential should stop working. |
@@ -110,7 +164,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. Import never modifies the image: the bytes written to storage are the bytes that were verified. A guest that needs to trust the deployment CA for in-guest SDK-over-TLS calls fetches it at runtime from `http://169.254.169.254/spinifex/ca.pem` (see the IMDS section), so a CA rotation reaches guests without re-importing anything. |
+| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify`, `--ami-name` (override the registered `DescribeImages` name, which otherwise defaults to `ami-{distro}-{version}-{arch}`; used for locally-built appliances such as `--ami-name spinifex-eks-node`), `--tmp-dir` (default: `/tmp`) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. Import never modifies the image: the bytes written to storage are the bytes that were verified. A guest that needs to trust the deployment CA for in-guest SDK-over-TLS calls fetches it at runtime from `http://169.254.169.254/spinifex/ca.pem` (see the IMDS section), so a CA rotation reaches guests without re-importing anything. |
 | `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
 | `spx admin images promote` | `--image-id` (required), `--yes` | Reads `ami-<id>/config.json`, validates the AMI is account-owned, then rewrites `ImageOwnerAlias` to `"system"` in-place. No block data is copied. The change takes effect immediately — the AMI becomes visible to all accounts via `DescribeImages`. Prompts for confirmation (skipped with `--yes`). Already-system AMIs are refused. |
 | `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. |
@@ -127,27 +181,6 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
 | `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
 
-### Ochre Weights Management
-
-| Command | Flags | Description |
-|---------|-------|-------------|
-| `spx admin ochre weights stage` | `--model-id` (required), `--s3-uri` (required, `s3://bucket/prefix/`), `--tmp-dir` (default: OS temp dir) | Stages a self-host model's weights for serving. Refuses first if `--model-id` is not a self-host catalog entry (unknown or provider-served), or if `bedrock-weights` already has this model staged from the same `--s3-uri` and that snapshot still exists (no-op). If the recorded snapshot is gone (host rebuild, volume GC), it re-stages instead of refusing, so no manual `remove` is needed. Then validates the prefix holds `config.json`, `tokenizer_config.json`, at least one `*.safetensors` file, and at least one of `tokenizer.json` or `tokenizer.model`, failing before downloading anything if any are missing. Downloads the prefix's objects from predastore into `--tmp-dir`, packs them into an ext4 image (`mkfs.ext4 -d`), imports it into a new viperblock volume via `v_utils.ImportDiskImage` with `AMIMetadata` left empty (plain volume, no AMI registration), snapshots the closed volume, and records the source URI and snapshot ID against `--model-id` in the `bedrock-weights` KV bucket. Re-staging a different `--s3-uri` replaces the entry and reports the previous snapshot ID for separate reclamation. |
-| `spx admin ochre weights list` | — | Lists every staged model with its source URI, snapshot ID and a STATUS column, so an operator can see why a model is (or isn't) advertised via `ListFoundationModels`. STATUS reads `OK` when the recorded snapshot still exists, or `DANGLING (snapshot missing)` when the KV record survives but its snapshot is gone. |
-| `spx admin ochre weights remove` | `--model-id` (required) | Drops `--model-id`'s entry from `bedrock-weights`, hiding it from `ListFoundationModels` again. Refuses if the model has no staged entry. Never deletes the underlying snapshot or source S3 objects; reclaiming that storage is a separate, explicit act. |
-
-### Ochre Serving Endpoints
-
-Operator surface over the daemon's `bedrock.endpoint.*` subjects. The gateway does not request endpoints itself yet, so until it does these are the only way to start a serving VM.
-
-| Command | Flags | Description |
-|---------|-------|-------------|
-| `spx admin ochre endpoint ensure` | `--model-id` (required), `--wait`, `--timeout` (default: 15m) | Asks the daemon to bring up a serving VM for `--model-id`, which must already have staged weights. Idempotent: a model already `STARTING` or `READY` returns the current record rather than launching a second VM. The daemon replies `STARTING` as soon as it has claimed the model and the launch continues in the background, so `--wait` polls `describe` until `READY` and reports the elapsed cold start. A launch that fails deletes its record, so a return to `ABSENT` is reported as an abort. A `--timeout` leaves the endpoint running rather than tearing it down. |
-| `spx admin ochre endpoint describe` | `--model-id` (required) | Shows the model's current endpoint record: state, instance and node IDs, base URL, weights volume, and the derived startup time once `READY`. A model with no record reads as `ABSENT`. |
-| `spx admin ochre endpoint list` | — | Lists every endpoint record with its state, instance ID and base URL. |
-| `spx admin ochre endpoint delete` | `--model-id` (required) | Moves a `READY` endpoint to `DRAINING` and tears its VM down, releasing the GPU. Idempotent: an already-absent endpoint reports success. |
-
-Self-hosted models are served from the `ubuntu-26.04-vllm-serving-x86_64` system image (`spx admin images import --name ubuntu-26.04-vllm-serving-x86_64`), built by `scripts/mkosi-build.sh --image ubuntu-vllm-serving` (`gpu-nvidia` + `vllm` profiles). It carries the tags `spinifex:managed-by=bedrock` and `spinifex:bedrock-role=vllm-serving`, which is what `LaunchServingVM` filters on rather than resolving by name. The image bakes vLLM's OpenAI-compatible server into a uv-managed venv and TEI's engine binary + CUDA runtime libraries (extracted from HuggingFace's own published CUDA image, no Docker daemon on the guest either way) side by side, one hand-stood image serving a whole co-served bundle rather than a single model. `LaunchServingVM` writes one `/etc/conf.d/bedrock-bundle/<instance>.env` file per bundle member; `bedrock-bundle-init.service` enumerates them at boot and starts one `bedrock-serve@<instance>.service` per member, which resolves and mounts that member's own cloned weights volume read-only and execs `vllm serve` (port 8000, the generative member) or `text-embeddings-router` directly (port 8001+, embedding/rerank members) — each engine's readiness is probed on the route it actually serves (`/v1/models` for vLLM, `/health` for TEI). A standalone model is the one-member case of the same path: it boots exactly as before, on port 8000.
-
 ### EKS Control-Plane Disaster Recovery
 
 | Command | Flags | Description |
@@ -166,10 +199,12 @@ Requests are SigV4-signed with `service=spinifex` and the gateway's configured r
 | `POST /admin/DeleteAccount` | `accountId` (12 digits, required), `accountName` (must match the stored record), `clientToken` (32–128 chars), `force`, `dryRun` | `deletionId`, `accountId`, `state`; plus `inventory` when `dryRun` | Marks the account `TERMINATING` and tears down everything it owns in dependency order, then deletes the account record. Returns as soon as the teardown starts — it takes minutes, so progress comes from `DescribeAccountDeletion`. `dryRun` needs neither the name nor a token, answers inline and changes nothing. One job per account: a second call with a different token gets `OperationInProgress` while one is running. `000000000000` and `000000000001` return `AccessDenied`, the same answer an unauthorized caller gets. |
 | `POST /admin/DescribeAccountDeletion` | `accountId` (12 digits, required) | `deletionId`, `state` (`RUNNING`, `COMPLETED`, `FAILED`), `startedAt`, `updatedAt`, `finishedAt`, `stages[]` with per-stage `deleted[]`, `stuck[]` and `elapsed`, `error` | The record outlives the account, so this stays answerable after the deletion completes. `NoSuchEntity` means no teardown was ever started for that account. A `RUNNING` job whose `updatedAt` has not advanced for five minutes is treated as abandoned — the gateway running it restarted — and a fresh `DeleteAccount` takes it over. |
 | `POST /admin/ListAccounts` | — | `accounts[]` of `accountId`, `accountName`, `status`, `createdAt`; `count` | Enumerates every account. No credential material, and no paging: `[signup] max_accounts` bounds the listing. `status` is how a teardown stuck in `TERMINATING` is noticed. |
+| `POST /admin/GetAccountQuota` | `accountId` (12 digits, required) | per-dimension limits, each with the source it came from | Reads the limits in force for one account, reporting for each dimension whether the value came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Driven by `spx admin account quota get`. |
+| `POST /admin/PutAccountQuota` | `accountId` (12 digits, required), the limit dimensions to override, or a clear directive | the resulting limits | Overrides individual limits for one account, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. The account must already exist. Driven by `spx admin account quota set`. |
 
 Errors are `{"error":{"code":…,"message":…},"requestId":…}`. `OperationInProgress` (409), `ServiceUnavailable` (503) and `InternalError` (500) are retryable with backoff, **always reusing the same `clientToken`** — a fresh token after a failure creates a duplicate account. `AccountAlreadyExists` (409), `IdempotentParameterMismatch` (400), `LimitExceeded` (400, `[signup] max_accounts` reached), `InvalidParameterValue`, `MissingParameter`, `InvalidRequest`, `InvalidAction`, `MethodNotAllowed` (405) and `AccessDenied` (403) are not.
 
-Each method is a separate grant, and `spx admin principal create` decides which a credential holds. `spx admin principal create operator` grants all four — the operator and load-test credential. `spx admin principal create signup --grant CreateAccount` grants one, which is what an internet-facing signup form should hold: that key cannot reach deletion, so a compromise of the form's host cannot remove a tenant. Separate principals are the point rather than an accident: revoking either costs nothing else.
+Each method is a separate grant, and `spx admin principal create` decides which a credential holds. `spx admin principal create operator` grants all six — the operator and load-test credential. `spx admin principal create signup --grant CreateAccount` grants one, which is what an internet-facing signup form should hold: that key cannot reach deletion, so a compromise of the form's host cannot remove a tenant. Separate principals are the point rather than an accident: revoking either costs nothing else.
 
 The surface is unreachable until an operator runs `spx admin principal create`; there is no config toggle and no default credential for it. The super-admin account's own `admin` user does hold these grants already, through the `AdministratorAccess` policy every account's administrator carries (`Action: "*"`), so its bootstrap key can call the surface without any further setup — but it is unscoped and unrotatable without locking out the operator, which is why automation should use a named principal. Revocation is `spx admin principal revoke <name>` (or `aws iam delete-access-key`), which is immediate and cluster-wide. `[signup] max_accounts` in `awsgw.toml` caps how many accounts may exist; it defaults to 128 when the section is absent, and 0 means uncapped.
 
@@ -1114,6 +1149,50 @@ Clusters, services and tasks run on EC2 container instances; there is no Fargate
 
 ---
 
+## EKS (Elastic Kubernetes Service)
+
+REST-JSON, dispatched by HTTP method + path rather than by an action parameter,
+so the routing table in `gateway/eks.go` is the authoritative list. Clusters run
+K3s on control-plane VMs; nodegroups are EC2 instances joined to them.
+
+| Command | Status |
+|---------|--------|
+| `create-cluster`, `describe-cluster`, `list-clusters`, `delete-cluster` | **DONE** |
+| `create-nodegroup`, `describe-nodegroup`, `list-nodegroups`, `delete-nodegroup`, `update-nodegroup-config`, `update-nodegroup-version` | **DONE** |
+| `create-addon`, `describe-addon`, `describe-addon-versions`, `list-addons`, `update-addon`, `delete-addon` | **DONE** |
+| `create-access-entry`, `describe-access-entry`, `list-access-entries`, `update-access-entry`, `delete-access-entry` | **DONE** |
+| `associate-access-policy`, `disassociate-access-policy`, `list-associated-access-policies`, `list-access-policies` | **DONE** |
+| `tag-resource`, `untag-resource`, `list-tags-for-resource` | **DONE for cluster ARNs**; other EKS resource ARNs return `NotImplemented` |
+| `update-cluster-config`, `update-cluster-version` | **NOT STARTED** (`NotImplemented`) |
+| Identity provider configs: `associate-identity-provider-config`, `describe-identity-provider-config`, `list-identity-provider-configs`, `disassociate-identity-provider-config` | **NOT STARTED** (`NotImplemented`) |
+
+Four routes on this surface are **internal, not AWS actions**, and are not
+callable with an ordinary tenant credential: `PublishInternal` (the control-plane
+VM broker, relaying bootstrap/state POSTs onto `eks.bus.*`/`eks.state.*`),
+`WebhookTokenReview` (the `eks-token-webhook` posting bearer tokens for host-side
+resolution), `ListInternalAddons` (the on-VM addon-sync agent fetching its staged
+manifests) and `GetRecoveryDirective` (the on-VM k3s-recovery agent reading its
+per-member directive at boot). They name the target account in the path rather
+than taking it from the credential, so each requires a control-plane instance-role
+session bound to the named cluster — a broad `eks:*` grant does not reach them.
+
+Disaster recovery for a lost single control plane is an operator command, not an
+API call: see `spx admin eks restore-snapshot` above.
+
+---
+
+## Resource Groups Tagging API (`tagging`)
+
+JSON-1.1, dispatched from the action suffix of
+`X-Amz-Target: ResourceGroupsTaggingAPI_20170126.<Action>`.
+
+| Command | Status |
+|---------|--------|
+| `get-resources` | **DONE** — authorizes against `"*"`, which is correct here rather than a fail-open: the Resource Groups Tagging API defines no resource types of its own. A two-way completeness test pins that, so a later action cannot be added to this surface with a silent account-wide grant |
+| `get-tag-keys`, `get-tag-values`, `tag-resources`, `untag-resources` | **NOT STARTED** (`InvalidAction`) |
+
+---
+
 ## CloudWatch (Basic Monitoring)
 
 | Command | Implemented Flags | Missing Flags | Status |

docs/aws-model-operation-coverage.md

@@ -7,7 +7,7 @@ Implemented means a modelled operation is registered to a real handler. Stub and
 | Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |
 |---|---:|---:|---:|---:|---:|---:|---:|---:|
 | acm | 2015-12-08 | 15 | 9 | 9 | 0 | 0 | 6 | 0 |
-| ec2 | 2016-11-15 | 625 | 119 | 119 | 0 | 0 | 506 | 0 |
+| ec2 | 2016-11-15 | 625 | 121 | 121 | 0 | 0 | 504 | 0 |
 | ecr | 2015-09-21 | 47 | 39 | 19 | 13 | 7 | 11 | 3 |
 | ecs | 2014-11-13 | 56 | 39 | 31 | 5 | 0 | 20 | 3 |
 | elasticloadbalancingv2 | 2015-12-01 | 46 | 37 | 33 | 0 | 0 | 13 | 4 |
@@ -56,9 +56,9 @@ None.
 
 ## ec2
 
-Implements **119 of 625** modelled operations (19.0%).
+Implements **121 of 625** modelled operations (19.4%).
 
-<details><summary>Implemented (119)</summary>
+<details><summary>Implemented (121)</summary>
 
 `AllocateAddress`, `AssociateAddress`, `AssociateIamInstanceProfile`, `AssociateRouteTable`, `AttachInternetGateway`, `AttachNetworkInterface`, `AttachVolume`, `AuthorizeSecurityGroupEgress`
 
@@ -86,14 +86,16 @@ Implements **119 of 625** modelled operations (19.0%).
 
 `ImportKeyPair`, `ModifyImageAttribute`, `ModifyInstanceAttribute`, `ModifyInstanceMetadataOptions`, `ModifyLaunchTemplate`, `ModifyNetworkInterfaceAttribute`, `ModifySubnetAttribute`, `ModifyVolume`
 
-`ModifyVpcAttribute`, `RebootInstances`, `RegisterImage`, `ReleaseAddress`, `ReplaceIamInstanceProfileAssociation`, `ReplaceRoute`, `ReplaceRouteTableAssociation`, `RequestSpotInstances`
+`ModifyVpcAttribute`, `MonitorInstances`, `RebootInstances`, `RegisterImage`, `ReleaseAddress`, `ReplaceIamInstanceProfileAssociation`, `ReplaceRoute`, `ReplaceRouteTableAssociation`
 
-`ResetImageAttribute`, `RevokeSecurityGroupEgress`, `RevokeSecurityGroupIngress`, `RunInstances`, `StartInstances`, `StopInstances`, `TerminateInstances`
+`RequestSpotInstances`, `ResetImageAttribute`, `RevokeSecurityGroupEgress`, `RevokeSecurityGroupIngress`, `RunInstances`, `StartInstances`, `StopInstances`, `TerminateInstances`
+
+`UnmonitorInstances`
 
 
 </details>
 
-<details><summary>Missing from dispatch (506)</summary>
+<details><summary>Missing from dispatch (504)</summary>
 
 `AcceptAddressTransfer`, `AcceptReservedInstancesExchangeQuote`, `AcceptTransitGatewayMulticastDomainAssociations`, `AcceptTransitGatewayPeeringAttachment`, `AcceptTransitGatewayVpcAttachment`, `AcceptVpcEndpointConnections`, `AcceptVpcPeeringConnection`, `AdvertiseByoipCidr`
 
@@ -207,21 +209,19 @@ Implements **119 of 625** modelled operations (19.0%).
 
 `ModifyVpcEndpointConnectionNotification`, `ModifyVpcEndpointServiceConfiguration`, `ModifyVpcEndpointServicePayerResponsibility`, `ModifyVpcEndpointServicePermissions`, `ModifyVpcPeeringConnectionOptions`, `ModifyVpcTenancy`, `ModifyVpnConnection`, `ModifyVpnConnectionOptions`
 
-`ModifyVpnTunnelCertificate`, `ModifyVpnTunnelOptions`, `MonitorInstances`, `MoveAddressToVpc`, `MoveByoipCidrToIpam`, `ProvisionByoipCidr`, `ProvisionIpamByoasn`, `ProvisionIpamPoolCidr`
+`ModifyVpnTunnelCertificate`, `ModifyVpnTunnelOptions`, `MoveAddressToVpc`, `MoveByoipCidrToIpam`, `ProvisionByoipCidr`, `ProvisionIpamByoasn`, `ProvisionIpamPoolCidr`, `ProvisionPublicIpv4PoolCidr`
 
-`ProvisionPublicIpv4PoolCidr`, `PurchaseCapacityBlock`, `PurchaseHostReservation`, `PurchaseReservedInstancesOffering`, `PurchaseScheduledInstances`, `RegisterInstanceEventNotificationAttributes`, `RegisterTransitGatewayMulticastGroupMembers`, `RegisterTransitGatewayMulticastGroupSources`
+`PurchaseCapacityBlock`, `PurchaseHostReservation`, `PurchaseReservedInstancesOffering`, `PurchaseScheduledInstances`, `RegisterInstanceEventNotificationAttributes`, `RegisterTransitGatewayMulticastGroupMembers`, `RegisterTransitGatewayMulticastGroupSources`, `RejectTransitGatewayMulticastDomainAssociations`
 
-`RejectTransitGatewayMulticastDomainAssociations`, `RejectTransitGatewayPeeringAttachment`, `RejectTransitGatewayVpcAttachment`, `RejectVpcEndpointConnections`, `RejectVpcPeeringConnection`, `ReleaseHosts`, `ReleaseIpamPoolAllocation`, `ReplaceNetworkAclAssociation`
+`RejectTransitGatewayPeeringAttachment`, `RejectTransitGatewayVpcAttachment`, `RejectVpcEndpointConnections`, `RejectVpcPeeringConnection`, `ReleaseHosts`, `ReleaseIpamPoolAllocation`, `ReplaceNetworkAclAssociation`, `ReplaceNetworkAclEntry`
 
-`ReplaceNetworkAclEntry`, `ReplaceTransitGatewayRoute`, `ReplaceVpnTunnel`, `ReportInstanceStatus`, `RequestSpotFleet`, `ResetAddressAttribute`, `ResetEbsDefaultKmsKeyId`, `ResetFpgaImageAttribute`
+`ReplaceTransitGatewayRoute`, `ReplaceVpnTunnel`, `ReportInstanceStatus`, `RequestSpotFleet`, `ResetAddressAttribute`, `ResetEbsDefaultKmsKeyId`, `ResetFpgaImageAttribute`, `ResetInstanceAttribute`
 
-`ResetInstanceAttribute`, `ResetNetworkInterfaceAttribute`, `ResetSnapshotAttribute`, `RestoreAddressToClassic`, `RestoreImageFromRecycleBin`, `RestoreManagedPrefixListVersion`, `RestoreSnapshotFromRecycleBin`, `RestoreSnapshotTier`
+`ResetNetworkInterfaceAttribute`, `ResetSnapshotAttribute`, `RestoreAddressToClassic`, `RestoreImageFromRecycleBin`, `RestoreManagedPrefixListVersion`, `RestoreSnapshotFromRecycleBin`, `RestoreSnapshotTier`, `RevokeClientVpnIngress`
 
-`RevokeClientVpnIngress`, `RunScheduledInstances`, `SearchLocalGatewayRoutes`, `SearchTransitGatewayMulticastGroups`, `SearchTransitGatewayRoutes`, `SendDiagnosticInterrupt`, `StartNetworkInsightsAccessScopeAnalysis`, `StartNetworkInsightsAnalysis`
+`RunScheduledInstances`, `SearchLocalGatewayRoutes`, `SearchTransitGatewayMulticastGroups`, `SearchTransitGatewayRoutes`, `SendDiagnosticInterrupt`, `StartNetworkInsightsAccessScopeAnalysis`, `StartNetworkInsightsAnalysis`, `StartVpcEndpointServicePrivateDnsVerification`
 
-`StartVpcEndpointServicePrivateDnsVerification`, `TerminateClientVpnConnections`, `UnassignIpv6Addresses`, `UnassignPrivateIpAddresses`, `UnassignPrivateNatGatewayAddress`, `UnlockSnapshot`, `UnmonitorInstances`, `UpdateSecurityGroupRuleDescriptionsEgress`
-
-`UpdateSecurityGroupRuleDescriptionsIngress`, `WithdrawByoipCidr`
+`TerminateClientVpnConnections`, `UnassignIpv6Addresses`, `UnassignPrivateIpAddresses`, `UnassignPrivateNatGatewayAddress`, `UnlockSnapshot`, `UpdateSecurityGroupRuleDescriptionsEgress`, `UpdateSecurityGroupRuleDescriptionsIngress`, `WithdrawByoipCidr`
 
 
 </details>

docs/compute/vpc-networking/README.md

@@ -795,6 +795,168 @@ aws ec2 run-instances --image-id $AMI --instance-type t3.small \
 
 Rule changes take effect immediately — no instance restart needed.
 
+## Platform Default Egress Restrictions (Outbound SMTP)
+
+Like AWS, Spinifex blocks outbound SMTP from instances **by default** so a
+compromised guest cannot turn a fresh account into a spam relay. Connections to
+the mail ports are dropped at the OVS datapath before they reach the wire:
+
+| Port | Protocol | Purpose |
+| --- | --- | --- |
+| 25 | TCP | SMTP relay |
+| 465 | TCP | SMTP over implicit TLS |
+| 587 | TCP | SMTP submission |
+
+This is a **platform default, not a security-group rule**. It is enforced as an
+OVN egress ACL on every guest's port group at a priority **above** tenant SG
+allows, so a tenant **cannot** open these ports by adding a security-group rule —
+matching AWS, where lifting the block is an operator action, not a tenant one.
+Only **public** destinations are blocked; mail to private ranges (`10/8`,
+`172.16/12`, `192.168/16`, `100.64/10`, link-local) is exempt, so an in-VPC or
+on-prem relay still works. Dropped attempts are logged for abuse triage.
+
+It sits alongside the security-group ACLs and the host firewall (the `nft`
+policy that scopes the node's own ports) as a third datapath control — this one
+applies to guest egress specifically.
+
+**Operator controls** (cluster-wide, in `spinifex.toml`):
+
+```toml
+[network]
+# Omit for the default [25, 465, 587]; set [] to disable entirely.
+blocked_ports_wan = [25, 465, 587]
+
+# Workaround to let a specific tenant/VPC send mail, until per-account
+# exceptions exist: list the VPC IDs to exempt from the block.
+egress_block_exempt_vpcs = ["vpc-0abc123..."]
+```
+
+Exempting a VPC removes the block for **all** guests in that VPC, so scope it
+narrowly. Add every VPC ID you want exempt to the one list —
+`egress_block_exempt_vpcs = ["vpc-a", "vpc-b", ...]`; the match is by VPC ID, so
+one entry covers every security group in that VPC.
+
+### Multi-node clusters — keep the value identical on every node, and restart vpcd
+
+`[network]` is a cluster-wide *setting*, but it physically lives in each node's
+own `spinifex.toml`, and there is no live distribution of it. Two properties of
+the current implementation make the operator responsible for consistency:
+
+- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd
+  leader, which programs the shared OVN northbound DB. Whichever node holds the
+  lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in
+  force. Leadership moves on restart or crash, so if the node configs disagree,
+  the effective policy **changes when the leader changes** and the drift pass
+  flaps the ACLs between the two states. Edit the value **identically on every
+  node**.
+- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when
+  vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the
+  config change to all nodes and restart vpcd cluster-wide (take the target down
+  and confirm no `spx` process survives — a selective single-service restart is
+  not reliable on the shared binary).
+
+This is the current implementation and is deliberately minimal. A future revision
+will move the exemption into shared cluster state (so a single edit propagates
+and cannot drift between nodes) and make it a per-account control rather than a
+per-VPC operator edit; until then, the two rules above are load-bearing.
+
+## The Instance-to-Host Plane (Metadata and VPC DNS)
+
+Security groups govern traffic **between instances** and **to the outside**. There
+is a third path they do not touch, and it is the one most easily left exposed:
+traffic from an instance to the **host it runs on**.
+
+Every instance reaches two link-local addresses served by the hypervisor:
+
+| Address | Service | Port |
+| --- | --- | --- |
+| `169.254.169.254` | Instance metadata (IMDS) — cloud-init, instance-role credentials | TCP 80 |
+| `169.254.169.253` | VPC DNS resolver | UDP/TCP 53 |
+
+These are not guest addresses and not OVN routed. Each instance's tap has a
+capture rule on the hypervisor that intercepts any packet addressed to `.254` or
+`.253` and delivers it to a per-ENI internal port in the host network namespace
+(named `ime-*`). That interception is **by destination address only — it does not
+match the port.** A packet to `169.254.169.254:22` is delivered to the host just
+as readily as one to `169.254.169.254:80`.
+
+That matters because host services — SSH, the AWS gateway on 9999, the console on
+3000, the DNS server — bind the wildcard address, so they also answer on these
+link-local addresses. Without a control on the `ime-*` path, an instance can
+reach every one of them, with none of the source scoping that protects the same
+services on the network. A resolver query sent straight to the host's DNS port
+this way also bypasses the per-instance DNS rate limit.
+
+### This plane is invisible to a network firewall
+
+The critical point for anyone securing a deployment: **this traffic never crosses
+the physical NIC.** It is intra-host, on the OVS bridge between the instance's tap
+and the `ime-*` port. A cloud security group, an upstream firewall, `ufw`, or any
+rule written against the WAN interface is not in this path and cannot filter it.
+The only place to enforce it is on the `ime-*` interface in the host itself.
+
+### How it is protected
+
+Spinifex's host firewall carries a rule that scopes the `ime-*` path to exactly
+the two legitimate endpoints and ports — IMDS on `.254:80`, DNS on `.253:53` —
+and drops everything else. Whether you have that rule depends entirely on whether
+the host firewall is on:
+
+| Install path | Host firewall | This plane |
+| --- | --- | --- |
+| **From the ISO** | on | protected |
+| **Binary installer** (`curl \| bash`) or `setup.sh` | **off** | **exposed** |
+| **Binary installer with `--firewall=on`** | on | protected |
+
+The ISO ships the firewall armed, so ISO deployments are protected out of the box.
+**The binary installer ships it off** — deliberately, because it runs on servers
+that may already have services the installer knows nothing about, and a
+default-deny policy could cut them off. The consequence is that a binary install
+left at its default has this plane wide open, and so does every host service on
+the WAN besides.
+
+If you install any way other than the ISO, turn the host firewall on. At install
+time:
+
+```bash
+curl -fsSL https://install.mulgadc.com | bash -s -- --firewall=on
+# or, from a source checkout:
+sudo /usr/local/share/spinifex/setup.sh --firewall=on
+```
+
+or afterwards, in `/etc/spinifex/spinifex.toml`, followed by a daemon restart:
+
+```toml
+[network]
+firewall_enabled = true
+```
+
+Before enabling it, check what else the machine is serving — anything listening
+outside the public port group stops accepting new connections. See
+[Firewall and Cluster Membership](/docs/install-multi-node#firewall-and-cluster-membership)
+for the full port policy and the cluster-formation steps.
+
+### Verifying it
+
+On the hypervisor, the `ime-*` accept should be scoped, not blanket:
+
+```bash
+sudo nft list chain inet spinifex_filter input | grep ime
+# Expect three rules naming 169.254.169.254 tcp dport 80 and 169.254.169.253
+# udp/tcp dport 53 — not a bare `iifname "ime-*" accept`.
+```
+
+From an instance, the two service ports work and nothing else does:
+
+```bash
+curl -s -o /dev/null -w '%{http_code}\n' http://169.254.169.254/latest/meta-data/   # 401 (IMDSv2)
+dig +short @169.254.169.253 example.com A                                            # resolves
+nc -vz 169.254.169.254 22                                                            # must fail
+```
+
+A missing `spinifex_filter` table means the host firewall is off and this plane
+is unprotected regardless of the rule above.
+
 ## Elastic IPs
 
 Elastic IPs are static public IPs that persist across instance stop/start cycles. Unlike auto-assigned public IPs (which change on stop/start), an Elastic IP stays with your instance.

docs/identity/iam-roles-and-instance-profiles/README.md

@@ -281,7 +281,7 @@ aws ec2 disassociate-iam-instance-profile \
 
 ## Assuming a Role
 
-Users and services can assume a role directly with STS, provided the trust policy allows their principal:
+Users and services can assume a role directly with STS, provided **both** gates pass: the role's trust policy admits their principal, and their own identity policy grants `sts:AssumeRole` on the role's ARN. See [STS](/docs/sts) for the identity-side grant.
 
 ```bash
 aws sts assume-role \

docs/identity/sts/README.md

@@ -42,7 +42,7 @@ STS (Security Token Service) issues **temporary credentials**: an `ASIA`-prefixe
 
 Spinifex supports the three main ways to obtain them:
 
-- **`assume-role`** — exchange your IAM user credentials for a [role's](/docs/iam-roles-and-instance-profiles) permissions, gated by the role's trust policy.
+- **`assume-role`** — exchange your IAM user credentials for a [role's](/docs/iam-roles-and-instance-profiles) permissions, gated by both the role's trust policy and your own `sts:AssumeRole` grant.
 - **`get-session-token`** — get a time-boxed copy of your own [IAM user's](/docs/iam-users-and-policies) permissions.
 - **`assume-role-with-web-identity`** — exchange an OIDC ID token (a Kubernetes ServiceAccount token) for role credentials, with no IAM credentials at all.
 
@@ -79,7 +79,14 @@ Run with temporary role credentials, the ARN switches to the assumed-role form
 
 ## Assuming a Role
 
-`assume-role` works when the target role's trust policy allows your principal. Create a role that trusts an IAM user (see [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles) for the full role lifecycle):
+`assume-role` needs two independent grants, and a missing one denies the call:
+
+1. The target role's **trust policy** must admit your principal.
+2. Your own **identity policy** must grant `sts:AssumeRole` on the role's ARN.
+
+The second is why `"Principal": {"AWS": "arn:aws:iam::000000000001:root"}` in a trust policy is safe: it delegates the decision to identity policies in that account rather than opening the role to everyone in it.
+
+Create a role that trusts an IAM user (see [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles) for the full role lifecycle):
 
 ```bash
 cat > /tmp/deploy-trust.json << 'EOF'
@@ -102,6 +109,15 @@ aws iam put-role-policy --role-name deploy --policy-name ec2-read \
   --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["ec2:Describe*"],"Resource":"*"}]}'
 ```
 
+That policy is what the *session* may do. The caller still needs its own grant to make the call at all:
+
+```bash
+aws iam put-user-policy --user-name admin --policy-name assume-deploy \
+  --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:iam::000000000001:role/deploy"}]}'
+```
+
+Scope `Resource` to the role ARN exactly as `create-role` returned it, path included — a wildcard such as `arn:aws:iam::000000000001:role/*` grants assumption of every role the account's trust policies admit you to.
+
 Then assume it:
 
 ```bash
@@ -128,7 +144,7 @@ aws sts assume-role \
 
 The session name tags every session so audit logs can tell *who* used the role; it appears in the assumed-role ARN and the `UserId`.
 
-If the trust policy does not allow your principal, `assume-role` fails with `AccessDenied` — no matter what IAM permissions you hold.
+If the trust policy does not allow your principal, `assume-role` fails with `AccessDenied` — no matter what IAM permissions you hold. It fails the same way when your own identity policy carries no `sts:AssumeRole` grant on the role, no matter what the trust policy says.
 
 ### Session Duration
 
@@ -303,7 +319,13 @@ Everything the validator accepts is evaluated at assume time; explicit `Deny` st
 
 ### AccessDenied When Assuming a Role
 
-The role's trust policy does not allow your principal, or the role does not exist — a missing role is deliberately indistinguishable from a denied one, matching AWS. Check who you are, that the role exists, and what it trusts:
+Either gate can produce this. A denial from your own identity policy names the principal, action and resource:
+
+```
+User: arn:aws:iam::000000000001:user/admin is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::000000000001:role/deploy
+```
+
+A bare `AccessDenied` means the role's trust policy does not allow your principal, or the role does not exist — a missing role is deliberately indistinguishable from a denied one, matching AWS. Check who you are, that the role exists, what it trusts, and what you are granted:
 
 ```bash
 aws sts get-caller-identity

docs/security/network-connections/README.md

@@ -102,6 +102,7 @@ behavior in other words.
 | 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
 | 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
 | 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
+| 8660 | predastore (admin) | HTTP | Cluster | `/healthz` and `/readyz`. Readiness names the blob peers a gate cannot reach, so it must not face the WAN. Binds the host's cluster `bind_addr`, never the wildcard. | None — unauthenticated by design; restrict by firewall |
 | 6641 | OVN Northbound DB (client) | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd. Binds `127.0.0.1` plus the node's lan-plane address (`--lan-addr`), never the wildcard address. On a node with no separate lan plane that address is the public one, and a host firewall is the only remaining control. | Cluster network only; TLS planned |
 | 6642 | OVN Southbound DB (client) | OVSDB/TCP | Cluster | Chassis / port / MAC binding state. Binds `127.0.0.1` plus the node's lan-plane address (`--lan-addr`), never the wildcard address. On a node with no separate lan plane that address is the public one, and a host firewall is the only remaining control. | Cluster network only; TLS planned |
 | 6643 | OVN Northbound DB (RAFT) | OVSDB/TCP | Cluster | NB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
@@ -207,7 +208,7 @@ table inet spinifex_filter {
 
     # Cluster, from peer nodes only. Replace with your nodes' lan-plane
     # addresses — a CIDR does not generalise to nodes on different subnets.
-    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644 } accept
+    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
     ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } udp dport { 5300, 6660, 7660 } accept
 
     # Encap, from peer chassis only. Replace with your nodes' vpc-plane

docs/service-interfaces.yaml

@@ -137,6 +137,7 @@ services:
       - "ochre.vector.describeJob"
       - "ochre.vector.query"
       - "ochre.vector.listJobs"
+      - "ochre.vector.stopJob"
     publishes:
       # → vpcd
       - "vpc.create"

go.mod

@@ -4,28 +4,28 @@ go 1.27.0
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.43.6
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18
-	github.com/aws/aws-sdk-go-v2/config v1.32.37
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.36
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37
+	github.com/aws/aws-sdk-go-v2 v1.45.1
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20
+	github.com/aws/aws-sdk-go-v2/config v1.33.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.2
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/containerd/containerd/v2 v2.3.4
 	github.com/distribution/reference v0.6.0
-	github.com/go-chi/chi/v5 v5.3.1
+	github.com/go-chi/chi/v5 v5.3.2
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/klauspost/cpuid/v2 v2.4.0
 	github.com/miekg/dns v1.1.73
-	github.com/mulgadc/bluebottle v1.18.0
-	github.com/mulgadc/northstar v1.18.0
-	github.com/mulgadc/predastore v1.18.0
-	github.com/mulgadc/viperblock v1.18.0
-	github.com/nats-io/nats-server/v2 v2.14.5
+	github.com/mulgadc/bluebottle v1.19.0
+	github.com/mulgadc/northstar v1.19.0
+	github.com/mulgadc/predastore v1.19.0
+	github.com/mulgadc/viperblock v1.19.0
+	github.com/nats-io/nats-server/v2 v2.14.6
 	github.com/nats-io/nats.go v1.53.1
 	github.com/opencontainers/runtime-spec v1.3.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
@@ -34,17 +34,18 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
-	go.opentelemetry.io/otel v1.45.0
-	go.opentelemetry.io/otel/metric v1.45.0
-	go.opentelemetry.io/otel/sdk v1.45.0
-	go.opentelemetry.io/otel/sdk/metric v1.45.0
-	go.opentelemetry.io/otel/trace v1.45.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
+	go.opentelemetry.io/otel v1.46.0
+	go.opentelemetry.io/otel/metric v1.46.0
+	go.opentelemetry.io/otel/sdk v1.46.0
+	go.opentelemetry.io/otel/sdk/metric v1.46.0
+	go.opentelemetry.io/otel/trace v1.46.0
 	go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b
 	go.yaml.in/yaml/v3 v3.0.5
-	golang.org/x/crypto v0.55.0
+	golang.org/x/crypto v0.56.0
 	golang.org/x/sync v0.22.0
 	golang.org/x/sys v0.47.0
+	golang.org/x/time v0.15.0
 	golang.org/x/tools v0.49.0
 	gopkg.in/ini.v1 v1.67.3
 )
@@ -58,23 +59,23 @@ require (
 	github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect
-	github.com/aws/smithy-go v1.27.8 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.110.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.36.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.41.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.48.0 // indirect
+	github.com/aws/smithy-go v1.28.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/buraksezer/consistent v1.0.0 // indirect
+	github.com/buraksezer/consistent v1.1.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cenkalti/hub v1.0.2 // indirect
 	github.com/cenkalti/rpc2 v1.0.5 // indirect
@@ -118,7 +119,7 @@ require (
 	github.com/google/go-tpm v0.9.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gookit/color v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-metrics v0.5.4 // indirect
@@ -169,7 +170,7 @@ require (
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.20.1 // indirect
-	github.com/quic-go/quic-go v0.61.0 // indirect
+	github.com/quic-go/quic-go v0.62.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.12.0 // indirect
 	github.com/sirupsen/logrus v1.9.4 // indirect
@@ -183,14 +184,14 @@ require (
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 // indirect
-	go.opentelemetry.io/otel/log v0.21.0 // indirect
-	go.opentelemetry.io/otel/sdk/log v0.21.0 // indirect
+	go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
+	go.opentelemetry.io/otel/log v0.22.0 // indirect
+	go.opentelemetry.io/otel/sdk/log v0.22.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
@@ -200,12 +201,11 @@ require (
 	golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect
 	golang.org/x/term v0.45.0 // indirect
 	golang.org/x/text v0.41.0 // indirect
-	golang.org/x/time v0.15.0 // indirect
 	golang.org/x/vuln v1.6.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
-	google.golang.org/grpc v1.83.0 // indirect
-	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
+	google.golang.org/grpc v1.83.1 // indirect
+	google.golang.org/protobuf v1.36.12 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect
 	tags.cncf.io/container-device-interface v1.1.0 // indirect

go.sum

@@ -524,78 +524,78 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw
 github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
-github.com/aws/aws-sdk-go-v2 v1.43.6 h1:RrmFcqCBxkJuf7g1axVo5krB4jM/AO8r5e5oujrgdoQ=
-github.com/aws/aws-sdk-go-v2 v1.43.6/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18/go.mod h1:4e5xhuXHx1e4U9EthvbPP1r/DIMp5c2823OL8karzcM=
+github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks=
+github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
 github.com/aws/aws-sdk-go-v2/config v1.18.14/go.mod h1:0pI6JQBHKwd0JnwAZS3VCapLKMO++UL2BOkWwyyzTnA=
 github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g=
 github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI=
-github.com/aws/aws-sdk-go-v2/config v1.32.37 h1:Ljl7LOJB6ym0liuEl0+TZ3d7f5I8MEZN1Cj9PINlj/g=
-github.com/aws/aws-sdk-go-v2/config v1.32.37/go.mod h1:WJ7pe7ZPpmG8Q5kKS53zeypIV4FBGACxmte8Uc6SgUc=
+github.com/aws/aws-sdk-go-v2/config v1.33.2 h1:Pj4+nF2kc4Z+1BJysVPnX9d5dMN7IYFXR4UJaWK2IpA=
+github.com/aws/aws-sdk-go-v2/config v1.33.2/go.mod h1:Igw+HTwbR2tsTU/ydifAS9EHAFJ2s/FCgkwQWFnAdE4=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.14/go.mod h1:85ckagDuzdIOnZRwws1eLKnymJs3ZM1QwVC1XcuNGOY=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.36 h1:84s5xMme6ENYEdKG8rsbSFFg/8+lbHBeM9QYSO0gnDk=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.36/go.mod h1:c46BLdagDLIswjgt+GeQOslXgeS0E6wCacs5yZbxPGk=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.2 h1:VQjZODPNfdikCX2ZZrltw4zNLkcwjyUFDUl2vT9yTwg=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.2/go.mod h1:OmeHCn28vZylsBvalLDf7t8fuJ2rHYQprJs+7WuxniI=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 h1:b5tb+CZItBkydC7r3hTNdSO3pszG1R2EtnA+7TePQPk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37/go.mod h1:ZQ+6SU9X0oz6+7MUCSswv9Mjci4eaqZr21HI2RVy/yA=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1 h1:YIEBqcqRnpi4Pfv0YHImtgi6czGCwKHANC7SwmUAVD0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1/go.mod h1:imEf0oufgAo8KAkCHhrOdqGEC0YWx1PPBQH82shSxGw=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 h1:lznzIOvvbqjfe8UAaciCRJgBgJsxuTROKlhZuXQWfv8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37/go.mod h1:otfkzyfQeMMLZAqX59GSXTL3o22BR/l6HFaRzzbWSqA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 h1:pc138gM1CW+XPc60rEwUlwwuwWFQK16CI1T7v1F9Oec=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1/go.mod h1:1+koxpPIbfBdfzP6vojm5/zTpTQ/micYwlxIiNB3TxI=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 h1:zCEORWo0eU0gDjG+IyApE/2B+ZGG1m+GU7B263XV8ds=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37/go.mod h1:i6c0PEl3TNOWxRbQ++KQcVenPWS/GoQeiklKhNuqzJ8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 h1:K0JsbZQj+1h208Ro1zHeA4l7bMp0NvRffHQ91q8Ol1s=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1/go.mod h1:W3/vL6EtCIatICGy9ab29QhMuae+cOKPWcMxv02CO+Q=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 h1:A3UAuCmx7LyUcrixBTzKJYYIUZ2yTvn6ZhT8PB+7APk=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38/go.mod h1:1PDUYG9Z+JrbbsobsAZHjWOm9QBT/djiK3QbykTL5Z4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1 h1:yhw5KD1phVyP9vijxOUzDfEtJx+bt+L63k+VfuiYFAA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1/go.mod h1:ZW2e0d7DYlRxlS9hEiMXE47gTdX5KRN4byUiNbUpG+Q=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 h1:5437eMoOwqqQpZn2XJy74mlDCuPYL81texMT3mXqgtU=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30/go.mod h1:xfu2m3dOpvW8lj98wQYa8V9ku/Rta59hsbireGzhh3A=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1 h1:s67hBfG5t9rn1NCvDuB4E3QIep3UFhHPtaIqFDjV3N8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1/go.mod h1:FpvjBMXtSNMLPmDJsWwcY5cRnqJlpS2y1R6n4pvzs4k=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 h1:a3D4AjrOrTrP8+d9ILBthqrElf0z1JNol09Xvnwcys8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37/go.mod h1:ky0gTu+ukvUTuUKFIpp6Wid4oninrkCyvbFkVs0kpHM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 h1:gX8B8y3Ho30B1LPxefDKMi/HZqWEb47U9ogs3DtSG0M=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38/go.mod h1:l5WblZlcmGPe4/O7JY2HO25Z+xqTBvyfTyFbRMf8gYw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 h1:GNU0/xtPEXMKilJZ/a8BedeuQnvu+Usi6qVm9EFfncc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2/go.mod h1:4jYWUecEsQtE73jPl7p3jrbYXH5ffcR4gegyCygagfg=
-github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 h1:i68sFvXidKlkiSvI7d7Ilc1/UvW4CtBOaivH7jhG4fs=
-github.com/aws/aws-sdk-go-v2/service/signin v1.5.6/go.mod h1:/h7Obr9WTtzbjTHGASRQwLN7Bupw+TC3x8x7fyx39hE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1 h1:RmmWQPREQdk9U+PfqeHW3MqZaBaNK7TpV9W3RY+b+7g=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1/go.mod h1:0A3W4F+68ZnNk5XcNL/e9HFMwnP8RlEicFfy6eOEDyw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1 h1:ZMbtPZZQRca+3+XYQne9PBvRiYpHZlNJJOZfE9WNfT0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1/go.mod h1:YAGWQdCYlVCoqrzvfv3RLxO6zKwti7gsAULOGWPLYv4=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.110.0 h1:He8vaTTqAAJrux/KdpjFXNWueLJZyKqE49QEXoqAu4I=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.110.0/go.mod h1:CUr46sCpGAg/rHaclRyhJX0LJAmH73uWSJPPSaMUrSk=
+github.com/aws/aws-sdk-go-v2/service/signin v1.8.0 h1:bSvKIoLuRGFqGwASgeCQncCJDi9YKKBDEmCEZzOX1uU=
+github.com/aws/aws-sdk-go-v2/service/signin v1.8.0/go.mod h1:9IqUlsJDbUPcg6cgx3WEzXdjrbWzLDQrak0aaSqlTcI=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 h1:tpfGChmjUmv3W9WlRvy+stwKDTbFFdq8Zk9DbFPrfMU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.33.6/go.mod h1:CSjiDzmG/lsKkTOYjbkM+duLmRlW+LOxD64Na44ijnI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.36.0 h1:iivsh357VnfIc18IFWSuoyQEluf8frfWf4cL2Y0JUQw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.36.0/go.mod h1:tWuiVBUtPBr8/rgRiYS8Uf85sHcAN+G7XS3D3CEoUh8=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.3/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 h1:49BBtY68A+KJCQ3a2F3eUe6ROsKucxUdfHKoqorc0wI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6/go.mod h1:ptG2hbs7QltE1GcQY0MpS4bfrc51KCnBXUr7OT1EEfE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.41.0 h1:wVxM3QzSKIK8tSN6OGgezp9OK91lCLH2zhmRInN9rFM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.41.0/go.mod h1:naFe83jSMuYkH+QjQPX8n1MLhBkeCFM5Lsnh5m5wz3c=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.4/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
-github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 h1:JvExZWabChDM0qJAirQYGfOYo0ndT3edXj+fqSPNjkE=
-github.com/aws/aws-sdk-go-v2/service/sts v1.45.6/go.mod h1:XZcaQkV2cItp6yEkrwljyaPOf22RuX7T43jxap/FOmM=
+github.com/aws/aws-sdk-go-v2/service/sts v1.48.0 h1:RzZVCzYM19vhJCT5s6vO2wN8ie770Li/TmbAZ9B6N7E=
+github.com/aws/aws-sdk-go-v2/service/sts v1.48.0/go.mod h1:mKo/CzaCz8qytGW70NG4vIIGAx1HXTlb5lHNkC5k3lk=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
-github.com/aws/smithy-go v1.27.8 h1:FR0dxZfIlV7Z8eh2iHfIofdunw382XsDV3Mxt9nUvRY=
-github.com/aws/smithy-go v1.27.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
+github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
@@ -610,8 +610,8 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
-github.com/buraksezer/consistent v1.0.0 h1:ggz8fN8QW95rM4D+NlNMSUUCtoubMra3x/4fmFSv+fo=
-github.com/buraksezer/consistent v1.0.0/go.mod h1:L/Bczo4XsqEvJLOadAlIhamyNMhIj0iGt0+exc/KeV8=
+github.com/buraksezer/consistent v1.1.0 h1:nbrOsL2mdeMLsGLpKSuIeoW3inpvV5nBATg0s/bm96w=
+github.com/buraksezer/consistent v1.1.0/go.mod h1:L/Bczo4XsqEvJLOadAlIhamyNMhIj0iGt0+exc/KeV8=
 github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
@@ -820,8 +820,8 @@ github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxv
 github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
 github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
-github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
+github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
+github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
 github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g=
 github.com/go-fonts/dejavu v0.3.2/go.mod h1:m+TzKY7ZEl09/a17t1593E4VYW8L1VaBXHzFZOIjGEY=
 github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks=
@@ -1105,8 +1105,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9K
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
 github.com/hamba/avro/v2 v2.17.2/go.mod h1:Q9YK+qxAhtVrNqOhwlZTATLgLA8qxG2vtvkhK8fJ7Jo=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
@@ -1327,14 +1327,14 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/bluebottle v1.18.0 h1:aNrogJBz59gJAxCwpERGBC5bGXkSoUCCCBbNvnamuRU=
-github.com/mulgadc/bluebottle v1.18.0/go.mod h1:KkUyzqfKxyI8PvKR27EVqJnVm4BSbajl1Q/u+PQxAYw=
-github.com/mulgadc/northstar v1.18.0 h1:ksRjIKE6qtrw/ZoYaJh8JRBh+xdnuDw8FsmFLFCiXbg=
-github.com/mulgadc/northstar v1.18.0/go.mod h1:z6iol77R1vsIQyIBc1JQboUPhSJ6eG4bipzfbZZbLEM=
-github.com/mulgadc/predastore v1.18.0 h1:RW1Qw1SM2iYxqi6HaHLv5Qb1h1vhZSO3x6pOhPAb9mw=
-github.com/mulgadc/predastore v1.18.0/go.mod h1:+VBT3ofH/YmdTg0ug9EQwHDb8bQbHaMvWUyh5cIpBeo=
-github.com/mulgadc/viperblock v1.18.0 h1:u6yu5lPfZ+RSmfzlg1OXI3whlRBnGp8t685xUzzxlns=
-github.com/mulgadc/viperblock v1.18.0/go.mod h1:OjqMFZba0TJl9zcW2oKmz2/MrApIxRIFNcafiv5vTkM=
+github.com/mulgadc/bluebottle v1.19.0 h1:KjaH4TbAAZAEfGVd47rgsBW4fLBm0eF3+kzE3mF95lo=
+github.com/mulgadc/bluebottle v1.19.0/go.mod h1:sg+gqQmHRdrS5ObBGfIRhl9sBvGbqMhLZElnC4otG/c=
+github.com/mulgadc/northstar v1.19.0 h1:iC3RaAPPBcsNzIKQKGMBKOMQgpUh2ZCArZK2Yzjf21E=
+github.com/mulgadc/northstar v1.19.0/go.mod h1:m23lGk5aHv/yTbR0XZT4rZobj6EZ4Cu8p1yzNA5npxU=
+github.com/mulgadc/predastore v1.19.0 h1:y47JrsOcBZb8391ndY8muT2uODIHUpeNP7c2Gs1TuqI=
+github.com/mulgadc/predastore v1.19.0/go.mod h1:mGMHNTPIsKFMC0CegmXYc5tHqQgVvDDCof3K3hL48pI=
+github.com/mulgadc/viperblock v1.19.0 h1:8gHHDy8pKtv2iqp6UmEpsHitDK/gxNLVHZ1GPqR6Bic=
+github.com/mulgadc/viperblock v1.19.0/go.mod h1:bjsD4znZgK3t5RkSlPrHE81FrDijy8vBcW2ESTnkNvo=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1344,8 +1344,8 @@ github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL
 github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
 github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
 github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
-github.com/nats-io/nats-server/v2 v2.14.5 h1:M6yeo/Xb7khi97RSEVELof3DForDqmYza3P4tHCPFWw=
-github.com/nats-io/nats-server/v2 v2.14.5/go.mod h1:1D3iocrisKvWaD1B/imqarTqmaGrWMqALMLbEDo3v7Q=
+github.com/nats-io/nats-server/v2 v2.14.6 h1:dHjEHa49lUTxIb5jrsk5kz/AmOyOQ4T9ti8djcBumDQ=
+github.com/nats-io/nats-server/v2 v2.14.6/go.mod h1:u/0bNBzN/m5orQGz7q2NNzxCQ6PWdwaOSwPNDADZn/A=
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nats.go v1.53.1 h1:Otsq3uLc/kLdjmkNHkXH0jBqwUquwdKFoe3fq6/3/Xo=
 github.com/nats-io/nats.go v1.53.1/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
@@ -1487,8 +1487,8 @@ github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA=
 github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk=
 github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
 github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
-github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA=
-github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs=
+github.com/quic-go/quic-go v0.62.0 h1:ZHDjCk5OacATwGvs8PWE97CTvX7AqZiVoW7++ZOXTf8=
+github.com/quic-go/quic-go v0.62.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
@@ -1639,8 +1639,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 h1:oEl2Pw/i4OQwhAuda2pAHFAcOMivA+Xa+iTccBfab/g=
-go.opentelemetry.io/contrib/bridges/otelslog v0.20.0/go.mod h1:yMSQaiiq5dpfrSJCYLBcqFeJkFFI67seT4ngvx6jfVo=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=
 go.opentelemetry.io/contrib/detectors/gcp v1.28.0/go.mod h1:9BIqH22qyHWAiZxQh0whuJygro59z+nbMVuc7ciiGug=
 go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00=
@@ -1676,10 +1676,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
-go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0 h1:1+WLVYezXA9tkuVzKQri8zgB1cEIVYKUSoYIRjsBiMU=
-go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0/go.mod h1:rbAXUUXqQDMxpSnmof4VtcZ+7YpZQEtjXSCIfdvR0Go=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0 h1:v4KkRLVvE1cWqTJDfZZkTCG+Z4aolsa6RVos0FX7vqE=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0/go.mod h1:g/xbuPC0XbgwMdKuyF5sKOUUEsorSkN6APydyFP/H9E=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
 go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
 go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
@@ -1702,23 +1702,23 @@ go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIl
 go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
-go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
-go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 h1:WseeVYf5dJZTsyPiyW5L14k5qsSibqXAMTSiFEDiWr0=
-go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0/go.mod h1:SiLZnQS6Qk2eCpvr2CH/XMAOa64TWGXxEZJZCpD2Lmc=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 h1:klTViGcsvLCd1xN3rZzfZ12NslC/OimbmR+k+A006RI=
-go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0/go.mod h1:jRsK04CWmXuY8A0O+wMpSf+t90RHZ53o5Qmxn2PQPfk=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
+go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
+go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
 go.opentelemetry.io/otel/exporters/prometheus v0.57.0/go.mod h1:QpFWz1QxqevfjwzYdbMb4Y1NnlJvqSGwyuU0B4iuc9c=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
-go.opentelemetry.io/otel/log v0.21.0 h1:SLsVDGmtyBrdw8/a2Z0bOIxou/+bN4z56GebH7T0LvA=
-go.opentelemetry.io/otel/log v0.21.0/go.mod h1:iReetQrZL9Wyg84cCkOoCmqDHS5RCFfyxC7J+r8fn8g=
+go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=
+go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
@@ -1741,11 +1741,11 @@ go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UD
 go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
 go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
 go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
-go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
+go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
 go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
-go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
-go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI=
+go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU=
+go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
@@ -1767,12 +1767,12 @@ go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx
 go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
 go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
 go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
-go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
-go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
-go.opentelemetry.io/otel/sdk/log v0.21.0 h1:QsE7XSR0ktQdKmRKGnR+f1ObGF32WG+7MER/P9KgmYc=
-go.opentelemetry.io/otel/sdk/log v0.21.0/go.mod h1:m9mApjCoD2/1QuKCAptjv+BrG9WKOvQLVdNx+iBldTo=
-go.opentelemetry.io/otel/sdk/log/logtest v0.21.0 h1:X+JBBgKlswCGYsmgL0CnoUUtlE//VB345c84jYAYkdQ=
-go.opentelemetry.io/otel/sdk/log/logtest v0.21.0/go.mod h1:HD1575K8e6sIFBBDd5tZB3t9DlMytWXq9FuR+Y4rfjE=
+go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
+go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
+go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=
+go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=
+go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg=
+go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s=
 go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
 go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=
 go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y=
@@ -1788,8 +1788,8 @@ go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2
 go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
 go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
 go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
-go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
-go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
+go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
+go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
@@ -1812,8 +1812,8 @@ go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTq
 go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
 go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
-go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
-go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
+go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
+go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
@@ -1905,8 +1905,8 @@ golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVo
 golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
-golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
-golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
+golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
+golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2849,8 +2849,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260511170946-3700d4141b60/go.
 google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY=
 google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
 google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:Z4WJ5pJOYWFWcHEQUelD5QaZDknIQkpIL/+fyJOT9+A=
-google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
-google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
+google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI=
+google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:+34luvCflYKiKylNwGJfn9cFBbcL/WrkciMmDmsTQ/A=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4405/go.mod h1:GRUCuLdzVqZte8+Dl/D4N25yLzcGqqWaYkeVOwulFqw=
@@ -3031,8 +3031,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68/go.
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -3126,8 +3126,8 @@ google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhH
 google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
 google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
-google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
-google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
+google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
+google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y=
 google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM=
@@ -3171,8 +3171,8 @@ google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXn
 google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
-google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
-google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

images/mkosi.profiles/eks-server/mkosi.extra/etc/spinifex-eks/otelcol-config.yaml

@@ -0,0 +1,121 @@
+# Control-plane telemetry for the EKS server guest.
+#
+# k3s already serves embedded-etcd metrics on 127.0.0.1:2381 — the daemon sets
+# etcd-expose-metrics in the k3s config it writes as cloud-init user-data. Until
+# this collector existed nothing read them, so an etcd stall was only ever
+# visible as "etcd:unreachable" in the state report, which cannot separate a
+# slow disk from a dead one. etcd_disk_wal_fsync_duration_seconds is the metric
+# that does.
+#
+# The endpoint comes from /etc/spinifex-eks/otelcol.env, written when the guest
+# is told where the sink is. The unit will not start without that file, so an
+# image with no sink configured runs no collector rather than a failing one.
+
+receivers:
+  # Embedded etcd. Its fsync and commit histograms are the whole point of this
+  # file; the rest of the k3s metrics come along at no extra scrape cost.
+  prometheus/etcd:
+    config:
+      scrape_configs:
+        - job_name: etcd
+          scrape_interval: 15s
+          static_configs:
+            - targets: ["127.0.0.1:2381"]
+
+  # Guest-side pressure. The state report already carries loadavg and
+  # MemAvailable as scalars, which is enough to rule starvation in or out at one
+  # instant and useless for seeing it build.
+  hostmetrics:
+    collection_interval: 15s
+    scrapers:
+      cpu:
+        metrics:
+          system.cpu.utilization:
+            enabled: true
+          system.cpu.logical.count:
+            enabled: true
+      load:
+      memory:
+        metrics:
+          system.memory.utilization:
+            enabled: true
+      disk:
+      filesystem:
+        exclude_mount_points:
+          match_type: regexp
+          mount_points:
+            - "/proc/*"
+            - "/sys/*"
+            - "/run/*"
+            - "/var/lib/kubelet/*"
+        exclude_fs_types:
+          match_type: strict
+          fs_types:
+            - tmpfs
+            - devtmpfs
+            - overlay
+            - squashfs
+            - proc
+            - sysfs
+            - cgroup
+            - cgroup2
+      network:
+      paging:
+
+processors:
+  resourcedetection/system:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+
+  # apm-server routes non-APM-agent OTLP metrics into
+  # metrics-apm.app.<service.name>-default, so an unset service.name puts etcd
+  # and host metrics in the shared "unknown" bucket with everything else.
+  resource/etcd:
+    attributes:
+      - key: service.name
+        value: "eks-etcd"
+        action: upsert
+      - key: eks.cluster
+        value: "${env:EKS_CLUSTER_NAME}"
+        action: upsert
+
+  resource/system:
+    attributes:
+      - key: service.name
+        value: "eks-control-plane"
+        action: upsert
+      - key: eks.cluster
+        value: "${env:EKS_CLUSTER_NAME}"
+        action: upsert
+
+  # Sheds data under memory pressure rather than queueing without bound when the
+  # sink is unreachable. Must be FIRST in every pipeline: it only applies
+  # backpressure to not-yet-processed data. Budgeted well under the unit's
+  # MemoryMax so it fires before the kernel does — this guest is 4 GB and the
+  # control plane is what matters on it.
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 64
+    spike_limit_mib: 16
+
+  batch:
+    send_batch_size: 1024
+    timeout: 2s
+
+exporters:
+  otlp:
+    endpoint: "${env:EKS_OTLP_ENDPOINT}"
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    metrics/etcd:
+      receivers: [prometheus/etcd]
+      processors: [memory_limiter, resourcedetection/system, resource/etcd, batch]
+      exporters: [otlp]
+    metrics:
+      receivers: [hostmetrics]
+      processors: [memory_limiter, resourcedetection/system, resource/system, batch]
+      exporters: [otlp]

images/mkosi.profiles/eks-server/mkosi.extra/etc/systemd/system/mulga-eks-otelcol.service

@@ -0,0 +1,39 @@
+[Unit]
+Description=Mulga EKS control-plane telemetry (OpenTelemetry Collector)
+Documentation=https://opentelemetry.io/docs/collector/
+After=network-online.target k3s.service
+Wants=network-online.target
+# No sink configured means no collector. A condition that is not met leaves the
+# unit inactive and says so, where a missing endpoint would otherwise be a
+# restart loop against an exporter that can never resolve.
+ConditionPathExists=/etc/spinifex-eks/otelcol.env
+
+[Service]
+Type=simple
+# EKS_OTLP_ENDPOINT lives here; EKS_CLUSTER_NAME comes from the first-boot env
+# the rest of the node services already read, so the cluster tag matches what
+# the state report publishes.
+EnvironmentFile=/etc/spinifex-eks/otelcol.env
+EnvironmentFile=-/etc/spinifex-eks/first-boot.env
+ExecStart=/usr/local/bin/otelcol-contrib --config=/etc/spinifex-eks/otelcol-config.yaml
+Restart=on-failure
+RestartSec=10
+
+# The control plane owns this guest. Three tiers, each below the next:
+# memory_limiter sheds data, GOMEMLIMIT makes the Go GC work harder, MemoryMax
+# is a backstop that should never fire. On a 4 GB node an agent that competes
+# with etcd for memory causes the outage it was installed to explain.
+Environment=GOMEMLIMIT=96MiB
+MemoryHigh=128M
+MemoryMax=192M
+CPUQuota=20%
+Nice=10
+IOSchedulingClass=idle
+
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

images/mkosi.profiles/eks-server/mkosi.extra/usr/lib/systemd/system-preset/40-mulga-eks-role.preset

@@ -16,6 +16,11 @@
 enable eks-node-role.service
 enable mulga-eks-etcd-snapshot-daily.timer
 enable mulga-eks-etcd-snapshot-15min.timer
+# Unconditional like the timers, and for the same reason: it self-guards. Its
+# ConditionPathExists means a guest with no sink configured never starts it, so
+# there is no role for the selector to decide. An agent that is given a sink
+# still reports host metrics; only the etcd scrape goes unanswered.
+enable mulga-eks-otelcol.service
 
 disable k3s.service
 disable k3s-agent.service

images/mkosi.profiles/eks-server/mkosi.postinst.chroot

@@ -77,4 +77,38 @@ if got != want:
 print(f"[eks-server] cloud-init datasource_list assertion passed: {got!r}")
 PY
 
-echo "[eks-server] postinst complete: eks-node-role enabled, etcd-snapshot timers enabled, datasource pin verified"
+# Telemetry collector. Pinned to the same version the cluster hosts' obs-agent
+# role runs, so a metric means the same thing wherever it was collected.
+# Verified against upstream's published checksums rather than a hash pinned
+# here: a hash in two places drifts, and the release is immutable.
+OTELCOL_VERSION="0.115.1"
+OTELCOL_URL_BASE="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTELCOL_VERSION}"
+OTELCOL_TAR="otelcol-contrib_${OTELCOL_VERSION}_linux_amd64.tar.gz"
+
+# The checksums file is named for the release repo and the distribution, with
+# no version in it — unlike the tarball beside it, which carries the version.
+echo "[eks-server] fetching otelcol-contrib checksums ${OTELCOL_VERSION}"
+curl -fsSL -o /tmp/otelcol-checksums.txt \
+    "${OTELCOL_URL_BASE}/opentelemetry-collector-releases_otelcol-contrib_checksums.txt"
+OTELCOL_SHA256=$(awk -v f="${OTELCOL_TAR}" '$2 == f {print $1; exit}' /tmp/otelcol-checksums.txt)
+if [[ -z "${OTELCOL_SHA256}" ]]; then
+    echo "[eks-server] could not parse ${OTELCOL_TAR} sha256 from upstream checksums" >&2
+    exit 1
+fi
+
+echo "[eks-server] downloading otelcol-contrib ${OTELCOL_VERSION} (sha256=${OTELCOL_SHA256})"
+curl -fsSL -o "/tmp/${OTELCOL_TAR}" "${OTELCOL_URL_BASE}/${OTELCOL_TAR}"
+echo "${OTELCOL_SHA256}  /tmp/${OTELCOL_TAR}" > /tmp/otelcol.sha256
+if ! sha256sum -c /tmp/otelcol.sha256; then
+    echo "[eks-server] otelcol-contrib checksum mismatch" >&2
+    exit 1
+fi
+# --no-same-owner: the archive records upstream's build uid, which does not
+# exist here, and restoring it fails outright rather than degrading. Extracted
+# as root instead, which is what owns the rest of /usr/local/bin.
+tar -xzf "/tmp/${OTELCOL_TAR}" --no-same-owner -C /usr/local/bin otelcol-contrib
+chown root:root /usr/local/bin/otelcol-contrib
+chmod 0755 /usr/local/bin/otelcol-contrib
+rm -f /tmp/otelcol-checksums.txt /tmp/otelcol.sha256 "/tmp/${OTELCOL_TAR}"
+
+echo "[eks-server] postinst complete: eks-node-role enabled, etcd-snapshot timers enabled, datasource pin verified, otelcol-contrib ${OTELCOL_VERSION} installed"

internal/ecrauth/ecrauth.go

@@ -19,7 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ecr"
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
+	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
 )
 
 // defaultRegion is used when the caller supplies no region; SigV4 still needs a

internal/ecsgw/client.go

@@ -20,7 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
 
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
+	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
 )
 
 // targetPrefix is the X-Amz-Target service prefix the gateway's ECS dispatch

internal/eksgw/client.go

@@ -17,8 +17,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
 	"github.com/mulgadc/spinifex/internal/gwsign"
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )
 
 // Client posts SigV4-signed requests to the AWS gateway for the "eks" service.

internal/fipsboot/fipsboot.go

@@ -1,16 +0,0 @@
-// Package fipsboot enforces FIPS 140-3 at startup via blank import in every binary.
-// Requires GOFIPS140=v1.0.0 at build time; panics if GODEBUG=fips140=off disables
-// the runtime FIPS mode.
-package fipsboot
-
-import (
-	"crypto/fips140"
-)
-
-func init() {
-	if !fips140.Enabled() {
-		//nolint:forbidigo // Refuse to start when the required FIPS mode is disabled.
-		panic("fipsboot: FIPS 140-3 mode is not enabled at runtime — refusing to start. " +
-			"Build with GOFIPS140=v1.0.0 and do not set GODEBUG=fips140=off.")
-	}
-}

internal/imdscreds/imdscreds.go

@@ -53,9 +53,35 @@ func Fetch(client *http.Client, base string) (Credentials, error) {
 // client's default HTTP client.
 func NewProvider(client *http.Client, base string) aws.CredentialsProvider {
 	imdsClient := NewClient(client, base)
-	return ec2rolecreds.New(func(o *ec2rolecreds.Options) {
+	return noExtendOnFailure{provider: ec2rolecreds.New(func(o *ec2rolecreds.Options) {
 		o.Client = imdsClient
-	})
+	})}
+}
+
+// noExtendOnFailure forwards everything ec2rolecreds offers except
+// HandleFailToRefresh, which answers a failed refresh by returning the same
+// expired key with a fresh Expires so a caller never learns the refresh failed.
+type noExtendOnFailure struct {
+	provider *ec2rolecreds.Provider
+}
+
+// aws.CredentialsCache picks its strategies by asserting on the provider's
+// dynamic type, so hiding the method is what disarms it — returning an
+// interface from NewProvider is not enough.
+var (
+	_ aws.CredentialsProvider                     = noExtendOnFailure{}
+	_ aws.AdjustExpiresByCredentialsCacheStrategy = noExtendOnFailure{}
+)
+
+func (p noExtendOnFailure) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	return p.provider.Retrieve(ctx)
+}
+
+// AdjustExpiresBy is forwarded deliberately. It only fires when the cache is
+// built with an ExpiryWindow, which no caller does today, but dropping it
+// silently would change that caller's behaviour without them touching this.
+func (p noExtendOnFailure) AdjustExpiresBy(creds aws.Credentials, dur time.Duration) (aws.Credentials, error) {
+	return p.provider.AdjustExpiresBy(creds, dur)
 }
 
 // NewClient builds an SDK IMDS client against base, an IMDS root URL such as

internal/imdscreds/imdscreds_test.go

@@ -6,6 +6,8 @@ import (
 	"net/http/httptest"
 	"testing"
 	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
 )
 
 // imdsStub serves the IMDSv2 token + role + credentials endpoints under the
@@ -116,3 +118,17 @@ func TestFetch_BadExpirationErrors(t *testing.T) {
 		t.Fatal("expected error on unparseable Expiration")
 	}
 }
+
+// The provider must not carry ec2rolecreds' extend-on-failure strategy.
+// aws.CredentialsCache selects it by asserting on the dynamic type, so this
+// assertion is the property that keeps a failed refresh an error.
+func TestNewProvider_WithholdsExtendOnFailure(t *testing.T) {
+	p := NewProvider(nil, "http://169.254.169.254/latest")
+
+	if _, ok := p.(aws.HandleFailRefreshCredentialsCacheStrategy); ok {
+		t.Error("provider implements HandleFailRefreshCredentialsCacheStrategy, so a failed refresh will silently extend a dead credential's expiry")
+	}
+	if _, ok := p.(aws.AdjustExpiresByCredentialsCacheStrategy); !ok {
+		t.Error("provider dropped AdjustExpiresBy, which a caller using an ExpiryWindow relies on")
+	}
+}

internal/rdsgw/client.go

@@ -22,8 +22,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
 	"github.com/mulgadc/spinifex/internal/gwsign"
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )
 
 const (

internal/tlsconfig/curves.go

@@ -1,16 +0,0 @@
-// Package tlsconfig pins the cluster-wide TLS 1.3 curve preference list.
-// Hardcoded to prevent silent downgrade via misconfiguration; the FIPS build
-// is the per-deployment choice.
-package tlsconfig
-
-import "crypto/tls"
-
-// Curves is the fixed TLS 1.3 curve allowlist: two ML-KEM hybrids (PQ-capable
-// peers always negotiate these) followed by two classical fallbacks for SDKs
-// without ML-KEM. Weak Go defaults (SecP256r1MLKEM768, P-256, P-521) excluded.
-var Curves = []tls.CurveID{
-	tls.X25519MLKEM768,
-	tls.SecP384r1MLKEM1024,
-	tls.X25519,
-	tls.CurveP384,
-}

internal/tlsconfig/curves_test.go

@@ -1,165 +0,0 @@
-package tlsconfig_test
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"io"
-	"math/big"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"slices"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
-)
-
-func TestCurves_OnlyApprovedEntries(t *testing.T) {
-	want := []tls.CurveID{
-		tls.X25519MLKEM768,
-		tls.SecP384r1MLKEM1024,
-		tls.X25519,
-		tls.CurveP384,
-	}
-	if !slices.Equal(tlsconfig.Curves, want) {
-		t.Errorf("Curves = %v, want %v", tlsconfig.Curves, want)
-	}
-}
-
-func TestCurves_ExcludesWeakPrimitives(t *testing.T) {
-	forbidden := []tls.CurveID{
-		tls.SecP256r1MLKEM768, // weakest ML-KEM hybrid (P-256 component)
-		tls.CurveP256,         // not CNSA 2.0
-		tls.CurveP521,         // niche, no current peers
-	}
-	for _, c := range forbidden {
-		if slices.Contains(tlsconfig.Curves, c) {
-			t.Errorf("Curves includes forbidden curve %v", c)
-		}
-	}
-}
-
-// TestIntegration_NegotiatesPQHybrid verifies that a Curves-configured server
-// negotiates a PQ hybrid at TLS 1.3; checks for any approved hybrid to stay
-// stable across Go stdlib reorderings.
-func TestIntegration_NegotiatesPQHybrid(t *testing.T) {
-	cert := selfSignedCert(t)
-	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		_, _ = io.WriteString(w, "ok")
-	}))
-	srv.TLS = &tls.Config{
-		Certificates:     []tls.Certificate{cert},
-		MinVersion:       tls.VersionTLS13,
-		CurvePreferences: tlsconfig.Curves,
-	}
-	srv.StartTLS()
-	defer srv.Close()
-
-	pool := x509.NewCertPool()
-	pool.AddCert(srv.Certificate())
-
-	client := &http.Client{
-		Timeout: 5 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				RootCAs:          pool,
-				CurvePreferences: tlsconfig.Curves,
-			},
-		},
-	}
-	resp, err := client.Get(srv.URL)
-	if err != nil {
-		t.Fatalf("https get: %v", err)
-	}
-	defer resp.Body.Close()
-
-	state := resp.TLS
-	if state == nil {
-		t.Fatal("response has no TLS state")
-	}
-	if state.Version != tls.VersionTLS13 {
-		t.Errorf("negotiated version = 0x%x, want TLS 1.3 (0x%x)", state.Version, tls.VersionTLS13)
-	}
-	pqHybrids := []tls.CurveID{tls.X25519MLKEM768, tls.SecP384r1MLKEM1024}
-	if !slices.Contains(pqHybrids, state.CurveID) {
-		t.Errorf("negotiated curve = %v, want one of %v (PQ hybrid)", state.CurveID, pqHybrids)
-	}
-}
-
-// TestIntegration_TLS12ClientRejected confirms MinVersion TLS 1.3 is enforced:
-// a TLS 1.2-pinned client must get a handshake error, not a silent downgrade.
-func TestIntegration_TLS12ClientRejected(t *testing.T) {
-	cert := selfSignedCert(t)
-	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		_, _ = io.WriteString(w, "ok")
-	}))
-	srv.TLS = &tls.Config{
-		Certificates:     []tls.Certificate{cert},
-		MinVersion:       tls.VersionTLS13,
-		CurvePreferences: tlsconfig.Curves,
-	}
-	srv.StartTLS()
-	defer srv.Close()
-
-	pool := x509.NewCertPool()
-	pool.AddCert(srv.Certificate())
-
-	client := &http.Client{
-		Timeout: 5 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				RootCAs:    pool,
-				MinVersion: tls.VersionTLS12,
-				MaxVersion: tls.VersionTLS12,
-			},
-		},
-	}
-	_, err := client.Get(srv.URL)
-	if err == nil {
-		t.Fatal("TLS 1.2-pinned client succeeded; MinVersion not enforced")
-	}
-	if !strings.Contains(err.Error(), "protocol version") &&
-		!strings.Contains(err.Error(), "tls:") {
-		t.Errorf("expected protocol-version handshake error, got: %v", err)
-	}
-}
-
-func selfSignedCert(t *testing.T) tls.Certificate {
-	t.Helper()
-	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		t.Fatalf("generate key: %v", err)
-	}
-	tmpl := &x509.Certificate{
-		SerialNumber: big.NewInt(1),
-		Subject:      pkix.Name{CommonName: "tlsconfig-test"},
-		NotBefore:    time.Now().Add(-time.Hour),
-		NotAfter:     time.Now().Add(time.Hour),
-		KeyUsage:     x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		DNSNames:     []string{"localhost"},
-		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
-	}
-	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
-	if err != nil {
-		t.Fatalf("create cert: %v", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(key)
-	if err != nil {
-		t.Fatalf("marshal key: %v", err)
-	}
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
-	pair, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		t.Fatalf("X509KeyPair: %v", err)
-	}
-	return pair
-}

scripts/check-test-package.sh

@@ -50,9 +50,10 @@ if ! git rev-parse --verify "$BASE_REF" &>/dev/null; then
 fi
 
 # e2e and integration suites share a package-level fixture built in
-# main_test.go, so their files cannot move individually.
+# main_test.go, so their files cannot move individually. export_test.go is the
+# remedy this check recommends, and it is in-package by definition.
 ADDED=$(git diff "$BASE_REF" HEAD --name-only --diff-filter=A -- '*_test.go' \
-    ':!tests/e2e/*' ':!tests/integration/*' || true)
+    ':!tests/e2e/*' ':!tests/integration/*' ':!*export_test.go' || true)
 
 if [[ -z "$ADDED" ]]; then
     [[ -z "$QUIET" ]] && echo "No new test files — skipping test package check."

scripts/diff-coverage.sh

@@ -70,7 +70,10 @@ trap 'rm -rf "$TMPDIR"' EXIT
 # -C enables copy detection for the same reason (file split = partial copy).
 # cmd/installer/ui is excluded: interactive TUI rendering has no meaningful unit
 # test, and the installer is validated by the ISO hardware run instead.
-git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' ':!spinifex/migrate/*' ':!cmd/installer/ui/*' | awk '
+# spinifex/testutil is excluded because it is fakes and harnesses used only from
+# _test.go files. Coverage is recorded per package under test, so a fake another
+# package's tests drive reads as zero-covered however hard it is exercised.
+git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' ':!spinifex/migrate/*' ':!cmd/installer/ui/*' ':!spinifex/testutil/*' | awk '
     /^\+\+\+ / {
         file = substr($2, 3)
         next

scripts/images/eks-node/mulga-eks-state-report.sh

@@ -42,13 +42,19 @@ DISK_MIN_KB=${DISK_MIN_KB:-262144}
 
 # diagnose emits a compact, JSON-safe reason for an unhealthy apiserver by
 # reading the ground truth only reachable from inside the guest: the failing
-# /readyz subchecks, isolated etcd reachability, and the etcd-disk free space.
+# /readyz subchecks, etcd reachability, etcd-disk free space, and host pressure.
 diagnose() {
     # apiserver prints "[+]name ok" for passing subchecks and "[-]name failed"
-    # for failing ones; collect the failing names (etcd, poststarthook/*, ...).
-    failed=$(kubectl get --raw='/readyz?verbose' 2>/dev/null \
-        | awk '/^\[-\]/{sub(/^\[-\]/,""); sub(/[[:space:]].*$/,""); printf "%s%s", sep, $0; sep=" "}')
-    [ -n "${failed}" ] || failed=none
+    # for failing ones. An empty body means the probe never answered, which is a
+    # different fault from "every subcheck passed" and must not read the same.
+    readyz=$(kubectl get --raw='/readyz?verbose' 2>/dev/null)
+    if [ -z "${readyz}" ]; then
+        failed=unreachable
+    else
+        failed=$(printf '%s\n' "${readyz}" \
+            | awk '/^\[-\]/{sub(/^\[-\]/,""); sub(/[[:space:]].*$/,""); printf "%s%s", sep, $0; sep=" "}')
+        [ -n "${failed}" ] || failed=none
+    fi
 
     if kubectl get --raw='/healthz/etcd' 2>/dev/null | grep -q '^ok$'; then
         etcd=ok
@@ -63,7 +69,51 @@ diagnose() {
         disk=ok
     fi
 
-    printf 'readyz:[%s]; etcd:%s; disk:%s' "${failed}" "${etcd}" "${disk}"
+    # etcd stalls under CPU or memory pressure long before it faults on its own,
+    # so an unhealthy CP reporting disk:ok is uninterpretable without these two.
+    load=$(awk '{print $1}' "${LOADAVG_FILE:-/proc/loadavg}" 2>/dev/null)
+    memavail_kb=$(awk '/^MemAvailable:/{print $2}' "${MEMINFO_FILE:-/proc/meminfo}" 2>/dev/null)
+    [ -n "${load}" ] || load=unknown
+    [ -n "${memavail_kb}" ] || memavail_kb=unknown
+
+    printf 'readyz:[%s]; etcd:%s; disk:%s; fsync:%sms; load:%s; memavail:%sk' \
+        "${failed}" "${etcd}" "${disk}" "$(etcd_fsync_ms)" "${load}" "${memavail_kb}"
+}
+
+# etcd_fsync_ms prints the mean WAL fsync in milliseconds, or "nometrics" when
+# etcd served none, or "warmup" when it has not done enough to average.
+#
+# disk:ok in diagnose() is free space, which says nothing about how long a write
+# takes. etcd stalls on fsync latency long before it runs out of room, and this
+# guest's disk is an EBS volume served by viperblock, so a fsync here is a
+# network round trip. Healthy is single-digit milliseconds.
+#
+# Mean rather than a percentile: the histogram's _sum and _count are two greps,
+# where a bucket percentile is an interpolation this script has no business
+# doing, and a stall shows as an order of magnitude rather than a tail.
+etcd_fsync_ms() {
+    # `|| metrics=` is load-bearing under set -e: a scrape against a port
+    # nothing answers exits non-zero, and a bare assignment would abort the
+    # caller — dropping the diagnosis exactly when it is wanted.
+    metrics=$(curl -fsS --max-time 2 "${ETCD_METRICS_URL:-http://127.0.0.1:2381/metrics}" 2>/dev/null) || metrics=
+    # "etcd served nothing" and "etcd has not done enough work yet" have
+    # different fixes — the first is a missing endpoint, the second is a guest
+    # that just booted — so they must not read the same. Collapsing both to one
+    # word is the mistake readyz:[none] made.
+    if [ -z "${metrics}" ]; then
+        echo nometrics
+        return
+    fi
+    # A minimum sample count, not just a non-zero one. A mean over the handful
+    # of fsyncs a just-started etcd has done is a number with no meaning, and
+    # reporting it invites the same mistake as judging a throughput floor on a
+    # 297-byte object.
+    fsync=$(printf '%s\n' "${metrics}" | awk -v min="${FSYNC_MIN_COUNT:-100}" '
+        /^etcd_disk_wal_fsync_duration_seconds_sum/ {sum=$2}
+        /^etcd_disk_wal_fsync_duration_seconds_count/ {count=$2}
+        END {if (count >= min) printf "%.1f", (sum / count) * 1000}')
+    [ -n "${fsync}" ] || fsync=warmup
+    echo "${fsync}"
 }
 
 publish_report() {
@@ -100,12 +150,27 @@ publish_report() {
             echo "=== end mulga-eks CP diag ==="
         } > "${CONSOLE:-/dev/console}" 2>&1 || true
     fi
+    # fsync goes out on every report, healthy or not. A figure that only appears
+    # once the control plane is already failing has no baseline to be read
+    # against, which is the same gap a passing cell that uploads no journal
+    # leaves. Emitted as a bare JSON number, or omitted entirely when unknown so
+    # a missing sample is never mistaken for a fast one.
+    fsync_field=
+    fsync_ms=$(etcd_fsync_ms)
+    # Tested for being a number rather than against the sentinel words: this
+    # emits a bare JSON value, so anything non-numeric reaching it produces a
+    # payload the daemon cannot parse at all. A new sentinel must not be able
+    # to break the report by being added.
+    case "${fsync_ms}" in
+        '' | *[!0-9.]*) : ;;
+        *) fsync_field=$(printf '"fsync_ms":%s,' "${fsync_ms}") ;;
+    esac
     if [ -n "${reason}" ]; then
-        payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,"reason":"%s","ts":%s}' \
-            "${health}" "${node_count}" "${nodegroup_ready}" "${reason}" "$(date +%s)")
+        payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,%s"reason":"%s","ts":%s}' \
+            "${health}" "${node_count}" "${nodegroup_ready}" "${fsync_field}" "${reason}" "$(date +%s)")
     else
-        payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,"ts":%s}' \
-            "${health}" "${node_count}" "${nodegroup_ready}" "$(date +%s)")
+        payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,%s"ts":%s}' \
+            "${health}" "${node_count}" "${nodegroup_ready}" "${fsync_field}" "$(date +%s)")
     fi
     printf '%s' "${payload}" | eks-gateway-publish -channel state 2>&1 \
         | logger -t mulga-eks-state-report

scripts/images/eks-node/mulga-eks-state-report_test.sh

@@ -49,6 +49,15 @@ cat > "${STUBBIN}/logger" <<'EOF'
 :
 EOF
 
+# curl stub: serves the etcd /metrics scrape. Empty ETCD_METRICS_BODY exits
+# non-zero like a real -fsS against a port nothing is listening on, which is the
+# case the fsync field has to survive.
+cat > "${STUBBIN}/curl" <<'EOF'
+#!/bin/sh
+[ -n "${ETCD_METRICS_BODY:-}" ] || exit 7
+printf '%s\n' "${ETCD_METRICS_BODY}"
+EOF
+
 chmod +x "${STUBBIN}"/*
 PATH="${STUBBIN}:${PATH}"
 export PATH
@@ -61,6 +70,14 @@ EKS_ACCOUNT_ID=000000000001
 EKS_CLUSTER_NAME=demo3
 EOF
 
+# Pressure fixtures: the agent reads these instead of /proc so the load and
+# memavail fields are deterministic here.
+LOADAVG_FILE="${WORK}/loadavg"
+MEMINFO_FILE="${WORK}/meminfo"
+echo "7.25 6.10 5.00 3/512 9001" > "${LOADAVG_FILE}"
+printf 'MemTotal:        4193768 kB\nMemAvailable:     262144 kB\n' > "${MEMINFO_FILE}"
+export LOADAVG_FILE MEMINFO_FILE
+
 FAILS=0
 fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
 pass() { echo "ok: $*"; }
@@ -105,7 +122,7 @@ run_agent
 P=$(cat "${WORK}/payload.json")
 case "${P}" in *'"healthz":"fail"'*) pass "unhealthy: healthz fail" ;; *) fail "unhealthy: healthz wrong: ${P}" ;; esac
 case "${P}" in *'"nodegroup_ready":{}'*) pass "unhealthy: nodegroup_ready empty (no kubectl node query on a failing apiserver)" ;; *) fail "unhealthy: nodegroup_ready wrong: ${P}" ;; esac
-case "${P}" in *'"reason":"readyz:[etcd poststarthook/start-service-ip-repair-controllers]; etcd:unreachable; disk:ok"'*) pass "unhealthy: reason names failing subchecks + etcd + disk" ;; *) fail "unhealthy: reason wrong: ${P}" ;; esac
+case "${P}" in *'"reason":"readyz:[etcd poststarthook/start-service-ip-repair-controllers]; etcd:unreachable; disk:ok; fsync:nometricsms; load:7.25; memavail:262144k"'*) pass "unhealthy: reason names failing subchecks + etcd + disk + fsync + pressure" ;; *) fail "unhealthy: reason wrong: ${P}" ;; esac
 case "${P}" in *'"reason":"'*'"'*'"'*) : ;; esac
 C=$(cat "${WORK}/console.out")
 case "${C}" in *'mulga-eks CP unhealthy'*) pass "unhealthy: console banner emitted" ;; *) fail "unhealthy: console banner missing: ${C}" ;; esac
@@ -121,6 +138,96 @@ run_agent
 P=$(cat "${WORK}/payload.json")
 case "${P}" in *'etcd:ok; disk:low:1024k'*) pass "disk-low: reason flags low etcd disk" ;; *) fail "disk-low: reason wrong: ${P}" ;; esac
 
+# --- Case 4: readyz itself never answers -> unreachable, not "none" ---
+# A wedged apiserver returns an empty body; reporting that as "none" reads as
+# "no subcheck failed", which is the opposite of what happened.
+HEALTHZ_BODY=fail
+READYZ_BODY=
+ETCD_BODY=error
+DF_AVAIL_KB=9000000
+export HEALTHZ_BODY READYZ_BODY ETCD_BODY DF_AVAIL_KB
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'readyz:[unreachable]'*) pass "wedged: empty readyz reads unreachable, not none" ;; *) fail "wedged: reason wrong: ${P}" ;; esac
+case "${P}" in *'load:7.25; memavail:262144k'*) pass "wedged: pressure captured alongside" ;; *) fail "wedged: pressure missing: ${P}" ;; esac
+
+# --- Case 5: etcd serves its metrics -> fsync mean reported in ms ---
+# The case the field exists for: memory and free space both fine, so a slow
+# fsync is the only remaining reading. 4.2s over 700 fsyncs is 6.0ms.
+HEALTHZ_BODY=fail
+READYZ_BODY=
+ETCD_BODY=error
+DF_AVAIL_KB=9000000
+ETCD_METRICS_BODY=$(printf 'etcd_disk_wal_fsync_duration_seconds_sum 4.2\netcd_disk_wal_fsync_duration_seconds_count 700\n')
+export HEALTHZ_BODY READYZ_BODY ETCD_BODY DF_AVAIL_KB ETCD_METRICS_BODY
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'fsync:6.0ms'*) pass "fsync: mean derived from histogram sum/count" ;; *) fail "fsync: wrong: ${P}" ;; esac
+
+# --- Case 6: a stalled disk is legible against the healthy case above ---
+# 63s over 700 fsyncs is 90ms — fifteen times case 5 on the same shape of input.
+ETCD_METRICS_BODY=$(printf 'etcd_disk_wal_fsync_duration_seconds_sum 63\netcd_disk_wal_fsync_duration_seconds_count 700\n')
+export ETCD_METRICS_BODY
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'fsync:90.0ms'*) pass "fsync: a stalled disk reads far above a healthy one" ;; *) fail "fsync: stall wrong: ${P}" ;; esac
+
+# --- Case 7: no fsyncs recorded yet -> unknown, never a divide by zero ---
+ETCD_METRICS_BODY=$(printf 'etcd_disk_wal_fsync_duration_seconds_sum 0\netcd_disk_wal_fsync_duration_seconds_count 0\n')
+export ETCD_METRICS_BODY
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'fsync:warmupms'*) pass "fsync: zero count reads warmup, not a division error" ;; *) fail "fsync: zero-count wrong: ${P}" ;; esac
+
+# --- Case 8: too few fsyncs to mean anything -> unknown, not a fast reading ---
+# 3 fsyncs on a just-started etcd averages to a number with no meaning. Same
+# mistake as judging a throughput floor on a 297-byte object.
+ETCD_METRICS_BODY=$(printf 'etcd_disk_wal_fsync_duration_seconds_sum 0.012\netcd_disk_wal_fsync_duration_seconds_count 3\n')
+export ETCD_METRICS_BODY
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'fsync:warmupms'*) pass "fsync: a sample below the minimum count reads warmup" ;; *) fail "fsync: min-count wrong: ${P}" ;; esac
+
+# --- Case 9: fsync is published on a HEALTHY report, not only a failing one ---
+# A figure that appears only once the CP is already failing has no baseline to
+# be read against.
+HEALTHZ_BODY=ok
+NODES_BODY=$(printf 'ip-1 Ready <none> 1d v1.32.5\n')
+NODES_LABELED_BODY=$(printf 'ip-1 Ready <none> 1d v1.32.5 ng-a\n')
+ETCD_METRICS_BODY=$(printf 'etcd_disk_wal_fsync_duration_seconds_sum 4.2\netcd_disk_wal_fsync_duration_seconds_count 700\n')
+export HEALTHZ_BODY NODES_BODY NODES_LABELED_BODY ETCD_METRICS_BODY
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'"healthz":"ok"'*) pass "healthy-fsync: report is healthy" ;; *) fail "healthy-fsync: not healthy: ${P}" ;; esac
+case "${P}" in *'"fsync_ms":6.0'*) pass "healthy-fsync: fsync published on a healthy report" ;; *) fail "healthy-fsync: fsync missing: ${P}" ;; esac
+case "${P}" in *'"reason"'*) fail "healthy-fsync: healthy report must carry no reason: ${P}" ;; *) pass "healthy-fsync: no reason on a healthy report" ;; esac
+
+# --- Case 10: an unknown fsync is omitted, never emitted as a fast number ---
+ETCD_METRICS_BODY=
+export ETCD_METRICS_BODY
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'"fsync_ms"'*) fail "omitted: unknown fsync must not appear in the payload: ${P}" ;; *) pass "omitted: unknown fsync is absent, not reported as fast" ;; esac
+# The payload must still be valid JSON with the field gone.
+case "${P}" in *'"nodegroup_ready":{"ng-a":1},"ts":'*) pass "omitted: payload stays well-formed without the field" ;; *) fail "omitted: payload malformed: ${P}" ;; esac
+
+# --- Case 11: no metrics endpoint reads differently from a warming etcd ---
+# Both leave the payload field out, but they have different fixes — a missing
+# endpoint against a guest that only just booted — so the reason must separate
+# them. This is the readyz:[none] mistake, not repeated.
+HEALTHZ_BODY=fail
+READYZ_BODY=
+ETCD_BODY=error
+ETCD_METRICS_BODY=
+export HEALTHZ_BODY READYZ_BODY ETCD_BODY ETCD_METRICS_BODY
+run_agent
+P=$(cat "${WORK}/payload.json")
+case "${P}" in *'fsync:nometricsms'*) pass "nometrics: an unreachable scrape is distinct from a warming etcd" ;; *) fail "nometrics: wrong: ${P}" ;; esac
+
+# A sentinel must never reach the payload as a bare JSON value. The guard is
+# numeric, so a new sentinel cannot silently produce unparseable output.
+case "${P}" in *'"fsync_ms":nometrics'*) fail "nometrics: sentinel leaked into the payload as a number: ${P}" ;; *) pass "nometrics: sentinel kept out of the JSON value" ;; esac
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all mulga-eks-state-report cases"
     exit 0

scripts/images/rds-mariadb/rds-agent.initd

@@ -40,6 +40,28 @@ start_pre() {
     fi
 }
 
+# The agent's log is inside a guest nothing can reach, so anything that is not
+# mirrored here is lost. Used by both the failed start and the handoff timeout.
+mirror_agent_log() {
+    if [ -s "${output_log}" ]; then
+        einfo "last ${handoff_tail} lines of ${output_log}:"
+        tail -n "${handoff_tail}" "${output_log}" 2>/dev/null | while IFS= read -r _line; do
+            einfo "  [rds-agent] ${_line}"
+        done
+    else
+        einfo "${output_log} is empty — the agent produced no output at all"
+    fi
+}
+
+# OpenRC skips start_post when the start itself fails, which is the one case
+# with no diagnostic at all: the console reports only that the process died.
+start() {
+    default_start && return 0
+    eerror "rds-agent exited during start"
+    mirror_agent_log
+    return 1
+}
+
 start_post() {
     # Resolve this after start_pre loads agent.env so custom handoff directories
     # are shared by the agent, this wait, and rds-init.
@@ -53,18 +75,10 @@ start_post() {
     while [ ! -f "${HANDOFF_ENV}" ]; do
         if [ "${_waited}" -ge "${handoff_timeout}" ]; then
             eend 1 "no bootstrap handoff after ${handoff_timeout}s"
-            # Mirror the agent's own tail onto the console. The log lives inside a
-            # guest nothing can reach when this fails — an unreachable gateway is
-            # the usual cause — so the host-side console capture is the only way
-            # the dial error survives, and without it the failure reads as silence.
-            if [ -s "${output_log}" ]; then
-                einfo "last ${handoff_tail} lines of ${output_log}:"
-                tail -n "${handoff_tail}" "${output_log}" 2>/dev/null | while IFS= read -r _line; do
-                    einfo "  [rds-agent] ${_line}"
-                done
-            else
-                einfo "${output_log} is empty — the agent produced no output at all"
-            fi
+            # An unreachable gateway is the usual cause, and the host-side console
+            # capture is the only way that dial error survives — without it the
+            # failure reads as silence.
+            mirror_agent_log
             # The agent keeps retrying, so it is left running: rds-init fails on
             # the missing handoff and the engine stays down, which is what the
             # control plane observes.
@@ -73,5 +87,11 @@ start_post() {
         sleep 1
         _waited=$((_waited + 1))
     done
+
+    # A handoff means the agent read agent.env, which cloud-init wrote seconds
+    # ago and only into page cache. A reboot is a QMP reset with no guest
+    # shutdown, so anything not yet written back is gone and the next boot has
+    # no gateway URL to reach the control plane with.
+    sync
     eend 0
 }

scripts/images/rds-mariadb/rds-agent_initd_test.sh

@@ -22,9 +22,16 @@ export AGENT_ENV="${CASE}/agent.env"
 export handoff_timeout=0
 
 EEND_STATUS=
+# A file rather than a variable: the mirror pipes tail into a while loop, which
+# POSIX sh runs in a subshell, so an assignment there would not survive it.
+CONSOLE="${CASE}/console"
+: >"${CONSOLE}"
 ebegin() { :; }
-einfo() { :; }
+einfo() { printf '%s\n' "$*" >>"${CONSOLE}"; }
+eerror() { printf '%s\n' "$*" >>"${CONSOLE}"; }
 eend() { EEND_STATUS=$1; }
+SYNCED=
+sync() { SYNCED=1; }
 
 start_pre
 start_post
@@ -45,4 +52,28 @@ if [ "${RDS_ENGINE:-}" != "mariadb" ]; then
     exit 1
 fi
 
+# cloud-init writes agent.env once and never again, so the copy the next boot
+# reads is whatever reached the disk. A reboot is a reset with no guest
+# shutdown, and an unflushed agent.env leaves the agent with no gateway URL.
+if [ "${SYNCED}" != "1" ]; then
+    echo "FAIL: the handoff wait did not flush the config cloud-init wrote" >&2
+    exit 1
+fi
+
+# A start that fails is the one case OpenRC runs no start_post for, so without
+# this the console carries nothing but "died" and the reason is lost with the
+# guest.
+export output_log="${CASE}/agent.log"
+printf 'rds-agent: startup failed err="build IMDS signer: no credentials"\n' >"${output_log}"
+: >"${CONSOLE}"
+default_start() { return 1; }
+if start; then
+    echo "FAIL: start reported success when the agent died" >&2
+    exit 1
+fi
+if ! grep -q "build IMDS signer" "${CONSOLE}"; then
+    echo "FAIL: a failed start did not mirror the agent log to the console" >&2
+    exit 1
+fi
+
 echo "rds-agent.initd: all tests passed"

scripts/images/rds-postgres/rds-agent.initd

@@ -40,6 +40,28 @@ start_pre() {
     fi
 }
 
+# The agent's log is inside a guest nothing can reach, so anything that is not
+# mirrored here is lost. Used by both the failed start and the handoff timeout.
+mirror_agent_log() {
+    if [ -s "${output_log}" ]; then
+        einfo "last ${handoff_tail} lines of ${output_log}:"
+        tail -n "${handoff_tail}" "${output_log}" 2>/dev/null | while IFS= read -r _line; do
+            einfo "  [rds-agent] ${_line}"
+        done
+    else
+        einfo "${output_log} is empty — the agent produced no output at all"
+    fi
+}
+
+# OpenRC skips start_post when the start itself fails, which is the one case
+# with no diagnostic at all: the console reports only that the process died.
+start() {
+    default_start && return 0
+    eerror "rds-agent exited during start"
+    mirror_agent_log
+    return 1
+}
+
 start_post() {
     # Resolve this after start_pre loads agent.env so custom handoff directories
     # are shared by the agent, this wait, and rds-init.
@@ -53,18 +75,10 @@ start_post() {
     while [ ! -f "${HANDOFF_ENV}" ]; do
         if [ "${_waited}" -ge "${handoff_timeout}" ]; then
             eend 1 "no bootstrap handoff after ${handoff_timeout}s"
-            # Mirror the agent's own tail onto the console. The log lives inside a
-            # guest nothing can reach when this fails — an unreachable gateway is
-            # the usual cause — so the host-side console capture is the only way
-            # the dial error survives, and without it the failure reads as silence.
-            if [ -s "${output_log}" ]; then
-                einfo "last ${handoff_tail} lines of ${output_log}:"
-                tail -n "${handoff_tail}" "${output_log}" 2>/dev/null | while IFS= read -r _line; do
-                    einfo "  [rds-agent] ${_line}"
-                done
-            else
-                einfo "${output_log} is empty — the agent produced no output at all"
-            fi
+            # An unreachable gateway is the usual cause, and the host-side console
+            # capture is the only way that dial error survives — without it the
+            # failure reads as silence.
+            mirror_agent_log
             # The agent keeps retrying, so it is left running: rds-init fails on
             # the missing handoff and the engine stays down, which is what the
             # control plane observes.
@@ -73,5 +87,11 @@ start_post() {
         sleep 1
         _waited=$((_waited + 1))
     done
+
+    # A handoff means the agent read agent.env, which cloud-init wrote seconds
+    # ago and only into page cache. A reboot is a QMP reset with no guest
+    # shutdown, so anything not yet written back is gone and the next boot has
+    # no gateway URL to reach the control plane with.
+    sync
     eend 0
 }

scripts/images/rds-postgres/rds-agent_initd_test.sh

@@ -22,9 +22,16 @@ export AGENT_ENV="${CASE}/agent.env"
 export handoff_timeout=0
 
 EEND_STATUS=
+# A file rather than a variable: the mirror pipes tail into a while loop, which
+# POSIX sh runs in a subshell, so an assignment there would not survive it.
+CONSOLE="${CASE}/console"
+: >"${CONSOLE}"
 ebegin() { :; }
-einfo() { :; }
+einfo() { printf '%s\n' "$*" >>"${CONSOLE}"; }
+eerror() { printf '%s\n' "$*" >>"${CONSOLE}"; }
 eend() { EEND_STATUS=$1; }
+SYNCED=
+sync() { SYNCED=1; }
 
 start_pre
 start_post
@@ -45,4 +52,28 @@ if [ "${RDS_ENGINE:-}" != "postgres" ]; then
     exit 1
 fi
 
+# cloud-init writes agent.env once and never again, so the copy the next boot
+# reads is whatever reached the disk. A reboot is a reset with no guest
+# shutdown, and an unflushed agent.env leaves the agent with no gateway URL.
+if [ "${SYNCED}" != "1" ]; then
+    echo "FAIL: the handoff wait did not flush the config cloud-init wrote" >&2
+    exit 1
+fi
+
+# A start that fails is the one case OpenRC runs no start_post for, so without
+# this the console carries nothing but "died" and the reason is lost with the
+# guest.
+export output_log="${CASE}/agent.log"
+printf 'rds-agent: startup failed err="build IMDS signer: no credentials"\n' >"${output_log}"
+: >"${CONSOLE}"
+default_start() { return 1; }
+if start; then
+    echo "FAIL: start reported success when the agent died" >&2
+    exit 1
+fi
+if ! grep -q "build IMDS signer" "${CONSOLE}"; then
+    echo "FAIL: a failed start did not mirror the agent log to the console" >&2
+    exit 1
+fi
+
 echo "rds-agent.initd: all tests passed"

scripts/images/rds-postgres/rds-init

@@ -579,6 +579,33 @@ EOF
     mv -f "${_tmp}" "${PGDATA}/conf.d/90-rds-init.conf"
 }
 
+# postgresql.conf is on the data volume, so the hook is asserted every boot
+# rather than only at initdb — a restored volume may predate it.
+#
+# Rewritten and renamed rather than appended to. It is the only file rds-init
+# puts on that volume without the engine's knowledge, so a torn tail is the one
+# corruption the engine cannot repair from its own WAL: the postmaster stops on
+# a syntax error and the instance never comes up.
+install_include_hook() {
+    _conf="${PGDATA}/postgresql.conf"
+    _tmp="${PGDATA}/.postgresql.conf.new"
+
+    if grep -q "^include_dir = 'conf.d'" "${_conf}" 2>/dev/null; then
+        return 0
+    fi
+    # Any half-written hook this script left behind goes rather than being kept:
+    # the guard above does not match one, so appending after its unterminated
+    # quote would stop the postmaster on a file it could not have started from.
+    grep -v "^include_dir = 'conf" "${_conf}" 2>/dev/null >"${_tmp}" || :
+    printf "\n# Managed by rds-init: parameter-group values (10-) then platform settings (90-).\ninclude_dir = 'conf.d'\n" \
+        >>"${_tmp}"
+
+    chown "${PG_USER}:${PG_USER}" "${_tmp}"
+    chmod 0600 "${_tmp}"
+    mv -f "${_tmp}" "${_conf}" ||
+        fail "the include hook could not be installed into ${_conf}"
+}
+
 # Everything refreshed on every boot: the include hook, the client
 # authentication rules, the resolved parameters, the serving cert, the TLS
 # enforcement those parameters ask for and the platform's own connectivity
@@ -588,12 +615,7 @@ install_runtime_config() {
     install -d -m 1775 -o "${PG_USER}" -g "${PG_USER}" "${SOCKET_DIR}"
     install -d -m 0755 -o "${PG_USER}" -g "${PG_USER}" "${LOG_DIR}"
 
-    # postgresql.conf is on the data volume, so the hook is asserted every boot
-    # rather than only at initdb — a restored volume may predate it.
-    if ! grep -q "^include_dir = 'conf.d'" "${PGDATA}/postgresql.conf"; then
-        printf "\n# Managed by rds-init: parameter-group values (10-) then platform settings (90-).\ninclude_dir = 'conf.d'\n" \
-            >> "${PGDATA}/postgresql.conf"
-    fi
+    install_include_hook
 
     write_pg_hba
     install_parameters

scripts/images/rds-postgres/rds-init_test.sh

@@ -1267,6 +1267,42 @@ if run_ok "restore-scope-setup"; then
     fi
 fi
 
+# --- Case 14: a half-written include hook is repaired, not appended after ---
+# postgresql.conf is the one file rds-init puts on the data volume without the
+# engine's knowledge, so a torn tail is the one corruption no WAL replay repairs:
+# the guard does not match an unterminated hook, and appending after it leaves an
+# open quote the postmaster stops on.
+reset_state
+write_handoff initialize 's3cr3t' appdb
+write_parameters
+if run_ok "torn-hook-setup"; then
+    # The shape a snapshot of an unflushed append restores: the hook cut short.
+    sed "s/^include_dir = 'conf.d'$/include_dir = 'conf/" "${PGDATA}/postgresql.conf" \
+        >"${PGDATA}/postgresql.conf.torn"
+    mv -f "${PGDATA}/postgresql.conf.torn" "${PGDATA}/postgresql.conf"
+    write_handoff attach '' appdb
+    write_parameters
+    if run_ok "torn-hook"; then
+        grep -q "^include_dir = 'conf.d'$" "${PGDATA}/postgresql.conf" \
+            && pass "torn-hook: the hook is back" || fail "torn-hook: no usable hook"
+        grep -c "^include_dir = 'conf" "${PGDATA}/postgresql.conf" | grep -qx 1 \
+            && pass "torn-hook: the torn line is gone rather than appended after" \
+            || fail "torn-hook: the torn line survived beside the new one"
+    fi
+fi
+
+# --- Case 14a: the hook is written whole or not at all ---
+# Appended in place, a snapshot could catch it half-written; renamed over, the
+# file a restore reads is either the old one or the complete new one.
+reset_state
+write_handoff initialize 's3cr3t' appdb
+write_parameters
+if run_ok "hook-atomic"; then
+    [ -e "${PGDATA}/.postgresql.conf.new" ] \
+        && fail "hook-atomic: the temp file was left behind" \
+        || pass "hook-atomic: no temp file survives the install"
+fi
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all rds-init cases"
     exit 0

scripts/setup.sh

@@ -105,12 +105,12 @@ detect_os() {
             ;;
         ubuntu)
             major="${VERSION_ID%%.*}"
-            if [ "$major" -lt 22 ] 2>/dev/null; then
-                fatal "Ubuntu $VERSION_ID is not supported. Minimum: Ubuntu 22.04"
+            if [ "$major" -lt 24 ] 2>/dev/null; then
+                fatal "Ubuntu $VERSION_ID is not supported. Minimum: Ubuntu 24.04"
             fi
             ;;
         *)
-            fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 22.04+"
+            fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 24.04+"
             ;;
     esac
 
@@ -371,7 +371,16 @@ table inet spinifex_filter {
         # here. Without this, cloud-init, instance-role credentials and all
         # guest DNS break. The ports exist only while instances run, so this
         # matches the prefix rather than an enumerated list.
-        iifname "ime-*" accept
+        #
+        # Scoped to the two link-local endpoints and their ports, not a blanket
+        # accept: the OVS demux delivers any guest packet to 169.254.169.254/.253
+        # onto an ime-* port regardless of L4 port, so `iifname "ime-*" accept`
+        # would expose every wildcard-bound host service (sshd, the AWS gateway,
+        # northstar, the UI) to any tenant. IMDS is HTTP on .254:80; VPC DNS is
+        # 53 on .253. Everything else on these ports falls through to policy drop.
+        iifname "ime-*" ip daddr 169.254.169.254 tcp dport 80 accept
+        iifname "ime-*" ip daddr 169.254.169.253 udp dport 53 accept
+        iifname "ime-*" ip daddr 169.254.169.253 tcp dport 53 accept
 
         # PMTUD is not optional under a Geneve overlay.
         icmp type { echo-request, destination-unreachable, time-exceeded, parameter-problem } accept
@@ -391,29 +400,34 @@ table inet spinifex_filter {
         # gateway, never NATS, so no cluster port needs to face the guests.
         # 443 has no listener yet and is held for the nginx TLS edge, so the
         # rollout does not need a firewall change on every node.
-        tcp dport { 443, 3000, 8443, 9999 } accept
+        # nfproto ipv4: these accepts carry no source scope, so without a family
+        # qualifier they would also serve on any global IPv6 a node autoconfigures
+        # from an upstream RA. The scoped rules are v4-only via `ip saddr` already.
+        meta nfproto ipv4 tcp dport { 443, 3000, 8443, 9999 } accept
 
         # Transiently open to any source, for cluster formation. A node dialling
         # in to join is not a peer yet, so the peer-scoped rule below cannot let
         # it in; `spx admin init` opens the formation port here for the length of
         # the formation window and closes it again afterwards. Normally holds
         # only the sentinel, which no packet can match.
-        tcp dport $spinifex_open_ports accept
+        meta nfproto ipv4 tcp dport $spinifex_open_ports accept
 
         # 53 is open on every node, deliberately: northstar serves public
         # authoritative DNS and binds its advertise address. A node running no
         # public zone answers here too, which is the accepted cost of not
         # templating the policy per node.
-        tcp dport 53 accept
-        udp dport 53 accept
+        meta nfproto ipv4 tcp dport 53 accept
+        meta nfproto ipv4 udp dport 53 accept
 
         # Cluster plane, peer-scoped. On a single-NIC node the planes collapse
         # onto the public address, which is why these are peer addresses rather
         # than an interface or a CIDR. 4432 stays here rather than joining the
         # public rule above: outside formation it is the daemon cluster manager,
         # whose /health and /local/* routes report node topology, service
-        # inventory and running instances to anyone who asks.
-        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644 } accept
+        # inventory and running instances to anyone who asks. 8660 is
+        # predastore's unauthenticated /healthz and /readyz, which name the
+        # peers a gate cannot reach, so it belongs here for the same reason.
+        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
         ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept
 
         # Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a

spinifex/accountteardown/reapers_ec2.go

@@ -76,7 +76,7 @@ func (r *instanceReaper) Stage() Stage { return StageCompute }
 func (r *instanceReaper) List(ctx context.Context, accountID string) ([]Resource, error) {
 	// Checked: a partial answer here would read as "no instances left" and let
 	// storage deletion start while a guest still holds a volume.
-	out, err := gateway_ec2_instance.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{}, r.nc, r.expectedNodes, accountID)
+	out, err := gateway_ec2_instance.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{}, r.nc, r.expectedNodes, nil, accountID)
 	if err != nil {
 		return nil, err
 	}
@@ -396,6 +396,10 @@ func (r *igwReaper) Delete(ctx context.Context, accountID string, resource Resou
 			return err
 		}
 	}
+	// A gateway whose attach is not yet confirmed reports no attachment, so
+	// Detail is empty and the delete is refused. DeleteVpc rejects while the
+	// gateway is attached, so the VPC survives and the next sweep, once the
+	// attachment is confirmed, takes the branch above.
 	_, err := gateway_ec2_igw.DeleteInternetGateway(ctx, &ec2.DeleteInternetGatewayInput{
 		InternetGatewayId: aws.String(resource.ID),
 	}, r.nc, accountID)

spinifex/accountteardown/reapers_ec2_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
@@ -403,6 +404,23 @@ func TestIGWReaperDeletesADetachedGatewayDirectly(t *testing.T) {
 	assert.Equal(t, []string{"ec2.DeleteInternetGateway"}, cluster.called())
 }
 
+// A gateway whose attach is not yet confirmed reports no attachment, so the
+// listing carries no VPC and the delete is refused. That is not a wedge: the
+// VPC survives because DeleteVpc rejects while the gateway is attached, and the
+// stage re-sweeps until the confirmation lands.
+func TestIGWReaperReportsARefusedDeleteForAnUnconfirmedAttach(t *testing.T) {
+	cluster := newFakeCluster(t)
+	cluster.fail("ec2.DeleteInternetGateway", awserrors.ErrorDependencyViolation)
+
+	reaper := &igwReaper{nc: cluster.nc}
+	err := reaper.Delete(testCtx(t), "000000000042",
+		Resource{Kind: "internet-gateway", ID: "igw-1"}, false)
+
+	require.Error(t, err)
+	assert.Equal(t, []string{"ec2.DeleteInternetGateway"}, cluster.called(),
+		"the reaper must not guess at the VPC: the next sweep sees the confirmed attachment")
+}
+
 // A VPC's main route table is deleted with the VPC and cannot be deleted on
 // its own, so listing it would guarantee a stuck report every time.
 func TestRouteTableReaperSkipsTheMainTable(t *testing.T) {