release: update go.mod for v1.18.0

The local helper masked x-amz-security-token by line prefix, which matched the
header form and missed the presigned query form entirely, leaking the token into
the exported logs on every signature mismatch. Defer to bluebottle's
RedactedCanonicalRequest, which knows the structure and masks both forms.

.github/actions/junit-to-es/action.yml

@@ -0,0 +1,43 @@
+name: JUnit to Elasticsearch
+description: >-
+  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
+  so per-test pass rates outlive the 90-day artifact retention. Call it once
+  per job right after the JUnit XML is produced (with `if: always()`), next to
+  the junit-report action that renders the same files into the job summary.
+
+inputs:
+  junit-glob:
+    description: Shell glob matching the JUnit XML files to ship.
+    required: true
+  cell:
+    description: >-
+      Cell label stored as labels.ci_cell — the same value the job puts in
+      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
+      Without it, results from different topologies collapse into one bucket.
+    required: true
+  es-sink:
+    description: host:port of the Elasticsearch sink.
+    required: false
+    default: "192.168.1.13:9200"
+  index:
+    description: Target index.
+    required: false
+    default: mulga-ci-test-results
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        JUNIT_GLOB: ${{ inputs.junit-glob }}
+        JUNIT_CELL: ${{ inputs.cell }}
+        MULGA_ES_SINK: ${{ inputs.es-sink }}
+        ES_INDEX: ${{ inputs.index }}
+        RUN_ID: ${{ github.run_id }}
+        RUN_ATTEMPT: ${{ github.run_attempt }}
+        WORKFLOW: ${{ github.workflow }}
+        BRANCH: ${{ github.head_ref || github.ref_name }}
+        SHA: ${{ github.sha }}
+        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Ship per-test JUnit results to the Elasticsearch sink.
+
+One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
+mulga-ci-runs summary says which suite failed; this says which test, with the
+failure text, so a per-test pass-rate trend outlives artifact retention.
+
+Inputs (env):
+    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
+                    a leg that never ran produces none.
+    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
+    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
+                    there, so no credentials are involved.
+    ES_INDEX        Target index; defaults to mulga-ci-test-results.
+    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+
+# Go test output carries colour codes into the JUnit CDATA (harness.Phase
+# emits them), which are noise in a stored failure message.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+# Enough failure text to recognise a failure; the artifact log stays the full
+# record.
+_MAX_MESSAGE = 4000
+
+# go-junit-report emits this when a `=== RUN` line has no matching result line,
+# which means the log it was fed was truncated or filtered. It is a parse gap,
+# not a test outcome, so it is stored under its own status and left out of every
+# pass rate rather than counted as a failure.
+_NO_RESULT = "No test result found"
+
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+
+def _truncate(s: str) -> str:
+    s = s.strip()
+    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
+
+
+def _suite_from_path(path: str) -> str:
+    base = os.path.basename(path)
+    if base.endswith(".xml"):
+        base = base[:-4]
+    if base.startswith("junit-"):
+        base = base[len("junit-") :]
+    return base
+
+
+def _cases(path: str) -> list[dict]:
+    """Parse one JUnit file into leaf test cases."""
+    try:
+        root = ET.parse(path).getroot()
+    except ET.ParseError as exc:
+        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
+        return []
+
+    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
+    # several. .iter() handles both.
+    out: list[dict] = []
+    for s in root.iter("testsuite"):
+        for tc in s.findall("testcase"):
+            status = "pass"
+            fail_type = ""
+            message = ""
+            for tag in ("failure", "error"):
+                node = tc.find(tag)
+                if node is not None:
+                    status = "fail" if tag == "failure" else "error"
+                    fail_type = tag
+                    message = _strip_ansi(
+                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
+                    )
+                    if (node.get("message") or "").strip() == _NO_RESULT:
+                        status = "unknown"
+                    break
+            if status == "pass" and tc.find("skipped") is not None:
+                status = "skip"
+            out.append(
+                {
+                    "name": tc.get("name", ""),
+                    "classname": tc.get("classname", ""),
+                    "time": float(tc.get("time", 0) or 0),
+                    "status": status,
+                    "fail_type": fail_type,
+                    "message": message,
+                }
+            )
+
+    # Drop container cases: Go reports a parent t.Run as a rollup of its
+    # children, so keeping both double-counts one outcome. Same rule the job
+    # summary renderer applies.
+    names = {c["name"] for c in out}
+    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
+    return [c for c in out if c["name"] not in containers]
+
+
+def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
+    full = case["name"]
+    top, _, sub = full.partition("/")
+    doc = {
+        "@timestamp": ts,
+        "suite": suite,
+        "status": case["status"],
+        "duration_seconds": case["time"],
+        "artifact_url": ci["artifact_url"],
+        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
+        "test": {
+            "name": top,
+            "subtest": sub,
+            "full_name": full,
+            "classname": case["classname"],
+        },
+        "ci": {
+            "workflow": ci["workflow"],
+            "branch": ci["branch"],
+            "sha": ci["sha"],
+            "run_attempt": ci["run_attempt"],
+        },
+    }
+    if case["fail_type"]:
+        doc["failure"] = {
+            "type": case["fail_type"],
+            "message": _truncate(case["message"]),
+        }
+    return doc
+
+
+def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
+    # Deterministic so a replayed post overwrites instead of double-counting.
+    # run_attempt is part of it because a re-run is a distinct result.
+    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
+
+
+def _bulk(sink: str, index: str, body: str) -> int:
+    req = urllib.request.Request(
+        f"http://{sink}/{index}/_bulk",
+        data=body.encode("utf-8"),
+        headers={"Content-Type": "application/x-ndjson"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            payload = json.loads(resp.read().decode("utf-8"))
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
+        return 1
+
+    if payload.get("errors"):
+        for item in payload.get("items", []):
+            err = next(iter(item.values())).get("error")
+            if err:
+                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
+                return 1
+    return 0
+
+
+def main() -> int:
+    pattern = os.environ.get("JUNIT_GLOB", "").strip()
+    if not pattern:
+        print("::error::junit-to-es: JUNIT_GLOB is empty")
+        return 1
+    cell = os.environ.get("JUNIT_CELL", "").strip()
+    if not cell:
+        print("::error::junit-to-es: JUNIT_CELL is empty")
+        return 1
+    sink = os.environ.get("MULGA_ES_SINK", "").strip()
+    if not sink:
+        print("::error::junit-to-es: MULGA_ES_SINK is empty")
+        return 1
+    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
+
+    ci = {
+        "run_id": os.environ.get("RUN_ID", ""),
+        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
+        "workflow": os.environ.get("WORKFLOW", ""),
+        "branch": os.environ.get("BRANCH", ""),
+        "sha": os.environ.get("SHA", ""),
+        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
+    }
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    lines: list[str] = []
+    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
+    for path in sorted(glob.glob(pattern)):
+        suite = _suite_from_path(path)
+        for case in _cases(path):
+            doc = _doc(cell, suite, case, ts, ci)
+            counts[case["status"]] = counts.get(case["status"], 0) + 1
+            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
+            lines.append(json.dumps(meta))
+            lines.append(json.dumps(doc))
+
+    if not lines:
+        # Not an error on its own: the leg may have been skipped, or died
+        # before any test ran. The run summary doc records that separately.
+        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
+        return 0
+
+    rc = _bulk(sink, index, "\n".join(lines) + "\n")
+    if rc == 0:
+        print(
+            f"junit-to-es: posted {sum(counts.values())} test docs "
+            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
+            f"{counts['fail']} fail, {counts['error']} error, "
+            f"{counts['skip']} skip, {counts['unknown']} unknown"
+        )
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/suite-sets.sh

@@ -24,10 +24,14 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+# quota belongs to both topologies because the gate it asserts is gateway code
+# that runs identically on either, and because a quota left switched on would
+# cap every suite sharing the environment — so it must be proved to restore
+# itself wherever it runs, not only on one node.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert"
+E2E_SUITES_MULTI="multinode lb cert quota"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"

.github/workflows/e2e-baremetal.yml

@@ -143,6 +143,16 @@ jobs:
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: "20"
+
       - name: Analyze e2e failures
         if: always()
         uses: ./.github/actions/e2e-analyze

.github/workflows/e2e-ddil.yml

@@ -180,6 +180,16 @@ jobs:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
+          cell: ddil
+
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/e2e-nightly.yml

@@ -342,6 +342,14 @@ jobs:
               | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
             ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
+            # ovn-controller and ovs-vswitchd log to files, never to journald:
+            # without these a cross-chassis black-hole cannot be attributed to
+            # IPsec versus SB port binding.
+            mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
+              | tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
           done
 
           cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
@@ -363,6 +371,16 @@ jobs:
             go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
           done
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.selected == 'true'
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: ${{ matrix.cell }}
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.selected == 'true'
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -496,12 +514,13 @@ jobs:
 
       - uses: actions/checkout@v7
         with:
+          ref: ${{ env.GIT_BRANCH }}
           path: mulga/spinifex
 
       - name: Resolve sibling refs
         id: siblings
         env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ env.GIT_BRANCH }}
         run: bash .github/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
@@ -727,12 +746,13 @@ jobs:
 
       - uses: actions/checkout@v7
         with:
+          ref: ${{ env.GIT_BRANCH }}
           path: mulga/spinifex
 
       - name: Resolve sibling refs
         id: siblings
         env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ env.GIT_BRANCH }}
         run: bash .github/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 

.github/workflows/e2e-suites.yml

@@ -394,6 +394,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -796,6 +806,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/e2e.yml

@@ -464,6 +464,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -903,6 +913,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/gpu-e2e.yml

@@ -287,6 +287,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.paused != 'true'
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: gpu
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
         uses: ./.github/actions/e2e-analyze

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.6"
+          go-version: "1.27.0"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image

.github/workflows/spinifex.yml

@@ -377,7 +377,7 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.12.2
+          version: v2.13.0
           working-directory: spinifex
 
       - name: Govulncheck

.gitignore

@@ -34,9 +34,10 @@ Thumbs.db
 # Environment
 .env
 .venv/
+.cache/
+
+# VScode
+.vscode/
 
 # Test reports
 *-report.txt
-
-# Per-worktree golangci-lint cache
-/.cache/

.golangci.yml

@@ -26,7 +26,7 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forbidigo # blocks the legacy JetStream/KV symbols
+    - forbidigo # blocks forbidden identifiers
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
@@ -92,6 +92,8 @@ linters:
     forbidigo:
       analyze-types: true
       forbid:
+        - pattern: '^panic$'
+          msg: panic terminates the process; return an error or explicitly justify the panic.
         - pattern: '\.JetStream$'
           pkg: '^github\.com/nats-io/nats\.go$'
           msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -116,6 +118,7 @@ linters:
 
     modernize:
       disable:
+        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
@@ -166,6 +169,7 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
+          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions

Makefile

@@ -1,11 +1,6 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
-# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
-# path, not worktree path, so parallel worktrees of this module collide and
-# surface each other's issues. Scope it per worktree.
-export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
-
 # GOFIPS140 is a global build-config change: it alters every package's action ID,
 # stdlib included, so a target that omits it shares no build-cache entry with one
 # that sets it. Exported once here rather than per-recipe so the two can't drift.
@@ -344,9 +339,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.6 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.6.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
@@ -13,9 +13,10 @@
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
   <a href="#aws-compatibility">AWS Compatibility</a> ·
+  <a href="#live-demo">Live Demo</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#installation">Installation</a> ·
+  <a href="#self-hosted-installation">Installation</a> ·
   <a href="https://docs.mulgadc.com">Docs</a>
 </p>
 
@@ -29,11 +30,9 @@ Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
 
 Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
+Built for teams that need:
 
-It's built for teams that need:
-
-- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
 - Full control of the stack: your hardware, your network, your data, your keys
 - A cloud that keeps running offline, through disconnection, with no external control plane
 - An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
@@ -50,15 +49,15 @@ From commodity hardware up to unmodified AWS tooling, every layer is replaceable
 
 ## Three Ways To Deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
+The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
+Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
+Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
 
 ### Edge
 
@@ -66,7 +65,7 @@ Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factorie
 
 ## AWS Compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
+Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
@@ -84,15 +83,33 @@ Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: every
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
+## Live Demo
+
+Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
+
+* Live in under a minute. No card, no install.
+* Yours for 72 hours to test and experiment.
+* Real AWS-compatible endpoint, access keys and console.
+* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
+* Point your existing AWS CLI or Terraform straight at it.
+
+Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
+
 ## Core Components
 
-### Spinifex (Compute Service – EC2 Alternative)
+### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
 
-- EC2-like VM management on bare metal
-- Launches with cloud-init metadata support
-- Works with standard AWS tooling
+The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
+
+- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
+- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
+- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
+- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
+- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
+- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
+- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
 
 ### Viperblock (Block Storage – EBS Alternative)
 
@@ -101,12 +118,12 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Fast, durable virtual disks
 - Replication for resilience
 - Exposed over NBD or embedded in VMs
-- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
-- In memory read/write block cache for blazing performance.
+- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
+- In-memory read/write block cache
 
 ### Predastore (Object Storage – S3-Compatible)
 
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
+[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
 
 - S3-compatible API and auth
 - Multipart uploads, streaming reads/writes
@@ -128,16 +145,7 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 
 Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Key Features
-
-- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
-- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
-- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
-- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
-- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
-- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
-
-## Installation
+## Self-hosted Installation
 
 Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
 
@@ -173,31 +181,21 @@ aws ec2 describe-instance-types
 
 For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
 
-### Component Repositories
-
-Spinifex coordinates these independent components:
-
-- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
-- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
-- **[Northstar](https://github.com/mulgadc/northstar)** - Authoritative DNS server
-
-Each component can be developed independently. See component-specific documentation for focused development guides.
-
 ## Spinifex UI
 
-Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
 </p>
 
-The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
 
-- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
-- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
+- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
 
-For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
 
 ## Development Philosophy
 
@@ -205,16 +203,9 @@ For the full walkthrough — first-time TLS certificate trust, login, and featur
 
 Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
 
-**Real-World Experience:**
-
-- Production AWS service development and operations
-- Large-scale infrastructure deployment and management
-- Edge computing and resource-constrained environments
-- Enterprise security and compliance requirements
-
 ### AI-Assisted Development
 
-While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
+Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
 
 **How We Use Claude Code:**
 
@@ -230,12 +221,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 - **Performance Optimization**: Real-world performance tuning and benchmarking
 - **Production Operations**: Deployment strategies and operational procedures
 
-This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
-
 ## Trademarks
 
 "AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.26-bookworm AS builder
+FROM golang:1.27-bookworm AS builder
 
 ARG VERSION
 

build/systemd/spinifex-predastore.service

@@ -1,4 +1,4 @@
-# spinifex-unit-version: 2
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -7,9 +7,9 @@ Wants=spinifex-nats.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
-# Guest block I/O arrives with no caller trace; sample those roots at 5% so
-# they don't drown request traces. Request-linked spans are always kept.
-Environment=MULGA_ROOT_TRACE_RATIO=0.05
+# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
+# instead, and it can see whether a trace errored. A head ratio decides before
+# the error exists, so it discarded failures at the same rate as successes.
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage

cmd/ecs-agent/runtime/runtime_test.go

@@ -1,71 +1,9 @@
 package runtime
 
 import (
-	"context"
-	"errors"
 	"testing"
 )
 
-// staticResolver is a test Resolver returning fixed creds or an error.
-type staticResolver struct {
-	user, pass, endpoint string
-	err                  error
-}
-
-func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
-	return s.user, s.pass, s.endpoint, s.err
-}
-
-func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
-	f := &FakePuller{}
-	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
-	if err != nil {
-		t.Fatalf("Pull: %v", err)
-	}
-	if img.Ref != "repo/app:1" {
-		t.Errorf("Ref defaulted wrong: %q", img.Ref)
-	}
-	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
-		t.Errorf("Pulls = %+v", f.Pulls)
-	}
-	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
-		t.Errorf("Authzd = %+v", f.Authzd)
-	}
-}
-
-func TestFakePuller_ResolverError(t *testing.T) {
-	f := &FakePuller{}
-	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
-	if err == nil {
-		t.Fatal("expected resolver error to propagate")
-	}
-}
-
-func TestFakePuller_ProgrammedError(t *testing.T) {
-	f := &FakePuller{Err: errors.New("pull failed")}
-	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
-		t.Fatal("expected programmed error")
-	}
-}
-
-func TestFakePuller_OnPullOverrides(t *testing.T) {
-	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
-	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
-		return want, nil
-	}}
-	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
-	if err != nil || got != want {
-		t.Fatalf("got %+v, %v; want %+v", got, err, want)
-	}
-}
-
-func TestFakePuller_Close(t *testing.T) {
-	f := &FakePuller{}
-	if err := f.Close(); err != nil || !f.Closed {
-		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
-	}
-}
-
 func TestNormalizeRef(t *testing.T) {
 	cases := map[string]string{
 		"nginx:alpine":      "docker.io/library/nginx:alpine",

cmd/installer/install/steps_test.go

@@ -887,13 +887,6 @@ func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
 	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
 }
 
-func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
-	// Log-only, but it is the operator's only record of the disk-replacement
-	// command, so both branches are walked.
-	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
-	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
-}
-
 func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	root := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
@@ -913,14 +906,6 @@ func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	}
 }
 
-func TestCopyAssetsAreBestEffort(t *testing.T) {
-	root := t.TempDir()
-	// Neither asset exists outside the live ISO; a missing one must not fail
-	// the install, only the branding.
-	copyGrubFont(root)
-	copySplashImage(root)
-}
-
 func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
 	root := withMountRoot(t)
 	targetSkeleton(t, root)

cmd/installer/install/zfs.go

@@ -182,8 +182,7 @@ func ImportablePools() ([]string, error) {
 		// zpool exits non-zero when there is nothing to import, which is the
 		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
 		// a machine with an existing pool looks empty to the caller.
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) {
+		if _, ok := errors.AsType[*exec.ExitError](err); ok {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("zpool import: %w", err)

cmd/rds-agent/agent.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 )
 
 // Overridable via -ldflags "-X main.version=...".
@@ -406,10 +407,10 @@ func retryObserved(ctx context.Context, what string, fn func(context.Context) er
 		}
 		if attempt >= retryErrorAttempt {
 			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		} else {
 			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		}
 
 		select {

cmd/rds-agent/agent_test.go

@@ -207,6 +207,8 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 		MasterUsername:       "master",
 		MasterUserPassword:   password,
 		Port:                 5432,
+		ListenAddress:        testListenAddress,
+		ClientCIDR:           testClientCIDR,
 		DataVolumeID:         "vol-data-01",
 		DataVolumeSerial:     "voldata01",
 		VMGeneration:         1,
@@ -217,6 +219,13 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 
 const testPayloadID = "bp-0123456789abcdef"
 
+// The customer ENI address the engine binds, and the VPC range its client
+// authentication admits. writeHandoff refuses a config without either.
+const (
+	testListenAddress = "10.0.1.20"
+	testClientCIDR    = "10.0.0.0/16"
+)
+
 // Writes the completion receipt rds-init leaves behind, so the acknowledgement
 // step finds what a finished initialization would have produced.
 func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {

cmd/rds-agent/bootstrap.go

@@ -96,6 +96,15 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 	if cfg.MasterUsername == "" {
 		return fmt.Errorf("bootstrap config carries no master username")
 	}
+	// Half a handoff would leave rds-init deciding what to bind and who to admit,
+	// and the only answers it could reach on its own are the wildcards this
+	// contract exists to remove.
+	if cfg.ListenAddress == "" {
+		return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
+	}
+	if cfg.ClientCIDR == "" {
+		return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
+	}
 	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
 		return fmt.Errorf("create handoff dir %s: %w", dir, err)
 	}
@@ -134,6 +143,8 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
 	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
 	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
+	writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
 	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))

cmd/rds-agent/bootstrap_test.go

@@ -18,6 +18,8 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
 		Port:               5433,
@@ -85,6 +87,8 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
@@ -116,6 +120,8 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		Port:               5432,
 	}); err != nil {
@@ -137,6 +143,8 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeAttach,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		Port:               5432,
 		ServingCertificate: "CERT",
 	}); err != nil {
@@ -177,6 +185,8 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)

cmd/rds-agent/config.go

@@ -13,7 +13,6 @@ const (
 	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
-	defaultEngineHost = "127.0.0.1"
 	// Where setup.sh stamps the engine the image bakes. The agent builds its
 	// engine implementation from this file rather than from anything delivered.
 	defaultEngineFile         = "/etc/spinifex-rds/engine"
@@ -38,7 +37,6 @@ type config struct {
 	// version alone rather than clearing it.
 	EngineVersion string
 	HandoffDir    string
-	EngineHost    string
 	EnginePort    int
 	PollWait      time.Duration
 
@@ -89,7 +87,6 @@ func loadConfig(envFile string) config {
 		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
-		EngineHost:           get("RDS_ENGINE_HOST"),
 		EngineBinDir:         get("RDS_ENGINE_BIN"),
 		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
@@ -108,9 +105,6 @@ func loadConfig(envFile string) config {
 	if cfg.HandoffDir == "" {
 		cfg.HandoffDir = defaultHandoffDir
 	}
-	if cfg.EngineHost == "" {
-		cfg.EngineHost = defaultEngineHost
-	}
 	if cfg.EngineFile == "" {
 		cfg.EngineFile = defaultEngineFile
 	}

cmd/rds-agent/config_test.go

@@ -26,8 +26,8 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
-		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
+	if cfg.PollWait != defaultPollWait {
+		t.Errorf("pollWait = %v, want the built-in default", cfg.PollWait)
 	}
 	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
 		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)

cmd/rds-agent/engine.go

@@ -108,9 +108,7 @@ var engineLayouts = map[string]engineLayout{
 		dataMount: "/var/lib/postgresql",
 		port:      5432,
 		newProbe:  newPostgresProbe,
-		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
-			return newPostgresEngine(cfg, run, startSess, probe), nil
-		},
+		newEngine: newPostgresEngineFromCatalog,
 	},
 	engineMariaDB: {
 		binDir:    "/usr/bin",

cmd/rds-agent/engine_mariadb_test.go

@@ -129,10 +129,11 @@ func newScriptedTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngin
 // that is registered: what a spec says about a parameter is what the guest must
 // see, and an unlisted name must read as static rather than as dynamic.
 func TestControlPlaneRulesFrom_TakesTheClassificationFromTheCatalog(t *testing.T) {
-	rules := controlPlaneRulesFrom(postgresEngineMeta)
+	meta := testPostgresEngineMeta(t)
+	rules := controlPlaneRulesFrom(meta)
 
 	for name, want := range map[string]bool{"shared_buffers": true, "work_mem": false} {
-		spec, ok := postgresEngineMeta.LookupParameter(name)
+		spec, ok := meta.LookupParameter(name)
 		if !ok {
 			t.Fatalf("the catalog no longer carries %s", name)
 		}

cmd/rds-agent/engine_postgres.go

@@ -41,8 +41,13 @@ var _ engine = (*postgresEngine)(nil)
 // is not there. Resolved on PATH, where the client package puts it.
 const postgresProbeBinary = "pg_isready"
 
+// The socket directory rather than loopback: the engine binds the customer ENI
+// and nothing else, so there is no TCP path for the probe to take. It also keeps
+// liveness independent of the network path, of the pg_hba scope and of the TLS
+// enforcement rule — libpq reads a host beginning with / as a socket directory,
+// and the generated `local ... peer` line covers it.
 func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
-	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, run))
 }
 
 func postgresProbeState(host string, run probeRunner) probeStateFn {
@@ -65,19 +70,6 @@ func postgresProbeState(host string, run probeRunner) probeStateFn {
 	}
 }
 
-// Resolved once, under the name the control plane knows this implementation by.
-// A lookup failure is a mismatch between the agent and the control plane it was
-// built against, so it fails at startup rather than at the first rotation.
-var postgresEngineMeta = mustLookupEngine(enginePostgres)
-
-func mustLookupEngine(name string) handlers_rds.Engine {
-	engine, err := handlers_rds.LookupEngine(name)
-	if err != nil {
-		panic("rds-agent: " + err.Error())
-	}
-	return engine
-}
-
 // The include the resolved set is rendered to, and the copy of the last one the
 // engine accepted. Both live beside the data rather than in /etc, matching
 // rds-init: a class change boots a fresh root volume and would revert them.
@@ -109,9 +101,18 @@ hostnossl all all ::/0 reject
 	postgresForceSSLRuleCount = 2
 )
 
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+// The layout's factory resolves the control plane metadata during startup.
+func newPostgresEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", enginePostgres, err)
+	}
+	return newPostgresEngine(cfg, meta, run, startSess, probe), nil
+}
+
+func newPostgresEngine(cfg config, meta handlers_rds.Engine, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
 	return &postgresEngine{
-		meta:      postgresEngineMeta,
+		meta:      meta,
 		run:       run,
 		startSess: startSess,
 		psql:      filepath.Join(cfg.EngineBinDir, "psql"),

cmd/rds-agent/engine_test.go

@@ -140,10 +140,19 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 	return r.out, r.err
 }
 
+func testPostgresEngineMeta(t *testing.T) handlers_rds.Engine {
+	t.Helper()
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		t.Fatalf("LookupEngine(%s): %v", enginePostgres, err)
+	}
+	return meta
+}
+
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
 	cfg := testEngineConfig(t)
-	return newPostgresEngine(cfg, withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
 }
 
 // The same engine with the read-backs left to the runner, for the cases that are
@@ -152,7 +161,7 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 func newScriptedTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
 	cfg := testEngineConfig(t)
-	return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), run, nil, newPostgresProbe(cfg, staticProbe(0)))
 }
 
 func testEngineConfig(t *testing.T) config {
@@ -479,10 +488,11 @@ func (r *tlsReadBackRunner) run(ctx context.Context, c command) (string, error)
 	return r.recordingRunner.run(ctx, c)
 }
 
-func forceSSLParameter(value string) []handlers_rds.Parameter {
+func forceSSLParameter(t *testing.T, value string) []handlers_rds.Parameter {
+	t.Helper()
 	return []handlers_rds.Parameter{
 		{Name: "work_mem", Value: "4096"},
-		{Name: postgresEngineMeta.TLSEnforcementParameter(), Value: value},
+		{Name: testPostgresEngineMeta(t).TLSEnforcementParameter(), Value: value},
 	}
 }
 
@@ -492,7 +502,7 @@ func TestPostgresEngine_ApplyParametersWritesTheEnforcementRule(t *testing.T) {
 	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
 		t.Fatalf("ApplyParameters: %v", err)
 	}
 	rule, err := os.ReadFile(engine.forceSSLRulePath())
@@ -516,7 +526,7 @@ func TestPostgresEngine_ApplyParametersRemovesAStaleEnforcementRule(t *testing.T
 		t.Fatalf("write the rule the restored volume carried: %v", err)
 	}
 
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
 		t.Fatalf("ApplyParameters: %v", err)
 	}
 	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
@@ -547,7 +557,7 @@ func TestPostgresEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *te
 	runner := &tlsReadBackRunner{ssl: "on"}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("yes"))
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "yes"))
 	if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
 		t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
 	}
@@ -562,7 +572,7 @@ func TestPostgresEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T)
 	runner := &tlsReadBackRunner{ssl: "off"}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1"))
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1"))
 	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
 		t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
 	}
@@ -592,7 +602,7 @@ func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing
 			runner := &tlsReadBackRunner{ssl: "on", ruleRows: tc.ruleRows, brokenRows: tc.brokenRows}
 			engine := newScriptedTestEngine(t, runner.run)
 
-			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(tc.value))
+			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, tc.value))
 			if err == nil || !strings.Contains(err.Error(), tc.want) {
 				t.Fatalf("ApplyParameters error = %v, want %q", err, tc.want)
 			}
@@ -606,13 +616,13 @@ func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing
 func TestPostgresEngine_ApplyParametersWithdrawsBothFilesTogether(t *testing.T) {
 	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
 	engine := newScriptedTestEngine(t, runner.run)
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
 		t.Fatalf("first ApplyParameters: %v", err)
 	}
 
 	// Turning enforcement off, verified against a pg_hba that still carries the
 	// rule — so the apply fails after both files have already been replaced.
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err == nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err == nil {
 		t.Fatal("ApplyParameters succeeded against an unverifiable enforcement")
 	}
 	installed, err := os.ReadFile(engine.params.installedPath())
@@ -639,14 +649,14 @@ func TestPostgresEngine_RestoreLastKnownGoodReDerivesEnforcement(t *testing.T) {
 	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
 		t.Fatalf("ApplyParameters(enforcing): %v", err)
 	}
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("RecordServingParameters: %v", err)
 	}
 	runner.ruleRows = 0
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
 		t.Fatalf("ApplyParameters(not enforcing): %v", err)
 	}
 	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {

cmd/rds-agent/health.go

@@ -28,8 +28,7 @@ func execProbeRunner(ctx context.Context, name string, args ...string) (int, str
 	if ctxErr := ctx.Err(); ctxErr != nil {
 		return -1, out, ctxErr
 	}
-	var exitErr *exec.ExitError
-	if errors.As(err, &exitErr) {
+	if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
 		return exitErr.ExitCode(), out, nil
 	}
 	return -1, out, err

cmd/rds-agent/health_test.go

@@ -110,6 +110,37 @@ func TestExecProbeRunner_CapturesTheClientsStderr(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
+// The engine binds the customer ENI and nothing else, so a loopback probe would
+// report a serving engine as absent. libpq reads a host beginning with / as a
+// socket directory, which is also what keeps liveness off the network path.
+func TestPostgresProbe_ProbesTheUnixSocket(t *testing.T) {
+	cfg := testProbeConfig()
+	var probed []string
+	probe := newPostgresProbe(cfg, func(_ context.Context, name string, args ...string) (int, string, error) {
+		probed = append([]string{name}, args...)
+		return 0, "", nil
+	})
+
+	if state, _ := probe.state(context.Background()); state != engineServing {
+		t.Fatalf("probe state = %v, want engineServing", state)
+	}
+	host := ""
+	for i, arg := range probed {
+		if arg == "-h" && i+1 < len(probed) {
+			host = probed[i+1]
+		}
+	}
+	if host != cfg.SocketDir {
+		t.Errorf("probe host = %q, want the socket directory %q", host, cfg.SocketDir)
+	}
+	if !strings.HasPrefix(host, "/") {
+		t.Errorf("probe host = %q, which libpq reads as a TCP host rather than a socket directory", host)
+	}
+}
+
+func testProbeConfig() config {
+	return config{
+		SocketDir:  engineLayouts[enginePostgres].socketDir,
+		EnginePort: engineLayouts[enginePostgres].port,
+	}
 }

cmd/rds-agent/paramrecovery.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 
@@ -66,7 +67,7 @@ func (g *paramGuard) Run(ctx context.Context) {
 	}
 
 	message := handlers_rds.ParameterRollbackMessage
-	slog.Error("rds-agent: "+message, "after", g.after)
+	slog.Error("rds-agent: "+message, "after_ms", otelsetup.Millis(g.after))
 	if g.cp != nil {
 		if _, err := g.cp.SubmitState(ctx, g.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
 			slog.Error("rds-agent: reporting the parameter rollback failed", "err", err)

cmd/rds-agent/quiesce.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"io"
 	"log/slog"
 	"os/exec"
@@ -72,7 +73,7 @@ func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold
 		session: session,
 		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
 	}
-	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
+	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold_ms", otelsetup.Millis(hold))
 }
 
 // Takes the hold off the engine, if one is still there, and disarms its

cmd/rds-agent/register.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 )
@@ -20,7 +21,7 @@ func (a *Agent) register(ctx context.Context) error {
 		a.hb.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
 
 		slog.Info("rds-agent: registered",
-			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat", a.hb.interval)
+			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat_ms", otelsetup.Millis(a.hb.interval))
 		return nil
 	})
 }

cmd/spinifex/cmd/admin.go

@@ -25,8 +25,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"uuid"
 
-	"github.com/google/uuid"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
@@ -415,15 +415,11 @@ func init() {
 	imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
 	imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
 	imagesRemoveCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesRemoveCmd.MarkFlagRequired("image-id")
 
 	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
 	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesPromoteCmd.MarkFlagRequired("image-id")
 }
 
 const bytesPerGiB = 1024 * 1024 * 1024
@@ -2649,7 +2645,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)
 	}
-	reservationOwner := "spx-cli-" + uuid.NewString()
+	reservationOwner := "spx-cli-" + uuid.NewV4().String()
 	switch err := names.Reserve(ctx, name, reservationOwner); {
 	case errors.Is(err, handlers_iam.ErrNameTaken):
 		fmt.Fprintf(os.Stderr, "Error: account %q already exists\n", name)

cmd/spinifex/cmd/admin_account_quota.go

@@ -0,0 +1,185 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+)
+
+// quotaDimensions maps each CLI flag to the override field it sets. One table
+// drives the flag registration and the parse, so a new dimension cannot be
+// added to one and forgotten in the other.
+var quotaDimensions = []struct {
+	flag  string
+	usage string
+	field func(*handlers_quota.Overrides) **int
+}{
+	{"vcpus", "Total vCPUs across running and stopped instances", func(o *handlers_quota.Overrides) **int { return &o.VCPUs }},
+	{"vpcs", "VPCs", func(o *handlers_quota.Overrides) **int { return &o.VPCs }},
+	{"subnets", "Subnets, counted per account rather than per VPC", func(o *handlers_quota.Overrides) **int { return &o.Subnets }},
+	{"eips", "Elastic IP addresses", func(o *handlers_quota.Overrides) **int { return &o.EIPs }},
+	{"volumes", "EBS volumes", func(o *handlers_quota.Overrides) **int { return &o.Volumes }},
+	{"volumes-gib", "Total EBS capacity in GiB", func(o *handlers_quota.Overrides) **int { return &o.VolumesGiB }},
+	{"rds-instances", "RDS database instances", func(o *handlers_quota.Overrides) **int { return &o.RDSInstances }},
+	{"load-balancers", "Load balancers, ALB and NLB together", func(o *handlers_quota.Overrides) **int { return &o.LoadBalancers }},
+}
+
+var accountQuotaCmd = &cobra.Command{
+	Use:   "quota",
+	Short: "Inspect and adjust an account's service quotas",
+	Long: `Inspect and adjust the service quotas applied to one account.
+
+Every account inherits the [quota] limits from awsgw.toml. An override raises or
+lowers individual dimensions for one account without affecting any other, so a
+customer can be given more vCPUs while every other tenant stays on the baseline.
+
+Only the dimensions passed as flags are changed; the rest keep inheriting. Use
+-1 for no limit on a dimension, and --clear to drop every override.`,
+}
+
+var accountQuotaGetCmd = &cobra.Command{
+	Use:   "get <account-id>",
+	Short: "Show the limits in force for an account",
+	Long: `Show the limits in force for an account and where each came from:
+"config" for an inherited baseline, "override" for one set on this account.`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaGet,
+}
+
+var accountQuotaSetCmd = &cobra.Command{
+	Use:   "set <account-id>",
+	Short: "Override one or more limits for an account",
+	Long: `Override one or more limits for an account. Dimensions not named keep
+inheriting the configured baseline.
+
+  spx admin account quota set 000000000005 --vcpus 32
+  spx admin account quota set 000000000005 --clear`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaSet,
+}
+
+// Both subcommands are served only by the /admin surface, so unlike the account
+// commands there is no NATS path and no --remote to opt into.
+func init() {
+	accountCmd.AddCommand(accountQuotaCmd)
+	accountQuotaCmd.AddCommand(accountQuotaGetCmd)
+	accountQuotaCmd.AddCommand(accountQuotaSetCmd)
+
+	for _, c := range []*cobra.Command{accountQuotaGetCmd, accountQuotaSetCmd} {
+		c.Flags().String("endpoint", "", "Gateway endpoint (default: this node's AWS gateway)")
+		c.Flags().String("region", "", "SigV4 region (default: this node's region)")
+		c.Flags().String("ca-bundle", "", "CA certificate (default: this node's CA)")
+	}
+
+	for _, d := range quotaDimensions {
+		accountQuotaSetCmd.Flags().Int(d.flag, 0, d.usage+" (-1 for no limit)")
+	}
+	accountQuotaSetCmd.Flags().Bool("clear", false, "Remove every override, returning the account to the configured limits")
+}
+
+func runAccountQuotaGet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0]}
+	if err := callAdmin(ctx, target, "GetAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+func runAccountQuotaSet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	if err != nil {
+		return err
+	}
+	clearAll, err := cmd.Flags().GetBool("clear")
+	if err != nil {
+		return err
+	}
+	// Refusing both is not pedantry: --clear silently winning would discard
+	// limits the caller believed it was setting in the same breath.
+	if clearAll && changed {
+		return fmt.Errorf("--clear cannot be combined with a dimension flag")
+	}
+	if !clearAll && !changed {
+		return fmt.Errorf("no dimension given; pass at least one limit flag, or --clear to remove every override")
+	}
+	if clearAll {
+		over = handlers_quota.Overrides{}
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0], Overrides: over}
+	if err := callAdmin(ctx, target, "PutAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+// quotaOverridesFromFlags builds the override set from the flags the caller
+// actually passed. An unset flag is left nil so the dimension keeps inheriting,
+// which is why Changed is consulted rather than the flag's value.
+func quotaOverridesFromFlags(cmd *cobra.Command) (handlers_quota.Overrides, bool, error) {
+	var over handlers_quota.Overrides
+	changed := false
+	for _, d := range quotaDimensions {
+		if !cmd.Flags().Changed(d.flag) {
+			continue
+		}
+		value, err := cmd.Flags().GetInt(d.flag)
+		if err != nil {
+			return over, false, err
+		}
+		if value < handlers_quota.Unlimited {
+			return over, false, fmt.Errorf("--%s must be 0 or greater, or %d for no limit", d.flag, handlers_quota.Unlimited)
+		}
+		*d.field(&over) = &value
+		changed = true
+	}
+	return over, changed, nil
+}
+
+func printAccountQuota(resp *gateway.AccountQuotaResponse) {
+	fmt.Printf("Account %s\n\n", resp.AccountID)
+	fmt.Printf("%-16s %10s  %s\n", "DIMENSION", "LIMIT", "SOURCE")
+	fmt.Printf("%-16s %10s  %s\n", "---------", "-----", "------")
+
+	names := make([]string, 0, len(resp.Limits))
+	for name := range resp.Limits {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		limit := strconv.Itoa(resp.Limits[name])
+		if resp.Limits[name] == handlers_quota.Unlimited {
+			limit = "unlimited"
+		}
+		fmt.Printf("%-16s %10s  %s\n", name, limit, resp.Source[name])
+	}
+
+	if resp.Overrides.UpdatedAt != "" {
+		fmt.Printf("\nOverride set by %s at %s\n", resp.Overrides.UpdatedBy, resp.Overrides.UpdatedAt)
+	}
+}

cmd/spinifex/cmd/admin_account_quota_test.go

@@ -0,0 +1,140 @@
+//test:in-package — drives the unexported flag table, parser and printer the quota subcommands are built from.
+package cmd
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+)
+
+// newQuotaSetCmd builds a command carrying the same flags as the real one, so
+// the parse is tested without mutating the package-level command between tests.
+func newQuotaSetCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "set"}
+	for _, d := range quotaDimensions {
+		cmd.Flags().Int(d.flag, 0, d.usage)
+	}
+	cmd.Flags().Bool("clear", false, "")
+	return cmd
+}
+
+// Only the flags actually passed become overrides. Every other dimension must
+// stay nil so it keeps inheriting, which is the whole point of a sparse record.
+func TestQuotaOverridesFromFlagsIsSparse(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "32", "--rds-instances", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, 32, *over.VCPUs)
+	require.NotNil(t, over.RDSInstances)
+	require.Equal(t, 8, *over.RDSInstances)
+
+	require.Nil(t, over.VPCs)
+	require.Nil(t, over.EIPs)
+	require.Nil(t, over.Volumes)
+	require.Nil(t, over.VolumesGiB)
+	require.Nil(t, over.Subnets)
+	require.Nil(t, over.LoadBalancers)
+}
+
+// A flag left unset must not be read as an explicit 0, which would deny every
+// request on that dimension instead of inheriting it.
+func TestQuotaOverridesFromFlagsUnsetIsNotZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vpcs", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.Nil(t, over.VCPUs, "an unset flag must not become an explicit zero")
+}
+
+// Passing 0 deliberately is a real limit and must be preserved.
+func TestQuotaOverridesFromFlagsExplicitZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--eips", "0"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.NotNil(t, over.EIPs)
+	require.Equal(t, 0, *over.EIPs)
+}
+
+func TestQuotaOverridesFromFlagsUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-1"}))
+
+	over, _, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, handlers_quota.Unlimited, *over.VCPUs)
+}
+
+// Anything below the Unlimited sentinel is a typo, not a smaller limit.
+func TestQuotaOverridesFromFlagsRejectsBelowUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-2"}))
+
+	_, _, err := quotaOverridesFromFlags(cmd)
+	require.Error(t, err)
+}
+
+// No flags at all means nothing changed, which the caller turns into an error
+// rather than a write that silently does nothing.
+func TestQuotaOverridesFromFlagsNoneChanged(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags(nil))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.False(t, changed)
+	require.True(t, over.Empty())
+}
+
+// The table must name every dimension, label its source, and render the
+// Unlimited sentinel as a word rather than as -1, which reads like a bug.
+func TestPrintAccountQuota(t *testing.T) {
+	resp := &gateway.AccountQuotaResponse{
+		AccountID: "000000000042",
+		Limits:    map[string]int{"vcpus": 32, "vpcs": handlers_quota.Unlimited, "eips": 4},
+		Source:    map[string]string{"vcpus": "override", "vpcs": "override", "eips": "config"},
+		Overrides: handlers_quota.Overrides{UpdatedBy: "operator", UpdatedAt: "2026-08-20T01:00:00Z"},
+	}
+
+	out := captureStdout(t, func() { printAccountQuota(resp) })
+
+	require.Contains(t, out, "000000000042")
+	require.Contains(t, out, "vcpus")
+	require.Contains(t, out, "32")
+	require.Contains(t, out, "override")
+	require.Contains(t, out, "config")
+	require.Contains(t, out, "unlimited")
+	require.NotContains(t, out, "-1")
+	require.Contains(t, out, "operator")
+}
+
+// Every dimension the override record carries must be reachable from the CLI,
+// or a limit exists that an operator cannot set.
+func TestQuotaDimensionsCoverEveryOverrideField(t *testing.T) {
+	var over handlers_quota.Overrides
+	for _, d := range quotaDimensions {
+		v := 1
+		*d.field(&over) = &v
+	}
+	require.False(t, over.Empty())
+
+	seen := map[string]bool{}
+	for _, d := range quotaDimensions {
+		require.False(t, seen[d.flag], "duplicate flag %q", d.flag)
+		seen[d.flag] = true
+	}
+	require.Len(t, quotaDimensions, 8)
+}

cmd/spinifex/cmd/admin_acm.go

@@ -60,16 +60,12 @@ func init() {
 	certCreateTenantCACmd.Flags().StringSlice("domain", nil, "Domain the tenant CA is permitted to sign for (repeatable)")
 	certCreateTenantCACmd.Flags().Bool("regenerate", false, "Replace an existing tenant CA with a fresh root (invalidates every device's trust store — requires confirmation)")
 	certCreateTenantCACmd.Flags().Bool("yes", false, "Skip the --regenerate confirmation prompt")
-	if err := certCreateTenantCACmd.MarkFlagRequired("domain"); err != nil {
-		panic(err)
-	}
+	_ = certCreateTenantCACmd.MarkFlagRequired("domain")
 
 	certCmd.AddCommand(certForceRenewCmd)
 	certForceRenewCmd.Flags().String("arn", "", "Certificate ARN to force-renew (required)")
 	certForceRenewCmd.Flags().String("account", "", "Account ID that owns the certificate; defaults to the bootstrap account")
-	if err := certForceRenewCmd.MarkFlagRequired("arn"); err != nil {
-		panic(err)
-	}
+	_ = certForceRenewCmd.MarkFlagRequired("arn")
 }
 
 func runCertForceRenew(cmd *cobra.Command, _ []string) {

cmd/spinifex/cmd/admin_ochre.go

@@ -134,12 +134,11 @@ func init() {
 	_ = ochreWeightsRemoveCmd.MarkFlagRequired("model-id")
 
 	ochreWeightsPullCmd.Flags().String("model-id", "", "Self-host catalog model ID this pull is for (required)")
-	ochreWeightsPullCmd.Flags().String("hf-repo", "", "Hugging Face repo, e.g. meta-llama/Llama-3.2-1B-Instruct (required)")
-	ochreWeightsPullCmd.Flags().String("revision", "main", "Hugging Face branch, tag or commit SHA to resolve and pin")
+	ochreWeightsPullCmd.Flags().String("hf-repo", "", "Hugging Face repo, e.g. meta-llama/Llama-3.2-1B-Instruct (default: the catalog entry's repo)")
+	ochreWeightsPullCmd.Flags().String("revision", "main", "Hugging Face branch, tag or commit SHA to resolve and pin (default: the catalog entry's pinned revision)")
 	ochreWeightsPullCmd.Flags().String("s3-uri", "", "predastore destination prefix, e.g. s3://bucket/prefix/ (default: s3://ochre-weights/<repo>/<sha>/)")
 	ochreWeightsPullCmd.Flags().String("hf-token", "", "Hugging Face token for a gated repo (falls back to HF_TOKEN, then a stored platform credential)")
 	_ = ochreWeightsPullCmd.MarkFlagRequired("model-id")
-	_ = ochreWeightsPullCmd.MarkFlagRequired("hf-repo")
 
 	ochreCredentialsSetCmd.Flags().String("vendor", "", "Vendor to store a credential for, e.g. huggingface (required)")
 	ochreCredentialsSetCmd.Flags().String("account", "", "Account ID to store the credential under (default: platform account)")
@@ -180,20 +179,25 @@ type ochrePullManifest struct {
 	PulledAt    time.Time `json:"pulled_at"`
 }
 
-// allowedPullFileNames are the fixed-name Hugging Face artefacts pull fetches
-// alongside safetensors shards -- config and tokenizer files needed to serve,
-// never weights themselves. Matched by basename so a nested path (e.g.
-// Llama's original/tokenizer.model) still qualifies.
-var allowedPullFileNames = map[string]bool{
+// rootOnlyPullFileNames are the fixed-name config artefacts pull fetches only
+// at the repo root. A same-named file in a subdir is a component config, not
+// the model's -- e.g. sentence-transformers' 1_Pooling/config.json -- and must
+// never flatten onto the root config.json, so nested matches are dropped.
+var rootOnlyPullFileNames = map[string]bool{
 	"config.json":                  true,
 	"tokenizer_config.json":        true,
-	"tokenizer.json":               true,
-	"tokenizer.model":              true,
 	"generation_config.json":       true,
 	"special_tokens_map.json":      true,
 	"model.safetensors.index.json": true,
 }
 
+// nestablePullFileNames are the tokenizer artefacts matched by basename at any
+// depth: some repos keep them in a subdir (e.g. Llama's original/tokenizer.model).
+var nestablePullFileNames = map[string]bool{
+	"tokenizer.json":  true,
+	"tokenizer.model": true,
+}
+
 // selectPullFiles filters a Hugging Face tree listing down to the
 // safetensors-only set pull downloads (D4): every *.safetensors file plus
 // the fixed config/tokenizer set. Anything else -- notably .bin/.pt pickle
@@ -205,7 +209,14 @@ func selectPullFiles(entries []hfhub.TreeEntry) []hfhub.TreeEntry {
 		if e.Type != "file" {
 			continue
 		}
-		if strings.HasSuffix(e.Path, ".safetensors") || allowedPullFileNames[path.Base(e.Path)] {
+		base := path.Base(e.Path)
+		nested := strings.Contains(e.Path, "/")
+		switch {
+		case strings.HasSuffix(e.Path, ".safetensors"):
+			selected = append(selected, e)
+		case nestablePullFileNames[base]:
+			selected = append(selected, e)
+		case rootOnlyPullFileNames[base] && !nested:
 			selected = append(selected, e)
 		}
 	}
@@ -324,8 +335,13 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.
 	defer os.RemoveAll(tmpDir)
 
 	var uploaded []string
+	seen := make(map[string]string, len(files))
 	for _, f := range files {
 		key := prefix + path.Base(f.Path)
+		if prev, dup := seen[key]; dup {
+			return uploaded, fmt.Errorf("refusing to overwrite %s: both %s and %s flatten to it", key, prev, f.Path)
+		}
+		seen[key] = f.Path
 		if err := pullOneFile(ctx, hf, store, hfRepo, sha, f.Path, bucket, key, tmpDir); err != nil {
 			return uploaded, fmt.Errorf("pull %s: %w", f.Path, err)
 		}
@@ -340,14 +356,28 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.
 // into predastore (D6), and finally the ochre-pull.json manifest. It never
 // touches the weights KV store; 'stage' consumes the printed s3:// URI
 // separately. A failure removes every object already written (D5).
-func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision, s3URIFlag string) (string, error) {
-	if _, found, selfHost := gateway_bedrock.LookupServingSpec(modelID); !found || !selfHost {
+func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision string, revisionExplicit bool, s3URIFlag string) (string, error) {
+	spec, found, selfHost := gateway_bedrock.LookupServingSpec(modelID)
+	if !found || !selfHost {
 		if !found {
 			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
 		}
 		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights pull does not apply", modelID)
 	}
 
+	// The catalog carries the canonical repo and a pinned revision so a bare
+	// pull cannot silently grab a regressed upstream default; an explicit flag
+	// still overrides either.
+	if hfRepo == "" {
+		hfRepo = spec.HFRepo
+	}
+	if hfRepo == "" {
+		return "", fmt.Errorf("no --hf-repo given and %q has no catalog HF repo", modelID)
+	}
+	if !revisionExplicit && spec.HFRevision != "" {
+		revision = spec.HFRevision
+	}
+
 	fmt.Printf("Resolving %s@%s ...\n", hfRepo, revision)
 	sha, err := hf.ResolveRevision(ctx, hfRepo, revision)
 	if err != nil {
@@ -989,6 +1019,7 @@ func runOchreWeightsPull(cmd *cobra.Command, _ []string) {
 	modelID, _ := cmd.Flags().GetString("model-id")
 	hfRepo, _ := cmd.Flags().GetString("hf-repo")
 	revision, _ := cmd.Flags().GetString("revision")
+	revisionExplicit := cmd.Flags().Changed("revision")
 	s3URI, _ := cmd.Flags().GetString("s3-uri")
 
 	appConfig, nc, err := loadConfigAndConnectFn()
@@ -1006,13 +1037,13 @@ func runOchreWeightsPull(cmd *cobra.Command, _ []string) {
 	hf := hfhub.NewClient(token)
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 
-	finalURI, err := runPullWeights(ctx, hf, store, modelID, hfRepo, revision, s3URI)
+	finalURI, err := runPullWeights(ctx, hf, store, modelID, hfRepo, revision, revisionExplicit, s3URI)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)
 		return
 	}
-	fmt.Printf("✅ Pulled %s@%s into %s\n", hfRepo, revision, finalURI)
+	fmt.Printf("✅ Pulled into %s\n", finalURI)
 	fmt.Println(finalURI)
 }
 
@@ -1138,15 +1169,11 @@ func init() {
 		c.Flags().String("account-id", "", "12-digit account ID to change (required)")
 		c.Flags().String("model-id", "", "Model ID to change (e.g. meta.llama3-2-1b-instruct-v1:0)")
 		c.Flags().Bool("all-models", false, "Apply to every model in the platform catalog")
-		if err := c.MarkFlagRequired("account-id"); err != nil {
-			panic(err)
-		}
+		_ = c.MarkFlagRequired("account-id")
 	}
 
 	adminOchreAccessListCmd.Flags().String("account-id", "", "12-digit account ID to inspect (required)")
-	if err := adminOchreAccessListCmd.MarkFlagRequired("account-id"); err != nil {
-		panic(err)
-	}
+	_ = adminOchreAccessListCmd.MarkFlagRequired("account-id")
 }
 
 // ochreAccessStore connects to the cluster and returns the grant store along

cmd/spinifex/cmd/admin_ochre_appliance.go

@@ -29,10 +29,13 @@ var ochreApplianceTeardownCmd = &cobra.Command{
 instance, this node's VPC host port, and its KV record.
 
 This is a destructive, cluster-wide operator action -- every account's vector
-index data stored in the appliance is lost with it, and it is not
-automatically rebuilt. A future daemon startup re-provisions a fresh
-appliance (a new instance, a new password) only if Ochre remains enabled;
-existing index registry records are not restored.
+index data stored in the appliance is lost with it. By default the index
+registry and KB/DataSource metadata survive, exactly as they survive a
+restart: a future daemon startup re-provisions a fresh appliance (a new
+instance, a new password) and reconciles each surviving index by re-creating
+it and re-ingesting from its recorded source, with no operator action and no
+change to its id. --purge-metadata additionally wipes that metadata for an
+intentional full destroy, after which nothing is auto-restored.
 
 Requires --confirm.`,
 	Run: runOchreApplianceTeardown,
@@ -44,40 +47,47 @@ func init() {
 
 	ochreApplianceTeardownCmd.Flags().Bool("confirm", false,
 		"Required: confirms you intend to destroy the shared platform appliance and every account's vector index data with it")
+	ochreApplianceTeardownCmd.Flags().Bool("purge-metadata", false,
+		"Also wipe the index registry and KB/DataSource records, disabling auto-restore on the next re-provision")
 }
 
 // applianceTeardownFn indirects the NATS call so runApplianceTeardownGuarded
 // is testable without a live daemon, mirroring vectorServiceFn/
 // endpointServiceFn.
-var applianceTeardownFn = func(ctx context.Context) error {
+var applianceTeardownFn = func(ctx context.Context, purgeMetadata bool) error {
 	_, nc, err := loadConfigAndConnectFn()
 	if err != nil {
 		return err
 	}
 	defer nc.Close()
 	_, err = utils.NATSRequest[handlers_ochrevector.TeardownApplianceResponse](ctx, nc,
-		handlers_ochrevector.SubjectTeardownAppliance, &handlers_ochrevector.TeardownApplianceRequest{}, applianceTeardownTimeout, utils.GlobalAccountID)
+		handlers_ochrevector.SubjectTeardownAppliance, &handlers_ochrevector.TeardownApplianceRequest{PurgeMetadata: purgeMetadata},
+		applianceTeardownTimeout, utils.GlobalAccountID)
 	return err
 }
 
 // runApplianceTeardownGuarded is the testable core of 'ochre appliance
 // teardown': it refuses without --confirm before making any NATS call, so a
 // missing flag never even opens a connection to the cluster.
-func runApplianceTeardownGuarded(ctx context.Context, confirmed bool, teardown func(context.Context) error) (string, error) {
+func runApplianceTeardownGuarded(ctx context.Context, confirmed, purgeMetadata bool, teardown func(context.Context, bool) error) (string, error) {
 	if !confirmed {
 		return "", errors.New("refusing to tear down the shared platform appliance without --confirm: " +
 			"this destroys every account's vector index data and is not automatically rebuilt")
 	}
-	if err := teardown(ctx); err != nil {
+	if err := teardown(ctx, purgeMetadata); err != nil {
 		return "", err
 	}
+	if purgeMetadata {
+		return "✅ Platform appliance torn down, index/KB metadata purged.", nil
+	}
 	return "✅ Platform appliance torn down.", nil
 }
 
 func runOchreApplianceTeardown(cmd *cobra.Command, _ []string) {
 	confirmed, _ := cmd.Flags().GetBool("confirm")
+	purgeMetadata, _ := cmd.Flags().GetBool("purge-metadata")
 
-	msg, err := runApplianceTeardownGuarded(context.Background(), confirmed, applianceTeardownFn)
+	msg, err := runApplianceTeardownGuarded(context.Background(), confirmed, purgeMetadata, applianceTeardownFn)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)

cmd/spinifex/cmd/admin_ochre_appliance_test.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/testutil"
 	"github.com/stretchr/testify/require"
 )
 
@@ -17,12 +18,12 @@ import (
 // reach a NATS call, let alone destroy the appliance.
 func TestRunApplianceTeardownGuarded_RefusesWithoutConfirm(t *testing.T) {
 	called := false
-	teardown := func(context.Context) error {
+	teardown := func(context.Context, bool) error {
 		called = true
 		return nil
 	}
 
-	_, err := runApplianceTeardownGuarded(context.Background(), false, teardown)
+	_, err := runApplianceTeardownGuarded(context.Background(), false, false, teardown)
 	require.Error(t, err)
 	require.ErrorContains(t, err, "--confirm")
 	require.False(t, called, "teardown must not run without --confirm")
@@ -32,12 +33,12 @@ func TestRunApplianceTeardownGuarded_RefusesWithoutConfirm(t *testing.T) {
 // drives the teardown call through and reports success.
 func TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown(t *testing.T) {
 	called := false
-	teardown := func(context.Context) error {
+	teardown := func(context.Context, bool) error {
 		called = true
 		return nil
 	}
 
-	msg, err := runApplianceTeardownGuarded(context.Background(), true, teardown)
+	msg, err := runApplianceTeardownGuarded(context.Background(), true, false, teardown)
 	require.NoError(t, err)
 	require.True(t, called)
 	require.Contains(t, msg, "torn down")
@@ -47,19 +48,49 @@ func TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown(t *testing.T) {
 // failure (e.g. the daemon reporting no appliance up) surfaces to the caller
 // rather than being swallowed behind a generic success message.
 func TestRunApplianceTeardownGuarded_TeardownErrorSurfaces(t *testing.T) {
-	teardown := func(context.Context) error {
+	teardown := func(context.Context, bool) error {
 		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
 	}
 
-	_, err := runApplianceTeardownGuarded(context.Background(), true, teardown)
+	_, err := runApplianceTeardownGuarded(context.Background(), true, false, teardown)
 	require.Error(t, err)
 	require.ErrorContains(t, err, "not enabled or not up")
 }
 
+// TestRunApplianceTeardownGuarded_PurgeMetadataThreadsThrough proves the
+// --purge-metadata flag reaches the teardown call and is reflected in the
+// success message, so an operator can tell which mode ran.
+func TestRunApplianceTeardownGuarded_PurgeMetadataThreadsThrough(t *testing.T) {
+	var gotPurge bool
+	teardown := func(_ context.Context, purgeMetadata bool) error {
+		gotPurge = purgeMetadata
+		return nil
+	}
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), true, true, teardown)
+	require.NoError(t, err)
+	require.True(t, gotPurge, "purgeMetadata must reach the teardown call")
+	require.Contains(t, msg, "metadata purged")
+}
+
+// TestApplianceTeardownFn_ThreadsPurgeMetadataOverNATS drives the real
+// (unstubbed) applianceTeardownFn through runApplianceTeardownGuarded, so the
+// actual NATSRequest call -- including the PurgeMetadata field on the wire
+// request -- executes rather than a test double standing in for it. No
+// daemon is subscribed to the teardown subject in this test, so the call
+// fails fast; the point is that it's reached and built with PurgeMetadata set.
+func TestApplianceTeardownFn_ThreadsPurgeMetadataOverNATS(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	stubConnect(t, fakeClusterConfig(), nc, nil)
+
+	_, err := runApplianceTeardownGuarded(context.Background(), true, true, applianceTeardownFn)
+	require.Error(t, err, "no daemon is subscribed to the teardown subject in this test")
+}
+
 // withApplianceTeardown swaps applianceTeardownFn for one that never dials
 // NATS, so the Run wrapper's real flag/exit control flow is exercised
 // without a live daemon.
-func withApplianceTeardown(t *testing.T, fn func(context.Context) error) {
+func withApplianceTeardown(t *testing.T, fn func(context.Context, bool) error) {
 	t.Helper()
 	orig := applianceTeardownFn
 	t.Cleanup(func() { applianceTeardownFn = orig })
@@ -67,7 +98,7 @@ func withApplianceTeardown(t *testing.T, fn func(context.Context) error) {
 }
 
 func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
-	withApplianceTeardown(t, func(context.Context) error {
+	withApplianceTeardown(t, func(context.Context, bool) error {
 		t.Fatal("teardown must not be called without --confirm")
 		return nil
 	})
@@ -78,7 +109,7 @@ func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
 }
 
 func TestRunOchreApplianceTeardown_ConfirmedPrintsSuccess(t *testing.T) {
-	withApplianceTeardown(t, func(context.Context) error { return nil })
+	withApplianceTeardown(t, func(context.Context, bool) error { return nil })
 
 	cmd := *ochreApplianceTeardownCmd
 	require.NoError(t, cmd.Flags().Set("confirm", "true"))
@@ -92,7 +123,7 @@ func TestRunOchreApplianceTeardown_ConfirmedPrintsSuccess(t *testing.T) {
 }
 
 func TestRunOchreApplianceTeardown_TeardownErrorExits1(t *testing.T) {
-	withApplianceTeardown(t, func(context.Context) error {
+	withApplianceTeardown(t, func(context.Context, bool) error {
 		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
 	})
 
@@ -102,3 +133,21 @@ func TestRunOchreApplianceTeardown_TeardownErrorExits1(t *testing.T) {
 	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
 	require.Equal(t, 1, code)
 }
+
+// TestRunOchreApplianceTeardown_PurgeMetadataFlagReachesTeardownFn proves
+// the --purge-metadata flag is read and passed through the Run wrapper.
+func TestRunOchreApplianceTeardown_PurgeMetadataFlagReachesTeardownFn(t *testing.T) {
+	var gotPurge bool
+	withApplianceTeardown(t, func(_ context.Context, purgeMetadata bool) error {
+		gotPurge = purgeMetadata
+		return nil
+	})
+
+	cmd := *ochreApplianceTeardownCmd
+	require.NoError(t, cmd.Flags().Set("confirm", "true"))
+	require.NoError(t, cmd.Flags().Set("purge-metadata", "true"))
+
+	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	require.Equal(t, -1, code)
+	require.True(t, gotPurge, "--purge-metadata must reach applianceTeardownFn")
+}

cmd/spinifex/cmd/admin_ochre_endpoint.go

@@ -79,6 +79,7 @@ func init() {
 	_ = ochreEndpointDescribeCmd.MarkFlagRequired("model-id")
 
 	ochreEndpointDeleteCmd.Flags().String("model-id", "", "Model ID whose endpoint to tear down (required)")
+	ochreEndpointDeleteCmd.Flags().String("account", "", "Account ID scoping the target (defaults to the shared platform account; set to tear down a pinned provisioned-throughput endpoint)")
 	_ = ochreEndpointDeleteCmd.MarkFlagRequired("model-id")
 }
 
@@ -331,8 +332,35 @@ func runOchreEndpointList(_ *cobra.Command, _ []string) {
 	fmt.Println(msg)
 }
 
+// deleteEndpointOutput is the testable core of 'ochre endpoint delete': it
+// tears the endpoint down and returns an honest message — a no-op that found
+// no record must not claim a teardown, and it points the operator at --account
+// so a pinned, account-scoped record they can see in 'list' is reachable.
+func deleteEndpointOutput(ctx context.Context, svc handlers_bedrock.EndpointService, modelID, accountID string) (string, error) {
+	out, err := svc.Delete(ctx, &handlers_bedrock.DeleteEndpointInput{ModelID: modelID, AccountID: accountID}, utils.GlobalAccountID)
+	if err != nil {
+		return "", err
+	}
+	if !out.Removed {
+		return fmt.Sprintf("No serving endpoint for %s under account %s; nothing to tear down. "+
+			"If 'endpoint list' shows one under a different account, pass --account.\n",
+			modelID, resolveAccountLabel(accountID)), nil
+	}
+	return fmt.Sprintf("Endpoint for %s torn down.\n", modelID), nil
+}
+
+// resolveAccountLabel renders the account an empty flag resolves to, so the
+// no-op message names the shared platform account rather than an empty string.
+func resolveAccountLabel(accountID string) string {
+	if accountID == "" {
+		return utils.GlobalAccountID + " (shared platform)"
+	}
+	return accountID
+}
+
 func runOchreEndpointDelete(cmd *cobra.Command, _ []string) {
 	modelID, _ := cmd.Flags().GetString("model-id")
+	accountID, _ := cmd.Flags().GetString("account")
 
 	svc, closeFn, err := endpointServiceFn()
 	if err != nil {
@@ -342,10 +370,11 @@ func runOchreEndpointDelete(cmd *cobra.Command, _ []string) {
 	}
 	defer closeFn()
 
-	if _, err := svc.Delete(context.Background(), &handlers_bedrock.DeleteEndpointInput{ModelID: modelID}, utils.GlobalAccountID); err != nil {
+	msg, err := deleteEndpointOutput(context.Background(), svc, modelID, accountID)
+	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)
 		return
 	}
-	fmt.Printf("Endpoint for %s torn down.\n", modelID)
+	fmt.Print(msg)
 }

cmd/spinifex/cmd/admin_ochre_endpoint_test.go

@@ -14,17 +14,19 @@ import (
 // fakeEndpointService serves scripted Describe responses so the wait loop can
 // be exercised without a daemon, a VM or a real cold start.
 type fakeEndpointService struct {
-	ensure      handlers_bedrock.EndpointRecord
-	ensureErr   error
-	describes   []handlers_bedrock.EndpointRecord
-	describeErr error
-	list        []handlers_bedrock.EndpointRecord
-	listErr     error
-	deleteErr   error
+	ensure        handlers_bedrock.EndpointRecord
+	ensureErr     error
+	describes     []handlers_bedrock.EndpointRecord
+	describeErr   error
+	list          []handlers_bedrock.EndpointRecord
+	listErr       error
+	deleteErr     error
+	deleteRemoved bool
 
 	describeCalls  int
 	deleteCalls    int
 	describeInputs []*handlers_bedrock.DescribeEndpointInput
+	deleteInputs   []*handlers_bedrock.DeleteEndpointInput
 }
 
 func (f *fakeEndpointService) Ensure(_ context.Context, _ *handlers_bedrock.EnsureEndpointInput, _ string) (*handlers_bedrock.EnsureEndpointOutput, error) {
@@ -53,12 +55,13 @@ func (f *fakeEndpointService) List(_ context.Context, _ *handlers_bedrock.ListEn
 	return &handlers_bedrock.ListEndpointsOutput{Endpoints: f.list}, nil
 }
 
-func (f *fakeEndpointService) Delete(_ context.Context, _ *handlers_bedrock.DeleteEndpointInput, _ string) (*handlers_bedrock.DeleteEndpointOutput, error) {
+func (f *fakeEndpointService) Delete(_ context.Context, in *handlers_bedrock.DeleteEndpointInput, _ string) (*handlers_bedrock.DeleteEndpointOutput, error) {
 	f.deleteCalls++
+	f.deleteInputs = append(f.deleteInputs, in)
 	if f.deleteErr != nil {
 		return nil, f.deleteErr
 	}
-	return &handlers_bedrock.DeleteEndpointOutput{}, nil
+	return &handlers_bedrock.DeleteEndpointOutput{Removed: f.deleteRemoved}, nil
 }
 
 var _ handlers_bedrock.EndpointService = (*fakeEndpointService)(nil)
@@ -409,7 +412,7 @@ func TestRunOchreEndpointList_ConnectFailureExits1(t *testing.T) {
 }
 
 func TestRunOchreEndpointDelete_ReportsTeardown(t *testing.T) {
-	svc := &fakeEndpointService{}
+	svc := &fakeEndpointService{deleteRemoved: true}
 	withEndpointService(t, svc, nil)
 
 	cmd := *ochreEndpointDeleteCmd
@@ -424,6 +427,45 @@ func TestRunOchreEndpointDelete_ReportsTeardown(t *testing.T) {
 	require.Contains(t, out, "torn down")
 }
 
+// TestRunOchreEndpointDelete_NoRecordDoesNotClaimTeardown is 6z9vg's honesty
+// guard: a delete that removed nothing must not print "torn down", and must
+// point the operator at --account so a pinned, account-scoped record they can
+// see in 'list' is reachable.
+func TestRunOchreEndpointDelete_NoRecordDoesNotClaimTeardown(t *testing.T) {
+	svc := &fakeEndpointService{deleteRemoved: false}
+	withEndpointService(t, svc, nil)
+
+	cmd := *ochreEndpointDeleteCmd
+	require.NoError(t, cmd.Flags().Set("model-id", testModelID))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreEndpointDelete(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.NotContains(t, out, "torn down")
+	require.Contains(t, out, "--account")
+}
+
+// TestRunOchreEndpointDelete_AccountFlagReachesInput is 6z9vg's core seam: the
+// --account flag must reach DeleteEndpointInput.AccountID, which is how an
+// operator tears down a pinned endpoint the bare GlobalAccountID key misses.
+func TestRunOchreEndpointDelete_AccountFlagReachesInput(t *testing.T) {
+	svc := &fakeEndpointService{deleteRemoved: true}
+	withEndpointService(t, svc, nil)
+
+	cmd := *ochreEndpointDeleteCmd
+	require.NoError(t, cmd.Flags().Set("model-id", testModelID))
+	require.NoError(t, cmd.Flags().Set("account", testAccountID))
+
+	code := withOchreExitCapture(t, func() {
+		_ = captureStdout(t, func() { runOchreEndpointDelete(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Len(t, svc.deleteInputs, 1)
+	require.Equal(t, testAccountID, svc.deleteInputs[0].AccountID)
+}
+
 func TestRunOchreEndpointDelete_ErrorExits1(t *testing.T) {
 	withEndpointService(t, &fakeEndpointService{deleteErr: errors.New("ModelNotReadyException")}, nil)
 

cmd/spinifex/cmd/admin_ochre_pull_test.go

@@ -32,10 +32,11 @@ import (
 // --- selectPullFiles / anySafetensors (D4) ---
 
 // TestSelectPullFiles_SafetensorsAndConfigOnly covers D4's filter: every
-// *.safetensors file, the fixed config/tokenizer set, and the shard index are
-// kept (matched by basename, so a nested path like Llama's
-// original/tokenizer.model still qualifies); a pickle .bin checkpoint and
-// directory entries are dropped.
+// *.safetensors file, the fixed config set (root only), tokenizer files (at
+// any depth, so Llama's original/tokenizer.model qualifies), and the shard
+// index are kept; a pickle .bin, a directory entry, and a nested config.json
+// (a sentence-transformers component config that must never flatten onto the
+// model's own config.json) are dropped.
 func TestSelectPullFiles_SafetensorsAndConfigOnly(t *testing.T) {
 	entries := []hfhub.TreeEntry{
 		{Type: "file", Path: "config.json"},
@@ -47,6 +48,7 @@ func TestSelectPullFiles_SafetensorsAndConfigOnly(t *testing.T) {
 		{Type: "file", Path: "model.safetensors.index.json"},
 		{Type: "directory", Path: "original"},
 		{Type: "file", Path: "original/tokenizer.model"},
+		{Type: "file", Path: "1_Pooling/config.json"},
 	}
 
 	selected := selectPullFiles(entries)
@@ -64,6 +66,7 @@ func TestSelectPullFiles_SafetensorsAndConfigOnly(t *testing.T) {
 		"model.safetensors.index.json",
 		"original/tokenizer.model",
 	}, paths)
+	assert.NotContains(t, paths, "1_Pooling/config.json", "a nested config.json must be dropped, never flattened onto the model config")
 }
 
 // TestAnySafetensors_TrueWhenPresent covers the pass case: at least one
@@ -227,7 +230,7 @@ const (
 // TestRunPullWeights_UnknownModelRefused covers the catalog gate: an
 // unrecognised model ID must refuse before any Hugging Face or S3 work.
 func TestRunPullWeights_UnknownModelRefused(t *testing.T) {
-	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, "not-a-real-model", testHFRepo, "main", "")
+	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, "not-a-real-model", testHFRepo, "main", true, "")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "not present in the Ochre catalog")
 }
@@ -235,7 +238,7 @@ func TestRunPullWeights_UnknownModelRefused(t *testing.T) {
 // TestRunPullWeights_ProviderModelRefused covers the self-host gate: a
 // provider-served model must refuse before any network work, same as stage.
 func TestRunPullWeights_ProviderModelRefused(t *testing.T) {
-	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, providerModelID, testHFRepo, "main", "")
+	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, providerModelID, testHFRepo, "main", true, "")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "not self-host")
 }
@@ -256,7 +259,7 @@ func TestRunPullWeights_ResolvesUploadsAndWritesManifest(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := objectstore.NewMemoryObjectStore()
 
-	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 	require.NoError(t, err)
 	assert.Equal(t, defaultPullPrefix(testHFRepo, testHFSHA), finalURI)
 
@@ -287,7 +290,7 @@ func TestRunPullWeights_ExplicitS3URIOverridesDefault(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := objectstore.NewMemoryObjectStore()
 
-	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "s3://custom-bucket/my-prefix")
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "s3://custom-bucket/my-prefix")
 	require.NoError(t, err)
 	assert.Equal(t, "s3://custom-bucket/my-prefix/", finalURI)
 }
@@ -300,7 +303,7 @@ func TestRunPullWeights_NoSafetensorsAborts(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := objectstore.NewMemoryObjectStore()
 
-	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "no *.safetensors")
 	assert.Equal(t, 0, store.Count(), "a refused pull must land nothing")
@@ -315,7 +318,7 @@ func TestRunPullWeights_GatedRepoAbortsCleanBeforeAnyUpload(t *testing.T) {
 		hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 		store := objectstore.NewMemoryObjectStore()
 
-		_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+		_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 		require.Error(t, err)
 		assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorAccessDeniedException), "status %d: got %v", status, err)
 		assert.Equal(t, 0, store.Count(), "a gated pull must land nothing")
@@ -356,11 +359,57 @@ func TestRunPullWeights_MidStreamFailureCleansUpPartialPrefix(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := &failAfterNPutsStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), n: 2}
 
-	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 	require.Error(t, err)
 	assert.Equal(t, 0, store.Count(), "objects uploaded before the failure must be cleaned up")
 }
 
+// TestRunPullWeights_DefaultsRepoFromCatalog covers the catalog repo default:
+// with no --hf-repo, pull uses the self-host entry's canonical HFRepo rather
+// than refusing, so an operator need not restate a repo the catalog knows.
+func TestRunPullWeights_DefaultsRepoFromCatalog(t *testing.T) {
+	files := []fakeHFFile{{"config.json", "{}"}, {"tokenizer_config.json", "{}"}, {"tokenizer.json", "{}"}, {"model.safetensors", "x"}}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, "", "main", false, "")
+	require.NoError(t, err)
+	assert.Equal(t, defaultPullPrefix(testHFRepo, testHFSHA), finalURI)
+}
+
+// TestRunPullWeights_DefaultsRevisionFromCatalogPin covers the pinned-revision
+// default: a bare pull of an entry carrying HFRevision resolves that pin, not
+// the "main" flag default, so a regressed upstream main cannot be picked up.
+func TestRunPullWeights_DefaultsRevisionFromCatalogPin(t *testing.T) {
+	const nomicModelID = "nomic-embed-text-v1.5"
+	const nomicRepo = "nomic-ai/nomic-embed-text-v1.5"
+	const pinnedRevision = "e5cf08a"
+	files := []fakeHFFile{{"config.json", "{}"}, {"tokenizer_config.json", "{}"}, {"tokenizer.json", "{}"}, {"model.safetensors", "x"}}
+	srv := newFakeHFServer(t, nomicRepo, pinnedRevision, testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	finalURI, err := runPullWeights(context.Background(), hf, store, nomicModelID, "", "main", false, "")
+	require.NoError(t, err)
+	assert.Equal(t, defaultPullPrefix(nomicRepo, testHFSHA), finalURI)
+}
+
+// TestPullFilesToPrefix_RefusesDuplicateFlattenedKey covers the flatten guard:
+// two source paths sharing a basename would collide at one destination key, so
+// the pull aborts rather than silently overwriting one with the other.
+func TestPullFilesToPrefix_RefusesDuplicateFlattenedKey(t *testing.T) {
+	files := []fakeHFFile{{"a/tokenizer.model", "first"}, {"b/tokenizer.model", "second"}}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	entries := []hfhub.TreeEntry{{Type: "file", Path: "a/tokenizer.model"}, {Type: "file", Path: "b/tokenizer.model"}}
+	_, err := pullFilesToPrefix(context.Background(), hf, store, testHFRepo, testHFSHA, "ochre-weights", "prefix/", entries)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "flatten to it")
+}
+
 // --- resolveHFToken (D2) ---
 
 func newOchreWeightsPullTestCmd(t *testing.T, modelID, hfRepo, revision, s3URI, hfToken string) *cobra.Command {

cmd/spinifex/cmd/admin_ochre_vector.go

@@ -96,6 +96,26 @@ truncated in the default table; pass --json for the full response.`,
 	Run: runOchreVectorQuery,
 }
 
+var ochreVectorBackupCmd = &cobra.Command{
+	Use:   "backup",
+	Short: "Back up an account's vector-store schema to predastore",
+	Long: `backup runs pg_dump against the platform appliance, scoped to --account's own
+kb_<account> schema, gzips the output, and uploads it to predastore under an
+account-scoped, timestamped object key. Backup is an explicit operator
+action -- it never runs automatically on appliance teardown.`,
+	Run: runOchreVectorBackup,
+}
+
+var ochreVectorRestoreCmd = &cobra.Command{
+	Use:   "restore",
+	Short: "Restore an account's vector-store schema from a predastore backup",
+	Long: `restore imports --from (an object key a prior backup wrote) back into
+--account's schema, replacing its existing tables cleanly, then repairs the
+account role's grants so query/ingest access resumes exactly as before the
+backup. Safe to run against an existing schema.`,
+	Run: runOchreVectorRestore,
+}
+
 func init() {
 	ochreCmd.AddCommand(ochreVectorCmd)
 	ochreVectorCmd.AddCommand(ochreVectorIndexCmd)
@@ -106,6 +126,8 @@ func init() {
 	ochreVectorCmd.AddCommand(ochreVectorJobCmd)
 	ochreVectorJobCmd.AddCommand(ochreVectorJobDescribeCmd)
 	ochreVectorCmd.AddCommand(ochreVectorQueryCmd)
+	ochreVectorCmd.AddCommand(ochreVectorBackupCmd)
+	ochreVectorCmd.AddCommand(ochreVectorRestoreCmd)
 
 	ochreVectorIndexCreateCmd.Flags().String("name", "", "Human-readable index name (required)")
 	ochreVectorIndexCreateCmd.Flags().Int("dimension", 0, "Embedding vector dimension (required)")
@@ -129,6 +151,14 @@ func init() {
 	ochreVectorQueryCmd.Flags().Bool("json", false, "Print full, untruncated results as JSON")
 	_ = ochreVectorQueryCmd.MarkFlagRequired("index")
 	_ = ochreVectorQueryCmd.MarkFlagRequired("text")
+
+	ochreVectorBackupCmd.Flags().String("account", "", "Account ID to back up (required)")
+	_ = ochreVectorBackupCmd.MarkFlagRequired("account")
+
+	ochreVectorRestoreCmd.Flags().String("account", "", "Account ID to restore into (required)")
+	ochreVectorRestoreCmd.Flags().String("from", "", "Backup object key to restore from, as printed by 'backup' (required)")
+	_ = ochreVectorRestoreCmd.MarkFlagRequired("account")
+	_ = ochreVectorRestoreCmd.MarkFlagRequired("from")
 }
 
 // vectorServiceFn indirects the NATS-backed client so the Run functions'
@@ -142,6 +172,59 @@ var vectorServiceFn = func() (handlers_ochrevector.VectorService, func(), error)
 	return handlers_ochrevector.NewNATSVectorService(nc), nc.Close, nil
 }
 
+// vectorBackupRestoreTimeout bounds the backup/restore NATS round trip: it
+// covers the appliance's own pg_dump/psql run, not just a KV read/write, so
+// it is far more generous than the other vector subjects' timeouts.
+const vectorBackupRestoreTimeout = 10 * time.Minute
+
+// backupAccountFn indirects the NATS call so runBackupAccount is testable
+// without a live daemon, mirroring vectorServiceFn/applianceTeardownFn.
+// Backup/restore are operator-only (unlike VectorService's methods): the
+// target account is named in the request payload, not derived from the
+// caller's own header identity, since the operator CLI always connects as
+// the global account but must be able to target any tenant's schema.
+var backupAccountFn = func(ctx context.Context, accountID string) (*handlers_ochrevector.BackupAccountResponse, error) {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, err
+	}
+	defer nc.Close()
+	return utils.NATSRequest[handlers_ochrevector.BackupAccountResponse](ctx, nc,
+		handlers_ochrevector.SubjectBackupAccount, &handlers_ochrevector.BackupAccountRequest{AccountID: accountID},
+		vectorBackupRestoreTimeout, utils.GlobalAccountID)
+}
+
+// restoreAccountFn indirects the NATS call so runRestoreAccount is testable
+// without a live daemon, mirroring backupAccountFn.
+var restoreAccountFn = func(ctx context.Context, accountID, objectKey string) error {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return err
+	}
+	defer nc.Close()
+	_, err = utils.NATSRequest[handlers_ochrevector.RestoreAccountResponse](ctx, nc,
+		handlers_ochrevector.SubjectRestoreAccount, &handlers_ochrevector.RestoreAccountRequest{AccountID: accountID, ObjectKey: objectKey},
+		vectorBackupRestoreTimeout, utils.GlobalAccountID)
+	return err
+}
+
+// runBackupAccount is the testable core of 'ochre vector backup'.
+func runBackupAccount(ctx context.Context, accountID string, backup func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error)) (string, error) {
+	out, err := backup(ctx, accountID)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("✅ Backed up account %s to %s (%d bytes).", accountID, out.ObjectKey, out.SizeBytes), nil
+}
+
+// runRestoreAccount is the testable core of 'ochre vector restore'.
+func runRestoreAccount(ctx context.Context, accountID, objectKey string, restore func(context.Context, string, string) error) (string, error) {
+	if err := restore(ctx, accountID, objectKey); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("✅ Restored account %s from %s.", accountID, objectKey), nil
+}
+
 // formatIndexRecord renders one index record as aligned key/value lines.
 func formatIndexRecord(rec handlers_ochrevector.Record) string {
 	rows := [][2]string{
@@ -448,3 +531,28 @@ func runOchreVectorQuery(cmd *cobra.Command, _ []string) {
 	}
 	fmt.Println(out)
 }
+
+func runOchreVectorBackup(cmd *cobra.Command, _ []string) {
+	accountID, _ := cmd.Flags().GetString("account")
+
+	msg, err := runBackupAccount(context.Background(), accountID, backupAccountFn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}
+
+func runOchreVectorRestore(cmd *cobra.Command, _ []string) {
+	accountID, _ := cmd.Flags().GetString("account")
+	objectKey, _ := cmd.Flags().GetString("from")
+
+	msg, err := runRestoreAccount(context.Background(), accountID, objectKey, restoreAccountFn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}

cmd/spinifex/cmd/admin_ochre_vector_test.go

@@ -382,3 +382,123 @@ func TestRunOchreVectorQuery_ConnectFailureExits1(t *testing.T) {
 	code := withOchreExitCapture(t, func() { runOchreVectorQuery(&cmd, nil) })
 	require.Equal(t, 1, code)
 }
+
+const testBackupAccountID = "123456789012"
+
+func TestRunBackupAccount_PrintsObjectKeyAndSize(t *testing.T) {
+	backup := func(_ context.Context, accountID string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		require.Equal(t, testBackupAccountID, accountID)
+		return &handlers_ochrevector.BackupAccountResponse{ObjectKey: testBackupAccountID + "/backup.sql.gz", SizeBytes: 4096}, nil
+	}
+
+	msg, err := runBackupAccount(context.Background(), testBackupAccountID, backup)
+	require.NoError(t, err)
+	require.Contains(t, msg, testBackupAccountID+"/backup.sql.gz")
+	require.Contains(t, msg, "4096")
+}
+
+func TestRunBackupAccount_ErrorSurfaces(t *testing.T) {
+	backup := func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		return nil, errors.New("ochrevector: appliance not available")
+	}
+
+	_, err := runBackupAccount(context.Background(), testBackupAccountID, backup)
+	require.ErrorContains(t, err, "not available")
+}
+
+func TestRunRestoreAccount_ReportsSuccess(t *testing.T) {
+	var gotAccount, gotKey string
+	restore := func(_ context.Context, accountID, objectKey string) error {
+		gotAccount, gotKey = accountID, objectKey
+		return nil
+	}
+
+	msg, err := runRestoreAccount(context.Background(), testBackupAccountID, testBackupAccountID+"/backup.sql.gz", restore)
+	require.NoError(t, err)
+	require.Equal(t, testBackupAccountID, gotAccount)
+	require.Equal(t, testBackupAccountID+"/backup.sql.gz", gotKey)
+	require.Contains(t, msg, testBackupAccountID)
+}
+
+func TestRunRestoreAccount_ErrorSurfaces(t *testing.T) {
+	restore := func(context.Context, string, string) error {
+		return errors.New("ochrevector: object key does not belong to account")
+	}
+
+	_, err := runRestoreAccount(context.Background(), testBackupAccountID, "wrong/backup.sql.gz", restore)
+	require.ErrorContains(t, err, "does not belong to account")
+}
+
+// withBackupAccount swaps backupAccountFn for one that never dials NATS, so
+// the Run wrapper's flag/exit control flow is exercised without a live
+// daemon, mirroring withApplianceTeardown.
+func withBackupAccount(t *testing.T, fn func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error)) {
+	t.Helper()
+	orig := backupAccountFn
+	t.Cleanup(func() { backupAccountFn = orig })
+	backupAccountFn = fn
+}
+
+// withRestoreAccount mirrors withBackupAccount for restoreAccountFn.
+func withRestoreAccount(t *testing.T, fn func(context.Context, string, string) error) {
+	t.Helper()
+	orig := restoreAccountFn
+	t.Cleanup(func() { restoreAccountFn = orig })
+	restoreAccountFn = fn
+}
+
+func TestRunOchreVectorBackup_PrintsSuccess(t *testing.T) {
+	withBackupAccount(t, func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		return &handlers_ochrevector.BackupAccountResponse{ObjectKey: testBackupAccountID + "/backup.sql.gz", SizeBytes: 1024}, nil
+	})
+
+	cmd := *ochreVectorBackupCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorBackup(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, testBackupAccountID+"/backup.sql.gz")
+}
+
+func TestRunOchreVectorBackup_ErrorExits1(t *testing.T) {
+	withBackupAccount(t, func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		return nil, errors.New("ochrevector: appliance not available")
+	})
+
+	cmd := *ochreVectorBackupCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+
+	code := withOchreExitCapture(t, func() { runOchreVectorBackup(&cmd, nil) })
+	require.Equal(t, 1, code)
+}
+
+func TestRunOchreVectorRestore_PrintsSuccess(t *testing.T) {
+	withRestoreAccount(t, func(context.Context, string, string) error { return nil })
+
+	cmd := *ochreVectorRestoreCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+	require.NoError(t, cmd.Flags().Set("from", testBackupAccountID+"/backup.sql.gz"))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorRestore(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, testBackupAccountID)
+}
+
+func TestRunOchreVectorRestore_ErrorExits1(t *testing.T) {
+	withRestoreAccount(t, func(context.Context, string, string) error {
+		return errors.New("ochrevector: restore requires an object key")
+	})
+
+	cmd := *ochreVectorRestoreCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+	require.NoError(t, cmd.Flags().Set("from", testBackupAccountID+"/backup.sql.gz"))
+
+	code := withOchreExitCapture(t, func() { runOchreVectorRestore(&cmd, nil) })
+	require.Equal(t, 1, code)
+}

cmd/spinifex/cmd/admin_upgrade_test.go

@@ -15,32 +15,27 @@ import (
 )
 
 // TestUpgradeSeesEveryConfigTarget pins the registrations `spx admin upgrade`
-// depends on. A migration whose package is no longer linked here would leave
-// the command reporting nothing pending on an install that needs migrating.
+// depends on. A target whose package is no longer linked here would leave the
+// command reporting nothing at all for a config file it installs.
 //
 // spinifex.toml is the only config target: predastore.toml is not migrated at
 // all, so a predastore config on disk must contribute neither a version nor a
-// pending step.
+// pending step. No migrations are registered, so nothing is ever pending.
 func TestUpgradeSeesEveryConfigTarget(t *testing.T) {
 	configDir := t.TempDir()
 
 	// A target must exist on disk to be reported at all.
 	require.NoError(t, os.MkdirAll(filepath.Join(configDir, "predastore"), 0750))
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	require.NoError(t, os.WriteFile(filepath.Join(configDir, "predastore", "predastore.toml"), []byte("version = 1\n"), 0640))
 
 	versions, err := migrate.DefaultRegistry.ConfigVersions(configDir)
 	require.NoError(t, err)
-	assert.Equal(t, map[string]int{"spinifex.toml": 3}, versions)
+	assert.Equal(t, map[string]int{"spinifex.toml": 4}, versions)
 
 	pending, err := migrate.DefaultRegistry.PendingConfig(configDir)
 	require.NoError(t, err)
-
-	targets := make(map[string][2]int, len(pending))
-	for _, p := range pending {
-		targets[p.Target] = [2]int{p.FromVersion, p.ToVersion}
-	}
-	assert.Equal(t, map[string][2]int{"spinifex.toml": {3, 4}}, targets)
+	assert.Empty(t, pending)
 }
 
 func TestValidateUpgradeFlags(t *testing.T) {
@@ -140,19 +135,6 @@ func TestReportConfigStatus_NoPendingMigrations(t *testing.T) {
 	assert.Contains(t, out, "spinifex.toml:")
 }
 
-func TestReportConfigStatus_PendingMigrationIsListed(t *testing.T) {
-	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
-
-	var pending []migrate.PendingMigration
-	out := captureStdout(t, func() { pending = reportConfigStatus(configDir) })
-
-	require.Len(t, pending, 1)
-	assert.Equal(t, "spinifex.toml", pending[0].Target)
-	assert.Contains(t, out, "Pending config migrations:")
-	assert.Contains(t, out, "spinifex.toml 3 → 4")
-}
-
 // captureStderr redirects os.Stderr for the duration of fn and returns what
 // was written. runAdminUpgrade's error paths print to stderr, not stdout.
 func captureStderr(t *testing.T, fn func()) string {
@@ -293,9 +275,9 @@ func TestRunAdminUpgrade_DryRunNothingPending(t *testing.T) {
 	assert.Contains(t, out, "Nothing pending — config and units are current.")
 }
 
-func TestRunAdminUpgrade_DryRunReportsBothPending(t *testing.T) {
+func TestRunAdminUpgrade_DryRunReportsUnitsPending(t *testing.T) {
 	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	withUnitRoot(t, t.TempDir()) // empty: every unit reports missing
 
 	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
@@ -307,15 +289,12 @@ func TestRunAdminUpgrade_DryRunReportsBothPending(t *testing.T) {
 	})
 	assert.Equal(t, -1, code)
 	assert.NotContains(t, out, "Nothing pending")
-	assert.Contains(t, out, "Pending config migrations:")
 	assert.Contains(t, out, "will install")
 }
 
 func TestRunAdminUpgrade_UnitsOnlySkipsConfigReporting(t *testing.T) {
 	configDir := t.TempDir()
-	// Version 3 would normally report a pending config migration; --units-only
-	// must never look at it.
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	withUnitRoot(t, t.TempDir())
 
 	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
@@ -332,7 +311,7 @@ func TestRunAdminUpgrade_UnitsOnlySkipsConfigReporting(t *testing.T) {
 
 func TestRunAdminUpgrade_SkipUnitsSkipsUnitReporting(t *testing.T) {
 	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	// Point systemdUnitRoot at a path Reconcile must never touch: if
 	// --skip-units leaked a call through, a nonexistent root would still
 	// "work" for a dry run, so the real proof is that its output never
@@ -348,31 +327,7 @@ func TestRunAdminUpgrade_SkipUnitsSkipsUnitReporting(t *testing.T) {
 	})
 	assert.Equal(t, -1, code)
 	assert.NotContains(t, out, "Reading installed systemd unit versions", "--skip-units must skip unit reporting")
-	assert.Contains(t, out, "Pending config migrations:")
-}
-
-func TestRunAdminUpgrade_ApplyRunsConfigMigrationAndUnitReconcile(t *testing.T) {
-	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
-	// Every unit already current: the real (non-dry-run) Reconcile call
-	// below finds nothing to change, so it never touches daemon-reload.
-	unitRoot := t.TempDir()
-	seedUnitRoot(t, unitRoot)
-	withUnitRoot(t, unitRoot)
-
-	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
-	setUpgradeFlags(t, cmd, map[string]string{"yes": "true"})
-
-	var code int
-	out := captureStdout(t, func() {
-		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
-	})
-	assert.Equal(t, -1, code, "the apply must succeed without exiting")
-	assert.Contains(t, out, "Done.")
-
-	migrated, err := os.ReadFile(filepath.Join(configDir, "spinifex.toml"))
-	require.NoError(t, err)
-	assert.Contains(t, string(migrated), `version = "4"`, "spinifex.toml must have been migrated to the latest version")
+	assert.Contains(t, out, "Reading current config versions")
 }
 
 func TestRunAdminUpgrade_ApplyWithNonWritableUnitRootExitsWithRootHint(t *testing.T) {

cmd/spinifex/cmd/cluster.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/daemon"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/nats-io/nats.go"
 	"github.com/spf13/cobra"
@@ -333,7 +334,7 @@ func runNodeDrainLocal(cmd *cobra.Command, args []string) {
 			// `systemctl --failed`) without blocking the target's teardown;
 			// exiting 0 would hide guests left running as storage vanishes.
 			slog.Error("drain phase failed: no ACK from local daemon before timeout; guests may be left running while storage is about to be torn down",
-				"phase", phase, "node", node, "timeout", timeout, "error", err)
+				"phase", phase, "node", node, "timeout_ms", otelsetup.Millis(timeout), "error", err)
 			reportGuestsLeftRunning(true, fmt.Sprintf("[%s] drain failed: guests left running with storage about to be torn down", strings.ToUpper(phase)))
 			os.Exit(1)
 		}

cmd/spinifex/cmd/templates/awsgw.toml

@@ -40,11 +40,17 @@ burst = 5
 [signup]
 max_accounts = 128
 
-# Per-account service quotas (default off)
+# Per-account service quotas (default off). These values are the baseline every
+# account inherits; raise an individual account with
+# `spx admin account quota set`, which stores an override in NATS KV.
+# A dimension set to -1 is uncapped for accounts that inherit it.
 [quota]
 enabled = false
-vcpus = 8
-vpcs = 8
+vcpus = 16
+vpcs = 4
 subnets = 16
 eips = 4
-volumes_gib = 100
+volumes = 16
+volumes_gib = 200
+rds_instances = 2
+load_balancers = 2

docs/COMMANDS.md

@@ -95,6 +95,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin account create --remote` | `--name` (required), `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA), `--client-token` (default: generated), `--source` (default: `spx-cli`) | Same account, created through `POST /admin/CreateAccount` over SigV4 instead of NATS — the path the public signup form uses, so a shell can exercise it. Credentials come from the standard AWS chain (env vars or `AWS_PROFILE`), never from the cluster master key; the caller must be an IAM user in the super-admin account holding `spinifex:CreateAccount`. Reuse the printed `--client-token` to retry: a fresh token would create a second account. |
 | `spx admin account list` | — | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created |
 | `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
+| `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
+| `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
 | `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Bedrock model endpoints are reaped ahead of instances, because only their own delete releases the ENI and weights volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
 | `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
 | `spx admin account delete --remote` | as above, plus `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Same teardown driven through `POST /admin/DeleteAccount` over SigV4 instead of NATS, then follows `DescribeAccountDeletion` to completion, printing each stage as it lands. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. |
@@ -144,7 +146,7 @@ Operator surface over the daemon's `bedrock.endpoint.*` subjects. The gateway do
 | `spx admin ochre endpoint list` | — | Lists every endpoint record with its state, instance ID and base URL. |
 | `spx admin ochre endpoint delete` | `--model-id` (required) | Moves a `READY` endpoint to `DRAINING` and tears its VM down, releasing the GPU. Idempotent: an already-absent endpoint reports success. |
 
-Self-hosted models are served from the `ubuntu-26.04-vllm-serving-x86_64` system image (`spx admin images import --name ubuntu-26.04-vllm-serving-x86_64`), built by `scripts/mkosi-build.sh --image ubuntu-vllm-serving` (`gpu-nvidia` + `vllm` profiles). It carries the tags `spinifex:managed-by=bedrock` and `spinifex:bedrock-role=vllm-serving`, which is what `LaunchServingVM` filters on rather than resolving by name. The image bakes vLLM's OpenAI-compatible server into a uv-managed venv — no Docker daemon, no runtime container pull — and boots `vllm-serve.service`, which waits for cloud-init's `/etc/conf.d/vllm-serve` handoff, resolves and mounts the endpoint's cloned weights volume read-only, then execs `vllm serve` on port 8000.
+Self-hosted models are served from the `ubuntu-26.04-vllm-serving-x86_64` system image (`spx admin images import --name ubuntu-26.04-vllm-serving-x86_64`), built by `scripts/mkosi-build.sh --image ubuntu-vllm-serving` (`gpu-nvidia` + `vllm` profiles). It carries the tags `spinifex:managed-by=bedrock` and `spinifex:bedrock-role=vllm-serving`, which is what `LaunchServingVM` filters on rather than resolving by name. The image bakes vLLM's OpenAI-compatible server into a uv-managed venv and TEI's engine binary + CUDA runtime libraries (extracted from HuggingFace's own published CUDA image, no Docker daemon on the guest either way) side by side, one hand-stood image serving a whole co-served bundle rather than a single model. `LaunchServingVM` writes one `/etc/conf.d/bedrock-bundle/<instance>.env` file per bundle member; `bedrock-bundle-init.service` enumerates them at boot and starts one `bedrock-serve@<instance>.service` per member, which resolves and mounts that member's own cloned weights volume read-only and execs `vllm serve` (port 8000, the generative member) or `text-embeddings-router` directly (port 8001+, embedding/rerank members) — each engine's readiness is probed on the route it actually serves (`/v1/models` for vLLM, `/health` for TEI). A standalone model is the one-member case of the same path: it boots exactly as before, on port 8000.
 
 ### EKS Control-Plane Disaster Recovery
 
@@ -191,8 +193,8 @@ The surface is unreachable until an operator runs `spx admin principal create`;
 | `modify-instance-metadata-options` | `--instance-id`, `--http-put-response-hop-limit` (1–64), `--http-tokens` (`required`/`optional`), `--http-endpoint` (`enabled`), `--http-protocol-ipv6`/`--instance-metadata-tags` (`disabled`) — unmodelled values return `UnsupportedOperation` | `--dry-run` | **DONE** |
 | `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | **DONE** (stub — always returns `standard`) |
 | `describe-instance-status` | `--instance-ids`, `--include-all-instances`, `--filters` (availability-zone, instance-state-code, instance-state-name, tag:*) | `--max-results`, `--next-token`, `--dry-run`, event/instance-status/system-status filters | **DONE** (static health) |
-| `monitor-instances` | — | `--instance-ids` | **NOT STARTED** |
-| `unmonitor-instances` | — | `--instance-ids` | **NOT STARTED** |
+| `monitor-instances` | `--instance-ids` | `--dry-run` | **DONE** (60s detailed telemetry; state is `enabled`/`disabled` only, never `pending`/`disabling`) |
+| `unmonitor-instances` | `--instance-ids` | `--dry-run` | **DONE** (returns the instance to the 300s basic tier) |
 
 ### EC2 — Spot Instances
 
@@ -902,6 +904,8 @@ Spinifex both stores externally-issued certificates (`import-certificate`) and i
 
 Each DB instance is one dedicated system-owned VM running the engine directly, launched from the `spinifex-rds-postgres` or `spinifex-rds-mariadb` AMI, tagged `spinifex:managed-by=rds` and therefore hidden from the customer's EC2 API. The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so **the endpoint is private — reachable from inside the VPC only**, from any subnet of it rather than the endpoint ENI's own. `Endpoint.Address` is `{db-instance-identifier}.{account-id}.{region}.rds.{base-domain}` where northstar is configured, and the endpoint ENI's private IP where it is not; the IP is stable across VM replacement either way.
 
+**The endpoint is the only path to the engine.** The DB VM has two other NICs that no customer security group governs — a system NIC in a shared RDS system VPC, which is how the in-guest agent reaches the control plane, and a management NIC on the host's management bridge — and the engine binds neither. It binds the endpoint ENI's address alone, so the port is not open on them at all rather than open and gated. The system NIC additionally carries a dedicated region-wide security group that authorizes no ingress; agent traffic is outbound and its replies are matched by connection state. PostgreSQL's generated `pg_hba.conf` admits the customer VPC's address range and nothing wider, so a client outside the VPC is refused authentication even if it somehow reached the socket. MariaDB has no equivalent file: its analogue is the host pattern on a grant, which lives in the data volume and would follow a restore into another VPC, so for that engine the bind is the whole boundary.
+
 Everything below applies to both engines unless it names one. `Engine` is fixed at create — there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
 
 | | `postgres` | `mariadb` |

docs/README.md

@@ -3,3 +3,19 @@
 This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browsing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)
 
 The [AWS model operation coverage](aws-model-operation-coverage.md) report is generated from Spinifex's dispatch tables and the pinned AWS API models. Run `make aws-model-coverage` to regenerate and print it.
+
+## Frontmatter
+
+Each published page lives at `<category>/<slug>/README.md` and carries YAML frontmatter that the docs site turns into page metadata. Three fields have length rules, because search engines flag a title tag or meta description that falls outside them:
+
+| Field         | Renders as                                | Length             |
+| ------------- | ----------------------------------------- | ------------------ |
+| `title`       | Page heading, sidebar label, landing card | Short — it is UI text |
+| `seoTitle`    | The browser and search-result title        | 50-60 characters   |
+| `description` | The search-result snippet, not shown on the page | 150-160 characters |
+
+`seoTitle` is used exactly as written and must end with ` — Spinifex Docs`. Leave it out and the title falls back to `title` plus that suffix, which is normally too short to satisfy the rule.
+
+Keep the two titles distinct. `title` is navigation text in a narrow sidebar, so stretching it to reach 50 characters damages the layout to satisfy a rule that only applies to the title tag. Put the search-facing wording in `seoTitle`.
+
+A new page is only published once its path is added to `scripts/docs-config.json` in the `mulga-docs` repo. That repo's `scripts/build-docs.js` warns about any field outside its range.

docs/admin/host-lifecycle/README.md

@@ -11,7 +11,7 @@ resources:
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
   - title: "Updating Spinifex"
-    url: "/docs/admin/update"
+    url: "/docs/update"
 ---
 
 # Host and Guest Lifecycle

docs/admin/setting-up-your-cluster/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Setting Up Your Cluster"
-description: "Import an AMI, create a VPC with a public subnet, and launch your first EC2 instance on Spinifex."
+seoTitle: "Set Up Your First Spinifex Cluster — Spinifex Docs"
+description: "Import an AMI, create an SSH key pair and a VPC with a public subnet, then launch your first EC2 instance and connect to it on a fresh Spinifex cluster."
 category: "Admin"
 tags:
   - setup

docs/admin/spinifex-admin-cli/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Spinifex Admin CLI"
-description: "Complete reference for the Spinifex administration CLI. Manage accounts, nodes, VMs, and services."
+seoTitle: "spx: The Spinifex Administration CLI — Spinifex Docs"
+description: "Complete reference for spx, the Spinifex admin CLI: initialise a cluster, manage accounts and nodes, drive the VM lifecycle, and start or stop services."
 category: "Admin"
 tags:
   - cli
@@ -76,7 +77,7 @@ ACCOUNT ID     NAME                 STATUS     CREATED
 000000000002   myteam               ACTIVE     2026-07-03 03:32
 ```
 
-Deleting an account, and creating or removing accounts from your own signup or provisioning system over the private admin API, are covered in [Account Management](/docs/admin/account-management).
+Deleting an account, and creating or removing accounts from your own signup or provisioning system over the private admin API, are covered in [Account Management](https://github.com/mulgadc/spinifex/blob/main/docs/admin/account-management/README.md).
 
 ## Node Management
 

docs/admin/update/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Updating Spinifex"
-description: "Upgrade an existing Spinifex installation to the latest release."
+seoTitle: "Update Spinifex to the Latest Release — Spinifex Docs"
+description: "Upgrade an existing Spinifex install with the same installer used to deploy it, or take the manual path to review configuration migrations before applying them."
 category: "Admin"
 tags:
   - update
@@ -42,7 +43,7 @@ For operators who want to review migrations before they are applied, a manual up
 > [!WARNING]
 > **AMI metadata is exempt when upgrading from v1.16.0 or earlier to the release carrying the EBS-provider decoupling.** AMI metadata moved to `ebsmetadata` documents, and the legacy path that read it from `ami-<id>/config.json` was removed rather than migrated. There is no prefix scan, no fallback and no backfill at daemon start, so an AMI imported before the change becomes invisible to the control plane afterwards: `describe-images --image-ids` answers `InvalidAMIID.NotFound` and launches fail with `AMI has no snapshot ID, cannot perform zero-copy clone`. As with object storage above, `spx admin upgrade` reports nothing pending, because the gap is in stored data rather than in a config file.
 >
-> **Re-import affected AMIs after upgrading**, using [`spx admin images import`](/docs/admin/spinifex-admin-cli), which writes metadata in the new location. If the source images are no longer available, the installation has to be re-initialised. Verify before you rely on it — an AMI is only healthy if it resolves by ID, not merely if it appears in the list:
+> **Re-import affected AMIs after upgrading**, using [`spx admin images import`](/docs/spinifex-admin-cli), which writes metadata in the new location. If the source images are no longer available, the installation has to be re-initialised. Verify before you rely on it — an AMI is only healthy if it resolves by ID, not merely if it appears in the list:
 >
 > ```bash
 > aws ec2 describe-images --image-ids <ami-id>
@@ -112,7 +113,7 @@ Migrations modify config files on disk but do not restart running services, and
 sudo systemctl restart spinifex.target
 ```
 
-A restart preserves any running guests — they are not rebooted, and storage returns within seconds. See [Host and Guest Lifecycle](/docs/admin/host-lifecycle) for the full contract.
+A restart preserves any running guests — they are not rebooted, and storage returns within seconds. See [Host and Guest Lifecycle](https://github.com/mulgadc/spinifex/blob/main/docs/admin/host-lifecycle/README.md) for the full contract.
 
 ## Checking for Unit Drift
 
@@ -164,7 +165,7 @@ Migrations edit config files on disk but the running daemons continue to use the
 sudo systemctl restart spinifex.target
 ```
 
-A restart preserves any running guests — they are not rebooted, and storage returns within seconds. See [Host and Guest Lifecycle](/docs/admin/host-lifecycle) for the full contract.
+A restart preserves any running guests — they are not rebooted, and storage returns within seconds. See [Host and Guest Lifecycle](https://github.com/mulgadc/spinifex/blob/main/docs/admin/host-lifecycle/README.md) for the full contract.
 
 ### A Node Answers Some Requests as if It Were Still on the Old Build
 

docs/compute/gpu-passthrough/README.md

@@ -1,6 +1,7 @@
 ---
 title: "GPU Passthrough"
-description: "Configure VFIO GPU passthrough on a Spinifex node to expose NVIDIA or AMD GPUs to EC2 instances."
+seoTitle: "VFIO GPU Passthrough for EC2 Instances — Spinifex Docs"
+description: "Configure VFIO GPU passthrough on a Spinifex node to bind NVIDIA or AMD GPUs to guest VMs and expose GPU-enabled EC2 instance types to your workloads."
 category: "Compute"
 tags:
   - gpu

docs/compute/launching-instances/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Launching Instances"
-description: "Launch, manage, and connect to EC2-compatible virtual machines on Spinifex."
+seoTitle: "Launching AWS EC2 Instances on Spinifex — Spinifex Docs"
+description: "Launch, manage, and connect to EC2-compatible virtual machines on Spinifex, with cloud-init, SSH key injection, VPC networking, and AWS lifecycle operations."
 category: "Compute"
 tags:
   - ec2

docs/compute/placement-groups/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Placement Groups"
-description: "Create and manage spread and cluster placement groups for hardware-level instance placement control."
+seoTitle: "AWS EC2 Placement Groups on Spinifex — Spinifex Docs"
+description: "Create and manage spread and cluster placement groups to control how Spinifex places EC2 instances across physical hosts for fault isolation or low latency."
 category: "Compute"
 tags:
   - ec2

docs/compute/vpc-networking/README.md

@@ -1,6 +1,7 @@
 ---
 title: "VPC Networking"
-description: "How Spinifex implements AWS-compatible VPC networking with public and private subnets, security groups, and Elastic IPs using OVN."
+seoTitle: "AWS-Compatible VPC Networking with OVN — Spinifex Docs"
+description: "How Spinifex implements AWS-compatible VPC networking on bare metal with OVN: public and private subnets, security groups, route tables, and Elastic IPs."
 category: "Compute"
 tags:
   - vpc
@@ -359,6 +360,37 @@ This requests an IP from the **router's DHCP** (e.g., 192.168.1.1 serving addres
 
 If no Linux bridge owns the default route, `setup-ovn.sh` exits with guidance rather than silently breaking host connectivity.
 
+## OVN Control Plane on Multi-Node Clusters
+
+How OVN is deployed depends on the size of the cluster, and this is the main reason three servers is the recommended minimum for a multi-server deployment.
+
+| Cluster size | OVN databases | Tolerates |
+|---|---|---|
+| 1–2 servers | standalone, on the first node | nothing — that node is a single point of failure for the control plane |
+| 3 or more | clustered (RAFT) across the first three nodes | loss of any one database node |
+
+Servers beyond the third run the full set of Spinifex services, but do not join the OVN database cluster — they connect to it as clients. The quorum stays at three however large the cluster gets, which is what keeps control-plane write latency stable.
+
+### What a control-plane outage actually costs
+
+Less than it sounds. `ovn-controller` has already programmed the forwarding rules into each host, so **running instances keep full networking** — east-west, north-south, NAT and security groups all continue to work with the databases down.
+
+What stops is *change*. Creating a VPC, launching an instance, and updating a security group all need the control plane, because each has to write new logical topology before anything can program it.
+
+So a standalone OVN deployment is a reasonable choice for a lab or a single-server install. It is a poor one for production, where the inability to launch an instance during an outage is usually as bad as the instances being down.
+
+### Clustering the databases
+
+The three database nodes are set up with `--db-cluster-local-addr`, and nodes 2 and 3 additionally point at node 1 with `--db-cluster-remote-addr`. Compute nodes take `--ovn-remote` listing all three, so they survive any one of them failing. See [Multi-Node Install](/docs/install-multi-node) for the exact commands in sequence.
+
+A clustered database can only be created from scratch — the `ovn-central` package starts a standalone one on install, so the cluster setup passes `--recreate-db` to replace it. Confirm the result with:
+
+```bash
+sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+```
+
+Three servers should be listed with exactly one leader. A report of a standalone database means the cluster flags did not take.
+
 ## Per-Node Configuration
 
 Different nodes in a cluster can have different WAN bridges and NICs:

docs/containers/ecr/README.md

@@ -1,6 +1,7 @@
 ---
 title: "ECR (Container Registry)"
-description: "Store and serve container images from Spinifex's AWS-compatible Elastic Container Registry — create a repository, authenticate Docker, push and pull images, and let your EKS workers pull from it, using the AWS CLI, the Spinifex console, or Terraform."
+seoTitle: "AWS ECR Container Registry on Spinifex — Spinifex Docs"
+description: "Store and serve container images from Spinifex's AWS-compatible ECR: create a repository, authenticate Docker, push and pull images, and let EKS workers pull."
 category: "Containers"
 tags:
   - ecr

docs/containers/ecs/README.md

@@ -1,6 +1,7 @@
 ---
 title: "ECS (Elastic Container Service)"
-description: "Run AWS-compatible ECS workloads on Spinifex — create a cluster, register a task definition, boot container instances from the ECS node image, run tasks, and front a long-running service with an Application Load Balancer, all via the AWS CLI, the Spinifex console, or Terraform."
+seoTitle: "AWS ECS Elastic Container Service Guide — Spinifex Docs"
+description: "Create an ECS cluster on Spinifex, register a task definition, boot container instances, run tasks, and front a service with an Application Load Balancer."
 category: "Containers"
 tags:
   - ecs

docs/containers/eks/README.md

@@ -1,6 +1,7 @@
 ---
 title: "EKS (Managed Kubernetes)"
-description: "Run a managed, AWS-compatible Kubernetes cluster on Spinifex — provision the control plane and worker node group, wire up the VPC, IAM, and security groups it needs, and deploy your first workload via the AWS CLI, the Spinifex console, or Terraform."
+seoTitle: "AWS EKS Managed Kubernetes on Spinifex — Spinifex Docs"
+description: "Provision an AWS-compatible EKS control plane and managed node group on Spinifex, wire up the VPC, IAM, and security groups, then deploy your first workload."
 category: "Containers"
 tags:
   - eks

docs/install/install-airgapped/README.md

@@ -1,5 +1,6 @@
 ---
 title: "Air-Gapped Install"
+seoTitle: "Install Spinifex in an Air-Gapped Network — Spinifex Docs"
 description: "Deploy Spinifex in environments without internet connectivity. Covers using a pre-built release tarball, mirrored APT packages, and locally-staged cloud images."
 category: "Install"
 tags:
@@ -10,7 +11,7 @@ resources:
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
   - title: "External Connection Inventory"
-    url: "/docs/security/network-connections"
+    url: "/docs/network-connections"
 ---
 
 # Air-Gapped Installation

docs/install/install-multi-node/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Multi-Node Install"
-description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
+seoTitle: "Multi-Node Spinifex Cluster Install — Spinifex Docs"
+description: "Deploy Spinifex across three or more servers to form an availability zone with clustered OVN, replicated object storage, and automatic cluster formation."
 category: "Install"
 tags:
   - install
@@ -11,6 +12,8 @@ tags:
 resources:
   - title: "Bootable USB Install"
     url: "/docs/install-usb"
+  - title: "VPC Networking"
+    url: "/docs/vpc-networking"
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
   - title: "Predastore (S3)"
@@ -21,84 +24,117 @@ resources:
 
 # Multi-Node Installation
 
-> Deploy Spinifex across multiple servers to create an availability zone.
+> Deploy Spinifex across three or more servers to create an availability zone.
 
 ## Table of Contents
 
 - [Overview](#overview)
-- [Object Storage Layout](#object-storage-layout)
+- [Cluster sizing](#cluster-sizing)
+- [Hardware](#hardware)
+- [Network requirements](#network-requirements)
 - [Prerequisites](#prerequisites)
 - [Instructions](#instructions)
-- [Converting ISO-Installed Nodes](#converting-iso-installed-nodes)
-- [Firewall and Cluster Membership](#firewall-and-cluster-membership)
+- [Verifying the cluster](#step-6-verify-the-cluster)
+- [Firewall and cluster membership](#firewall-and-cluster-membership)
+- [How multi-node storage works](#how-multi-node-storage-works)
 - [Troubleshooting](#troubleshooting)
 
 ---
 
 ## Overview
 
-A Spinifex cluster distributes services across multiple servers for high availability, data durability, and fault tolerance. Cluster formation is automatic — the init node waits for peers to join, then distributes credentials, CA certificates, and configuration.
+> [!IMPORTANT]
+> **This guide builds a three-node cluster** — the minimum we recommend for any multi-server
+> deployment. Every step below is written for three servers and uses `$SPINIFEX_NODE1`,
+> `$SPINIFEX_NODE2` and `$SPINIFEX_NODE3`.
+>
+> **Running more than three?** Install the first three exactly as described, then repeat
+> Steps 1, 2, 3 and the join in Step 4 for each additional server. The only difference is in
+> Step 3: the OVN database cluster stays at three members, so servers four and beyond point at
+> those three instead of joining them.
 
-**Installing on bare metal?** Step 1 below can be done either with the binary installer or by booting each server from the Spinifex ISO — see [Bootable USB Install](/docs/install-usb). The ISO installs the operating system, disks and network configuration as well as Spinifex itself, which makes it the better option for servers with no existing OS. Either way, install every server first, then return here and continue from Step 2.
+A Spinifex cluster distributes services across multiple servers for high availability, data durability, and fault tolerance. Cluster formation is automatic — the init node waits for its peers to join, then distributes credentials, CA certificates, and configuration.
 
-**Cluster Sizing:**
+**Installing on bare metal?** Step 1 can be done either with the binary installer or by booting each server from the Spinifex ISO — see [Bootable USB Install](/docs/install-usb). The ISO installs the operating system, disks and network configuration as well as Spinifex itself, which makes it the better option for servers with no existing OS. Either way, install every server first, then return here and continue from Step 2.
 
-**Three servers is the minimum recommended for a multi-server deployment.** At three or more, the OVN control-plane databases run clustered across three nodes and the cluster tolerates the loss of any one of them. Below three, OVN runs standalone on the first node, which becomes a single point of failure for control-plane operations.
+### Cluster sizing
 
-A standalone OVN outage is less severe than it sounds. `ovn-controller` has already programmed the forwarding rules into each host, so **running instances keep full networking** — east-west, north-south, NAT and security groups all continue. What stops is change: creating VPCs, launching instances, and updating security groups all require the control plane.
+**Three servers is the minimum we recommend.** Three is the point at which every distributed layer can lose a node and keep running:
 
-Servers beyond the third join as compute nodes and do not run a database, which keeps write latency stable as the cluster grows.
+| Layer | What three servers gives you |
+|---|---|
+| **VPC networking** — OVN | Control-plane databases run clustered, surviving the loss of any one node. |
+| **Object storage** — Predastore (S3) | Objects are erasure coded `RS(2,1)`, surviving the loss of any one node's shards. |
+| **Block storage** — Viperblock (EBS) | Volumes are stored in Predastore, so they inherit the same durability. |
 
-**Network Requirements:**
+On one or two servers none of that holds. OVN runs standalone on the first node, and the storage metadata quorum has no majority to lose. If that node goes down, running instances keep full networking — but nothing can *change*: no new VPCs, no launches, no security group edits. See [OVN control plane on multi-node clusters](/docs/vpc-networking#ovn-control-plane-on-multi-node-clusters).
 
-- Minimum 1 NIC per server (2 recommended for production)
-- UDP port 6081 open between hosts (Geneve tunnels)
-- TCP ports 4222, 4248, 6641, 6642 open between hosts (NATS, OVN)
-- TCP ports 6643, 6644 open between database servers (OVN database clustering)
-- TCP port 8443 and UDP ports 6660, 7660 open between hosts (Predastore S3, object shards, metadata consensus)
+Servers beyond the third run the full set of services — storage, gateway and networking agents — and add their capacity to the pool. What they do not do is join the OVN database cluster, which stays at three members, so write latency there stays flat as the cluster grows.
 
-Predastore uses the same three ports on every server, so the surface does not widen as the cluster grows. See [Object Storage Layout](#object-storage-layout) below.
+### Hardware
 
-## Object Storage Layout
+Per server, for the three-node minimum:
 
-Predastore is configured for the whole cluster in `/etc/spinifex/predastore/predastore.toml`. Each server is one `[[host]]` — a single Predastore process owning that machine's data directory and TLS identity — carrying three nodes under `[[host.node]]`:
+| | Minimum | Recommended |
+|---|---|---|
+| **Nodes** | 3 | 3 or more |
+| **RAM** | 32 GB | 128 GB |
+| **CPU** | 16 cores | 32+ cores |
+| **OS / Spinifex disk** | SSD | NVMe |
+| **NICs** | 2 — WAN 1 GbE, LAN/VPC 10 GbE+ | 2 — WAN 10 GbE, LAN/VPC 25 GbE+ |
 
-| Role | Port | Purpose |
-|------|------|---------|
-| `gate` | TCP 8443 | Serves the S3 API. Every server runs one, so any of them answers an S3 request. |
-| `blob` | UDP 6660 | Holds erasure-coded object shards. One per machine. |
-| `meta` | UDP 7660 | Member of the Raft quorum over global state — buckets and the object index. |
+Two NICs matters more than the raw numbers. One carries WAN traffic; the other carries LAN and VPC traffic between nodes — Geneve tunnels, object shards and OVN replication all cross it, so it is the interface that wants the bandwidth. A single-NIC server will work, but inter-node storage and tunnel traffic then competes with everything going in and out of the cluster.
 
-Ports have to be unique within a host but not across the cluster, so every machine uses the same three. Blob and meta traffic between hosts runs over QUIC, authenticated by the cluster CA; nodes on the same machine talk over an in-process pipe and open no socket, which is why a single-server install listens on 8443 alone.
+### Network requirements
 
-Reed-Solomon parameters are chosen from the cluster size, since each machine contributes exactly one blob node: two servers get `RS(1,1)`, three or more get `RS(2,1)`. `RS(2,1)` survives the loss of any one server's shards.
+Open between all hosts:
 
-You do not configure any of this by hand. `spx admin init` and `spx admin join` build the topology from the servers that actually form the cluster in Step 4, and each machine gets the same file with its own host ID recorded in `spinifex.toml`.
+| Protocol and port | Used by |
+|---|---|
+| UDP 6081 | Geneve tunnels |
+| TCP 4222, 4248 | NATS |
+| TCP 6641, 6642 | OVN northbound and southbound |
+| TCP 8443 | Predastore S3 gate |
+| UDP 6660, 7660 | Object shards, metadata consensus |
 
-> [!WARNING]
-> **There is no upgrade path onto this layout.** A cluster installed before the object storage cutover cannot be migrated — the on-disk directory structure and the configuration schema both changed, and no migration rewrites either. Such a cluster has to be re-initialised from scratch, which discards its stored objects. Export anything you need first.
+Open between the three OVN database servers (servers 1 to 3) only:
+
+| Protocol and port | Used by |
+|---|---|
+| TCP 6643, 6644 | OVN database clustering |
+
+Predastore uses the same three ports on every server, so the surface does not widen as the cluster grows. See [How multi-node storage works](#how-multi-node-storage-works).
+
+Those are the ports your **network** has to permit between the servers — switches, upstream firewalls, and anything else in the path.
+
+Spinifex's own **host** firewall is separate, and already carries this policy. It ships armed on the ISO path and off on the binary installer path. Two things follow from that, both of which the ISO box in Step 2 acts on:
+
+- **The formation port needs nothing from you.** `spx admin init` opens 4432 to any source for the length of the formation window and closes it again afterwards, because a node dialling in to join is not a peer yet. The handshake behind it is TLS 1.3 with a bearer token.
+- **The rest of the cluster plane is peer-scoped**, and that includes the OVN database ports Step 3 uses. Nodes that do not yet know each other cannot reach them, which is why an ISO-installed node has its firewall taken down before Step 3 and re-armed after Step 6.
 
 ## Prerequisites
 
+**Installed from the ISO (Option B)?** The bridge is configured for you — the ISO sets up the host's network interfaces, `br-wan` included. Skip to [Instructions](#instructions).
+
 > [!IMPORTANT]
-> **Prerequisite — WAN bridge required on every node.**
+> **Binary installer only — a WAN bridge is required on every node.**
 >
 > Before running the installer on any server, that server's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
 >
-> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in a future release.
+> The binary installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in a future release.
 >
 > **Verify on every node before continuing:**
 >
 > - `ip -br link show br-wan` — bridge exists and is `UP`
 > - `ip route` — default route's `dev` is `br-wan`
 >
-> **Setup references:** [VPC Networking → Bridge Setup](/docs/vpc-networking#bridge-setup-physical-network-wiring) for the topology.
+> See [VPC Networking → Bridge Setup](/docs/vpc-networking#bridge-setup-physical-network-wiring) for the topology.
 
 ## Instructions
 
 ## Step 1. Install Spinifex on Each Server
 
-Choose one of the two methods below and apply it to **every** server in the cluster.
+Choose one method and apply it to **every** server in the cluster.
 
 **Option A — existing OS.** On a server already running Ubuntu 26.04 or Debian 13:
 
@@ -108,26 +144,11 @@ curl -fsSL https://install.mulgadc.com | bash
 
 **Option B — bare metal, from the ISO.** Boot each server from the Spinifex ISO and follow [Bootable USB Install](/docs/install-usb). This installs the operating system, partitions the disks, and configures the hostname and network interfaces alongside Spinifex. The ISO installer does not form a cluster — that is what the remaining steps do.
 
-Complete this step on all servers before continuing. The nodes must be installed and reachable from one another before the cluster is formed, because Step 4 requires every node to be available at the same time.
-
-### Shortcut: form the cluster with one command
-
-Steps 2 to 6 are a fixed sequence, and `scripts/install-node.sh` in the Spinifex repository runs it for you over SSH — from a workstation that can reach every server, not from the servers themselves:
-
-```bash
-scripts/install-node.sh \
-  --external-pool 10.0.1.100-10.0.1.150 \
-  --external-gateway 10.0.1.1 --external-prefix-len 24 \
-  server1 server2 server3
-```
-
-The first host initializes and the rest join. It resolves each node's plane addresses, rebuilds the OVN databases in clustered form, forms the cluster, restarts services and verifies the result — and `--dry-run` prints every command it would run without touching anything. It needs passwordless `sudo` on each host and requires all of them to be on the same Spinifex version.
-
-The remaining steps document what it does, and are the path to follow when installing by hand or when something needs to be adjusted mid-way.
+Complete this step on all three servers before continuing. Step 4 requires every node to be installed, reachable, and available at the same time.
 
 ## Step 2. Set Node IP Variables
 
-On **each server**, export the management IPs for all nodes:
+On **each server**, export the management IPs of all three nodes plus the region and AZ. The same values go on every server:
 
 ```bash
 export SPINIFEX_NODE1=192.168.1.10
@@ -137,40 +158,29 @@ export AWS_REGION=us-east-1
 export AWS_AZ=us-east-1a
 ```
 
-## Converting ISO-Installed Nodes
+Adding a fourth server or more? Export `SPINIFEX_NODE4` and so on alongside these — the first three stay as they are, because they remain the OVN database nodes.
 
-Skip this section if you installed with the binary installer (Option A).
+> [!NOTE]
+> **ISO installs only — skip this box if you used the binary installer (Option A).**
+>
+> The ISO brings each server up as a **running standalone single-node cluster** with its firewall armed, so forming a cluster is a conversion rather than a fresh setup. One thing is needed, on **every** server, before Step 3:
+>
+> ```bash
+> sudo systemctl stop spinifex.target
+> sudo /usr/local/lib/spinifex/spinifex-firewall-apply disable
+> ```
+>
+> Each node's firewall currently trusts only itself, because that is the whole cluster as far as it knows, and the cluster plane is peer-scoped — so the OVN database connections Step 3 makes between servers are dropped. Stopping `spinifex.target` is not enough: the firewall lives in the kernel and outlives the services.
+>
+> Turn it back on after Step 6 — see [Firewall and cluster membership](#firewall-and-cluster-membership).
 
-The ISO installs each server as a **running standalone single-node cluster** — it initializes Spinifex, starts a standalone OVN database, brings up `spinifex.target`, and arms the host firewall at first boot. That is the right behaviour for a single server, and it means forming a cluster is a conversion rather than a fresh setup. Three extra things are needed:
+## Step 3. Set Up OVN Networking
 
-**Before Step 3**, stop services on every server:
+Servers 1 to 3 run a clustered **OVN database** — the VPC networking control plane — so it survives losing any one of them. This is the only database limited to three members; storage and NATS run on every server. **Server 1 creates the cluster and must be set up first**; servers 2 and 3 then join it.
 
-```bash
-sudo systemctl stop spinifex.target
-```
+`--recreate-db` appears in each command below because `ovn-central` starts a standalone OVN database when the package installs, and a clustered one can only be created from scratch. It replaces that standalone database on both the binary and ISO paths.
 
-**Before Step 3**, also turn the firewall off on every server:
-
-```bash
-sudo /usr/local/lib/spinifex/spinifex-firewall-apply disable
-```
-
-Each server's firewall currently allows internal cluster traffic only from itself, because that is the whole cluster as far as it knows. Step 3 has servers 2 and 3 connect to server 1's OVN database, and Step 4 has them connect to its formation server — neither is recognised yet, so both are blocked until the firewall is out of the way. Stopping `spinifex.target` does not do this: the firewall lives in the kernel and outlives the services.
-
-Turn it back on after Step 6 — see [Firewall and cluster membership](#firewall-and-cluster-membership).
-
-**In Step 4**, pass `--force` to `spx admin init` and `spx admin join`. Servers 2 and 3 each arrived with their own CA and master key from the single-node install, and joining replaces them with server 1's. `--force` is how you confirm that.
-
-That discard is safe here — nothing has been sealed under those keys on a freshly installed node — but it is unrecoverable on a server that has been in service. `spx admin join` refuses without `--force` for exactly this reason.
-
-## Step 3. Setup OVN Networking
-
-All three servers run a clustered OVN database, so the control plane survives the loss of any one of them. Server 1 creates the cluster and must be set up first; servers 2 and 3 join it.
-
-If your WAN interface is already a bridge, setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC).
-
-> [!IMPORTANT]
-> **`--recreate-db` destroys all logical network state** — every logical switch, router, port and ACL, and so every VPC on that node. It is required here because the `ovn-central` package starts a standalone database on install, and a clustered database can only be created from scratch. That is safe on a freshly installed server and is **not** safe on one already running workloads. If you are converting a server that has been in service, back up its NB database first.
+If your WAN interface is already a bridge, `setup-ovn.sh` auto-detects it. Otherwise add `--wan-bridge=br-wan --wan-iface=eth1` for a dedicated WAN NIC.
 
 **Server 1 — create the cluster:**
 
@@ -204,7 +214,7 @@ sudo /usr/local/share/spinifex/setup-ovn.sh \
   --encap-ip=$SPINIFEX_NODE3
 ```
 
-**Servers 4 and beyond** are compute nodes and run no database. Point them at all three database servers so they survive any one of them failing:
+**Servers 4 and beyond** — repeat for each one, substituting its own address. They point at the three OVN database servers rather than joining the database cluster, so they survive any one of those failing:
 
 ```bash
 sudo /usr/local/share/spinifex/setup-ovn.sh \
@@ -212,30 +222,35 @@ sudo /usr/local/share/spinifex/setup-ovn.sh \
   --encap-ip=$SPINIFEX_NODE4
 ```
 
-Verify the database cluster formed, then that all chassis registered:
+Verify the OVN database cluster formed, then that every chassis registered with it:
 
 ```bash
 sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
 sudo ovn-sbctl show
 ```
 
-`cluster/status` should list three servers with one leader. If it reports a standalone database, the cluster did not form — re-check that `--db-cluster-local-addr` was passed on every database server.
+`cluster/status` should list three servers with one leader, and `ovn-sbctl show` a chassis for every node in the cluster. If `cluster/status` reports a standalone database, the OVN cluster did not form — re-check that `--db-cluster-local-addr` was passed on servers 1, 2 and 3.
 
 ## Step 4. Form the Cluster
 
-Run init and join concurrently — init blocks until all nodes join.
+Run init and join **concurrently** — init blocks until all nodes have joined.
 
-If you installed from the ISO, add `--force` to every command below (see [Converting ISO-Installed Nodes](#converting-iso-installed-nodes)).
+`--force` is in every command below so the sequence is identical whichever way you installed. It does the work on ISO-installed servers, which arrive as their own single-node cluster: joining replaces that server's CA and master key with server 1's, and `--force` is the confirmation. On `spx admin init` it is idempotent — existing keys, credentials and CA are preserved, and only the config files and server certificate are refreshed. On a freshly installed server there is nothing to lose either way.
 
-**Server 1 — Initialize:**
+> [!WARNING]
+> Do not point these commands at a server that has already been in service. Joining discards its master key, orphaning every volume and fragment sealed under it. That is what `--force` overrides, and it is unrecoverable.
+
+**Server 1 — initialize:**
 
 ```bash
-sudo spx admin init \
+sudo spx admin init --force \
   --node node1 --nodes 3 \
   --bind $SPINIFEX_NODE1 --cluster-bind $SPINIFEX_NODE1 \
   --port 4432 --region $AWS_REGION --az $AWS_AZ
 ```
 
+`--nodes 3` is the number of servers init waits for. Set it to your total node count if you are building a larger cluster.
+
 The init output displays the join command including the token:
 
 ```
@@ -247,25 +262,29 @@ The init output displays the join command including the token:
    sudo spx admin join --host 10.0.0.1:4432 --token spx_join_a8Bf3x9Kz2mN --node <name> --bind <ip>
 ```
 
-**Server 2 — Join** (while init is running):
+Take the **token** from that output, but run the commands below rather than the line it prints — they add `--force` and `--cluster-bind`.
+
+**Server 2 — join** (while init is still running):
 
 ```bash
-sudo spx admin join \
+sudo spx admin join --force \
   --node node2 --bind $SPINIFEX_NODE2 --cluster-bind $SPINIFEX_NODE2 \
   --host $SPINIFEX_NODE1:4432 --token <token-from-init-output> \
   --region $AWS_REGION --az $AWS_AZ
 ```
 
-**Server 3 — Join** (while init is running):
+**Server 3 — join** (while init is still running):
 
 ```bash
-sudo spx admin join \
+sudo spx admin join --force \
   --node node3 --bind $SPINIFEX_NODE3 --cluster-bind $SPINIFEX_NODE3 \
   --host $SPINIFEX_NODE1:4432 --token <token-from-init-output> \
   --region $AWS_REGION --az $AWS_AZ
 ```
 
-**Note:** The join token expires 30 minutes after init by default. For larger deployments with slower provisioning, use `--token-ttl 2h`
+Each additional server runs the same join command with its own `--node` name and `--bind` address.
+
+**Note:** the join token expires 30 minutes after init by default. For larger deployments with slower provisioning, use `--token-ttl 2h`.
 
 ## Step 5. Start Services
 
@@ -275,16 +294,77 @@ On **all servers**:
 sudo systemctl start spinifex.target
 ```
 
-## Step 6. Verify
+## Step 6. Verify the Cluster
 
-From any node:
+Run these from any node. Together they confirm that every server joined, that services are healthy on each, and that capacity is being pooled across the cluster.
+
+**1. Every node is present and Ready.**
+
+```bash
+spx get nodes
+```
+
+```
+spinifex@node1:~$ spx get nodes
+NAME  | STATUS | ROLES         | IP       | REGION    | AZ         | UPTIME | VMs | SERVICES
+node1 | Ready  | nats:follower | 10.2.0.2 | us-east-1 | us-east-1a | 21h27m | 0   | nats,predastore,viperblock,daemon,awsgw,vpcd,ui
+node2 | Ready  | nats:follower | 10.2.0.3 | us-east-1 | us-east-1a | 21h27m | 1   | nats,predastore,viperblock,daemon,awsgw,vpcd,ui
+node3 | Ready  | nats:leader   | 10.2.0.4 | us-east-1 | us-east-1a | 21h27m | 0   | nats,predastore,viperblock,daemon,awsgw,vpcd,ui
+```
+
+What to check:
+
+- **Every server you installed is listed.** A missing node never joined — see [Nodes not joining](#nodes-not-joining).
+- **`STATUS` is `Ready`** on all of them. `NotReady` means the node is in the cluster configuration but is not answering, so start with `spinifex.target` on that host.
+- **Exactly one `nats:leader`.** The rest are followers.
+- **`SERVICES` lists the same set on every node.** A short list means something failed to start there; check `systemctl status` for the missing unit.
+
+**2. Capacity is pooled across the cluster.**
+
+```bash
+spx top nodes
+```
+
+```
+spinifex@node1:~$ spx top nodes
+NAME  | CPU (used/total) | MEM (used/total) | GPU (used/total) | VMs
+node1 | 0/64             | 0Mi/220.2Gi      | -                | 0
+node2 | 2/64             | 2.8Gi/251.7Gi    | -                | 1
+node3 | 0/64             | 0Mi/251.7Gi      | -                | 0
+
+
+INSTANCE TYPE | AVAILABLE | VCPU | MEMORY
+c6a.12xlarge  | 3         | 48   | 96.0Gi
+c6a.16xlarge  | 0         | 64   | 128.0Gi
+c6a.24xlarge  | 0         | 96   | 192.0Gi
+c6a.2xlarge   | 21        | 8    | 16.0Gi
+c6a.4xlarge   | 9         | 16   | 32.0Gi
+c6a.8xlarge   | 3         | 32   | 64.0Gi
+c6a.large     | 92        | 2    | 4.0Gi
+c6a.xlarge    | 45        | 4    | 8.0Gi
+c6i.12xlarge  | 3         | 48   | 96.0Gi
+c6i.16xlarge  | 0         | 64   | 128.0Gi
+c6i.24xlarge  | 0         | 96   | 192.0Gi
+c6i.2xlarge   | 21        | 8    | 16.0Gi
+c6i.4xlarge   | 9         | 16   | 32.0Gi
+c6i.8xlarge   | 3         | 32   | 64.0Gi
+c6i.large     | 92        | 2    | 4.0Gi
+c6i.xlarge    | 45        | 4    | 8.0Gi
+m6a.12xlarge  | 3         | 48   | 192.0Gi
+```
+
+The top table is per-node CPU, memory and GPU usage. The bottom table is what the cluster can actually launch right now: `AVAILABLE` is the number of instances of that type that would currently fit across all nodes. A type showing `0` does not fit on any single node — instances are not split across servers, so the largest type you can launch is bounded by your biggest node, not by the cluster total.
+
+If capacity looks like a single server rather than the sum of your nodes, the others have not joined.
+
+**3. The AWS API answers.**
 
 ```bash
 export AWS_PROFILE=spinifex
 aws ec2 describe-instance-types
 ```
 
-If this returns a list of available instance types, your cluster is working.
+A list of instance types means the gateway, IAM and the cluster behind them are all working.
 
 **Congratulations! Your Spinifex cluster is installed.**
 
@@ -294,7 +374,7 @@ Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import a
 
 Spinifex ships an optional host firewall. It divides the node's ports into two groups:
 
-| | Ports | Who can reach them |
+| Group | Ports | Who can reach them |
 |---|---|---|
 | **Public** | SSH, 443, 3000 (console), 8443 (S3), 9999 (AWS gateway), 53 (DNS) | anyone |
 | **Internal** | OVN, NATS, formation, Geneve and the rest of the cluster plane | **cluster members only** |
@@ -308,7 +388,7 @@ The internal group is the point. Before this existed, OVN and NATS were reachabl
 | How the node was installed | Firewall |
 |---|---|
 | From the ISO | **on** |
-| `curl \| bash` or `setup.sh` | **off** |
+| Binary installer (`curl \| bash`) or `setup.sh` | **off** |
 | `setup.sh --firewall=on` | **on** |
 
 The binary installer defaults to off deliberately: it runs on servers that already have an operating system and services on them, and switching on a default-deny policy uninvited could cut off something Spinifex knows nothing about. **For production, turn it on** — either at install time:
@@ -343,10 +423,12 @@ Restarting the daemon is what re-arms it: the node rebuilds its peer list from t
 ### Checking it
 
 ```bash
-sudo nft list table inet spinifex_filter | grep spinifex_peers
+sudo nft list table inet spinifex_filter
 ```
 
-Every node's addresses should be listed — on a multi-NIC node, that is its WAN, LAN and VPC addresses, so expect several entries per node. A missing node means its cluster traffic is being dropped.
+The peer list is an nft variable, expanded when the rules load, so it does not appear under a name of its own. Look instead at the `ip saddr { ... }` addresses on the cluster-plane rules — the ones accepting 4222, 6641, 6642 and the rest.
+
+Every node's addresses should be there. On a multi-NIC node that means its WAN, LAN and VPC addresses, so expect several entries per node. A missing node means its cluster traffic is being dropped.
 
 Dropped packets are logged, rate-limited, so this tells you whether a connection problem is the firewall or something else:
 
@@ -354,9 +436,21 @@ Dropped packets are logged, rate-limited, so this tells you whether a connection
 sudo journalctl -k | grep 'spinifex-fw drop'
 ```
 
-### Adding a node later
+## How Multi-Node Storage Works
 
-Expanding an existing cluster is the same shape as forming one: the node you are adding is not a member yet. Turn the firewall off on the existing nodes and on the new one, add the node, then re-arm all of them. Verify with the `nft list` command above that every node now lists the newcomer.
+Background reading — you do not configure any of this by hand. `spx admin init` and `spx admin join` build the topology from the servers that actually form the cluster in Step 4, and each machine gets the same file with its own host ID recorded in `spinifex.toml`.
+
+Predastore is configured for the whole cluster in `/etc/spinifex/predastore/predastore.toml`. Each server is one `[[host]]` — a single Predastore process owning that machine's data directory and TLS identity — carrying three nodes under `[[host.node]]`:
+
+| Role | Port | Purpose |
+|---|---|---|
+| `gate` | TCP 8443 | Serves the S3 API. Every server runs one, so any of them answers an S3 request. |
+| `blob` | UDP 6660 | Holds erasure-coded object shards. One per machine. |
+| `meta` | UDP 7660 | Member of the Raft quorum over global state — buckets and the object index. |
+
+Ports have to be unique within a host but not across the cluster, so every machine uses the same three. Blob and meta traffic between hosts runs over QUIC, authenticated by the cluster CA; nodes on the same machine talk over an in-process pipe and open no socket, which is why a single-server install listens on 8443 alone.
+
+Reed-Solomon parameters are chosen from the cluster size, since each machine contributes exactly one blob node: two servers get `RS(1,1)`, three or more get `RS(2,1)`. `RS(2,1)` survives the loss of any one server's shards — another reason three nodes is the recommended minimum.
 
 ## Troubleshooting
 
@@ -368,7 +462,9 @@ The init command must still be running when join executes. If init exited, re-ru
 curl -sk https://$SPINIFEX_NODE1:4432/health
 ```
 
-If that command hangs rather than failing quickly, it is the firewall on node 1 — a refused connection means nothing is listening, but a hang means the packets are being dropped. Node 1 does not recognise the joining node as a cluster member yet. Confirm it on node 1:
+A hang rather than a quick failure means packets are being dropped; a refused connection means nothing is listening.
+
+If it hangs, check node 1's init output before blaming the firewall — `spx admin init` opens the formation port itself while it waits, so this is usually not the cause. It prints `⚠️ Could not open port 4432 in the host firewall` when that fails, which is the case where it is. Confirm on node 1:
 
 ```bash
 sudo journalctl -k | grep 'spinifex-fw drop'
@@ -388,7 +484,7 @@ Safe on a freshly installed node. On one that has been in service it orphans eve
 sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
 ```
 
-If this reports a standalone database rather than three servers, the database was created before the cluster flags were supplied. A clustered database can only be created from scratch — re-run Step 3 with `--recreate-db`, remembering that it discards all logical network state.
+If this reports a standalone database rather than three servers, the OVN database was created before the cluster flags were supplied. A clustered one can only be created from scratch — re-run Step 3 with `--recreate-db`.
 
 ### OVN Chassis Not Registering
 

docs/install/install-source/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Source Install"
-description: "Build Spinifex from source for development, custom builds, or contributing."
+seoTitle: "Build and Install Spinifex from Source — Spinifex Docs"
+description: "Build Spinifex from source on Ubuntu or Debian for development, custom builds, or contributing changes, then install and run the resulting binaries locally."
 category: "Install"
 tags:
   - install

docs/install/install-usb/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Bootable USB Install"
-description: "Install Spinifex on bare-metal hardware by flashing the Spinifex ISO to a USB drive and booting the target device from it."
+seoTitle: "Install Spinifex from a Bootable USB — Spinifex Docs"
+description: "Install Spinifex on bare-metal x86 hardware by flashing the Spinifex ISO to a USB drive, booting the target server from it, and wiping the disk you select."
 category: "Install"
 sections:
   - overview
@@ -134,7 +135,7 @@ AWS_PROFILE=spinifex aws ec2 describe-instance-types
 
 That profile is the operator account, with administrator access. Copy it to your workstation to drive the node remotely — you will also need the cluster CA, available unauthenticated from `https://<node-address>:3000/api/ca.pem`.
 
-From here, [Setting Up Your Cluster](/docs/admin/setting-up-your-cluster) walks through importing an AMI, creating a key pair and a VPC, and launching your first instance.
+From here, [Setting Up Your Cluster](/docs/setting-up-your-cluster) walks through importing an AMI, creating a key pair and a VPC, and launching your first instance.
 
 ### Building a Cluster
 
@@ -179,10 +180,10 @@ sudo systemctl restart spinifex-daemon
 The node rewrites its peer list from the cluster it is now part of and re-arms itself, including at boot. Confirm every server can see the others:
 
 ```bash
-sudo nft list table inet spinifex_filter | grep spinifex_peers
+sudo nft list table inet spinifex_filter
 ```
 
-The addresses of **all** your servers should appear. If one is missing, that server's cluster traffic will be blocked — see [Firewall and cluster membership](/docs/install-multi-node#firewall-and-cluster-membership).
+Check the `ip saddr { ... }` addresses on the cluster-plane rules — the peer list is an nft variable expanded at load time, so it has no name of its own in the output. The addresses of **all** your servers should appear. If one is missing, that server's cluster traffic will be blocked — see [Firewall and cluster membership](/docs/install-multi-node#firewall-and-cluster-membership).
 
 ### Setup Complete
 

docs/install/install/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Single-Node Install"
-description: "Install Spinifex on a single server using the binary installer."
+seoTitle: "Install Spinifex on a Single Server — Spinifex Docs"
+description: "Install Spinifex on one Ubuntu or Debian server with the binary installer and get an AWS-compatible EC2, S3, and VPC stack running on your own hardware."
 category: "Install"
 tags:
   - install

docs/migration/hybrid-sync/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Hybrid Sync"
-description: "Synchronize data between Spinifex and AWS when network connectivity is available."
+seoTitle: "Hybrid Sync Between Spinifex and AWS — Spinifex Docs"
+description: "Synchronise data bidirectionally between Spinifex and AWS when connectivity allows, so local infrastructure stays usable at intermittently connected sites."
 category: "Migration"
 tags:
   - hybrid

docs/migration/moving-aws-workload/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Moving an AWS Workload to Mulga"
-description: "Migrate existing AWS workloads to Spinifex using compatible APIs, SDKs, and Terraform."
+seoTitle: "Migrate an AWS Workload to Spinifex — Spinifex Docs"
+description: "Move existing AWS workloads onto Spinifex using compatible APIs, SDKs, and Terraform across EC2, VPC, EBS, S3, IAM, STS, ELBv2, ACM, ECR, ECS, and EKS."
 category: "Migration"
 tags:
   - migration

docs/reference-architectures/OnLogic-HX401-single-node/README.md

@@ -1,22 +1,26 @@
 ---
 title: "Single-Node Spinifex on OnLogic HX401"
-description: "Run a complete EC2, EBS and S3 stack on one fanless OnLogic edge node, and measure what the platform costs against bare metal."
+seoTitle: "Single-Node Spinifex on OnLogic HX401 — Spinifex Docs"
+description: "EC2, EBS and S3 on a single fanless OnLogic HX401 — the full Spinifex service set on a passive-cooled industrial node for factory and remote edge deployments."
 category: "Reference Architectures"
 tags:
   - onlogic
   - hx401
   - edge
   - single-node
-  - benchmarks
   - viperblock
   - predastore
 resources:
-  - title: "Installation"
+  - title: "Single-Node Install"
     url: "/docs/install"
+  - title: "VPC Networking"
+    url: "/docs/vpc-networking"
   - title: "Launching Instances"
     url: "/docs/launching-instances"
   - title: "OnLogic HX401"
     url: "https://www.onlogic.com/hx401/"
+  - title: "Benchmark data and platform notes (OnLogic-refarch)"
+    url: "https://github.com/tomnewton-mulga/OnLogic-refarch"
 sections:
   - overview
   - prerequisites
@@ -25,11 +29,9 @@ sections:
 
 ## Overview
 
-Spinifex is an open-source infrastructure platform that brings core AWS services — EC2, EBS and S3 — to bare-metal, edge and on-prem environments. It exposes an EC2-compatible API, so tooling that works against AWS (the `aws` CLI, Terraform, SDKs) works against a Spinifex node unchanged, with a single profile swap.
+The [OnLogic HX401](https://www.onlogic.com/hx401/) is a fanless, passive-cooled industrial edge computer that runs the complete Spinifex service set — EC2, EBS, S3 and VPC networking — in a form factor small enough to panel-mount inside a factory cabinet, attach to a DIN rail in a control room, or deploy unattended at a remote site without active cooling or rack infrastructure. Spinifex exposes EC2, EBS and S3-compatible APIs on the node, so the operational model is unchanged from AWS: use the `aws` CLI, SDKs, Terraform or OpenTofu, and CI/CD pipelines already used for cloud workloads. The change is the endpoint, not the workflow.
 
-This document covers a node deployment of Spinifex on the OnLogic HX401. The OnLogic HX401 is a fanless industrial box with 12 physical cores and 31 GiB of RAM — enough to run the full service set and still host a meaningful guest.
-
-The goal of this deployment is to measure what the platform costs against bare metal — CPU overhead from virtualisation, guest block storage throughput, and S3 data-path performance. The same benchmark suite was run against both Spinifex v1.15.0 and v1.16.0 on the same hardware, to help the team quantify the efficacy of the new releases's fixes and improvements.
+Single-node removes the distributed-storage guarantees of a multi-node cluster — Predastore runs RS(1,0) on one disk, so there is no erasure-coded redundancy. For edge deployments where data is generated locally, processed on-node, and shipped upstream on a schedule, or where the box provides compute capacity next to a sensor or control network rather than acting as a primary data store, this is the expected and appropriate configuration.
 
 <p align="center"><img src="../../../.github/assets/images/onlogic/HX401.png" alt="OnLogic HX401 fanless edge node"></p>
 
@@ -37,25 +39,33 @@ The goal of this deployment is to measure what the platform costs against bare m
 
 | Component | Specification |
 |---|---|
-| **Bare-metal host** | OnLogic HX401 |
+| **Bare-metal host** | [OnLogic HX401](https://www.onlogic.com/hx401/) — fanless, passive-cooled industrial edge computer |
 | **CPU** | Intel i5-1250PE — 12 physical cores / 16 threads |
 | **Memory** | 31 GiB |
 | **Storage** | Transcend TS256GMTE652T2, 256 GB NVMe (PCIe Gen3 x4) |
+| **Networking** | 2× Intel GbE (I210-IT + I219-LM) |
 | **Host OS** | Debian 13 |
-| **Orchestration** | Spinifex v1.16.0 — EC2-compatible bare-metal API |
 | **Guest OS** | Ubuntu 26.04 LTS |
 | **Instance type** | `c6i.2xlarge` — 8 vCPU, 16 GiB |
-| **Block storage** | Viperblock — EBS-compatible, local NVMe-backed |
-| **Object storage** | Predastore — S3-compatible, RS(1,0) on a single node |
-| **Networking** | OVN, standalone NB/SB on the node itself |
+| **Form factor** | Fanless, passive-cooled; DIN rail, wall and VESA mountable |
 
+## Architecture
+
+### AWS services exercised
+
+| Service | Role |
+|---|---|
+| **EC2** | Guest instances on the host NVMe; no spread placement groups on a single node |
+| **EBS (Viperblock)** | Local NVMe-backed block volumes per instance, surviving termination independently when `delete_on_termination = false` |
+| **S3 (Predastore)** | RS(1,0) single-node object storage — no erasure-coding overhead, no distributed redundancy |
+| **VPC (OVN)** | Standalone OVN northbound/southbound databases on the node itself |
 
 ## Prerequisites
 
 - OnLogic HX401 (or comparable x86 node) running **Debian 13** or **Ubuntu 26.04 LTS**
 - The WAN interface enslaved to a Linux bridge named `br-wan` — the host IP, default route and DHCP must live on the bridge, not the bare NIC
-- A reserved range of addresses on your LAN for guest instances (this deployment uses `192.168.157.201-250`)
-- AWS CLI installed
+- A reserved range of addresses on your LAN for guest instances — this deployment uses `192.168.157.201-250`
+- AWS CLI configured with `AWS_PROFILE=spinifex` pointing at the node's EC2-compatible endpoint (`https://<host>:9999`)
 
 Verify the bridge before installing:
 
@@ -66,11 +76,11 @@ ip route
 
 ## Instructions
 
-### 1. Install Spinifex and initialise the node
+### 1. Install Spinifex and verify the node
 
-Follow the [Single Node Install](/docs/install) guide, using `--nodes 1` to select the single-node templates — RS(1,0) storage and a one-member NATS.
+Follow the [Single-Node Install](/docs/install) guide, using `--nodes 1` to select the single-node templates — RS(1,0) storage and a single-member NATS cluster.
 
-**Set a static external pool.** `init` auto-detects the external network and will select `source = "dhcp"` when the host itself holds a DHCP lease. That does not work for guests on every network: the upstream server answers the host on `br-wan` but does not necessarily offer to guest ENI MAC addresses, and instance launches then fail with
+**Set a static external pool.** `init` auto-detects the external network and selects `source = "dhcp"` when the host holds a DHCP lease. That does not work for guests on every network: the upstream DHCP server answers the host on `br-wan` but does not necessarily offer addresses to guest ENI MAC addresses, causing instance launches to fail with:
 
 ```text
 PrepareRunInstances: public IP allocation failed — aborting launch
@@ -90,7 +100,7 @@ prefix_len  = 24
 dns_servers = ["1.1.1.1", "8.8.8.8"]
 ```
 
-See the [VPC Networking](/docs/vpc-networking) guide for full configuration options. Then start the platform and verify:
+See the [VPC Networking](/docs/vpc-networking) guide for full configuration options. Then start the platform and confirm the node is ready:
 
 ```bash
 sudo systemctl start spinifex.target
@@ -102,187 +112,156 @@ NAME  | STATUS | ROLES       | IP              | REGION         | AZ
 node1 | Ready  | nats:leader | 192.168.157.134 | ap-southeast-2 | ap-southeast-2a | 0   | nats,predastore,viperblock,daemon,awsgw,vpcd,ui
 ```
 
-### 2. Establish the bare-metal baseline
+All services — Predastore, Viperblock, and the OVN databases — run on the single node. There is no Raft group to converge and no leader election to wait for.
 
-Benchmark the host before launching anything, with the platform idle. Without this, a slow guest result cannot be attributed to the platform rather than the hardware.
+### 2. Launch instances through the EC2-compatible API
 
-This uses `spx-bench.sh` — the same script is run on the host and inside the guest so the two results diff directly. It measures CPU (sysbench), memory, and disk (fio: `randrw` 70/30 4k, `randread` 4k, `seqread` 1M, `seqwrite` 1M, all at 4 jobs / iodepth 32 / `direct=1`).
+Point an existing AWS profile at the Spinifex endpoint and use the standard EC2 instance lifecycle:
+
+```bash
+export AWS_PROFILE=spinifex
+
+aws ec2 run-instances \
+  --image-id "$AMI" --instance-type c6i.2xlarge \
+  --count 1 \
+  --subnet-id "$SUBNET" --security-group-ids "$SECURITY_GROUP" \
+  --key-name "$KEY_NAME"
+```
+
+`c6i.2xlarge` provisions 8 vCPU and 16 GiB from the host's 12 cores and 31 GiB, leaving headroom for the platform services. On a single node there are no spread placement groups — all instances land on the same physical host, which is expected.
+
+### 3. Attach Viperblock volumes and access Predastore object storage
+
+Create and attach a dedicated data volume. The default root volume on the stock Ubuntu AMI is 4 GiB — too small for most workloads. Attach a separate volume and benchmark or write data there:
+
+```bash
+VOLUME_ID=$(aws ec2 create-volume \
+  --availability-zone ap-southeast-2a \
+  --size 30 --volume-type gp2 \
+  --query VolumeId --output text)
+
+aws ec2 attach-volume \
+  --volume-id "$VOLUME_ID" \
+  --instance-id "$INSTANCE_ID" \
+  --device /dev/sdf
+```
+
+Inside the guest, format and mount:
+
+```bash
+sudo mkfs.ext4 -q -L data /dev/vdb
+sudo mkdir -p /mnt/data && sudo mount /dev/vdb /mnt/data
+```
+
+Setting `delete_on_termination = false` in Terraform keeps the volume alive across instance replacement — relaunching a terminated instance re-attaches the same volume rather than starting from empty storage:
+
+```hcl
+resource "aws_ebs_volume" "data" {
+  availability_zone = var.az
+  size              = 30
+  type              = "gp2"
+}
+
+resource "aws_volume_attachment" "data" {
+  device_name           = "/dev/sdf"
+  volume_id             = aws_ebs_volume.data.id
+  instance_id           = aws_instance.worker.id
+  delete_on_termination = false
+}
+```
+
+Predastore presents an S3-compatible endpoint — standard S3 tooling works without modification:
+
+```bash
+aws s3 mb s3://my-bucket
+aws s3 cp ./data.tar.gz s3://my-bucket/
+aws s3 sync ./results/ s3://my-bucket/results/
+```
+
+### 4. Storage benchmark
+
+Run this after setup or hardware changes to confirm both storage services are performing as expected. The methodology: fio against an ext4-formatted Viperblock data volume (four jobs, iodepth 32, direct I/O, 30-second run), with a bare-metal host baseline run first so the two results diff directly using the same `spx-bench.sh` script.
+
+**Host bare-metal baseline** (platform idle, no guests):
 
 ```bash
 ./spx-bench.sh --tag host-baremetal
 ```
 
 ```text
---- 2. CPU (sysbench, events/sec — higher is better) ------------
+--- CPU (sysbench, events/sec) ---------------------------------
   threads=1         1456.50 events/s   95th 0.70 ms
   threads=16       12822.85 events/s   95th 2.00 ms
 
---- 3. Memory --------------------------------------------------
-  random write, 16 threads: 1555.84 MiB/sec
-
---- 4. Disk (file-backed, NVMe root) ---------------------------
-  randrw 70/30 4k    read      66,169 IOPS    258 MiB/s   p99   10.42 ms
-  randrw 70/30 4k    write     28,426 IOPS    111 MiB/s   p99    0.72 ms
-  randread 4k        read     171,326 IOPS    669 MiB/s   p99    2.06 ms
-  seqread 1M         read       1,818 IOPS  1,818 MiB/s   p99  128.45 ms
-  seqwrite 1M        write        308 IOPS    308 MiB/s   p99 1115.68 ms
+--- Disk (fio, NVMe root) --------------------------------------
+  randrw 70/30 4k    read      66,169 IOPS    258 MiB/s
+  randrw 70/30 4k    write     28,426 IOPS    111 MiB/s
+  randread 4k        read     171,326 IOPS    669 MiB/s
+  seqread 1M         read       1,818 MiB/s
+  seqwrite 1M        write        308 MiB/s
 ```
 
-### 3. Launch a guest instance
-
-```bash
-aws ec2 run-instances \
-  --image-id "$AMI" --instance-type c6i.2xlarge \
-  --key-name onlogic --security-group-ids "$SG" --subnet-id "$SUBNET"
-```
-
-**Attach a data volume for the benchmark.** The stock `ubuntu-26.04-x86_64` AMI declares a 4 GiB root which fills just installing benchmark tools, and `run-instances` does not grow it without explicit `--block-device-mappings`. Every fio job fails with `ENOSPC` if you benchmark the root. Attach a gp3 volume instead, which is also the more representative target:
-
-```bash
-VOL=$(aws ec2 create-volume --availability-zone ap-southeast-2a \
-  --size 30 --volume-type gp3 --query VolumeId --output text)
-aws ec2 attach-volume --volume-id "$VOL" --instance-id "$ID" --device /dev/vdb
-```
-
-```bash
-# in the guest
-sudo mkfs.ext4 -q -L bench /dev/vdb
-sudo mkdir -p /mnt/bench && sudo mount /dev/vdb /mnt/bench
-```
-
-### 4. Measure the guest
+**Guest** (`c6i.2xlarge`, 8 vCPU, attached gp2 volume):
 
 ```bash
 BENCHDIR=/mnt/bench ./spx-bench.sh --tag guest --require-mount
 ```
 
-```text
---- 2. CPU (sysbench, events/sec — higher is better) ------------
-  threads=1         1357.98 events/s   95th 0.75 ms
-  threads=8         8608.31 events/s   95th 1.06 ms
-
---- 3. Memory --------------------------------------------------
-  random write, 8 threads: 1133.25 MiB/sec
-```
-
-### 5. Measure the S3 data path
-
-Predastore's S3 API listens on port `8443`. This is **not** the general AWS gateway endpoint on `9999`.
-
-```bash
-./s3-bench.sh 256    # 256 MB object, write + read + checksum verify
-```
-
-## Results
-
-### CPU and memory
-
-| | host | guest | guest/host |
-|---|---|---|---|
-| sysbench, 1 thread | 1,456 events/s | 1,358 events/s | **93.2%** |
-| sysbench, all threads | 12,823 (16t) | 8,608 (8t) | not comparable |
-| Memory random write | 1,556 MiB/s (16t) | 1,133 MiB/s (8t) | not comparable |
-
-Single thread is the only directly comparable row, and **93.2% of bare metal** is the headline CPU number — roughly 7% single-thread virtualisation overhead. The multi-thread and memory rows use different thread counts (16 on the host, 8 in the guest) and are listed for completeness, not as ratios.
-
-<p align="center"><img src="images/cpu-comparison.png" alt="sysbench single-thread: host 1,456 events/s vs guest 1,358 events/s (93.2%)"></p>
-
-### Storage
-
-Median of three runs. Guest measured on the attached gp3 volume.
-
-| Test | host | guest | guest/host |
-|---|---|---|---|
-| randrw 70/30 4k read | 66,169 IOPS | 6,860 IOPS | 10.4% |
-| randrw 70/30 4k write | 28,426 IOPS | 2,951 IOPS | 10.4% |
+| Test | Host | Guest | Guest/Host |
+|---|---:|---:|---:|
+| sysbench single-thread | 1,456 events/s | 1,358 events/s | **93.2%** |
+| randrw 4k read | 66,169 IOPS | 6,860 IOPS | 10.4% |
+| randrw 4k write | 28,426 IOPS | 2,951 IOPS | 10.4% |
 | randread 4k | 171,326 IOPS | 9,263 IOPS | 5.4% |
 | seqread 1M | 1,818 MiB/s | 484 MiB/s | 26.6% |
 | seqwrite 1M | 308 MiB/s | 73 MiB/s | 23.7% |
 
-The block path is where virtualisation costs real throughput. Sequential work retains roughly a quarter of bare metal; small random I/O retains between a twentieth and a tenth.
-
 <p align="center">
-<img src="images/storage-iops-comparison.png" alt="Random I/O IOPS: host vs guest">
-<img src="images/storage-seq-comparison.png" alt="Sequential throughput MiB/s: host vs guest">
+<img src="images/cpu-comparison.png" alt="sysbench single-thread: host 1,456 events/s vs guest 1,358 events/s (93.2%)">
 </p>
 
-**Read the guest storage figures with their variance, not as point values.** Across three runs:
+Guest CPU lands at **93.2% of bare metal** — roughly 7% single-thread virtualisation overhead, a result that reproduced on separate hardware. Block storage reflects the cost of the current single-queue virtio-blk path and unthreaded NBD backend sharing a single 256 GB NVMe between the host OS, platform services, and guest volumes; these are the same implementation constraints as on the Cisco cluster, expressed more sharply on constrained hardware.
 
-| Test | run 1 | run 2 | run 3 | spread |
-|---|---|---|---|---|
+Variance across three runs is notable for two metrics:
+
+| Test | Run 1 | Run 2 | Run 3 | Spread |
+|---|---:|---:|---:|---:|
 | randrw 4k read | 7,321 | 6,424 | 6,860 | 14% |
 | randread 4k | 9,959 | **3,813** | 9,263 | **2.6×** |
 | seqread 1M | 518 | 484 | 453 | 14% |
 | seqwrite 1M | 83 | 73 | **31** | **2.7×** |
 | seqwrite p99 | 7.95 s | 7.28 s | **15.90 s** | 2.2× |
 
-<p align="center"><img src="images/storage-variance.png" alt="Guest storage variance across 3 runs: randread and seqwrite show bimodal behaviour"></p>
+<p align="center">
+<img src="images/storage-iops-comparison.png" alt="Random I/O IOPS: host vs guest">
+<img src="images/storage-seq-comparison.png" alt="Sequential throughput MiB/s: host vs guest">
+<img src="images/storage-variance.png" alt="Guest storage variance across 3 runs: randread and seqwrite show bimodal behaviour">
+</p>
 
-`randread` and `seqwrite` are bimodal rather than noisy around a mean — one run in three falls to roughly a third of its neighbours. The likely mechanism is a viperblock WAL flush or a Predastore compaction cycle landing inside some runs and not others. Anything below about a 3× change in those two metrics is not resolvable at this sample size.
+`randread` and `seqwrite` are bimodal rather than noisy — one run in three falls to roughly a third of its neighbours, likely a Viperblock WAL flush or Predastore compaction cycle intersecting some runs but not others. Treat those two as directionally correct rather than stable point values at this sample size. The `seqwrite` p99 reaching 7–16 seconds is worth noting for write-heavy workloads.
 
-`seqwrite` p99 reaching 7-16 seconds is worth attention before committing this platform to write-heavy edge workloads.
+**Predastore S3 validation** (256 MiB objects, three runs, write + read + checksum verify):
 
-## Release comparison: v1.15.0 vs v1.16.0
-
-Both releases were built from their release tags and installed on the same machine, each with a full re-initialisation, an identical `c6i.2xlarge` guest, and an identical 30 GiB gp3 volume. Three runs each.
-
-### S3 / Predastore
-
-| | v1.15.0 | v1.16.0 | change |
-|---|---|---|---|
-| write | 77.7 / 75.5 / 72.0 → **75.5 MiB/s** | 146.6 / 142.0 / 148.2 → **146.6 MiB/s** | **1.94× faster** |
-| read | 249.9 / 234.9 / 243.8 → **243.8 MiB/s** | 298.1 / 301.8 / 290.9 → **298.1 MiB/s** | **1.22× faster** |
-
-Checksums matched on every run. Run-to-run spread is about 4%, so both differences are far outside the noise.
-
-**The mechanism is a changed default, not a faster code path.** On a single node:
-
-```text
-v1.15.0   [rs] data = 2, parity = 1     6 host entries
-v1.16.0   [rs] data = 1, parity = 0     1 host entry
+```bash
+./s3-bench.sh 256
 ```
 
-v1.15.0 splits every object into two data shards plus a parity shard and writes all three to the same disk — 1.5× write amplification plus the erasure-coding cost, buying no redundancy, because there is only one machine to lose. v1.16.0 ships a single-node storage template that does none of that.
-
-The write path gains most (1.94×), which is what removing amplification and parity computation predicts. Reads gain less (1.22×) because the read path never had to reconstruct anything, only reassemble shards.
-
-Anyone still running v1.15.0 on a single node should note that this is a configuration change as much as a software one.
+| | v1.15.0 | v1.16.0 | Change |
+|---|---:|---:|---:|
+| Write | 75.5 MiB/s | 146.6 MiB/s | **1.94×** |
+| Read | 243.8 MiB/s | 298.1 MiB/s | **1.22×** |
 
 <p align="center"><img src="images/s3-version-comparison.png" alt="S3 throughput: v1.15.0 vs v1.16.0 — 1.94× write, 1.22× read"></p>
 
-### Guest block path
+The v1.16.0 write improvement comes from the single-node storage template switching to RS(1,0): v1.15.0 split every object into two data shards plus a parity shard and wrote all three to the same disk, paying 1.5× write amplification for redundancy a single-node cluster cannot deliver. The read gain is smaller because the read path never had to reconstruct anything. All checksum validations passed on every run.
 
-Medians of three runs, with the full range in brackets.
+Raw fio output, S3 results, and the v1.15 / v1.16 annotated comparisons are in the
+[benchmark repository](https://github.com/tomnewton-mulga/OnLogic-refarch).
 
-| Test | v1.15.0 | v1.16.0 | v1.16 gain |
-|---|---|---|---|
-| randrw 70/30 4k read | **5,075** IOPS [4,104-6,236] | **6,860** IOPS [6,424-7,321] | 1.35× |
-| randrw 70/30 4k write | **2,191** IOPS [1,771-2,684] | **2,951** IOPS [2,764-3,147] | 1.35× |
-| randread 4k | **6,658** IOPS [4,881-6,813] | **9,263** IOPS [3,813-9,959] | 1.39× |
-| seqread 1M | **387** MiB/s [370-402] | **484** MiB/s [453-518] | 1.25× |
-| seqwrite 1M | **52** MiB/s [40-58] | **73** MiB/s [31-83] | 1.40× |
-| seqwrite p99 | **13.89 s** [13.76-17.11] | **7.95 s** [7.28-15.90] | 1.75× lower |
+## What this architecture unlocks
 
-<p align="center"><img src="images/block-path-version-comparison.png" alt="Guest block path: v1.15.0 vs v1.16.0 with full run ranges"></p>
-
-v1.16.0 is ahead on every median, by 1.25× to 1.40×. For `randrw` read, `randrw` write and `seqread` the two ranges **do not overlap at all**, so those differences are credible despite only three runs each. For `randread` and `seqwrite` the ranges do overlap — entirely because of v1.16.0's bimodal low outliers — so treat those two as directionally consistent rather than demonstrated.
-
-This is the indirect effect of the storage template change. Viperblock's own code is unchanged between the releases, but it writes through Predastore, so removing the 1.5× write amplification and parity computation underneath reduces the work behind every guest block flush as well.
-
-## Teardown
-
-```bash
-aws ec2 terminate-instances --instance-ids "$ID"
-aws ec2 delete-volume --volume-id "$VOL"      # after the instance is terminated
-aws ec2 delete-security-group --group-id "$SG"
-```
-
-Terminate instances before deleting attached volumes — the volume detaches with the instance, not before it.
-
-## Conclusion
-
-A single OnLogic HX401 runs the complete Spinifex service set — EC2, EBS, S3, VPC networking and DNS — and hosts an 8 vCPU guest with room to spare. Guest CPU performance lands at **93.2% of bare metal**, a figure that reproduced on separate hardware.
-
-The release comparison is the most actionable result here. Moving from v1.15.0 to v1.16.0 on identical hardware gave **1.94× S3 write throughput** and **1.25-1.40× on every guest block metric** — not because the data paths were rewritten, but because the single-node storage template stopped erasure-coding data across a machine that has only one disk. v1.15.0 wrote two data shards and a parity shard to the same NVMe, paying 1.5× write amplification and the encode cost for redundancy it could never deliver.
-
-Every operation in this document used standard `aws ec2` and `aws s3api` calls against Spinifex's EC2-compatible endpoint. Teams already operating AWS tooling can point it at an edge node with a profile change.
+- **Cloud tooling at the edge:** run EC2, EBS and S3 workflows on hardware that fits inside a panel, on a DIN rail, or in a small enclosure at a factory, retail site, branch or remote location — without a rack, a UPS, or active cooling.
+- **No-moving-parts reliability:** the fanless passive-cooled design tolerates dusty, vibration-prone, or thermally variable environments where server-class hardware is impractical to operate or maintain.
+- **Familiar operations:** keep AWS CLI profiles, Terraform or OpenTofu modules, SDKs and deployment pipelines rather than adopting a separate edge-only platform.
+- **Local processing with S3 access:** workloads running in EC2-compatible instances read from and write to Predastore over the standard S3 API — sensor data, inference outputs and logs remain on-node until explicitly synced upstream.
+- **A path back to cloud:** the same AWS API surface at the edge and on AWS makes workload movement, cloud burst, and consistent application packaging straightforward — the profile swap that points tooling at the HX401 is the same one that points it back at AWS.

docs/reference-architectures/RTX-Pro-6000-EKS/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Spinifex EKS AI Platform on Dual RTX Pro 6000 Baremetal"
-description: "Deploy a GPU-accelerated AI inference platform — an OpenAI-compatible LLM API and a real-time CV stream — in Kubernetes on bare-metal hardware, managed entirely with standard AWS tooling."
+seoTitle: "EKS AI Platform on Dual RTX Pro 6000 — Spinifex Docs"
+description: "Deploy a GPU-accelerated AI inference platform — an OpenAI-compatible LLM API and a real-time CV stream — on Kubernetes on bare metal with standard AWS tooling."
 category: "Reference Architectures"
 tags:
   - nvidia

docs/reference-architectures/SMCI-H14-orchestration/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Multi-Tenant AI on Supermicro H14 with AMD MI350X"
-description: "Provision isolated GPU VMs for simultaneous AI workloads on a Supermicro H14 bare-metal node using Spinifex's EC2-compatible API."
+seoTitle: "Multi-Tenant AI on Supermicro H14 MI350X — Spinifex Docs"
+description: "Provision isolated GPU VMs for simultaneous AI workloads on a Supermicro H14 bare-metal node with AMD MI350X GPUs, using Spinifex's EC2-compatible API."
 category: "Reference Architectures"
 tags:
   - supermicro

docs/reference-architectures/SMCI-MIG-orchestration/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Mixed AI Workloads on a Single H200 Chassis: Guest-Managed MIG"
-description: "Install Spinifex from source, configure host-local networking, attach Predastore storage, and run four concurrent AI workloads across MIG partitions managed inside guest VMs."
+seoTitle: "Guest-Managed MIG on an H200 Chassis — Spinifex Docs"
+description: "Install Spinifex from source, configure host-local networking, attach Predastore storage, and run four concurrent AI workloads across guest-managed MIG slices."
 category: "Reference Architectures"
 tags:
   - nvidia

docs/reference-architectures/arcee-trinity-mi350x/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Arcee Trinity 400B FP8 on AMD MI350X with Spinifex"
-description: "Provision, load, and benchmark arcee-ai/Trinity-Large-Preview-FP8 across two AMD MI350X GPUs on a Supermicro H14 using Spinifex's EC2-compatible API."
+seoTitle: "Arcee Trinity 400B FP8 on AMD MI350X — Spinifex Docs"
+description: "Provision, load, and benchmark arcee-ai/Trinity-Large-Preview-FP8 across two AMD MI350X GPUs on a Supermicro H14 node using Spinifex's EC2-compatible API."
 category: "Reference Architectures"
 tags:
   - supermicro

docs/reference-architectures/cisco-ucs-llm-serving/README.md

@@ -1,6 +1,7 @@
 ---
 title: "vLLM Serving on Cisco UCS: Intel AMX vs NVIDIA L4"
-description: "Qwen2.5-7B-Instruct served with vLLM on the same Cisco UCS Spinifex cluster, comparing Intel AMX-accelerated CPU serving against NVIDIA L4 GPU serving across input length and concurrency."
+seoTitle: "vLLM on Cisco UCS: Intel AMX vs NVIDIA L4 — Spinifex Docs"
+description: "Qwen2.5-7B-Instruct served with vLLM on a Cisco UCS Spinifex cluster, comparing Intel AMX-accelerated CPU serving with NVIDIA L4 GPU serving under concurrency."
 category: "Reference Architectures"
 tags:
   - cisco
@@ -17,12 +18,12 @@ resources:
     url: "/docs/launching-instances"
   - title: "GPU Passthrough"
     url: "/docs/gpu-passthrough"
-  - title: "Platform benchmark (companion document)"
-    url: "../cisco-ucs-platform-benchmark/README.md"
+  - title: "Cisco UCS: AWS-compatible cloud at the edge (companion document)"
+    url: "/docs/cisco-ucs-platform-benchmark"
   - title: "Vision pipeline (companion document)"
-    url: "../cisco-ucs-vision-pipeline/README.md"
-  - title: "cisco-ucs-llm-serving (benchmark scripts and results)"
-    url: "https://github.com/mulgadc/cisco-ucs-llm-serving"
+    url: "/docs/cisco-ucs-vision-pipeline"
+  - title: "Benchmark and workload materials (CISCO-refarch)"
+    url: "https://github.com/tomnewton-mulga/CISCO-refarch"
   - title: "vLLM"
     url: "https://docs.vllm.ai"
   - title: "Qwen2.5-7B-Instruct"
@@ -44,7 +45,7 @@ cluster, comparing Intel AMX-accelerated CPU serving on an `m8i.2xlarge`-class i
 against NVIDIA L4 GPU serving on a `g6.2xlarge`-class instance — a real,
 production-representative LLM-serving stack, with the two accelerators measured independently at matched model, version, and serving configuration.
 
-**Companion documents:** [Spinifex Platform Benchmark on Cisco UCS](../cisco-ucs-platform-benchmark/README.md) · [Spinifex Vision Pipeline on Cisco UCS](../cisco-ucs-vision-pipeline/README.md)
+**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/docs/cisco-ucs-platform-benchmark) · [Spinifex Vision Pipeline on Cisco UCS](/docs/cisco-ucs-vision-pipeline)
 
 ### Platform
 

docs/reference-architectures/cisco-ucs-platform-benchmark/README.md

@@ -1,13 +1,15 @@
 ---
-title: "Spinifex Platform Benchmark on Cisco UCS (3-Node)"
-description: "Measured CPU, disk, network, and S3 performance of Spinifex on a 3-node Cisco Unified Edge cluster — host versus guest, Intel AMX confirmed executing at the ISA level, and EC2 instance worker concurrency limits."
+title: "Cisco UCS: AWS-compatible cloud at the edge"
+seoTitle: "Cisco UCS Edge Cloud with Spinifex — Spinifex Docs"
+description: "Place EC2 instances, EBS volumes, S3 object storage and Kubernetes workloads on a resilient three-node Cisco UCS cluster, using familiar AWS APIs and tooling."
 category: "Reference Architectures"
 tags:
   - cisco
   - ucs
+  - edge
+  - aws-compatible
   - xeon-6
   - intel-amx
-  - benchmark
   - predastore
   - viperblock
   - raft
@@ -20,12 +22,8 @@ resources:
     url: "/docs/launching-instances"
   - title: "GPU Passthrough"
     url: "/docs/gpu-passthrough"
-  - title: "AWS: Accelerate CPU-based AI inference with Intel AMX on EC2"
-    url: "https://aws.amazon.com/blogs/compute/accelerate-cpu-based-ai-inference-workloads-using-intel-amx-on-amazon-ec2/"
-  - title: "cisco-ucs-platform-benchmark (benchmark scripts and results)"
-    url: "https://github.com/mulgadc/cisco-ucs-platform-benchmark"
-  - title: "Dell EMC PowerEdge R7525 Review (inspiration for methodology)"
-    url: "https://www.servethehome.com/dell-emc-poweredge-r7525-review-flagship-dell-dual-socket-server-amd-epyc/4/"
+  - title: "cisco-ucs-platform-benchmark (scripts and raw results)"
+    url: "https://github.com/tomnewton-mulga/CISCO-refarch"
 sections:
   - overview
   - prerequisites
@@ -34,16 +32,14 @@ sections:
 
 ## Overview
 
-Spinifex is an open-source infrastructure platform that brings core AWS services — EC2,
-S3, EBS and EKS — to bare-metal, edge, and on-prem deployments, exposing a fully
-AWS-compatible API.
+This reference architecture turns three Cisco Unified Edge servers into a small,
+resilient cloud that can run where data is produced: a factory, retail estate,
+branch, lab, sovereign environment, or disconnected site. Spinifex exposes EC2,
+EBS, S3 and EKS-compatible APIs on the cluster, so the operational model is
+familiar: use the AWS CLI, SDKs, Terraform or OpenTofu, Kubernetes manifests and
+CI/CD systems already used for AWS. The change is the endpoint, not the workflow.
 
-This document characterizes the platform itself on a 3-node Cisco Unified Edge cluster,
-independent of any specific application: CPU, disk, network, and S3 (Predastore)
-throughput measured identically on bare metal and inside guest instances, and how many
-concurrent worker processes each instance can sustain before latency degrades.
-
-**Companion documents:** [Spinifex Vision Pipeline on Cisco UCS](../cisco-ucs-vision-pipeline/README.md) · [vLLM Serving on Cisco UCS: Intel AMX vs NVIDIA L4](../cisco-ucs-llm-serving/README.md)
+**Companion architectures:** [Vision Pipeline on Cisco UCS](/docs/cisco-ucs-vision-pipeline) shows a local AI pipeline that streams inputs from S3; [vLLM Serving on Cisco UCS](/docs/cisco-ucs-llm-serving) shows CPU and GPU model serving on the same cluster.
 
 ### Platform
 
@@ -55,185 +51,247 @@ concurrent worker processes each instance can sustain before latency degrades.
 | **ISA** | AVX-512 (F/DQ/BW/VL/VNNI/BF16/FP16), **Intel AMX** (tile/BF16/INT8), VT-x, VT-d |
 | **Memory** | 499 GiB usable per node (≈1.5 TiB aggregate) |
 | **GPU** | 1× NVIDIA L4 (23,034 MiB) via VFIO PCIe passthrough on one node; the other two nodes are CPU-only |
-| **Storage** | 4× KIOXIA CD8P NVMe (1.92 TB each) per node, raw — Predastore/Viperblock claim these directly |
+| **Storage** | 4× KIOXIA CD8P NVMe (1.92 TB each) per node, raw — Predastore and Viperblock claim these directly |
 | **Boot** | Cisco SATA RAID VD (Marvell 88SE9230 controller) |
-| **Network** | 2× Intel E825-C 25GbE ports per node, each its own bond: one for management/WAN (VLAN 1337, 1 Gbps upstream uplink), one dedicated to storage/cluster traffic (VLAN 1336, full 25GbE) |
+| **Network** | 2× Intel E825-C 25GbE ports per node — one bond for management/WAN (VLAN 1337), one dedicated to storage/cluster traffic (VLAN 1336, full 25GbE) |
 | **Aggregate** | 96 cores / 192 threads, ~1.5 TiB RAM, ~23 TiB raw NVMe, 1× L4 GPU |
 
-Single-NUMA-per-node is a meaningful simplification versus a typical dual-socket
-reference design — no vCPU/memory locality tuning required for VM placement. The GPU is
-present on one node only — GPU-accelerated instances are bound to that node, while
-CPU-only instances can schedule across all three.
+Single-NUMA-per-node simplifies placement — no vCPU/memory locality tuning required.
+GPU workloads schedule to the L4 node; CPU instances can use all three. Storage and
+object data are distributed across the cluster rather than tied to the node that
+launched an instance.
+
+### Hardware Chassis
+
+<img src="../../../.github/assets/images/cisco-ucs-platform-benchmark/reference-architectures-cisco-ucs-platform-hardware1.png" alt="Cisco UCS x Spinifex">
+
+## Architecture
+
+### AWS services exercised
+
+| Service | Role |
+|---|---|
+| **EC2** | Guest instances across all three nodes; spread placement groups distribute across physical nodes |
+| **EBS (Viperblock)** | Root and data volumes per instance, surviving termination independently when `delete_on_termination = false` |
+| **S3 (Predastore)** | Cluster-wide object storage for datasets, model weights, artifacts and backups |
+| **VPC (OVN)** | Overlay networking between instances |
+| **EKS** | (Optional) Kubernetes control plane with pod rescheduling on node failure |
 
 ## Prerequisites
 
 - 3-node Spinifex cluster installed and services healthy on all nodes — follow the [Multi-Node Install](/docs/install-multi-node) guide.
-- VPC, network pool, SSH key pair, and security group configured — see [VPC Networking](/docs/vpc-networking) and [Launching Instances](/docs/launching-instances)
-- GPU passthrough configured on the one node with the NVIDIA L4 — see [GPU Passthrough](/docs/gpu-passthrough):
+- VPC, subnet, network address pool, SSH key pair, and security group configured — see [VPC Networking](/docs/vpc-networking) and [Launching Instances](/docs/launching-instances).
+- GPU passthrough enabled on the L4 node when GPU instances are required — see [GPU Passthrough](/docs/gpu-passthrough):
 
 ```bash
 sudo spx admin gpu setup   # reboot required after this step
 sudo spx admin gpu enable
 ```
 
-- AWS CLI configured with `AWS_PROFILE=spinifex` pointing at the cluster's EC2-compatible endpoint
+- AWS CLI configured with `AWS_PROFILE=spinifex` pointing at the cluster's EC2-compatible endpoint (`https://<host>:9999`).
 
 ## Instructions
 
-Benchmark scripts and raw results for all sections below are available in the
-[platform benchmark repository](https://github.com/mulgadc/cisco-ucs-platform-benchmark).
+### 1. Verify the cluster and its control plane
 
-### 1. Matched-width CPU
-
-sysbench, identical thread counts, run on bare metal and inside a guest of the same vCPU
-width:
-
-| Scope | Target | Median events/s |
-|---|---|---:|
-| host | mulga-01 | 8,205.4 |
-| host | mulga-02 | 8,209.3 |
-| host | mulga-03 | 8,210.4 |
-| guest | cpu1 | 8,168.4 |
-| guest | cpu2 | 8,174.0 |
-| guest | gpu | 8,160.8 |
-
-Guests land within ~0.5% of host — virtualization overhead on this platform is
-negligible for CPU-bound work.
-
-### 2. Confirm Intel AMX is actually executing
-
-Intel AMX (Advanced Matrix Extensions) is a relatively new instruction set available on
-the Xeon 6543P-B, designed specifically to accelerate the matrix operations that dominate
-AI inference workloads. CPUID flags alone don't prove it's actually in use — the proof
-is oneDNN's own kernel-selection trace during a real inference run at each precision:
+Before placing workloads, confirm all three nodes are ready and that the storage and
+control-plane services are running on each:
 
 ```bash
-ONEDNN_VERBOSE=1 python3 <any BF16-hinted inference workload> 2> onednn-verbose.log
-grep -o 'avx10_1_512_amx[a-z_]*\|avx512_core[a-z_]*' onednn-verbose.log | sort -u
+spx get nodes
+systemctl is-active spinifex-predastore spinifex-viperblock spinifex-daemon
 ```
 
-| Precision | Kernel selected |
-|---|---|
-| FP32 (forced, negative control) | `avx512_core` only — **zero** AMX kernel selections |
-| BF16 | `avx10_1_512_amx` |
-| INT8 | `avx10_1_512_amx` (+ one `avx512_core` fallback for an unquantized matmul) |
+Predastore's Raft group and OVN's northbound/southbound databases are both distributed
+across all three nodes — losing one node triggers a clean leader re-election among the
+remaining two, and object storage and networking continue serving. Check that all three
+nodes are contributing to the Raft group before exposing the cluster to workload traffic.
 
-`avx10_1_512_amx` is the current kernel-naming scheme on this hardware generation — not
-the older `brgemm_avx512_amx*` string some documentation still references. Confirmed by
-direct trace inspection, not inferred from CPUID or throughput alone.
+### 2. Launch instances through the EC2-compatible API
 
-### 3. Disk — host versus guest
+Point an existing AWS profile at the Spinifex endpoint and use the standard EC2
+instance lifecycle. Terraform is the recommended path — Spinifex's spread placement
+group implementation holds node reservations that are not always released on destroy,
+so a fixed group name can cause a subsequent apply to hang. Using a per-deploy unique
+name (via `random_id` or equivalent) avoids this:
 
-fio, file-backed (non-destructive), identical parameters on bare metal and inside a
-guest:
+```hcl
+resource "random_id" "suffix" {
+  byte_length = 4
+}
 
-| Metric | Host (mulga-01) | Guest (cpu1, idle) | Ratio |
-|---|---:|---:|---:|
-| Random 4K read | 4,460.8 MiB/s | 11.6 MiB/s | ~1/385 |
-| Sequential 1M read | 7,122.6 MiB/s | 375.5 MiB/s | ~1/19 |
+resource "aws_placement_group" "spread" {
+  name     = "edge-spread-${random_id.suffix.hex}"
+  strategy = "spread"
+}
 
-<img src="../../../.github/assets/images/cisco-ucs-platform-benchmark/disk-host-vs-guest.png" alt="Host versus guest disk throughput, random 4K and sequential 1M read, log scale">
+resource "aws_instance" "worker" {
+  count                  = 2
+  ami                    = var.ami_id
+  instance_type          = "m8i.2xlarge"
+  subnet_id              = var.subnet_id
+  key_name               = var.key_name
+  placement_group        = aws_placement_group.spread.name
+  vpc_security_group_ids = [var.security_group_id]
+}
+```
 
-Guest I/O traverses Viperblock's NBD path; the random 4K gap is far larger than the sequential 1M gap because small requests pay the full per-request serialization overhead without amortizing it across a large transfer. This is a known area for improvement in upcoming Spinifex releases.
+For a one-off launch via the CLI, use a unique group name each time for the same reason:
 
-### 4. Network — physical fabric versus guest overlay
+```bash
+export AWS_PROFILE=spinifex
+GROUP="edge-spread-$(openssl rand -hex 4)"
 
-iperf3, matched methodology, on the physical storage fabric (bare metal) and between
-guest instances over the OVN/Geneve overlay:
+aws ec2 create-placement-group \
+  --group-name "$GROUP" \
+  --strategy spread
 
-| Path | Streams | Median Gbit/s |
+aws ec2 run-instances \
+  --image-id "$AMI" --instance-type m8i.2xlarge \
+  --count 2 \
+  --subnet-id "$SUBNET" --security-group-ids "$SECURITY_GROUP" \
+  --key-name "$KEY_NAME" \
+  --placement "{\"GroupName\":\"$GROUP\"}"
+```
+
+A `g6.2xlarge` instance type routes to the node with GPU passthrough configured;
+`m8i` types schedule across all three nodes.
+
+### 3. Attach Viperblock volumes and access Predastore object storage
+
+Create and attach an EBS-compatible data volume to a running instance:
+
+```bash
+VOLUME_ID=$(aws ec2 create-volume \
+  --availability-zone "$AZ" \
+  --size 100 --volume-type gp2 \
+  --query VolumeId --output text)
+
+aws ec2 attach-volume \
+  --volume-id "$VOLUME_ID" \
+  --instance-id "$INSTANCE_ID" \
+  --device /dev/sdf
+```
+
+Setting `delete_on_termination = false` in Terraform keeps the volume alive across
+instance replacement — relaunching a terminated instance re-attaches the same volume
+rather than starting from empty storage:
+
+```hcl
+resource "aws_ebs_volume" "data" {
+  availability_zone = var.az
+  size              = 100
+  type              = "gp2"
+}
+
+resource "aws_volume_attachment" "data" {
+  device_name           = "/dev/sdf"
+  volume_id             = aws_ebs_volume.data.id
+  instance_id           = aws_instance.worker[0].id
+  delete_on_termination = false
+}
+```
+
+Predastore presents an S3-compatible endpoint — standard S3 tooling works without
+modification:
+
+```bash
+aws s3 mb s3://my-bucket
+aws s3 cp ./dataset.tar.gz s3://my-bucket/
+aws s3 sync ./results/ s3://my-bucket/results/
+```
+
+All three nodes share the same bucket over the cluster's internal storage fabric, so
+datasets, model weights and pipeline outputs are accessible to any instance without
+mounting a shared filesystem.
+
+### 4. Optional: deploy Kubernetes workloads through the EKS-compatible API
+
+Spinifex also exposes an EKS-compatible control plane — this was not part of the Cisco
+exercise documented here, which used EC2, EBS and S3 directly. For workloads where
+automatic pod rescheduling matters, Spinifex supports the same `kubectl` workflows as
+a standard EKS cluster:
+
+```bash
+kubectl get nodes
+kubectl apply -f deployment.yaml
+kubectl get pods -n default
+```
+
+Kubernetes pod rescheduling is the platform's primary answer to node failure for
+scheduled workloads. The companion [vision pipeline](/docs/cisco-ucs-vision-pipeline)
+and [vLLM serving](/docs/cisco-ucs-llm-serving) architectures use raw EC2 instances;
+EKS is the pattern to reach for when automatic rescheduling matters over manual placement.
+
+### 5. Storage benchmark
+
+Predastore and Viperblock — Spinifex's S3 and EBS implementations — represent the
+most substantial engineering work in the platform. Implementing reliable erasure-coded
+object storage and a distributed block device stack on commodity NVMe, while exposing
+the AWS API surface that most workloads depend on, is where most of the hard problems
+live. Both subsystems are under active development, and a production-grade environment
+like this cluster is where integration-level issues surface before they reach users. Run this check after cluster setup or hardware
+changes to confirm both services are performing as expected.
+
+The methodology: one guest per physical node, fio against an ext4 Viperblock data
+volume (four jobs, iodepth 32, direct I/O, 512 MiB file, 30-second run), three
+sequential repetitions per guest.
+
+| Guest | 16K mixed read/write | 16K random read | 128K mixed read/write | 128K random read |
+|---|---:|---:|---:|---:|
+| GPU | 75 / 32 MiB/s | 75 MiB/s | 210 / 91 MiB/s | 332 MiB/s |
+| CPU1 | 74 / 32 MiB/s | 76 MiB/s | 218 / 94 MiB/s | 333 MiB/s |
+| CPU2 | 69 / 30 MiB/s | 107 MiB/s | 226 / 98 MiB/s | 405 MiB/s |
+
+The 128K guest random-read numbers (332–405 MiB/s) sit between the [AWS EBS gp3](https://docs.aws.amazon.com/ebs/latest/userguide/general-purpose.html)
+baseline (125 MiB/s) and its provisioned maximum (1,000 MiB/s) — a meaningful result
+given that gp3 baseline is what most AWS operators treat as the default floor for
+general-purpose block storage. The 16K numbers (75–107 MiB/s) land near that baseline.
+The gap to the raw NVMe host performance (the [KIOXIA CD8P](https://americas.kioxia.com/en-us/business/ssd/data-center-ssd/cd8p-r.html) delivers over 4,000 MiB/s
+host-side at comparable queue depths) is not architectural — it is the current
+single-queue virtio-blk attach path and unthreaded NBD backend, both of which have
+a clear source-level fix. These numbers are the baseline to improve against in future
+releases, not the ceiling.
+
+128K mixed I/O improves throughput but carries a high p99 latency envelope (roughly
+0.47–0.75 s); size queues and write patterns accordingly for latency-sensitive
+applications.
+
+Predastore S3 validation used 1 GiB objects with three sequential and three
+distributed-concurrent repetitions per host:
+
+| Workload | Write, median | Read, median |
 |---|---:|---:|
-| Physical storage fabric (bare metal) | 1 | 23.5 (≈94% of 25GbE line rate) |
-| Guest↔guest | 1 | 13.4–20.0 |
-| Guest↔guest | 2–4 | 14.5–19.0 |
-| Guest↔guest | 16 | 10.3–13.3 |
+| One host client | 121.9 MiB/s | 213.2 MiB/s |
+| Three-host aggregate | 184.2 MiB/s | 381.7 MiB/s |
 
-<img src="../../../.github/assets/images/cisco-ucs-platform-benchmark/network-guest-vs-fabric.png" alt="Guest-to-guest overlay throughput versus the physical storage fabric, by concurrent TCP stream count">
+The single-host sequential read (213 MiB/s) is a credible result for a single-node
+S3-compatible store at low-to-moderate concurrency with 1 GiB objects — throughput at
+this scale is typically network- and per-request-latency-bound rather than disk-bound,
+and 213 MiB/s uses roughly 17% of the 25GbE storage fabric. The three-node distributed read (381.7 MiB/s, 1.79× single-host) confirms
+Predastore is distributing reads across the cluster. The write scaling ratio (184.2 vs
+121.9 MiB/s, 1.51×) is narrower than read, so worth tracking as a leading indicator of
+backend contention as the cluster grows and write load increases.
 
-Guest-to-guest throughput lands 20–45% below the physical fabric depending on stream
-count and pairing.
+All sequential S3 checksum validations passed. Full methodology, raw fio JSON and S3
+metrics are in the [benchmark repository](https://github.com/tomnewton-mulga/CISCO-refarch).
 
-The gap is primarily single-queue virtio-net — `ethtool -l` on all three guest instances
-reports `Combined: 1` with no guest-configurable multiqueue support, concentrating all
-network work on a single core regardless of vCPU count. This explains why 16 concurrent
-streams perform *worse* than 1: they compete for the same queue rather than
-parallelizing. Geneve encapsulation reduces tenant MTU to 1408 (TCP MSS 1356), adding a
-smaller compounding overhead on top.
+## What this architecture unlocks
 
-### 5. Predastore (S3) throughput
+- **Cloud-to-edge placement:** run the same EC2, volume, S3 and Kubernetes patterns
+  at the site where latency, privacy, bandwidth or sovereignty requires it.
+- **Familiar operations:** keep AWS CLI profiles, Terraform/OpenTofu modules, SDKs,
+  CI/CD pipelines and deployment tooling rather than introducing a separate
+  edge-only platform.
+- **Local AI with shared data:** place GPU and AMX-capable CPU inference next to the
+  data source while keeping datasets and artifacts available to all instances through S3.
+- **A resilient foundation:** distribute control and storage services across three
+  physical Cisco nodes instead of concentrating the site on one server. Kubernetes
+  rescheduling and Raft-based storage both tolerate a single node loss without
+  operator intervention.
+- **A path back to cloud:** the same APIs at the edge and on AWS make workload
+  movement, burst strategies and consistent application packaging straightforward.
 
-Three simultaneous clients per side, one per physical node, over HTTPS to the cluster's
-S3 endpoint:
-
-| Direction | Guest-aggregate | Host-aggregate |
-|---|---:|---:|
-| Read | 175.1 MiB/s | 299.0 MiB/s |
-| Write | 293.1 MiB/s | 124.1 MiB/s |
-
-Reads run 1.71x faster on the host aggregate than guest. Write reverses — guests
-outperform the host (293.1 vs 124.1 MiB/s).
-
-### 6. Worker concurrency per instance — how many processes can each node sustain
-
-With one EC2 instance per physical node (3 instances fixed), we
-scale the number of concurrent worker processes within each instance from 1 to N, until
-p95 request latency exceeds 2x the single-worker baseline or aggregate throughput stops
-improving. Load generator: OpenVINO/YOLO11m inference against Predastore (CPU workers) and
-TensorRT/YOLO11m (GPU worker).
-
-| Workers/instance | Total workers | Aggregate img/s | cpu1 p95 (ms) | cpu2 p95 (ms) | gpu p95 (ms) |
-|---:|---:|---:|---:|---:|---:|
-| 1 | 3 | 49.0 | 90.9 | 88.1 | 51.0 |
-| 2 | 6 | 82.6 | 140.9 | 141.7 | 49.8 |
-| 3 | 9 | 103.3 | 239.0 | 232.2 | 53.0 |
-| 4 | 12 | 121.1 | 319.3 | 318.4 | 53.0 |
-| 5 | 15 | 133.1 | 402.1 | 413.6 | 57.4 |
-| 6 | 18 | 142.5 | 490.4 | 493.6 | 62.2 |
-
-
-<img src="../../../.github/assets/images/cisco-ucs-platform-benchmark/consolidation-throughput.png" alt="Aggregate throughput versus concurrent workers per instance, by worker type and cluster total">
-
-<img src="../../../.github/assets/images/cisco-ucs-platform-benchmark/consolidation-p95-latency.png" alt="p95 request latency versus concurrent workers per instance, by worker type">
-
-Both CPU workers track each other closely across the entire sweep (e.g. 18.05 vs.
-18.25 img/s at N=2, 490.4 vs. 493.6 ms p95 at N=6), consistent with identical hardware
-on identical nodes.
-
-**CPU workers' aggregate throughput peaks at 2 workers/instance (~18.1 img/s each) and
-declines steadily from there** (~16.6 at N=4, ~14.8 at N=6), while p95 latency climbs
-almost linearly with N (91–88 ms → ~490 ms, a ~5.4–5.6x increase over the sweep) — **the
-2x-baseline SLA is breached at 3 workers/instance** (239.0 ms cpu1 / 232.2 ms cpu2 vs. a
-181.8 ms / 176.2 ms threshold respectively) and never recovers. Both signals agree: **the
-practical ceiling for this CPU-bound workload is 2 workers/instance (6/cluster)** if
-throughput is the priority, or 2 if latency matters at all — by 3 it's already costing
-more than it gains.
-
-**The GPU worker shows no strain through the full sweep** — p95 stays essentially flat
-(51.0–62.2 ms, well under its own 102.0 ms breach threshold) and aggregate throughput
-keeps climbing almost linearly through 6 workers/instance (22.85 → 112.82 img/s, a ~4.9x
-increase for 6x the load) — consistent with the L4 sitting at only 22–35% utilisation under a single worker. The sweep was capped at 6
-workers/instance by design; the GPU's actual ceiling is higher and wasn't reached here.
-
-**Predastore itself is not the limiting subsystem for this result.** mulga-01's
-Predastore CPU% is already elevated (85.4%) at N=1, briefly exceeds 100% at N=2
-(119.0%, multi-threaded process accounting), and settles at 93.2% by N=6 — it fluctuates
-around a high baseline rather than climbing *with* N the way the observed CPU-worker
-degradation does, and the other two nodes' Predastore CPU% (27–50%) never comes close.
-
-The cause is CPU-compute contention rather than storage or network: inference latency
-grows ~11x across the sweep while S3 read/write latency grows only ~24–40%, and each
-OpenVINO worker consumes roughly 3.5 cores unpinned — oversubscribing the 8-vCPU guest
-by N=3, which lines up exactly with the observed SLA breach.
-
-### 7. Conclusion
-
-On matched-width CPU work, guests cost virtually nothing versus bare metal (~0.5%), and
-Intel AMX is confirmed executing at the kernel-selection level, not just present in
-CPUID. Network and disk both show real overhead versus bare metal — guest network at 20–45% below the physical fabric, guest disk far more so,
-currently the platform's clearest area for near-term improvement. The practical ceiling
-for CPU-bound concurrent work is 2 worker processes per 8-vCPU instance before latency
-degrades — confirmed as compute contention rather than a storage or network ceiling —
-while the GPU instance shows no strain through the full sweep, consistent with the L4
-sitting at 22–35% utilisation under a single worker.
-
-All benchmark instances were provisioned through Spinifex's EC2-compatible endpoint using standard `aws ec2` CLI calls — the same instance types, placement groups, and security groups that work on AWS work here unchanged. Teams already operating AWS infrastructure can point their existing tooling at a Spinifex node with a single profile swap.
+The raw measurements are evidence that the platform is functioning as designed; the
+larger outcome is a portable, resilient operating model for workloads that need cloud
+interfaces outside a public-cloud region.

docs/reference-architectures/cisco-ucs-vision-pipeline/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Spinifex Vision Pipeline on Cisco UCS"
-description: "A YOLO11m object-detection + Qwen2-VL scene-captioning pipeline on Spinifex, streaming from a shared Predastore bucket across two independent EC2-compatible instances, with Intel AMX and NVIDIA L4 acceleration compared head-to-head."
+seoTitle: "Spinifex Vision Pipeline on Cisco UCS — Spinifex Docs"
+description: "A YOLO11m detection and Qwen2-VL captioning pipeline streaming from a shared Predastore bucket across two EC2 instances, with Intel AMX and NVIDIA L4 compared."
 category: "Reference Architectures"
 tags:
   - cisco
@@ -13,8 +14,8 @@ tags:
   - predastore
   - terraform
 resources:
-  - title: "cisco-ucs-vision-pipeline (Terraform workbook, scripts)"
-    url: "https://github.com/mulgadc/cisco-ucs-vision-pipeline"
+  - title: "Benchmark and workload materials (CISCO-refarch)"
+    url: "https://github.com/tomnewton-mulga/CISCO-refarch"
   - title: "Multi-Node Install"
     url: "/docs/install-multi-node"
   - title: "VPC Networking"
@@ -23,8 +24,8 @@ resources:
     url: "/docs/launching-instances"
   - title: "GPU Passthrough"
     url: "/docs/gpu-passthrough"
-  - title: "Platform benchmark (companion document)"
-    url: "../cisco-ucs-platform-benchmark/README.md"
+  - title: "Cisco UCS: AWS-compatible cloud at the edge (companion document)"
+    url: "/docs/cisco-ucs-platform-benchmark"
   - title: "Ultralytics YOLO"
     url: "https://docs.ultralytics.com"
   - title: "AWS: Accelerate CPU-based AI inference with Intel AMX on EC2"
@@ -47,7 +48,7 @@ cluster running Spinifex: real-time object detection plus a vision-language mode
 producing plain-English scene descriptions, both streaming from the same shared Predastore
 (S3-compatible) bucket as two entirely independent EC2-compatible instances.
 
-**Companion documents:** [Spinifex Platform Benchmark on Cisco UCS](../cisco-ucs-platform-benchmark/README.md) · [vLLM Serving on Cisco UCS: Intel AMX vs NVIDIA L4](../cisco-ucs-llm-serving/README.md)
+**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/docs/cisco-ucs-platform-benchmark) · [vLLM Serving on Cisco UCS: Intel AMX vs NVIDIA L4](/docs/cisco-ucs-llm-serving)
 
 ### Platform
 

docs/security/flaw-remediation-policy/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Flaw Remediation Policy"
-description: "CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex"
+seoTitle: "Spinifex Flaw Remediation Policy (CVSS) — Spinifex Docs"
+description: "CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex and its direct dependencies, for maintainers and CMMC Level 1 operators."
 category: "Security"
 sections:
   - overview
@@ -38,7 +39,7 @@ resources:
 - [3. Remediation SLAs](#3-remediation-slas)
 - [4. Reporting Workflow](#4-reporting-workflow)
 - [5. Operator Responsibilities](#5-operator-responsibilities)
-- [6. Evidence](#6-evidence)
+- [6. Evidence](#6-evidence-and-record-keeping)
 - [7. Operator Checklist](#7-operator-checklist)
 
 ---
@@ -59,7 +60,7 @@ resources:
 
 Spinifex ships a short dependency chain and identification is already automated: Dependabot raises security PRs the moment GitHub links a CVE to a shipped dependency, `govulncheck` runs on every PR, and GitHub Security Advisories watch all three `mulgadc/*` repositories.
 
-Sections map to the three scored objectives: [§1](#1-identification) names the identification sources (objectives [a]/[b]); [§3](#3-remediation-slas) and [§4](#4-reporting-workflow) set the report and correct timeframes (objectives [c]–[f]). [§5](#5-operator-responsibilities) delineates layers the operator owns, and [§6](#6-evidence) covers the records needed to demonstrate compliance at assessment.
+Sections map to the three scored objectives: [§1](#1-identification) names the identification sources (objectives [a]/[b]); [§3](#3-remediation-slas) and [§4](#4-reporting-workflow) set the report and correct timeframes (objectives [c]–[f]). [§5](#5-operator-responsibilities) delineates layers the operator owns, and [§6](#6-evidence-and-record-keeping) covers the records needed to demonstrate compliance at assessment.
 
 ## 1. Identification
 
@@ -92,7 +93,7 @@ CVSS v3.1 base score from the upstream advisory; otherwise scored by maintainers
 
 ## 3. Remediation SLAs
 
-Clocks start at the moment of identification ([§1](#1-identification-sources)). All three objectives — identify [a], report [c], correct [e] — are time-bound.
+Clocks start at the moment of identification ([§1](#1-identification)). All three objectives — identify [a], report [c], correct [e] — are time-bound.
 
 | Severity | Identify | Report | Correct |
 |----------|----------|--------|---------|
@@ -129,7 +130,7 @@ Operators own remediation for layers Spinifex does not ship:
 | Host OS + kernel | Track distribution advisories (Debian DSA, Ubuntu USN) at matching cadence. |
 | OVN / OVS | Patched via the operator's package channel alongside the OS. |
 | QEMU / KVM / libvirt | As above — hypervisor CVEs affect tenant isolation. |
-| Host AV / EDR agent | See [Malware Protection §2](/docs/security/malware-protection#2-update-requirements-sil1-3144). |
+| Host AV / EDR agent | See [Malware Protection §2](/docs/malware-protection#2-update-requirements-sil1-3144). |
 
 Operators should subscribe to the `mulgadc/spinifex` release feed (Watch → Custom → Releases + Security Advisories) so Critical/High advisories arrive via GitHub's notification channel rather than polling.
 
@@ -150,4 +151,4 @@ These demonstrate objectives [b], [d], [f] — "within the specified time" — i
 - Change-management records patch-apply timestamps for every Spinifex release.
 - Operator subscribes to `mulgadc/spinifex`, `mulgadc/predastore`, `mulgadc/viperblock` releases and advisories.
 - OS / kernel / hypervisor patching cadence documented and at least as strict as [§3](#3-remediation-slas).
-- Annual review confirms at least one patch cycle completed within SLA in the prior 12 months, with evidence per [§6](#6-evidence).
+- Annual review confirms at least one patch cycle completed within SLA in the prior 12 months, with evidence per [§6](#6-evidence-and-record-keeping).

docs/security/malware-protection/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Malware Protection"
-description: "Operator guide for host-based malware protection and file integrity monitoring on Spinifex nodes"
+seoTitle: "Malware Protection for Spinifex Nodes — Spinifex Docs"
+description: "Operator guide to host-based malware protection and file integrity monitoring on the Linux hosts running Spinifex services, aligned to CMMC Level 1 needs."
 category: "Security"
 sections:
   - overview

docs/security/media-sanitization/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Media Sanitization and Disposal"
-description: "Operator guide for sanitizing and disposing of storage media used by Spinifex nodes"
+seoTitle: "Media Sanitization and Disposal Guide — Spinifex Docs"
+description: "Operator guide to sanitizing and disposing of storage media used by Spinifex nodes, covering system disks, Viperblock and Predastore volumes, and key tokens."
 category: "Security"
 sections:
   - overview
@@ -36,8 +37,8 @@ resources:
 - [Approach](#approach)
 - [1. Media in Scope](#1-media-in-scope)
 - [2. Sanitization Method Selection](#2-sanitization-method-selection)
-- [3. Volume-Level Sanitization — Before Reuse (MP.L1-3.8.3 [b])](#3-volume-level-sanitization--before-reuse-mpl1-383-b)
-- [4. Whole-Drive and Node Decommissioning — Before Disposal (MP.L1-3.8.3 [a])](#4-whole-drive-and-node-decommissioning--before-disposal-mpl1-383-a)
+- [3. Volume-Level Sanitization — Before Reuse (MP.L1-3.8.3 [b])](#3-volume-level-sanitization-before-reuse-mpl1-383-b)
+- [4. Whole-Drive and Node Decommissioning — Before Disposal (MP.L1-3.8.3 [a])](#4-whole-drive-and-node-decommissioning-before-disposal-mpl1-383-a)
 - [5. Key Destruction](#5-key-destruction)
 - [6. Removable and Backup Media](#6-removable-and-backup-media)
 - [7. Evidence and Record Keeping](#7-evidence-and-record-keeping)
@@ -91,7 +92,7 @@ The method depends on (a) media type and (b) whether the media remains inside th
 
 | Scenario | Minimum method | Reference |
 |----------|----------------|-----------|
-| Encrypted volume returning to free pool, DEK deletable | **Cryptographic Erase** (Purge) via DEK deletion | [§3](#3-volume-level-sanitization--before-reuse-mpl1-383-b) |
+| Encrypted volume returning to free pool, DEK deletable | **Cryptographic Erase** (Purge) via DEK deletion | [§3](#3-volume-level-sanitization-before-reuse-mpl1-383-b) |
 | SSD / NVMe leaving the node | **Cryptographic Erase** + **Purge** (SANITIZE block-erase or ATA Secure Erase) | [§4.1](#41-nvme-ssd) |
 | Magnetic HDD leaving the node, operational | **Purge** (single-pass overwrite with verify, or ATA Secure Erase on drives that support it) | [§4.3](#43-hdd) |
 | Magnetic HDD leaving the node, faulty (not writable) | **Destroy** (degauss followed by shred/incinerate) | [§4.3](#43-hdd) |
@@ -120,7 +121,7 @@ This is the sanitization-before-reuse path for every in-platform object. Operato
 
 ### 3.2 What Happens to the Physical Storage
 
-Cryptographic erase leaves ciphertext chunks in Predastore. These chunks are eventually overwritten as new volumes reuse the space. For CMMC purposes, the ciphertext without the DEK satisfies Purge. The underlying drive still requires whole-drive sanitization when it eventually leaves the cluster — see [§4](#4-whole-drive-and-node-decommissioning--before-disposal-mpl1-383-a).
+Cryptographic erase leaves ciphertext chunks in Predastore. These chunks are eventually overwritten as new volumes reuse the space. For CMMC purposes, the ciphertext without the DEK satisfies Purge. The underlying drive still requires whole-drive sanitization when it eventually leaves the cluster — see [§4](#4-whole-drive-and-node-decommissioning-before-disposal-mpl1-383-a).
 
 ## 4. Whole-Drive and Node Decommissioning — Before Disposal (MP.L1-3.8.3 [a])
 
@@ -210,11 +211,11 @@ Cryptographic erase relies on the key being unrecoverable. A drive that previous
 | `/etc/spinifex/master.key` | Node system disk (every node) | Overwrite the file (`shred -u /etc/spinifex/master.key`) before the drive is sanitized, in addition to drive sanitization. Once the master key is gone cluster-wide, every DEK it wrapped is effectively destroyed. |
 | Cluster CA private key (`/etc/spinifex/ca.key`) | Leader node system disk | Same treatment. Loss of the CA key does not sanitize data, but prevents impersonation of the cluster identity post-disposal. |
 | Per-node TLS private key (`/etc/spinifex/server.key`) | Each node system disk | `shred -u` before drive sanitization. |
-| Wrapped DEKs in NATS KV | `/var/lib/spinifex/nats/` on NATS-carrying nodes | Deleted via the `DeleteVolume` API; the KV record is compacted. Sanitization of the NATS JetStream disk per [§4](#4-whole-drive-and-node-decommissioning--before-disposal-mpl1-383-a) ensures no recoverable copies. |
+| Wrapped DEKs in NATS KV | `/var/lib/spinifex/nats/` on NATS-carrying nodes | Deleted via the `DeleteVolume` API; the KV record is compacted. Sanitization of the NATS JetStream disk per [§4](#4-whole-drive-and-node-decommissioning-before-disposal-mpl1-383-a) ensures no recoverable copies. |
 | Removable master-key media (USB) | USB flash token | Physically destroy the token when the cluster is decommissioned; alternatively retain inside a secured location as evidence of cluster-wide cryptographic erase. Do not reuse for another cluster without full sanitization (see [§6](#6-removable-and-backup-media)). |
 | Backup-copy keys | Any off-site or escrow copy | Destroy simultaneously with the primary. A surviving backup defeats the cryptographic erase. |
 
-If even one copy of the master key survives and the encrypted drives are recoverable, cryptographic erase has **not** been achieved. Track every key copy in the device register ([Physical Security Guide §5](/docs/security/physical-security-guide#5-manage-physical-access-devices-pel1-3105)).
+If even one copy of the master key survives and the encrypted drives are recoverable, cryptographic erase has **not** been achieved. Track every key copy in the device register ([Physical Security Guide §5](/docs/physical-security-guide#5-manage-physical-access-devices-pel1-3105)).
 
 ## 6. Removable and Backup Media
 
@@ -227,7 +228,7 @@ If even one copy of the master key survives and the encrypted drives are recover
 | Off-site cloud backup | Issue the cloud provider's delete-and-purge API; retain the provider's deletion-confirmation receipt. For cryptographically wrapped backups, destroy the wrapping key cluster-side and retain that deletion as evidence. |
 | Printed material (console photos, recovery codes, admin handover sheets) | Cross-cut shred. |
 
-Any removable medium that has entered the protected boundary and been written to must be tracked in the device register (see [Physical Security Guide §5](/docs/security/physical-security-guide#5-manage-physical-access-devices-pel1-3105)) and accounted for at decommissioning.
+Any removable medium that has entered the protected boundary and been written to must be tracked in the device register (see [Physical Security Guide §5](/docs/physical-security-guide#5-manage-physical-access-devices-pel1-3105)) and accounted for at decommissioning.
 
 ## 7. Evidence and Record Keeping
 
@@ -254,5 +255,5 @@ Use this list to confirm a Spinifex deployment meets MP.L1-3.8.3:
 - Every backup copy of any key is destroyed at the same time as the primary; surviving backup copies are tracked and accounted for.
 - Removable media (USB, tape, optical) is destroyed rather than erased unless the medium is an SED with documented cryptographic erase.
 - Decommissioning records capture method, tool output, verification, final disposition, and operator; certificates of destruction are reconciled against the asset register.
-- Drives awaiting sanitization or awaiting pickup by a destruction vendor are held inside the physical protection boundary ([Physical Security Guide §1](/docs/security/physical-security-guide#1-protected-assets)) with chain of custody logged.
+- Drives awaiting sanitization or awaiting pickup by a destruction vendor are held inside the physical protection boundary ([Physical Security Guide §1](/docs/physical-security-guide#1-protected-assets)) with chain of custody logged.
 - System security plan references this guide and names the sanitization tooling, destruction vendor (if any), retention period for records, and the security owner attesting annually.

docs/security/network-connections/README.md

@@ -1,6 +1,7 @@
 ---
 title: "External Connection Inventory"
-description: "Operator inventory of inbound listeners and outbound connections on Spinifex nodes"
+seoTitle: "Spinifex External Connection Inventory — Spinifex Docs"
+description: "Operator inventory of every inbound listener and outbound connection on Spinifex nodes, with ports, protocols, and purpose, for documented CMMC Level 1 sites."
 category: "Security"
 sections:
   - overview

docs/security/physical-security-guide/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Physical Security Operator Guide"
-description: "Operator guide for physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes"
+seoTitle: "Physical Security Guide for Operators — Spinifex Docs"
+description: "Operator guide to physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes and network gear."
 category: "Security"
 sections:
   - overview
@@ -144,7 +145,7 @@ Every entry into a protected space must produce a log entry capturing:
 
 ### 4.3 Integration with Spinifex Logs
 
-Forward physical-access events to the same SIEM or log collector used for Spinifex service logs (see [Malware Protection §3](/docs/security/malware-protection#3-scan-schedule-sil1-3145)) so a `master.key` access attempt can be correlated with a physical entry into the rack.
+Forward physical-access events to the same SIEM or log collector used for Spinifex service logs (see [Malware Protection §3](/docs/malware-protection#3-scan-schedule-sil1-3145)) so a `master.key` access attempt can be correlated with a physical entry into the rack.
 
 ## 5. Manage Physical Access Devices (PE.L1-3.10.5)
 
@@ -199,7 +200,7 @@ Use this list to confirm a site meets the four CMMC practices before admitting S
 - CCTV or equivalent monitoring covers the rack fronts, rears, and console positions, with 30-day minimum retention.
 - Physical access logs are produced for every entry, retained 12 months, and reviewed on the cadence in [§4.2](#42-retention-and-review).
 - Door-forced, door-held, and out-of-hours events alert the on-call operator.
-- Physical access events forwarded to the SIEM used for Spinifex service logs (see [Malware Protection §3](/docs/security/malware-protection#3-scan-schedule-sil1-3145)).
+- Physical access events forwarded to the SIEM used for Spinifex service logs (see [Malware Protection §3](/docs/malware-protection#3-scan-schedule-sil1-3145)).
 - Device register reconciles quarterly against the access-control system and physical key count; discrepancies are closed before sign-off.
 - Lost-device and termination procedures trigger immediate revocation per [§5.2](#52-control-b).
 - System security plan references this guide and records the facility, access-control product, CCTV retention, and log-forwarding destination.

docs/terraform-workbooks/bastion-private-subnet/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Bastion with Private Subnet"
-description: "Deploy a bastion host to access an isolated private compute instance with no internet connectivity."
+seoTitle: "Terraform Bastion Host and Private Subnet — Spinifex Docs"
+description: "Deploy a VPC with public and private subnets on Spinifex, then use a bastion host as the only route to an isolated EC2 instance that has no internet access."
 category: "Terraform Workbooks"
 tags:
   - terraform

docs/terraform-workbooks/demo-app/Dockerfile

@@ -4,7 +4,7 @@
 # distroless base (no shell, no package manager) for a small, production-shaped
 # image suitable for pushing to ECR and pulling onto EKS workers.
 
-FROM golang:1.23-alpine AS build
+FROM golang:1.27-alpine AS build
 WORKDIR /src
 COPY go.mod ./
 COPY main.go ./

docs/terraform-workbooks/demo-app/go.mod

@@ -1,3 +1,3 @@
 module github.com/mulgadc/spinifex-demo
 
-go 1.23
+go 1.27.0

docs/terraform-workbooks/ecs-quickstart/README.md

@@ -1,6 +1,7 @@
 ---
 title: "ECS Quickstart"
-description: "Stand up a complete AWS-compatible ECS stack on Spinifex with Terraform — a VPC, IAM roles, a cluster, a task definition with a task role, container instances launched from the ECS node image, and an awsvpc service fronted by an Application Load Balancer."
+seoTitle: "Terraform ECS Quickstart on Spinifex — Spinifex Docs"
+description: "Stand up a full AWS-compatible ECS stack with Terraform: a VPC, IAM roles, a cluster, a task definition, container instances, and a load-balanced service."
 category: "Terraform Workbooks"
 tags:
   - terraform

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -1,6 +1,7 @@
 ---
 title: "GitOps on EKS (Argo CD + EBS-CSI)"
-description: "Deliver a Spinifex-themed app to EKS with GitOps — the Argo CD addon syncs it from a git repo, an EBS-CSI (Viperblock) PersistentVolume holds its state, and it is served over HTTPS through the AWS Load Balancer Controller + ACM, using Terraform on Spinifex."
+seoTitle: "GitOps on EKS with Argo CD and EBS-CSI — Spinifex Docs"
+description: "Deliver an app to EKS with GitOps: the Argo CD addon syncs it from git, an EBS-CSI PersistentVolume holds its state, and HTTPS is served via LBC and ACM."
 category: "Terraform Workbooks"
 tags:
   - terraform
@@ -26,7 +27,7 @@ resources:
 
 # Terraform: GitOps on EKS (Argo CD + EBS-CSI)
 
-> The top rung of the EKS ladder. It extends [EKS HTTPS Ingress](../eks-https-ingress) and changes how the app is delivered: the **Argo CD** addon syncs a more elaborate Spinifex-themed app from a **git repo**, an **EBS-CSI** PersistentVolume (Viperblock-backed) holds its state, and it is still served over **HTTPS** through the AWS Load Balancer Controller + ACM.
+> The top rung of the EKS ladder. It extends [EKS HTTPS Ingress](/docs/eks-https-ingress) and changes how the app is delivered: the **Argo CD** addon syncs a more elaborate Spinifex-themed app from a **git repo**, an **EBS-CSI** PersistentVolume (Viperblock-backed) holds its state, and it is still served over **HTTPS** through the AWS Load Balancer Controller + ACM.
 
 ## Table of Contents
 
@@ -38,7 +39,7 @@ resources:
 
 ## Overview
 
-This workbook keeps everything from [EKS HTTPS Ingress](../eks-https-ingress) — the public/private VPC, the NAT gateway, the LBC addon, the ACM cert, the HTTPS Ingress — and adds the parts you reach for once an app is real: **GitOps delivery** and **persistent storage**.
+This workbook keeps everything from [EKS HTTPS Ingress](/docs/eks-https-ingress) — the public/private VPC, the NAT gateway, the LBC addon, the ACM cert, the HTTPS Ingress — and adds the parts you reach for once an app is real: **GitOps delivery** and **persistent storage**.
 
 Three changes over the previous rung:
 
@@ -85,7 +86,7 @@ Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress;
 
 - Spinifex running with an `eks-node` image carrying the LBC, Argo CD, and EBS-CSI bundles
 - OpenTofu or Terraform, plus `kubectl`, the AWS CLI, and Docker
-- The demo image built and pushed to ECR (see [`../demo-app/README.md`](../demo-app/README.md))
+- The demo image built and pushed to ECR (see [`docs/terraform-workbooks/demo-app`](https://github.com/mulgadc/spinifex/blob/main/docs/terraform-workbooks/demo-app/README.md))
 - A git repo with the app manifests ([`mulgadc/eks-demo-app`](https://github.com/mulgadc/eks-demo-app)); a read-only PAT if it is private
 
 ## Instructions

docs/terraform-workbooks/eks-https-ingress/README.md

@@ -1,6 +1,7 @@
 ---
 title: "EKS HTTPS Ingress (LBC + ACM)"
-description: "Serve a Spinifex-themed demo app over HTTPS on EKS using the AWS Load Balancer Controller addon and an ACM certificate — an internet-facing ALB provisioned from a Kubernetes Ingress, using Terraform on Spinifex."
+seoTitle: "EKS HTTPS Ingress with LBC and ACM — Spinifex Docs"
+description: "Serve a demo app over HTTPS on EKS using the AWS Load Balancer Controller addon and an ACM certificate, with an internet-facing ALB built from an Ingress."
 category: "Terraform Workbooks"
 tags:
   - terraform
@@ -36,7 +37,7 @@ resources:
 
 ## Overview
 
-This is the second rung of the EKS ladder. It takes the cluster + demo app from [EKS Quickstart](../eks-quickstart) and publishes the app the way you would in production: a Kubernetes `Ingress` reconciled by the **AWS Load Balancer Controller (LBC)** into an internet-facing **Application Load Balancer** that terminates TLS with an **ACM** certificate.
+This is the second rung of the EKS ladder. It takes the cluster + demo app from [EKS Quickstart](/docs/eks-quickstart) and publishes the app the way you would in production: a Kubernetes `Ingress` reconciled by the **AWS Load Balancer Controller (LBC)** into an internet-facing **Application Load Balancer** that terminates TLS with an **ACM** certificate.
 
 To make room for an internet-facing ALB with private workers, the network grows from the quickstart's two public subnets to a public/private split: the ALB and a **NAT gateway** sit in the public subnets, the workers sit in the private subnets and reach ECR through the NAT. The cluster is tagged `spinifex.io/managed-ingress = "false"`, which disables K3s' built-in traefik/servicelb so the LBC owns ingress.
 
@@ -73,13 +74,13 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 - **`spinifex.io/managed-ingress = "false"`** disables the K3s built-in traefik/servicelb so the LBC owns ingress, matching AWS parity. Omit the tag (or set it `true`) and the built-in ingress stays on.
 - The ALB targets the workers' **NodePort** (`target-type: instance`). The one SG rule admits the VPC CIDR on that port so the in-VPC ALB can reach the workers; the worker SG is otherwise auto-managed.
 - Spinifex ACM supports **ImportCertificate only** — the cert is generated by the `tls` provider and imported, not requested.
-- **Leave `addon_version` unset** (see the [quickstart/addons notes](../eks-quickstart)) — the AWS provider and the Spinifex catalog disagree on the version string format, so let the server choose.
+- **Leave `addon_version` unset** (see the [quickstart/addons notes](/docs/eks-quickstart)) — the AWS provider and the Spinifex catalog disagree on the version string format, so let the server choose.
 
 **Prerequisites:**
 
 - Spinifex installed and running, with the `eks-node` image (carrying the LBC bundle) available
 - OpenTofu or Terraform, plus `kubectl`, the AWS CLI, and Docker
-- The demo image built and pushed to ECR (see [`../demo-app/README.md`](../demo-app/README.md))
+- The demo image built and pushed to ECR (see [`docs/terraform-workbooks/demo-app`](https://github.com/mulgadc/spinifex/blob/main/docs/terraform-workbooks/demo-app/README.md))
 
 ## Instructions
 

docs/terraform-workbooks/eks-quickstart/README.md

@@ -1,6 +1,7 @@
 ---
 title: "EKS Quickstart"
-description: "Stand up a minimal managed-Kubernetes cluster and a browsable Spinifex-themed demo app — VPC, IAM roles, an EKS cluster, a one- or three-worker node group, an ECR repository, and a load-balanced web page — using Terraform on Spinifex."
+seoTitle: "Terraform EKS Quickstart on Spinifex — Spinifex Docs"
+description: "Stand up a minimal managed Kubernetes cluster with Terraform: a VPC, IAM roles, an EKS cluster, a worker node group, an ECR repository, and a demo web app."
 category: "Terraform Workbooks"
 tags:
   - terraform
@@ -108,7 +109,7 @@ Set the worker count with `node_desired_size` (`1` for a single node, `3` for an
 tofu apply -var node_desired_size=3
 ```
 
-Once the cluster is `ACTIVE`, **build and push the demo image** to the ECR repository this created (full commands in [`../demo-app/README.md`](../demo-app/README.md)):
+Once the cluster is `ACTIVE`, **build and push the demo image** to the ECR repository this created (full commands in [`docs/terraform-workbooks/demo-app`](https://github.com/mulgadc/spinifex/blob/main/docs/terraform-workbooks/demo-app/README.md)):
 
 ```bash
 cd ../demo-app

docs/terraform-workbooks/nginx-alb/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Nginx Web Server (Load Balanced)"
-description: "Deploy a VPC with two private EC2 instances running Nginx, fronted by an internet-facing ALB using Terraform on Spinifex."
+seoTitle: "Load-Balanced Nginx Behind an AWS ALB — Spinifex Docs"
+description: "Deploy a VPC with two private EC2 instances running Nginx behind an internet-facing Application Load Balancer on Spinifex, using Terraform or OpenTofu."
 category: "Terraform Workbooks"
 tags:
   - terraform

docs/terraform-workbooks/nginx-webserver/README.md

@@ -1,6 +1,7 @@
 ---
 title: "Nginx Web Server"
-description: "Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex."
+seoTitle: "Terraform Nginx Web Server on Spinifex — Spinifex Docs"
+description: "Provision a VPC, public subnet, internet gateway, route table, security group, and an EC2 instance that installs and starts Nginx from cloud-init user-data."
 category: "Terraform Workbooks"
 tags:
   - terraform

docs/terraform-workbooks/rds-quickstart/README.md

@@ -1,6 +1,7 @@
 ---
 title: "RDS Quickstart (PostgreSQL)"
-description: "Stand up a managed PostgreSQL database on Spinifex with Terraform — a VPC, a DB subnet group, a parameter group, an aws_db_instance, and a client VM inside the VPC that connects to it with psql."
+seoTitle: "Terraform RDS PostgreSQL Quickstart — Spinifex Docs"
+description: "Stand up a managed PostgreSQL database on Spinifex with Terraform: a VPC, DB subnet group, parameter group, aws_db_instance, and a client VM that runs psql."
 category: "Terraform Workbooks"
 tags:
   - terraform
@@ -53,6 +54,8 @@ The client subnet is public in that it has an internet-gateway route, which the
 
 The database security group admits `5432` from the client security group and nothing else, which compiles to an ACL on the endpoint ENI itself — the port is deny-by-default for everything else in the VPC.
 
+That holds for the instance, not just for the endpoint ENI. The DB VM has two further NICs the platform uses to manage it, and no customer security group governs either; the engine binds the endpoint ENI's address alone, so it is not listening on them. PostgreSQL's client authentication rules are scoped to this VPC's own range as well, so the endpoint really is the whole of the reachable surface.
+
 ## Prerequisites
 
 - **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and OpenTofu (or Terraform) installed.

docs/terraform-workbooks/s3-webapp/README.md

@@ -1,6 +1,7 @@
 ---
 title: "S3-Backed Web App"
-description: "Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex."
+seoTitle: "Terraform S3-Backed Flask Web App on EC2 — Spinifex Docs"
+description: "Deploy a Flask file-sharing app on EC2 backed by S3 (Predastore) with Terraform, using an IAM instance profile and short-lived STS credentials from IMDS."
 category: "Terraform Workbooks"
 tags:
   - terraform

go.mod

@@ -1,14 +1,14 @@
 module github.com/mulgadc/spinifex
 
-go 1.26.6
+go 1.27.0
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.43.5
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17
-	github.com/aws/aws-sdk-go-v2/config v1.32.36
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.35
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36
+	github.com/aws/aws-sdk-go-v2 v1.43.6
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18
+	github.com/aws/aws-sdk-go-v2/config v1.32.37
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.36
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
@@ -17,15 +17,14 @@ require (
 	github.com/distribution/reference v0.6.0
 	github.com/go-chi/chi/v5 v5.3.1
 	github.com/golang-jwt/jwt/v5 v5.3.1
-	github.com/google/uuid v1.6.0
 	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/klauspost/cpuid/v2 v2.4.0
-	github.com/miekg/dns v1.1.72
-	github.com/mulgadc/bluebottle v1.17.0
-	github.com/mulgadc/northstar v1.17.0
-	github.com/mulgadc/predastore v1.17.0
-	github.com/mulgadc/viperblock v1.17.0
+	github.com/miekg/dns v1.1.73
+	github.com/mulgadc/bluebottle v1.18.0
+	github.com/mulgadc/northstar v1.18.0
+	github.com/mulgadc/predastore v1.18.0
+	github.com/mulgadc/viperblock v1.18.0
 	github.com/nats-io/nats-server/v2 v2.14.5
 	github.com/nats-io/nats.go v1.53.1
 	github.com/opencontainers/runtime-spec v1.3.0
@@ -34,7 +33,7 @@ require (
 	github.com/pterm/pterm v0.12.83
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
-	github.com/stretchr/testify v1.11.1
+	github.com/stretchr/testify v1.12.1
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
 	go.opentelemetry.io/otel v1.45.0
 	go.opentelemetry.io/otel/metric v1.45.0
@@ -46,6 +45,7 @@ require (
 	golang.org/x/crypto v0.55.0
 	golang.org/x/sync v0.22.0
 	golang.org/x/sys v0.47.0
+	golang.org/x/tools v0.49.0
 	gopkg.in/ini.v1 v1.67.3
 )
 
@@ -58,23 +58,23 @@ require (
 	github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.5.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.33.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.45.5 // indirect
-	github.com/aws/smithy-go v1.27.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect
+	github.com/aws/smithy-go v1.27.8 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/buraksezer/consistent v0.10.0 // indirect
+	github.com/buraksezer/consistent v1.0.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cenkalti/hub v1.0.2 // indirect
 	github.com/cenkalti/rpc2 v1.0.5 // indirect
@@ -98,7 +98,6 @@ require (
 	github.com/containerd/plugin v1.1.0 // indirect
 	github.com/containerd/ttrpc v1.2.8 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgraph-io/badger/v4 v4.9.6 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -117,6 +116,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/go-tpm v0.9.8 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gookit/color v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
@@ -134,7 +134,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/klauspost/compress v1.19.2 // indirect
-	github.com/klauspost/reedsolomon v1.14.1 // indirect
+	github.com/klauspost/reedsolomon v1.14.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
@@ -165,7 +165,6 @@ require (
 	github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
@@ -196,13 +195,12 @@ require (
 	go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 // indirect
-	golang.org/x/mod v0.38.0 // indirect
-	golang.org/x/net v0.57.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect
+	golang.org/x/mod v0.39.0 // indirect
+	golang.org/x/net v0.58.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect
 	golang.org/x/term v0.45.0 // indirect
 	golang.org/x/text v0.41.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.48.0 // indirect
 	golang.org/x/vuln v1.6.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect

go.sum

@@ -524,78 +524,78 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw
 github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
-github.com/aws/aws-sdk-go-v2 v1.43.5 h1:yKT5GYnFWhuDo+DqKvE5ZPwVn3RjC4MAeBtZGlh6AVM=
-github.com/aws/aws-sdk-go-v2 v1.43.5/go.mod h1:wZjAJppCntyOGgVSmgVTfDyRJK5PHOasO6Wsy8U7Axk=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17 h1:mn+Vxb9zgz/FE/yDTcFim3DZ1qpcrxR+qBQkBrl6bzA=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.17/go.mod h1:eDfmEFxu+BSVsUGLbzJhWjpOurv1mqczClS97yI8wdk=
+github.com/aws/aws-sdk-go-v2 v1.43.6 h1:RrmFcqCBxkJuf7g1axVo5krB4jM/AO8r5e5oujrgdoQ=
+github.com/aws/aws-sdk-go-v2 v1.43.6/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18/go.mod h1:4e5xhuXHx1e4U9EthvbPP1r/DIMp5c2823OL8karzcM=
 github.com/aws/aws-sdk-go-v2/config v1.18.14/go.mod h1:0pI6JQBHKwd0JnwAZS3VCapLKMO++UL2BOkWwyyzTnA=
 github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g=
 github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI=
-github.com/aws/aws-sdk-go-v2/config v1.32.36 h1:mX6ietU7UlB4w/2IUaexJdsyUDvhTd+jYPjVePiyi6s=
-github.com/aws/aws-sdk-go-v2/config v1.32.36/go.mod h1:rMpV4xk7ZK59edraSaHP0jsWrztWTT5tbCwWY495hug=
+github.com/aws/aws-sdk-go-v2/config v1.32.37 h1:Ljl7LOJB6ym0liuEl0+TZ3d7f5I8MEZN1Cj9PINlj/g=
+github.com/aws/aws-sdk-go-v2/config v1.32.37/go.mod h1:WJ7pe7ZPpmG8Q5kKS53zeypIV4FBGACxmte8Uc6SgUc=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.14/go.mod h1:85ckagDuzdIOnZRwws1eLKnymJs3ZM1QwVC1XcuNGOY=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.35 h1:Cxua2RVdRwL0sfjHM/SnQoOnQ7xKng9m5EQBO8BnZlg=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.35/go.mod h1:9XQ+RSIGPkycr+oCJYnB1uTv5kMVVR+rd2vYK0Hxj2w=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.36 h1:84s5xMme6ENYEdKG8rsbSFFg/8+lbHBeM9QYSO0gnDk=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.36/go.mod h1:c46BLdagDLIswjgt+GeQOslXgeS0E6wCacs5yZbxPGk=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36 h1:gucL1KH/PAYbpTpBg09CiVpBdTu4qkCl8C7xOTBixUg=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36/go.mod h1:usTB+PHhNMhrx2dxUeHcM7OrT5pySvmjYI++IsefPN0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 h1:b5tb+CZItBkydC7r3hTNdSO3pszG1R2EtnA+7TePQPk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37/go.mod h1:ZQ+6SU9X0oz6+7MUCSswv9Mjci4eaqZr21HI2RVy/yA=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 h1:5CrzwxDqf4w3x1Vs3/NiZ0nsC34Hbm3pIDMWbsLebOE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36/go.mod h1:A3gHdKZIvG/QXERzZwcxNS3RNDFcRCuhhTFBYp+V/nw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 h1:lznzIOvvbqjfe8UAaciCRJgBgJsxuTROKlhZuXQWfv8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37/go.mod h1:otfkzyfQeMMLZAqX59GSXTL3o22BR/l6HFaRzzbWSqA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 h1:A4N2f4YPcST0v+dWtX+xrpPPCL9VTBhoIFFUWYqbacE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36/go.mod h1:B/Qr859uxWUEfZeGotK5KAEoof4Q9YWgNtPSwV6jcyk=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 h1:zCEORWo0eU0gDjG+IyApE/2B+ZGG1m+GU7B263XV8ds=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37/go.mod h1:i6c0PEl3TNOWxRbQ++KQcVenPWS/GoQeiklKhNuqzJ8=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 h1:oyd3ke4V9AhKcRR7rRgxk1VyI+DjK2CBQtbxh3OkdaA=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37/go.mod h1:aA9D7SqfG9IC1b7FLD7Iyc8Q4JN0a8gHhNjN4zPlIaI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 h1:A3UAuCmx7LyUcrixBTzKJYYIUZ2yTvn6ZhT8PB+7APk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38/go.mod h1:1PDUYG9Z+JrbbsobsAZHjWOm9QBT/djiK3QbykTL5Z4=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 h1:iE4NGbvqUZnHDqddQAauZzCILYtFjOHwRM5MOOKLB5A=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16/go.mod h1:VsjEgrP+ibcou8TlWA4tYaB+0OojuhirsmCe+U60hTA=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29 h1:E65Hj648dOV6FuUfI0mYXXhQRHbsi7n+B9h6fZPJO/E=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.29/go.mod h1:xLrF9yNTCs92VZSpdEd68EJbgcdw3SMR74RO6QDzWHE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 h1:5437eMoOwqqQpZn2XJy74mlDCuPYL81texMT3mXqgtU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30/go.mod h1:xfu2m3dOpvW8lj98wQYa8V9ku/Rta59hsbireGzhh3A=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 h1:fx2ujmozWn+C/GtfXfz5k6Ckzza40ElOpIW7d92fLWQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36/go.mod h1:QT2ufGVJ+xTRxtXPHTQ1kHkAdWIKPCmD+BqYAXWv8/4=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37 h1:KGHa9iZCrgtkOsFfXb0S4ywsjostA/hau7WE9aSb43E=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.37/go.mod h1:FV79f0DSnZIEGsQjWenENGtUycrasyAaJZO+zRanLHA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1 h1:VUTtUJMuRNMkb/7NIKmd8NQaeQLPGCMoTJxkYKre4qM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1/go.mod h1:WvUaO0lP5GNMs1R6cs6qvB3mqo16GLta8yfOuf55Rpc=
-github.com/aws/aws-sdk-go-v2/service/signin v1.5.5 h1:0VTFBfOgPJrUSpGMgzoi8qLcXF5dbmiBuxpo14eBWUw=
-github.com/aws/aws-sdk-go-v2/service/signin v1.5.5/go.mod h1:sNZYlBxoohYMBYl47BO/bFtAM6I8HSsPa1qwwPPRGoQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 h1:a3D4AjrOrTrP8+d9ILBthqrElf0z1JNol09Xvnwcys8=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37/go.mod h1:ky0gTu+ukvUTuUKFIpp6Wid4oninrkCyvbFkVs0kpHM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 h1:gX8B8y3Ho30B1LPxefDKMi/HZqWEb47U9ogs3DtSG0M=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38/go.mod h1:l5WblZlcmGPe4/O7JY2HO25Z+xqTBvyfTyFbRMf8gYw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 h1:GNU0/xtPEXMKilJZ/a8BedeuQnvu+Usi6qVm9EFfncc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2/go.mod h1:4jYWUecEsQtE73jPl7p3jrbYXH5ffcR4gegyCygagfg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 h1:i68sFvXidKlkiSvI7d7Ilc1/UvW4CtBOaivH7jhG4fs=
+github.com/aws/aws-sdk-go-v2/service/signin v1.5.6/go.mod h1:/h7Obr9WTtzbjTHGASRQwLN7Bupw+TC3x8x7fyx39hE=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.33.5 h1:jDQARFp1mJ2PEnllQf01nfFXGfWMJ59e0/HCHUTTZCk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.33.5/go.mod h1:OcT2AhgTuxGAwZk5hgxaNLGpS33W8s8dUQadGVDVY9I=
+github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 h1:tpfGChmjUmv3W9WlRvy+stwKDTbFFdq8Zk9DbFPrfMU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.33.6/go.mod h1:CSjiDzmG/lsKkTOYjbkM+duLmRlW+LOxD64Na44ijnI=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.3/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5 h1:8xo1q9ttkYqMJ6vOXX67FPSpVEI7BWKVTKh77g82w+8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5/go.mod h1:hbBeEUrZg6VddXYZpbKPyF0tl4XEnM+Dbx92RW3vmZI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 h1:49BBtY68A+KJCQ3a2F3eUe6ROsKucxUdfHKoqorc0wI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6/go.mod h1:ptG2hbs7QltE1GcQY0MpS4bfrc51KCnBXUr7OT1EEfE=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.4/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
-github.com/aws/aws-sdk-go-v2/service/sts v1.45.5 h1:eQ5BtXDrPg2wK0AjtVPzeBhUpYPeqHE/ptiH7xJRGek=
-github.com/aws/aws-sdk-go-v2/service/sts v1.45.5/go.mod h1:f9ImhnOISY7BuTZLM8qHepCYnglHBVLk5wVzatmP++w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 h1:JvExZWabChDM0qJAirQYGfOYo0ndT3edXj+fqSPNjkE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.45.6/go.mod h1:XZcaQkV2cItp6yEkrwljyaPOf22RuX7T43jxap/FOmM=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
-github.com/aws/smithy-go v1.27.7 h1:Zgj5z4LfcDYoQIVk+n/yGdTkP/2y6ZT5vYxe0fp7bqE=
-github.com/aws/smithy-go v1.27.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/smithy-go v1.27.8 h1:FR0dxZfIlV7Z8eh2iHfIofdunw382XsDV3Mxt9nUvRY=
+github.com/aws/smithy-go v1.27.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
@@ -610,8 +610,8 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
-github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU=
-github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
+github.com/buraksezer/consistent v1.0.0 h1:ggz8fN8QW95rM4D+NlNMSUUCtoubMra3x/4fmFSv+fo=
+github.com/buraksezer/consistent v1.0.0/go.mod h1:L/Bczo4XsqEvJLOadAlIhamyNMhIj0iGt0+exc/KeV8=
 github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
@@ -1223,8 +1223,8 @@ github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
 github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
-github.com/klauspost/reedsolomon v1.14.1 h1:swE9kzyWXD/wVG+l5Pe8bWnQ0giIY7D1GjCBKk3kG2U=
-github.com/klauspost/reedsolomon v1.14.1/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
+github.com/klauspost/reedsolomon v1.14.2 h1:SafJYwpBBQBI6amHUygcjxZjXeN2HpiENHQDwuPWCCQ=
+github.com/klauspost/reedsolomon v1.14.2/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
@@ -1286,8 +1286,8 @@ github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU
 github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
 github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
-github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
+github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
+github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
 github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
@@ -1327,14 +1327,14 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/bluebottle v1.17.0 h1:K+tGfqwI8nnjV2naG5EGpWMO3Y6XpCTl7/5d5HmROiA=
-github.com/mulgadc/bluebottle v1.17.0/go.mod h1:kBSgbql3n7FY/emDWP2clK0jyEpOaPSCap+wggILwFQ=
-github.com/mulgadc/northstar v1.17.0 h1:bUkcGpKw06FcWyibkqP/O5J2gnXkM8EAtGX1AZjrmzc=
-github.com/mulgadc/northstar v1.17.0/go.mod h1:Qg559eZgF8GIsxhSyfDaZCnvhg1Ml66/nnsAQxE9ybg=
-github.com/mulgadc/predastore v1.17.0 h1:1xMFrzQU7dBaC5Q6qiAJx5ipj/yDubjtOJY7rjpryYo=
-github.com/mulgadc/predastore v1.17.0/go.mod h1:RM3hLSVsJAfwPJnmqbmcDgfdnP/RntXe+r1GOTD+S9c=
-github.com/mulgadc/viperblock v1.17.0 h1:ybcjcBf10jA1SiUkH5PTGFUZ9ZhGMljBff7iV06Vkg8=
-github.com/mulgadc/viperblock v1.17.0/go.mod h1:XYRmXrJ+k1P5nCxxTihcBUVM/4Srxc8zkFQp+a7ALNk=
+github.com/mulgadc/bluebottle v1.18.0 h1:aNrogJBz59gJAxCwpERGBC5bGXkSoUCCCBbNvnamuRU=
+github.com/mulgadc/bluebottle v1.18.0/go.mod h1:KkUyzqfKxyI8PvKR27EVqJnVm4BSbajl1Q/u+PQxAYw=
+github.com/mulgadc/northstar v1.18.0 h1:ksRjIKE6qtrw/ZoYaJh8JRBh+xdnuDw8FsmFLFCiXbg=
+github.com/mulgadc/northstar v1.18.0/go.mod h1:z6iol77R1vsIQyIBc1JQboUPhSJ6eG4bipzfbZZbLEM=
+github.com/mulgadc/predastore v1.18.0 h1:RW1Qw1SM2iYxqi6HaHLv5Qb1h1vhZSO3x6pOhPAb9mw=
+github.com/mulgadc/predastore v1.18.0/go.mod h1:+VBT3ofH/YmdTg0ug9EQwHDb8bQbHaMvWUyh5cIpBeo=
+github.com/mulgadc/viperblock v1.18.0 h1:u6yu5lPfZ+RSmfzlg1OXI3whlRBnGp8t685xUzzxlns=
+github.com/mulgadc/viperblock v1.18.0/go.mod h1:OjqMFZba0TJl9zcW2oKmz2/MrApIxRIFNcafiv5vTkM=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1558,8 +1558,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -1575,8 +1576,9 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
+github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/substrait-io/substrait-go v0.4.2/go.mod h1:qhpnLmrcvAnlZsUyPXZRqldiHapPTXC3t7xFgDi3aQg=
@@ -2018,8 +2020,8 @@ golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
 golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
 golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
-golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
-golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
+golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
+golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2132,8 +2134,8 @@ golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
 golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
 golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
-golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
-golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2384,8 +2386,8 @@ golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3c
 golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
 golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
-golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxDllNhhf/fAx8qIm4t6A7vpm2A=
-golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg=
+golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q=
+golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2595,8 +2597,8 @@ golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg
 golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
-golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
-golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
 golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=

images/mkosi.profiles/vllm/bedrock-bundle-init_test.sh

@@ -0,0 +1,106 @@
+#!/bin/sh
+# Self-contained POSIX test for bedrock-bundle-init (mkosi.extra/usr/local/
+# sbin/bedrock-bundle-init): the bounded wait for the first member env file,
+# enumerating every *.env file present, and starting each one's
+# bedrock-serve@<instance>.service. systemctl is stubbed on PATH so this runs
+# with no real systemd. Not shipped into the image -- it sits beside the
+# profile, not under mkosi.extra/.
+#
+# Run: sh images/mkosi.profiles/vllm/bedrock-bundle-init_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${BEDROCK_TEST_SCRIPT:-${SCRIPT_DIR}/mkosi.extra/usr/local/sbin/bedrock-bundle-init}"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+cat > "${STUBBIN}/systemctl" <<'EOF'
+#!/bin/sh
+echo "systemctl $*" >> "${SYSTEMCTL_CALLS}"
+exit 0
+EOF
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+ENV_DIR="${WORK}/bedrock-bundle"
+SYSTEMCTL_CALLS="${WORK}/systemctl.calls"
+export SYSTEMCTL_CALLS
+
+reset_state() {
+    rm -rf "${ENV_DIR}"
+    mkdir -p "${ENV_DIR}"
+    : > "${SYSTEMCTL_CALLS}"
+}
+
+run() {
+    env BEDROCK_ENV_DIR="${ENV_DIR}" \
+        BEDROCK_BUNDLE_ENV_WAIT="${BEDROCK_BUNDLE_ENV_WAIT:-2}" \
+        sh "${SCRIPT}" </dev/null
+}
+
+run_ok() { run > "${WORK}/out" 2>&1 || { fail "$1: non-zero exit: $(cat "${WORK}/out")"; return 1; }; }
+run_fails() { run > "${WORK}/out" 2>&1 && fail "$1: expected a non-zero exit" || pass "$1: refused"; }
+
+# --- Case 1: a single member (bundle-of-one) starts exactly one instance ---
+reset_state
+: > "${ENV_DIR}/solo.env"
+if run_ok "bundle-of-one"; then
+    grep -q "systemctl start --no-block bedrock-serve@solo.service" "${SYSTEMCTL_CALLS}" \
+        && pass "bundle-of-one: started the sole member's instance" || fail "bundle-of-one: instance not started"
+    [ "$(grep -c '^systemctl start' "${SYSTEMCTL_CALLS}")" -eq 1 ] \
+        && pass "bundle-of-one: started exactly one instance" || fail "bundle-of-one: started more than one instance"
+fi
+
+# --- Case 2: a real bundle starts one instance per member, sanitized names
+# preserved verbatim from the filename (bedrock-serve's own INSTANCE arg
+# matching userdata.go's memberEnvPath/sanitizeMemberInstanceName) ---
+reset_state
+: > "${ENV_DIR}/meta_llama3-2-1b-instruct-v1_0.env"
+: > "${ENV_DIR}/nomic-embed-text-v1_5.env"
+: > "${ENV_DIR}/bge-reranker-v2-m3.env"
+if run_ok "three-member-bundle"; then
+    for n in meta_llama3-2-1b-instruct-v1_0 nomic-embed-text-v1_5 bge-reranker-v2-m3; do
+        grep -q "systemctl start --no-block bedrock-serve@${n}.service" "${SYSTEMCTL_CALLS}" \
+            && pass "three-member-bundle: started ${n}" || fail "three-member-bundle: ${n} not started"
+    done
+    [ "$(grep -c '^systemctl start' "${SYSTEMCTL_CALLS}")" -eq 3 ] \
+        && pass "three-member-bundle: started exactly three instances" || fail "three-member-bundle: wrong instance count"
+fi
+
+# --- Case 3: no env files ever appear ---
+reset_state
+_t0=$(date +%s)
+BEDROCK_BUNDLE_ENV_WAIT=3 run_fails "no-env-files"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -ge 3 ] \
+    && pass "no-env-files: waited the full ${_elapsed}s bound before giving up" \
+    || fail "no-env-files: returned after only ${_elapsed}s, expected >= 3s"
+grep -q . "${SYSTEMCTL_CALLS}" \
+    && fail "no-env-files: started something despite no handoff" || pass "no-env-files: nothing started"
+
+# --- Case 4: an env file appears mid-wait ---
+reset_state
+( sleep 1; : > "${ENV_DIR}/late.env" ) &
+BGPID=$!
+if BEDROCK_BUNDLE_ENV_WAIT=5 run_ok "env-appears-midwait"; then
+    grep -q "systemctl start --no-block bedrock-serve@late.service" "${SYSTEMCTL_CALLS}" \
+        && pass "env-appears-midwait: retried until the handoff showed up, then started it" \
+        || fail "env-appears-midwait: did not start the member once its env file appeared"
+fi
+wait "${BGPID}" 2>/dev/null || true
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all bedrock-bundle-init cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

images/mkosi.profiles/vllm/bedrock-serve_test.sh

@@ -0,0 +1,322 @@
+#!/bin/sh
+# Self-contained POSIX test for bedrock-serve (mkosi.extra/usr/local/sbin/
+# bedrock-serve): the env-file wait, engine dispatch (vllm vs tei), the
+# ordinal-indexed weights-device resolution across a multi-member bundle (the
+# three tiers: named device, by-id-sorted-by-target, ext4-scan), the bounded
+# waits' timeouts, and the final read-only mount + exec. No root and no real
+# block devices: blkid, mount and both engines' entrypoints are stubbed on
+# PATH, and every path the script touches is redirected into a temp dir via
+# its env knobs. Not shipped into the image -- it sits beside the profile,
+# not under mkosi.extra/.
+#
+# Run: sh images/mkosi.profiles/vllm/bedrock-serve_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+# BEDROCK_TEST_SCRIPT: override to point at an alternate copy of the wrapper
+# (e.g. a saved pre-fix version), so a regression case can be run against
+# both and its before/after behaviour compared directly.
+SCRIPT="${BEDROCK_TEST_SCRIPT:-${SCRIPT_DIR}/mkosi.extra/usr/local/sbin/bedrock-serve}"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+# blkid stub: prints a real blkid's line for any device listed in FS_TABLE as
+# "<dev> <fstype>", nothing (rc=2) for one that holds none.
+cat > "${STUBBIN}/blkid" <<'EOF'
+#!/bin/sh
+dev="$1"
+fstype=$(awk -v d="${dev}" '$1 == d { print $2 }' "${FS_TABLE}" 2>/dev/null)
+[ -n "${fstype}" ] || exit 2
+echo "${dev}: UUID=\"1234-5678\" TYPE=\"${fstype}\""
+EOF
+
+# mount stub: records the call and, on success, appends the mount to
+# MOUNTS_FILE so weights_mounted() sees the effect on a later invocation
+# (idempotent-restart case), the way a real kernel's /proc/mounts would.
+cat > "${STUBBIN}/mount" <<'EOF'
+#!/bin/sh
+echo "mount $*" >> "${MOUNT_CALLS}"
+[ "${MOUNT_FAIL:-0}" = "1" ] && exit 1
+dev=""; mnt=""
+for a in "$@"; do
+    case "$a" in -*) continue ;; esac
+    if [ -z "$dev" ]; then dev="$a"; else mnt="$a"; fi
+done
+echo "${dev} ${mnt} ext4 ro 0 0" >> "${MOUNTS_FILE}"
+exit 0
+EOF
+
+# vllm stub: the venv binary the wrapper execs into for a vllm-engine member.
+# Records its argv so the test can assert the mount point, port and
+# served-model-name reached it.
+cat > "${STUBBIN}/vllm" <<'EOF'
+#!/bin/sh
+echo "vllm $*" >> "${VLLM_CALLS}"
+exit 0
+EOF
+
+# tei-router-stub: stands in for text-embeddings-router, which the wrapper
+# now execs directly on BEDROCK_SERVE_PORT for a tei-engine member -- no
+# front-door in between. Records its argv (model id/path, port, extra args).
+cat > "${STUBBIN}/tei-router-stub" <<'EOF'
+#!/bin/sh
+echo "tei-router-stub $*" >> "${TEI_CALLS}"
+exit 0
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+ENV_DIR="${WORK}/bedrock-bundle"
+WEIGHTS_ROOT="${WORK}/weights-root"
+SYS_BLOCK="${WORK}/sys/block"
+DEV_DIR="${WORK}/dev"
+BYID_DIR="${WORK}/by-id"
+MOUNTS_FILE="${WORK}/mounts"
+FS_TABLE="${WORK}/fs.table"
+MOUNT_CALLS="${WORK}/mount.calls"
+VLLM_CALLS="${WORK}/vllm.calls"
+TEI_CALLS="${WORK}/tei.calls"
+export FS_TABLE MOUNT_CALLS VLLM_CALLS TEI_CALLS MOUNTS_FILE
+
+write_env() {
+    # write_env <instance> <engine> <device> [extra-args]: lay down the
+    # bootstrap handoff buildBundleUserData would have cloud-init write.
+    mkdir -p "${ENV_DIR}"
+    cat > "${ENV_DIR}/$1.env" <<EOF
+BEDROCK_GROUP_ID=test-group
+BEDROCK_ENGINE=$2
+BEDROCK_MODEL_ID=test-model-$1
+BEDROCK_ARGS=${4:-}
+BEDROCK_WEIGHTS_DEVICE=$3
+BEDROCK_SERVE_PORT=8000
+EOF
+}
+
+# add_disk <name>: materialise a whole disk in the fake sysfs + /dev.
+add_disk() {
+    mkdir -p "${SYS_BLOCK}/$1" "${DEV_DIR}"
+    : > "${DEV_DIR}/$1"
+}
+
+reset_state() {
+    rm -rf "${WORK}/sys" "${DEV_DIR}" "${BYID_DIR}" "${WEIGHTS_ROOT}" "${ENV_DIR}"
+    mkdir -p "${SYS_BLOCK}" "${DEV_DIR}" "${BYID_DIR}" "${ENV_DIR}"
+    # Root is vda2, so root_disk() must resolve to "vda" and never treat it as
+    # a weights candidate.
+    printf '/dev/vda2 / ext4 rw,relatime 0 0\n' > "${MOUNTS_FILE}"
+    : > "${FS_TABLE}"
+    : > "${MOUNT_CALLS}"
+    : > "${VLLM_CALLS}"
+    : > "${TEI_CALLS}"
+    add_disk vda
+}
+
+# run <instance>: invoke bedrock-serve for one member with every path knob
+# pointed into the temp dir and short bounded waits, so the "nothing ever
+# appears" case does not stall the suite.
+run() {
+    env BEDROCK_ENV_DIR="${ENV_DIR}" \
+        BEDROCK_ENV_WAIT="${BEDROCK_ENV_WAIT:-2}" \
+        BEDROCK_WEIGHTS_MOUNT="${WEIGHTS_ROOT}/$1" \
+        BEDROCK_VLLM_VENV_BIN="${STUBBIN}" \
+        BEDROCK_TEI_BIN="${STUBBIN}/tei-router-stub" \
+        BEDROCK_DEVICE_WAIT="${BEDROCK_DEVICE_WAIT:-2}" \
+        BEDROCK_SYS_BLOCK="${SYS_BLOCK}" \
+        BEDROCK_DEV_DIR="${DEV_DIR}" \
+        BEDROCK_BYID_DIR="${BYID_DIR}" \
+        BEDROCK_MOUNTS_FILE="${MOUNTS_FILE}" \
+        sh "${SCRIPT}" "$1" </dev/null
+}
+
+run_ok() { run "$1" > "${WORK}/out" 2>&1 || { fail "$1: non-zero exit: $(cat "${WORK}/out")"; return 1; }; }
+run_fails() { run "$1" > "${WORK}/out" 2>&1 && fail "$1: expected a non-zero exit" || pass "$1: refused"; }
+
+# --- Case 1: vllm engine, named device present ---
+reset_state
+add_disk sdf
+write_env m1 vllm "${DEV_DIR}/sdf"
+if run_ok "m1"; then
+    grep -q "mount -o ro ${DEV_DIR}/sdf ${WEIGHTS_ROOT}/m1" "${MOUNT_CALLS}" \
+        && pass "vllm-named-device: mounted read-only from the named device" || fail "vllm-named-device: not mounted from named device"
+    grep -q "vllm serve ${WEIGHTS_ROOT}/m1 --port 8000 --served-model-name test-model-m1" "${VLLM_CALLS}" \
+        && pass "vllm-named-device: vllm exec'd with mount point, port and model name" || fail "vllm-named-device: vllm not invoked correctly"
+fi
+
+# --- Case 2: tei engine, named device present, TEI exec'd directly on the
+# serve port, not vllm and not any front-door ---
+reset_state
+add_disk sdf
+write_env m1 tei "${DEV_DIR}/sdf"
+if run_ok "m1"; then
+    grep -q "mount -o ro ${DEV_DIR}/sdf ${WEIGHTS_ROOT}/m1" "${MOUNT_CALLS}" \
+        && pass "tei-named-device: mounted read-only from the named device" || fail "tei-named-device: not mounted from named device"
+    grep -q -- "--model-id ${WEIGHTS_ROOT}/m1 --port 8000" "${TEI_CALLS}" \
+        && pass "tei-named-device: TEI exec'd directly with mount point and port" || fail "tei-named-device: TEI not invoked correctly"
+    [ -s "${VLLM_CALLS}" ] && fail "tei-named-device: vllm was invoked for a tei member" || pass "tei-named-device: vllm never invoked"
+fi
+
+# --- Case 3: two-member bundle, by-id fallback disambiguated by ordinal ---
+# sdf (ordinal 0) and sdg (ordinal 1) both absent under their launcher names;
+# two by-id links exist, resolving to vdb and vdc respectively. Each
+# member's own wrapper invocation must pick its OWN ordinal-indexed device,
+# not just "the sole candidate" (which no longer holds once there is more
+# than one member).
+reset_state
+add_disk vdb
+add_disk vdc
+: > "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_volAAAAAAAAAAAAAAAAA"
+: > "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_volBBBBBBBBBBBBBBBBB"
+ln -sf "../dev/vdb" "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_volAAAAAAAAAAAAAAAAA"
+ln -sf "../dev/vdc" "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_volBBBBBBBBBBBBBBBBB"
+write_env m1 vllm "/dev/sdf"
+write_env m2 tei "/dev/sdg"
+# resolve_weights_device mounts THROUGH the by-id link path itself (same as
+# the single-model image always did), not a readlink-resolved target -- the
+# link is only used, sorted by its target name, to pick which member gets
+# which candidate.
+if run_ok "m1"; then
+    grep -q "mount -o ro ${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_volAAAAAAAAAAAAAAAAA ${WEIGHTS_ROOT}/m1" "${MOUNT_CALLS}" \
+        && pass "byid-ordinal: member 0 (sdf) mounted from the first by-id link (-> vdb)" || fail "byid-ordinal: member 0 mounted the wrong device: $(cat "${MOUNT_CALLS}")"
+fi
+: > "${MOUNT_CALLS}"
+if run_ok "m2"; then
+    grep -q "mount -o ro ${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_volBBBBBBBBBBBBBBBBB ${WEIGHTS_ROOT}/m2" "${MOUNT_CALLS}" \
+        && pass "byid-ordinal: member 1 (sdg) mounted from the second by-id link (-> vdc)" || fail "byid-ordinal: member 1 mounted the wrong device: $(cat "${MOUNT_CALLS}")"
+fi
+
+# --- Case 4: two-member bundle, ext4-scan fallback disambiguated by ordinal
+# (by-id bridge has not minted anything yet) ---
+reset_state
+add_disk vdb
+add_disk vdc
+echo "${DEV_DIR}/vdb ext4" > "${FS_TABLE}"
+echo "${DEV_DIR}/vdc ext4" >> "${FS_TABLE}"
+write_env m1 vllm "/dev/sdf"
+write_env m2 vllm "/dev/sdg"
+if run_ok "m1"; then
+    grep -q "mount -o ro ${DEV_DIR}/vdb ${WEIGHTS_ROOT}/m1" "${MOUNT_CALLS}" \
+        && pass "ext4-ordinal: member 0 (sdf) mounted from the first ext4 candidate (vdb)" || fail "ext4-ordinal: member 0 mounted the wrong device"
+fi
+: > "${MOUNT_CALLS}"
+if run_ok "m2"; then
+    grep -q "mount -o ro ${DEV_DIR}/vdc ${WEIGHTS_ROOT}/m2" "${MOUNT_CALLS}" \
+        && pass "ext4-ordinal: member 1 (sdg) mounted from the second ext4 candidate (vdc)" || fail "ext4-ordinal: member 1 mounted the wrong device"
+fi
+
+# --- Case 5: bundle-of-one degenerates to the old single-candidate behaviour
+# via the by-id tier (ordinal 0 == "the sole candidate") ---
+reset_state
+add_disk vdb
+: > "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef"
+ln -sf "../dev/vdb" "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef"
+write_env solo vllm "/dev/sdf"
+if run_ok "solo"; then
+    grep -q "mount -o ro ${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef ${WEIGHTS_ROOT}/solo" "${MOUNT_CALLS}" \
+        && pass "bundle-of-one: mounted from the sole by-id link, same as the single-model image" || fail "bundle-of-one: not mounted from the sole by-id link"
+    grep -q "vllm serve ${WEIGHTS_ROOT}/solo --port 8000 --served-model-name test-model-solo" "${VLLM_CALLS}" \
+        && pass "bundle-of-one: vllm exec'd on port 8000, unchanged from today's single-model serve" || fail "bundle-of-one: vllm invocation regressed"
+fi
+
+# --- Case 6: the root disk is never an ext4-scan candidate ---
+reset_state
+echo "${DEV_DIR}/vda ext4" > "${FS_TABLE}"
+write_env m1 vllm "/dev/sdf"
+run_fails "m1"
+grep -q . "${MOUNT_CALLS}" \
+    && fail "ext4-root-excluded: mounted the root disk as weights" || pass "ext4-root-excluded: root disk excluded"
+
+# --- Case 7: nothing ever appears for this member ---
+# Times the run to prove the wrapper actually spun in the retry loop for the
+# full DEVICE_WAIT_SECS bound rather than dying on the first "not found yet".
+reset_state
+write_env m1 vllm "/dev/sdf"
+_t0=$(date +%s)
+BEDROCK_DEVICE_WAIT=3 run_fails "m1"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -ge 3 ] \
+    && pass "nothing-appears: waited the full ${_elapsed}s bound before giving up" \
+    || fail "nothing-appears: returned after only ${_elapsed}s, expected >= 3s"
+
+# --- Case 8: a malformed BEDROCK_WEIGHTS_DEVICE suffix is refused, not
+# retried (weights_device_ordinal's tier-2 refusal) ---
+reset_state
+mkdir -p "${ENV_DIR}"
+cat > "${ENV_DIR}/m1.env" <<'EOF'
+BEDROCK_GROUP_ID=test-group
+BEDROCK_ENGINE=vllm
+BEDROCK_MODEL_ID=test-model-m1
+BEDROCK_ARGS=
+BEDROCK_WEIGHTS_DEVICE=/dev/sdzz
+BEDROCK_SERVE_PORT=8000
+EOF
+_t0=$(date +%s)
+BEDROCK_DEVICE_WAIT=30 run_fails "m1"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -lt 15 ] \
+    && pass "bad-device-suffix: failed fast rather than retrying an unresolvable suffix" \
+    || fail "bad-device-suffix: took ${_elapsed}s, expected a fast refusal"
+
+# --- Case 9: an unknown BEDROCK_ENGINE is refused, not silently defaulted ---
+reset_state
+add_disk sdf
+write_env m1 not-a-real-engine "${DEV_DIR}/sdf"
+run_fails "m1"
+[ -s "${VLLM_CALLS}" ] && fail "bad-engine: vllm was invoked despite an unknown engine" || pass "bad-engine: nothing invoked"
+
+# --- Case 10: already-mounted weights point is a no-op restart ---
+reset_state
+add_disk sdf
+write_env m1 vllm "${DEV_DIR}/sdf"
+printf '%s %s ext4 ro 0 0\n' "${DEV_DIR}/sdf" "${WEIGHTS_ROOT}/m1" >> "${MOUNTS_FILE}"
+if run_ok "m1"; then
+    grep -q . "${MOUNT_CALLS}" \
+        && fail "already-mounted: mounted over an existing mount" || pass "already-mounted: no-op, straight to exec"
+    grep -q "vllm serve" "${VLLM_CALLS}" \
+        && pass "already-mounted: still execs vllm" || fail "already-mounted: vllm never invoked"
+fi
+
+# --- Case 11: BEDROCK_ARGS passes each flag through as its own argument ---
+reset_state
+add_disk sdf
+write_env m1 vllm "${DEV_DIR}/sdf" "--dtype=bfloat16 --max-model-len=4096"
+if run_ok "m1"; then
+    grep -q -- "--dtype=bfloat16 --max-model-len=4096" "${VLLM_CALLS}" \
+        && pass "extra-args: BEDROCK_ARGS flags reached vllm serve" || fail "extra-args: flags missing"
+fi
+
+# --- Case 11b: BEDROCK_ARGS reaches TEI the same way, each flag its own arg ---
+reset_state
+add_disk sdf
+write_env m1 tei "${DEV_DIR}/sdf" "--pooling=cls --max-batch-tokens=8192"
+if run_ok "m1"; then
+    grep -q -- "--pooling=cls --max-batch-tokens=8192" "${TEI_CALLS}" \
+        && pass "extra-args-tei: BEDROCK_ARGS flags reached TEI" || fail "extra-args-tei: flags missing"
+fi
+
+# --- Case 12: env file never appears for this member ---
+reset_state
+_t0=$(date +%s)
+BEDROCK_ENV_WAIT=3 run_fails "m1"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -ge 3 ] \
+    && pass "env-never-appears: waited the full ${_elapsed}s bound before giving up" \
+    || fail "env-never-appears: returned after only ${_elapsed}s, expected >= 3s"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all bedrock-serve cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

images/mkosi.profiles/vllm/mkosi.conf

@@ -15,8 +15,12 @@
 # and mergemanydict() is first-source-wins, so a 90- name loses to cloud-init's
 # own 90_dpkg.cfg ('_' sorts above '-') and the pin is silently discarded.
 ExtraTrees=../../../scripts/images/eks-node/cloud-init-ec2.cfg:/etc/cloud/cloud.cfg.d/99-spinifex-ec2-imds.cfg
+           ../../staging/vllm:/
 
 [Build]
 # mkosi.postinst.chroot fetches the uv installer, a standalone CPython build,
 # and vLLM's own PyPI/CUDA wheels — all network, none of it in the archive.
+# The TEI engine staged under ../../staging/vllm above needs no network here:
+# stage_tei_binary() (scripts/mkosi-build.sh) extracts it from HuggingFace's
+# own published CUDA image on the host, before this container even starts.
 WithNetwork=yes