feat: version-aware host-asset preflight for binary-swap deploys (#750)

A binary-swap deploy (make deploy or a manual spx copy) ships a new binary
whose host-config prerequisites — the privileged helper scripts and their
sudoers grant, placed only by setup.sh / setup-ovn.sh — it does not ship.
The node looks healthy and the gap surfaces later as a cryptic launch
failure or a silent security regression.

Add a preflight package that checks the host against what this build
expects, comparing asset content, not existence, so a stale-but-present
asset is caught too. The managed helpers install from quoted heredocs, so
their on-disk bytes equal the heredoc body verbatim; the binary carries
the canonical sha256s and compares.

- spinifex/preflight: CheckHost / CheckHostAt, content hash of the two
  managed helpers and a NOPASSWD grant-line check for the sysctl helper.
- Generated canonicalHashes plus a drift guard that re-extracts the
  heredocs and fails if the committed manifest no longer matches.
- Three surfaces: spx admin preflight, a make deploy gate
  (SKIP_ASSET_PREFLIGHT=1 to bypass), and a non-fatal daemon boot warning.
This commit is contained in:
Josh Siviour
2026-08-10 07:52:39 +10:00
committed by GitHub
parent bc3d59c0c3
commit d178ef5992
8 changed files with 567 additions and 0 deletions
+14
View File
@@ -260,6 +260,20 @@ bench:
# `make install-microvm` explicitly the first time.
deploy: build
sudo install -m 755 bin/spx /usr/local/bin/spx
@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
if ! sudo /usr/local/bin/spx admin preflight; then \
echo ""; \
echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
echo " grants don't match the binary just installed. Remediate with:"; \
echo " scripts/update-nodes.sh (from the mulga root)"; \
echo " or:"; \
echo " make reinstall"; \
echo " Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \
exit 1; \
fi; \
else \
echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
fi
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
$(MAKE) install-microvm; \
else \
+50
View File
@@ -0,0 +1,50 @@
package cmd
import (
"fmt"
"os"
"text/tabwriter"
"github.com/mulgadc/spinifex/spinifex/preflight"
"github.com/spf13/cobra"
)
var adminPreflightQuiet bool
var adminPreflightCmd = &cobra.Command{
Use: "preflight",
Short: "Verify this host's managed assets against what this build expects",
Long: `spx knows its own build (spx version) but not whether this host already
has the privileged helper scripts and sudoers grants that build depends on —
setup.sh and setup-ovn.sh install them, and a binary-swap deploy (make
deploy or a manual binary copy) skips those steps.
Checks compare content, not mere existence, so a present-but-stale asset is
caught too. Exits non-zero if any asset is Missing, Stale, or Ungranted.`,
RunE: runAdminPreflight,
SilenceUsage: true,
}
func init() {
adminCmd.AddCommand(adminPreflightCmd)
adminPreflightCmd.Flags().BoolVar(&adminPreflightQuiet, "quiet", false, "print only non-OK rows")
}
func runAdminPreflight(_ *cobra.Command, _ []string) error {
results := preflight.CheckHost()
w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
fmt.Fprintln(w, "PATH\tKIND\tSTATUS\tDETAIL")
for _, r := range results {
if adminPreflightQuiet && r.Status == preflight.OK {
continue
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Path, r.Kind, r.Status, r.Detail)
}
w.Flush()
if preflight.HasProblem(results) {
return fmt.Errorf("host-asset preflight found problems — see table above")
}
return nil
}
+56
View File
@@ -0,0 +1,56 @@
#!/bin/bash
# gen-asset-manifest.sh — extracts the canonical bytes of each managed asset
# heredoc from the provisioning scripts, sha256s them, and (re)writes
# spinifex/preflight/manifest_gen.go.
#
# Run this after editing a managed heredoc in setup.sh or setup-ovn.sh.
# manifest_gen_test.go re-extracts the same heredocs and fails if the
# committed manifest no longer matches, catching drift in CI.
#
# Usage: scripts/gen-asset-manifest.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
OUT="$ROOT/spinifex/preflight/manifest_gen.go"
# extract_heredoc FILE ANCHOR
# Prints the body of the first quoted 'HELPER' heredoc starting after the
# first line containing ANCHOR — both scripts hold more than one such heredoc.
extract_heredoc() {
local file=$1 anchor=$2
awk -v anchor="$anchor" '
state == 0 && index($0, anchor) { state = 1; next }
state == 1 && index($0, "<<") && index($0, "HELPER") { state = 2; next }
state == 2 && $0 == "HELPER" { exit }
state == 2 { print }
' "$file"
}
sysctl_hash=$(extract_heredoc "$ROOT/scripts/setup.sh" 'install_endpoint_sysctl_helper()' | sha256sum | cut -d' ' -f1)
ovs_hash=$(extract_heredoc "$ROOT/scripts/setup-ovn.sh" 'Step 8: Grant the spinifex service users access to OVS/OVN' | sha256sum | cut -d' ' -f1)
if [ -z "$sysctl_hash" ] || [ -z "$ovs_hash" ]; then
echo "gen-asset-manifest.sh: failed to extract one or both heredocs — anchors may be stale" >&2
exit 1
fi
mkdir -p "$(dirname "$OUT")"
cat >"$OUT" <<EOF
// Code generated by scripts/gen-asset-manifest.sh; DO NOT EDIT.
package preflight
var canonicalHashes = map[string]string{
"/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl": "$sysctl_hash",
"/usr/local/lib/spinifex/ovs-socket-perms.sh": "$ovs_hash",
}
EOF
if command -v gofmt >/dev/null 2>&1; then
gofmt -w "$OUT"
fi
echo "wrote $OUT"
echo " spinifex-set-endpoint-sysctl: $sysctl_hash"
echo " ovs-socket-perms.sh: $ovs_hash"
+12
View File
@@ -65,6 +65,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/network/host"
"github.com/mulgadc/spinifex/spinifex/objectstore"
"github.com/mulgadc/spinifex/spinifex/otelsetup"
"github.com/mulgadc/spinifex/spinifex/preflight"
"github.com/mulgadc/spinifex/spinifex/types"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/mulgadc/spinifex/spinifex/vm"
@@ -1247,6 +1248,17 @@ func (d *Daemon) startLocal() error {
slog.Warn("Failed to set daemon OOM score", "err", err)
}
// Warn on host assets this build expects but that are missing, stale, or
// ungranted, so drift surfaces as a named log line rather than a later
// cryptic failure. Never fatal — unrelated asset problems must not ground a node.
for _, r := range preflight.CheckHost() {
if r.Status != preflight.OK {
slog.Warn("Host asset preflight problem", "path", r.Path, "kind", r.Kind,
"status", r.Status.String(), "detail", r.Detail,
"remediation", "run scripts/update-nodes.sh or make reinstall")
}
}
// Recover local instance state from disk. Fatal on corruption — orphaned VMs.
if err := d.LoadState(); err != nil {
return fmt.Errorf("load local instance state: %w", err)
+8
View File
@@ -0,0 +1,8 @@
// Code generated by scripts/gen-asset-manifest.sh; DO NOT EDIT.
package preflight
var canonicalHashes = map[string]string{
"/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl": "8f9bf3205d02b35e147924e3f6fa119ed3d5414375d325d905b80cbba349b613",
"/usr/local/lib/spinifex/ovs-socket-perms.sh": "11ad8fa79ca1544044afbf1bba944a8248ff0e0278ce6c4af7e19de5c61b9b9a",
}
+119
View File
@@ -0,0 +1,119 @@
package preflight
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
// repoRoot walks up from this test file's directory until scripts/setup.sh
// is found, so the drift guard locates the checkout regardless of where
// `go test` is invoked from.
func repoRoot(t *testing.T) string {
t.Helper()
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed to resolve this test file's path")
}
dir := filepath.Dir(file)
for {
if _, err := os.Stat(filepath.Join(dir, "scripts", "setup.sh")); err == nil {
return dir
}
parent := filepath.Dir(dir)
if parent == dir {
t.Fatal("could not locate repo root: no scripts/setup.sh found walking up from test file")
}
dir = parent
}
}
// extractHeredoc mirrors scripts/gen-asset-manifest.sh's extraction: the
// body of the first quoted 'HELPER' heredoc starting after the first line
// containing anchor. Both provisioning scripts hold more than one such heredoc.
func extractHeredoc(file, anchor string) ([]byte, error) {
data, err := os.ReadFile(file)
if err != nil {
return nil, err
}
lines := strings.Split(string(data), "\n")
anchorLine := -1
for i, line := range lines {
if strings.Contains(line, anchor) {
anchorLine = i
break
}
}
if anchorLine < 0 {
return nil, fmt.Errorf("anchor %q not found in %s", anchor, file)
}
startLine := -1
for i := anchorLine + 1; i < len(lines); i++ {
if strings.Contains(lines[i], "<<") && strings.Contains(lines[i], "HELPER") {
startLine = i
break
}
}
if startLine < 0 {
return nil, fmt.Errorf("no HELPER heredoc start found after anchor %q in %s", anchor, file)
}
var body []string
for i := startLine + 1; i < len(lines); i++ {
if lines[i] == "HELPER" {
return []byte(strings.Join(body, "\n") + "\n"), nil
}
body = append(body, lines[i])
}
return nil, fmt.Errorf("no HELPER heredoc terminator found after anchor %q in %s", anchor, file)
}
// TestManifestMatchesProvisioningScripts re-extracts each managed heredoc
// and asserts the committed canonicalHashes still matches, turning an
// un-regenerated heredoc edit into a CI failure instead of silent drift.
func TestManifestMatchesProvisioningScripts(t *testing.T) {
root := repoRoot(t)
cases := []struct {
path string
file string
anchor string
}{
{
path: "/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl",
file: filepath.Join(root, "scripts", "setup.sh"),
anchor: "install_endpoint_sysctl_helper()",
},
{
path: "/usr/local/lib/spinifex/ovs-socket-perms.sh",
file: filepath.Join(root, "scripts", "setup-ovn.sh"),
anchor: "Step 8: Grant the spinifex service users access to OVS/OVN",
},
}
for _, c := range cases {
body, err := extractHeredoc(c.file, c.anchor)
if err != nil {
t.Fatalf("%s: %v", c.path, err)
}
sum := sha256.Sum256(body)
got := hex.EncodeToString(sum[:])
want, ok := canonicalHashes[c.path]
if !ok {
t.Fatalf("%s: no entry in canonicalHashes; run scripts/gen-asset-manifest.sh", c.path)
}
if got != want {
t.Errorf("%s: heredoc in %s hashes to %s, manifest has %s — run scripts/gen-asset-manifest.sh",
c.path, c.file, got, want)
}
}
}
+156
View File
@@ -0,0 +1,156 @@
// Package preflight verifies a host's managed helper scripts and sudoers
// grants — which a binary-swap deploy does not ship — against what this
// build of spinifex expects, comparing content so a stale asset is caught.
package preflight
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/mulgadc/spinifex/spinifex/utils"
)
//go:generate ../../scripts/gen-asset-manifest.sh
// Status is the outcome of checking one managed asset against this build's manifest.
type Status int
const (
// OK means the on-host asset matches what this build expects.
OK Status = iota
// Missing means the asset is absent from the host.
Missing
// Stale means the asset is present but its content does not match this build's manifest.
Stale
// Ungranted means the sudoers grant for a helper is absent or missing its NOPASSWD line.
Ungranted
)
// String returns the human-readable name of the status, used in table output and log lines.
func (s Status) String() string {
switch s {
case OK:
return "OK"
case Missing:
return "Missing"
case Stale:
return "Stale"
case Ungranted:
return "Ungranted"
default:
return "Unknown"
}
}
// Result is the outcome of checking one managed asset against this build's manifest.
type Result struct {
Path string
Kind string // "helper" | "sudoers-grant"
Status Status
Detail string
}
// managedHelpers lists every helper script this build expects on the host,
// keyed by installed path. An explicit list, not a directory scan, so adding
// an asset here is a reviewed change and never accidental.
var managedHelpers = []string{
utils.EndpointSysctlHelper,
"/usr/local/lib/spinifex/ovs-socket-perms.sh",
}
// sudoersGrantFile is assembled with substitutions and multiple rules by
// setup.sh, so it is not hashable as a unit; the check below looks for the
// specific NOPASSWD line for the sysctl helper.
const sudoersGrantFile = "/etc/sudoers.d/spinifex-network"
// CheckHost verifies every managed asset on the real host against this build's manifest.
func CheckHost() []Result {
return CheckHostAt("/")
}
// CheckHostAt verifies every managed asset with paths resolved under root.
// It is the test seam CheckHost uses with root "/".
func CheckHostAt(root string) []Result {
results := make([]Result, 0, len(managedHelpers)+1)
for _, path := range managedHelpers {
results = append(results, checkHelper(root, path))
}
results = append(results, checkSudoersGrant(root))
return results
}
// HasProblem reports whether any result is not OK.
func HasProblem(results []Result) bool {
for _, r := range results {
if r.Status != OK {
return true
}
}
return false
}
func checkHelper(root, path string) Result {
want, ok := canonicalHashes[path]
if !ok {
// managedHelpers and canonicalHashes are both generated from the same
// asset list; a miss here means the manifest needs regenerating.
return Result{Path: path, Kind: "helper", Status: Missing, Detail: "no canonical hash for this build — run scripts/gen-asset-manifest.sh"}
}
data, err := os.ReadFile(filepath.Join(root, path))
if err != nil {
return Result{Path: path, Kind: "helper", Status: Missing, Detail: err.Error()}
}
got := hashHex(data)
if got != want {
return Result{
Path: path,
Kind: "helper",
Status: Stale,
Detail: fmt.Sprintf("installed %s, expected %s", shortHash(got), shortHash(want)),
}
}
return Result{Path: path, Kind: "helper", Status: OK}
}
func checkSudoersGrant(root string) Result {
data, err := os.ReadFile(filepath.Join(root, sudoersGrantFile))
if err != nil {
return Result{Path: sudoersGrantFile, Kind: "sudoers-grant", Status: Ungranted, Detail: err.Error()}
}
for line := range strings.SplitSeq(string(data), "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "#") {
continue
}
if strings.Contains(trimmed, "NOPASSWD") && strings.Contains(trimmed, utils.EndpointSysctlHelper) {
return Result{Path: sudoersGrantFile, Kind: "sudoers-grant", Status: OK}
}
}
return Result{
Path: sudoersGrantFile,
Kind: "sudoers-grant",
Status: Ungranted,
Detail: "no NOPASSWD grant line for " + utils.EndpointSysctlHelper,
}
}
func hashHex(data []byte) string {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
// shortHash truncates a hex digest for compact Detail messages.
func shortHash(h string) string {
const n = 12
if len(h) > n {
return h[:n]
}
return h
}
+152
View File
@@ -0,0 +1,152 @@
package preflight
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/mulgadc/spinifex/spinifex/utils"
)
// writeHelper writes the given content at the managed path under root,
// creating parent directories as needed.
func writeHelper(t *testing.T, root, path string, content []byte) {
t.Helper()
full := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", filepath.Dir(full), err)
}
if err := os.WriteFile(full, content, 0o644); err != nil {
t.Fatalf("write %s: %v", full, err)
}
}
func TestCheckHostAt_Helper(t *testing.T) {
path := utils.EndpointSysctlHelper
want, ok := canonicalHashes[path]
if !ok {
t.Fatalf("no canonical hash for %s — is manifest_gen.go generated?", path)
}
t.Run("absent", func(t *testing.T) {
root := t.TempDir()
r := findResult(t, CheckHostAt(root), path)
if r.Status != Missing {
t.Errorf("status = %v, want Missing", r.Status)
}
})
t.Run("wrong bytes", func(t *testing.T) {
root := t.TempDir()
writeHelper(t, root, path, []byte("not the canonical helper body\n"))
r := findResult(t, CheckHostAt(root), path)
if r.Status != Stale {
t.Errorf("status = %v, want Stale", r.Status)
}
gotShort := shortHash(hashHex([]byte("not the canonical helper body\n")))
wantShort := shortHash(want)
if !strings.Contains(r.Detail, gotShort) || !strings.Contains(r.Detail, wantShort) {
t.Errorf("Detail %q does not contain both short hashes (%s, %s)", r.Detail, gotShort, wantShort)
}
})
t.Run("correct bytes", func(t *testing.T) {
// Read the canonical body the same way the package does (via the
// generated manifest + the real provisioning script), so this
// exercises the real OK path rather than a hard-coded fixture.
root := t.TempDir()
body := canonicalHelperBody(t, path)
writeHelper(t, root, path, body)
r := findResult(t, CheckHostAt(root), path)
if r.Status != OK {
t.Errorf("status = %v, want OK (detail: %s)", r.Status, r.Detail)
}
})
}
func TestCheckHostAt_SudoersGrant(t *testing.T) {
t.Run("file absent", func(t *testing.T) {
root := t.TempDir()
r := findResult(t, CheckHostAt(root), sudoersGrantFile)
if r.Status != Ungranted {
t.Errorf("status = %v, want Ungranted", r.Status)
}
})
t.Run("file present without grant line", func(t *testing.T) {
root := t.TempDir()
writeHelper(t, root, sudoersGrantFile, []byte("spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip\n"))
r := findResult(t, CheckHostAt(root), sudoersGrantFile)
if r.Status != Ungranted {
t.Errorf("status = %v, want Ungranted", r.Status)
}
})
t.Run("file present with grant line", func(t *testing.T) {
root := t.TempDir()
writeHelper(t, root, sudoersGrantFile, []byte(
"spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip\n"+
"spinifex-daemon ALL=(root) NOPASSWD: "+utils.EndpointSysctlHelper+"\n"))
r := findResult(t, CheckHostAt(root), sudoersGrantFile)
if r.Status != OK {
t.Errorf("status = %v, want OK (detail: %s)", r.Status, r.Detail)
}
})
}
func TestHasProblem(t *testing.T) {
if HasProblem([]Result{{Status: OK}, {Status: OK}}) {
t.Error("HasProblem = true for all-OK results, want false")
}
if !HasProblem([]Result{{Status: OK}, {Status: Stale}}) {
t.Error("HasProblem = false with a Stale result present, want true")
}
}
func TestStatusString(t *testing.T) {
cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted"}
for status, want := range cases {
if got := status.String(); got != want {
t.Errorf("Status(%d).String() = %q, want %q", status, got, want)
}
}
}
// findResult returns the Result for path, failing the test if absent.
func findResult(t *testing.T, results []Result, path string) Result {
t.Helper()
for _, r := range results {
if r.Path == path {
return r
}
}
t.Fatalf("no Result for path %s in %+v", path, results)
return Result{}
}
// canonicalHelperBody re-extracts the canonical bytes for path from the
// provisioning scripts, so the OK-path test above is real rather than a
// fixture that could silently diverge from what CheckHostAt verifies.
func canonicalHelperBody(t *testing.T, path string) []byte {
t.Helper()
root := repoRoot(t)
var file, anchor string
switch path {
case utils.EndpointSysctlHelper:
file = filepath.Join(root, "scripts", "setup.sh")
anchor = "install_endpoint_sysctl_helper()"
case "/usr/local/lib/spinifex/ovs-socket-perms.sh":
file = filepath.Join(root, "scripts", "setup-ovn.sh")
anchor = "Step 8: Grant the spinifex service users access to OVS/OVN"
default:
t.Fatalf("canonicalHelperBody: unknown managed path %s", path)
}
body, err := extractHeredoc(file, anchor)
if err != nil {
t.Fatalf("extractHeredoc(%s, %q): %v", file, anchor, err)
}
return body
}