diff --git a/Makefile b/Makefile index fc866e625..7417df363 100644 --- a/Makefile +++ b/Makefile @@ -260,6 +260,20 @@ bench: # `make install-microvm` explicitly the first time. deploy: build sudo install -m 755 bin/spx /usr/local/bin/spx + @if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \ + if ! sudo /usr/local/bin/spx admin preflight; then \ + echo ""; \ + echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \ + echo " grants don't match the binary just installed. Remediate with:"; \ + echo " scripts/update-nodes.sh (from the mulga root)"; \ + echo " or:"; \ + echo " make reinstall"; \ + echo " Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \ + exit 1; \ + fi; \ + else \ + echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \ + fi @if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \ $(MAKE) install-microvm; \ else \ diff --git a/cmd/spinifex/cmd/admin_preflight.go b/cmd/spinifex/cmd/admin_preflight.go new file mode 100644 index 000000000..91e2121bf --- /dev/null +++ b/cmd/spinifex/cmd/admin_preflight.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/mulgadc/spinifex/spinifex/preflight" + "github.com/spf13/cobra" +) + +var adminPreflightQuiet bool + +var adminPreflightCmd = &cobra.Command{ + Use: "preflight", + Short: "Verify this host's managed assets against what this build expects", + Long: `spx knows its own build (spx version) but not whether this host already +has the privileged helper scripts and sudoers grants that build depends on — +setup.sh and setup-ovn.sh install them, and a binary-swap deploy (make +deploy or a manual binary copy) skips those steps. + +Checks compare content, not mere existence, so a present-but-stale asset is +caught too. Exits non-zero if any asset is Missing, Stale, or Ungranted.`, + RunE: runAdminPreflight, + SilenceUsage: true, +} + +func init() { + adminCmd.AddCommand(adminPreflightCmd) + adminPreflightCmd.Flags().BoolVar(&adminPreflightQuiet, "quiet", false, "print only non-OK rows") +} + +func runAdminPreflight(_ *cobra.Command, _ []string) error { + results := preflight.CheckHost() + + w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(w, "PATH\tKIND\tSTATUS\tDETAIL") + for _, r := range results { + if adminPreflightQuiet && r.Status == preflight.OK { + continue + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Path, r.Kind, r.Status, r.Detail) + } + w.Flush() + + if preflight.HasProblem(results) { + return fmt.Errorf("host-asset preflight found problems — see table above") + } + return nil +} diff --git a/scripts/gen-asset-manifest.sh b/scripts/gen-asset-manifest.sh new file mode 100755 index 000000000..4ee623714 --- /dev/null +++ b/scripts/gen-asset-manifest.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# gen-asset-manifest.sh — extracts the canonical bytes of each managed asset +# heredoc from the provisioning scripts, sha256s them, and (re)writes +# spinifex/preflight/manifest_gen.go. +# +# Run this after editing a managed heredoc in setup.sh or setup-ovn.sh. +# manifest_gen_test.go re-extracts the same heredocs and fails if the +# committed manifest no longer matches, catching drift in CI. +# +# Usage: scripts/gen-asset-manifest.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +OUT="$ROOT/spinifex/preflight/manifest_gen.go" + +# extract_heredoc FILE ANCHOR +# Prints the body of the first quoted 'HELPER' heredoc starting after the +# first line containing ANCHOR — both scripts hold more than one such heredoc. +extract_heredoc() { + local file=$1 anchor=$2 + awk -v anchor="$anchor" ' + state == 0 && index($0, anchor) { state = 1; next } + state == 1 && index($0, "<<") && index($0, "HELPER") { state = 2; next } + state == 2 && $0 == "HELPER" { exit } + state == 2 { print } + ' "$file" +} + +sysctl_hash=$(extract_heredoc "$ROOT/scripts/setup.sh" 'install_endpoint_sysctl_helper()' | sha256sum | cut -d' ' -f1) +ovs_hash=$(extract_heredoc "$ROOT/scripts/setup-ovn.sh" 'Step 8: Grant the spinifex service users access to OVS/OVN' | sha256sum | cut -d' ' -f1) + +if [ -z "$sysctl_hash" ] || [ -z "$ovs_hash" ]; then + echo "gen-asset-manifest.sh: failed to extract one or both heredocs — anchors may be stale" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUT")" +cat >"$OUT" </dev/null 2>&1; then + gofmt -w "$OUT" +fi + +echo "wrote $OUT" +echo " spinifex-set-endpoint-sysctl: $sysctl_hash" +echo " ovs-socket-perms.sh: $ovs_hash" diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 50c667c20..6009dbc4d 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -65,6 +65,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/host" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/otelsetup" + "github.com/mulgadc/spinifex/spinifex/preflight" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" @@ -1247,6 +1248,17 @@ func (d *Daemon) startLocal() error { slog.Warn("Failed to set daemon OOM score", "err", err) } + // Warn on host assets this build expects but that are missing, stale, or + // ungranted, so drift surfaces as a named log line rather than a later + // cryptic failure. Never fatal — unrelated asset problems must not ground a node. + for _, r := range preflight.CheckHost() { + if r.Status != preflight.OK { + slog.Warn("Host asset preflight problem", "path", r.Path, "kind", r.Kind, + "status", r.Status.String(), "detail", r.Detail, + "remediation", "run scripts/update-nodes.sh or make reinstall") + } + } + // Recover local instance state from disk. Fatal on corruption — orphaned VMs. if err := d.LoadState(); err != nil { return fmt.Errorf("load local instance state: %w", err) diff --git a/spinifex/preflight/manifest_gen.go b/spinifex/preflight/manifest_gen.go new file mode 100644 index 000000000..a3fb245e8 --- /dev/null +++ b/spinifex/preflight/manifest_gen.go @@ -0,0 +1,8 @@ +// Code generated by scripts/gen-asset-manifest.sh; DO NOT EDIT. + +package preflight + +var canonicalHashes = map[string]string{ + "/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl": "8f9bf3205d02b35e147924e3f6fa119ed3d5414375d325d905b80cbba349b613", + "/usr/local/lib/spinifex/ovs-socket-perms.sh": "11ad8fa79ca1544044afbf1bba944a8248ff0e0278ce6c4af7e19de5c61b9b9a", +} diff --git a/spinifex/preflight/manifest_gen_test.go b/spinifex/preflight/manifest_gen_test.go new file mode 100644 index 000000000..e0ec90c5c --- /dev/null +++ b/spinifex/preflight/manifest_gen_test.go @@ -0,0 +1,119 @@ +package preflight + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// repoRoot walks up from this test file's directory until scripts/setup.sh +// is found, so the drift guard locates the checkout regardless of where +// `go test` is invoked from. +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed to resolve this test file's path") + } + dir := filepath.Dir(file) + for { + if _, err := os.Stat(filepath.Join(dir, "scripts", "setup.sh")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not locate repo root: no scripts/setup.sh found walking up from test file") + } + dir = parent + } +} + +// extractHeredoc mirrors scripts/gen-asset-manifest.sh's extraction: the +// body of the first quoted 'HELPER' heredoc starting after the first line +// containing anchor. Both provisioning scripts hold more than one such heredoc. +func extractHeredoc(file, anchor string) ([]byte, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + lines := strings.Split(string(data), "\n") + + anchorLine := -1 + for i, line := range lines { + if strings.Contains(line, anchor) { + anchorLine = i + break + } + } + if anchorLine < 0 { + return nil, fmt.Errorf("anchor %q not found in %s", anchor, file) + } + + startLine := -1 + for i := anchorLine + 1; i < len(lines); i++ { + if strings.Contains(lines[i], "<<") && strings.Contains(lines[i], "HELPER") { + startLine = i + break + } + } + if startLine < 0 { + return nil, fmt.Errorf("no HELPER heredoc start found after anchor %q in %s", anchor, file) + } + + var body []string + for i := startLine + 1; i < len(lines); i++ { + if lines[i] == "HELPER" { + return []byte(strings.Join(body, "\n") + "\n"), nil + } + body = append(body, lines[i]) + } + return nil, fmt.Errorf("no HELPER heredoc terminator found after anchor %q in %s", anchor, file) +} + +// TestManifestMatchesProvisioningScripts re-extracts each managed heredoc +// and asserts the committed canonicalHashes still matches, turning an +// un-regenerated heredoc edit into a CI failure instead of silent drift. +func TestManifestMatchesProvisioningScripts(t *testing.T) { + root := repoRoot(t) + + cases := []struct { + path string + file string + anchor string + }{ + { + path: "/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl", + file: filepath.Join(root, "scripts", "setup.sh"), + anchor: "install_endpoint_sysctl_helper()", + }, + { + path: "/usr/local/lib/spinifex/ovs-socket-perms.sh", + file: filepath.Join(root, "scripts", "setup-ovn.sh"), + anchor: "Step 8: Grant the spinifex service users access to OVS/OVN", + }, + } + + for _, c := range cases { + body, err := extractHeredoc(c.file, c.anchor) + if err != nil { + t.Fatalf("%s: %v", c.path, err) + } + + sum := sha256.Sum256(body) + got := hex.EncodeToString(sum[:]) + + want, ok := canonicalHashes[c.path] + if !ok { + t.Fatalf("%s: no entry in canonicalHashes; run scripts/gen-asset-manifest.sh", c.path) + } + if got != want { + t.Errorf("%s: heredoc in %s hashes to %s, manifest has %s — run scripts/gen-asset-manifest.sh", + c.path, c.file, got, want) + } + } +} diff --git a/spinifex/preflight/preflight.go b/spinifex/preflight/preflight.go new file mode 100644 index 000000000..206db4552 --- /dev/null +++ b/spinifex/preflight/preflight.go @@ -0,0 +1,156 @@ +// Package preflight verifies a host's managed helper scripts and sudoers +// grants — which a binary-swap deploy does not ship — against what this +// build of spinifex expects, comparing content so a stale asset is caught. +package preflight + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/mulgadc/spinifex/spinifex/utils" +) + +//go:generate ../../scripts/gen-asset-manifest.sh + +// Status is the outcome of checking one managed asset against this build's manifest. +type Status int + +const ( + // OK means the on-host asset matches what this build expects. + OK Status = iota + // Missing means the asset is absent from the host. + Missing + // Stale means the asset is present but its content does not match this build's manifest. + Stale + // Ungranted means the sudoers grant for a helper is absent or missing its NOPASSWD line. + Ungranted +) + +// String returns the human-readable name of the status, used in table output and log lines. +func (s Status) String() string { + switch s { + case OK: + return "OK" + case Missing: + return "Missing" + case Stale: + return "Stale" + case Ungranted: + return "Ungranted" + default: + return "Unknown" + } +} + +// Result is the outcome of checking one managed asset against this build's manifest. +type Result struct { + Path string + Kind string // "helper" | "sudoers-grant" + Status Status + Detail string +} + +// managedHelpers lists every helper script this build expects on the host, +// keyed by installed path. An explicit list, not a directory scan, so adding +// an asset here is a reviewed change and never accidental. +var managedHelpers = []string{ + utils.EndpointSysctlHelper, + "/usr/local/lib/spinifex/ovs-socket-perms.sh", +} + +// sudoersGrantFile is assembled with substitutions and multiple rules by +// setup.sh, so it is not hashable as a unit; the check below looks for the +// specific NOPASSWD line for the sysctl helper. +const sudoersGrantFile = "/etc/sudoers.d/spinifex-network" + +// CheckHost verifies every managed asset on the real host against this build's manifest. +func CheckHost() []Result { + return CheckHostAt("/") +} + +// CheckHostAt verifies every managed asset with paths resolved under root. +// It is the test seam CheckHost uses with root "/". +func CheckHostAt(root string) []Result { + results := make([]Result, 0, len(managedHelpers)+1) + for _, path := range managedHelpers { + results = append(results, checkHelper(root, path)) + } + results = append(results, checkSudoersGrant(root)) + return results +} + +// HasProblem reports whether any result is not OK. +func HasProblem(results []Result) bool { + for _, r := range results { + if r.Status != OK { + return true + } + } + return false +} + +func checkHelper(root, path string) Result { + want, ok := canonicalHashes[path] + if !ok { + // managedHelpers and canonicalHashes are both generated from the same + // asset list; a miss here means the manifest needs regenerating. + return Result{Path: path, Kind: "helper", Status: Missing, Detail: "no canonical hash for this build — run scripts/gen-asset-manifest.sh"} + } + + data, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { + return Result{Path: path, Kind: "helper", Status: Missing, Detail: err.Error()} + } + + got := hashHex(data) + if got != want { + return Result{ + Path: path, + Kind: "helper", + Status: Stale, + Detail: fmt.Sprintf("installed %s, expected %s", shortHash(got), shortHash(want)), + } + } + return Result{Path: path, Kind: "helper", Status: OK} +} + +func checkSudoersGrant(root string) Result { + data, err := os.ReadFile(filepath.Join(root, sudoersGrantFile)) + if err != nil { + return Result{Path: sudoersGrantFile, Kind: "sudoers-grant", Status: Ungranted, Detail: err.Error()} + } + + for line := range strings.SplitSeq(string(data), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") { + continue + } + if strings.Contains(trimmed, "NOPASSWD") && strings.Contains(trimmed, utils.EndpointSysctlHelper) { + return Result{Path: sudoersGrantFile, Kind: "sudoers-grant", Status: OK} + } + } + return Result{ + Path: sudoersGrantFile, + Kind: "sudoers-grant", + Status: Ungranted, + Detail: "no NOPASSWD grant line for " + utils.EndpointSysctlHelper, + } +} + +func hashHex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// shortHash truncates a hex digest for compact Detail messages. +func shortHash(h string) string { + const n = 12 + if len(h) > n { + return h[:n] + } + return h +} diff --git a/spinifex/preflight/preflight_test.go b/spinifex/preflight/preflight_test.go new file mode 100644 index 000000000..49eddf633 --- /dev/null +++ b/spinifex/preflight/preflight_test.go @@ -0,0 +1,152 @@ +package preflight + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// writeHelper writes the given content at the managed path under root, +// creating parent directories as needed. +func writeHelper(t *testing.T, root, path string, content []byte) { + t.Helper() + full := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(full), err) + } + if err := os.WriteFile(full, content, 0o644); err != nil { + t.Fatalf("write %s: %v", full, err) + } +} + +func TestCheckHostAt_Helper(t *testing.T) { + path := utils.EndpointSysctlHelper + want, ok := canonicalHashes[path] + if !ok { + t.Fatalf("no canonical hash for %s — is manifest_gen.go generated?", path) + } + + t.Run("absent", func(t *testing.T) { + root := t.TempDir() + r := findResult(t, CheckHostAt(root), path) + if r.Status != Missing { + t.Errorf("status = %v, want Missing", r.Status) + } + }) + + t.Run("wrong bytes", func(t *testing.T) { + root := t.TempDir() + writeHelper(t, root, path, []byte("not the canonical helper body\n")) + r := findResult(t, CheckHostAt(root), path) + if r.Status != Stale { + t.Errorf("status = %v, want Stale", r.Status) + } + gotShort := shortHash(hashHex([]byte("not the canonical helper body\n"))) + wantShort := shortHash(want) + if !strings.Contains(r.Detail, gotShort) || !strings.Contains(r.Detail, wantShort) { + t.Errorf("Detail %q does not contain both short hashes (%s, %s)", r.Detail, gotShort, wantShort) + } + }) + + t.Run("correct bytes", func(t *testing.T) { + // Read the canonical body the same way the package does (via the + // generated manifest + the real provisioning script), so this + // exercises the real OK path rather than a hard-coded fixture. + root := t.TempDir() + body := canonicalHelperBody(t, path) + writeHelper(t, root, path, body) + r := findResult(t, CheckHostAt(root), path) + if r.Status != OK { + t.Errorf("status = %v, want OK (detail: %s)", r.Status, r.Detail) + } + }) +} + +func TestCheckHostAt_SudoersGrant(t *testing.T) { + t.Run("file absent", func(t *testing.T) { + root := t.TempDir() + r := findResult(t, CheckHostAt(root), sudoersGrantFile) + if r.Status != Ungranted { + t.Errorf("status = %v, want Ungranted", r.Status) + } + }) + + t.Run("file present without grant line", func(t *testing.T) { + root := t.TempDir() + writeHelper(t, root, sudoersGrantFile, []byte("spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip\n")) + r := findResult(t, CheckHostAt(root), sudoersGrantFile) + if r.Status != Ungranted { + t.Errorf("status = %v, want Ungranted", r.Status) + } + }) + + t.Run("file present with grant line", func(t *testing.T) { + root := t.TempDir() + writeHelper(t, root, sudoersGrantFile, []byte( + "spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip\n"+ + "spinifex-daemon ALL=(root) NOPASSWD: "+utils.EndpointSysctlHelper+"\n")) + r := findResult(t, CheckHostAt(root), sudoersGrantFile) + if r.Status != OK { + t.Errorf("status = %v, want OK (detail: %s)", r.Status, r.Detail) + } + }) +} + +func TestHasProblem(t *testing.T) { + if HasProblem([]Result{{Status: OK}, {Status: OK}}) { + t.Error("HasProblem = true for all-OK results, want false") + } + if !HasProblem([]Result{{Status: OK}, {Status: Stale}}) { + t.Error("HasProblem = false with a Stale result present, want true") + } +} + +func TestStatusString(t *testing.T) { + cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted"} + for status, want := range cases { + if got := status.String(); got != want { + t.Errorf("Status(%d).String() = %q, want %q", status, got, want) + } + } +} + +// findResult returns the Result for path, failing the test if absent. +func findResult(t *testing.T, results []Result, path string) Result { + t.Helper() + for _, r := range results { + if r.Path == path { + return r + } + } + t.Fatalf("no Result for path %s in %+v", path, results) + return Result{} +} + +// canonicalHelperBody re-extracts the canonical bytes for path from the +// provisioning scripts, so the OK-path test above is real rather than a +// fixture that could silently diverge from what CheckHostAt verifies. +func canonicalHelperBody(t *testing.T, path string) []byte { + t.Helper() + root := repoRoot(t) + + var file, anchor string + switch path { + case utils.EndpointSysctlHelper: + file = filepath.Join(root, "scripts", "setup.sh") + anchor = "install_endpoint_sysctl_helper()" + case "/usr/local/lib/spinifex/ovs-socket-perms.sh": + file = filepath.Join(root, "scripts", "setup-ovn.sh") + anchor = "Step 8: Grant the spinifex service users access to OVS/OVN" + default: + t.Fatalf("canonicalHelperBody: unknown managed path %s", path) + } + + body, err := extractHeredoc(file, anchor) + if err != nil { + t.Fatalf("extractHeredoc(%s, %q): %v", file, anchor, err) + } + return body +}