From bb9cd8ea23449e14c8f790d18d341c3fa47aec40 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:55:55 +1000 Subject: [PATCH 01/16] feat: formation join token (#141) --- cmd/spinifex/cmd/admin.go | 74 +++++++++- docs/COMMANDS.md | 4 +- docs/install-multi-node/README.md | 20 ++- scripts/create-multi-node.sh | 23 +++- scripts/iac/configure-cluster.sh | 9 ++ scripts/start-dev.sh | 6 +- spinifex/formation/formation.go | 63 ++++++++- spinifex/formation/formation_test.go | 193 +++++++++++++++++++++------ tests/e2e/TEST_COVERAGE.md | 10 +- tests/e2e/lib/multinode-helpers.sh | 11 ++ 10 files changed, 352 insertions(+), 61 deletions(-) diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 3251198a7..872455236 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -23,8 +23,10 @@ import ( _ "embed" "encoding/base64" "encoding/json" + "errors" "fmt" "io" + "log/slog" "maps" "net" "net/http" @@ -242,6 +244,7 @@ func init() { adminInitCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)") adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.") adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)") + adminInitCmd.Flags().String("token-ttl", "30m", "Join token validity duration (e.g. 30m, 1h, 2h)") adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name") adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)") adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui") @@ -266,10 +269,12 @@ func init() { adminJoinCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.2 for multi-node on single host)") adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)") adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)") + adminJoinCmd.Flags().String("token", "", "Join token from the init node (required)") adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)") adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)") adminJoinCmd.MarkFlagRequired("node") adminJoinCmd.MarkFlagRequired("host") + adminJoinCmd.MarkFlagRequired("token") imagesImportCmd.Flags().String("tmp-dir", os.TempDir(), "Temporary directory for image import processing") @@ -600,6 +605,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { } predastoreNodesStr, _ := cmd.Flags().GetString("predastore-nodes") formationTimeoutStr, _ := cmd.Flags().GetString("formation-timeout") + tokenTTLStr, _ := cmd.Flags().GetString("token-ttl") clusterName, _ := cmd.Flags().GetString("cluster-name") services, _ := cmd.Flags().GetStringSlice("services") @@ -889,7 +895,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName, configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind, - port, nodes, formationTimeoutStr, services, networkConfig) + port, nodes, formationTimeoutStr, tokenTTLStr, services, networkConfig) return } @@ -1073,13 +1079,29 @@ func runAdminInit(cmd *cobra.Command, args []string) { // then generates configs with complete cluster topology. func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName, configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind string, - port, expectedNodes int, formationTimeoutStr string, services []string, networkConfig *formation.NetworkConfig) { + port, expectedNodes int, formationTimeoutStr, tokenTTLStr string, services []string, networkConfig *formation.NetworkConfig) { formationTimeout, err := time.ParseDuration(formationTimeoutStr) if err != nil { fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err) os.Exit(1) } + tokenTTL, err := time.ParseDuration(tokenTTLStr) + if err != nil { + fmt.Fprintf(os.Stderr, "❌ Error: Invalid --token-ttl: %v\n", err) + os.Exit(1) + } + if tokenTTL < formationTimeout+1*time.Minute { + fmt.Fprintf(os.Stderr, "❌ Error: --token-ttl (%s) must be >= --formation-timeout + 1m (%s)\n", tokenTTL, formationTimeout+1*time.Minute) + os.Exit(1) + } + + joinToken, err := formation.GenerateJoinToken() + if err != nil { + fmt.Fprintf(os.Stderr, "❌ Error generating join token: %v\n", err) + os.Exit(1) + } + // Generate IAM master key for the cluster masterKey, err := handlers_iam.GenerateMasterKey() if err != nil { @@ -1118,7 +1140,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, AdminSecretKey: bootstrapResult.AdminSecretKey, } - fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig) + fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig, joinToken, tokenTTL) // Include master key in formation server for distribution to joining nodes fs.SetMasterKey(base64.StdEncoding.EncodeToString(masterKey)) @@ -1137,6 +1159,13 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, os.Exit(1) } + // Write join token to file for automated workflows + tokenPath := filepath.Join(configDir, "join-token") + if err := os.WriteFile(tokenPath, []byte(joinToken), 0600); err != nil { + fmt.Fprintf(os.Stderr, "❌ Error writing join token: %v\n", err) + os.Exit(1) + } + // Start formation server formationAddr := fmt.Sprintf("%s:%d", bindIP, port) if err := fs.Start(formationAddr); err != nil { @@ -1146,12 +1175,15 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, fmt.Printf("\n📡 Formation server started on %s\n", formationAddr) fmt.Printf(" Waiting for %d more node(s) to join...\n", expectedNodes-1) - fmt.Printf(" Other nodes should run: spx admin join --host %s --node --bind \n\n", formationAddr) + fmt.Printf(" Token expires in %s\n\n", tokenTTL) + fmt.Printf(" Other nodes should run:\n") + fmt.Printf(" spx admin join --host %s --token %s --node --bind \n\n", formationAddr, joinToken) // Wait for all nodes to register if err := fs.WaitForCompletion(formationTimeout); err != nil { fmt.Fprintf(os.Stderr, "❌ %v\n", err) fs.Shutdown(context.Background()) + os.Remove(tokenPath) os.Exit(1) } @@ -1295,6 +1327,9 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, // Shutdown formation server fs.Shutdown(context.Background()) + if err := os.Remove(tokenPath); err != nil && !errors.Is(err, os.ErrNotExist) { + slog.Warn("Failed to remove join token file", "path", tokenPath, "error", err) + } // Print cluster summary fmt.Println("\n🎉 Cluster formation complete!") @@ -1319,6 +1354,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) { node, _ := cmd.Flags().GetString("node") leaderHost, _ := cmd.Flags().GetString("host") + joinToken, _ := cmd.Flags().GetString("token") region, _ := cmd.Flags().GetString("region") az, _ := cmd.Flags().GetString("az") dataDir, _ := cmd.Flags().GetString("data-dir") @@ -1401,7 +1437,15 @@ func runAdminJoin(cmd *cobra.Command, args []string) { } joinURL := fmt.Sprintf("https://%s/formation/join", leaderHost) - resp, err := client.Post(joinURL, "application/json", bytes.NewBuffer(reqBody)) + req, err := http.NewRequest(http.MethodPost, joinURL, bytes.NewBuffer(reqBody)) + if err != nil { + fmt.Fprintf(os.Stderr, "❌ Error creating join request: %v\n", err) + os.Exit(1) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+joinToken) + + resp, err := client.Do(req) if err != nil { fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err) fmt.Fprintf(os.Stderr, "Make sure the leader node has run 'spx admin init' and is accessible at %s\n", leaderHost) @@ -1433,7 +1477,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) { var statusResp formation.StatusResponse for { - sResp, err := client.Get(statusURL) + statusReq, err := http.NewRequest(http.MethodGet, statusURL, nil) + if err != nil { + fmt.Fprintf(os.Stderr, "❌ Error creating status request: %v\n", err) + os.Exit(1) + } + statusReq.Header.Set("Authorization", "Bearer "+joinToken) + + sResp, err := client.Do(statusReq) if err != nil { fmt.Fprintf(os.Stderr, "❌ Error polling formation status: %v\n", err) os.Exit(1) @@ -1446,8 +1497,17 @@ func runAdminJoin(cmd *cobra.Command, args []string) { os.Exit(1) } + if sResp.StatusCode == http.StatusUnauthorized { + fmt.Fprintf(os.Stderr, "❌ Error: join token rejected by formation server (expired or invalid)\n") + os.Exit(1) + } + if sResp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "❌ Error: unexpected status %d from formation server\n", sResp.StatusCode) + os.Exit(1) + } + if err := json.Unmarshal(sBody, &statusResp); err != nil { - fmt.Fprintf(os.Stderr, "Error parsing status response: %v\n", err) + fmt.Fprintf(os.Stderr, "❌ Error parsing status response: %v\n", err) os.Exit(1) } diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index f3850d005..90410eeb0 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -51,8 +51,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status | |---------|-------|---------------|-------------|------------|--------| -| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files
2. Root credentials printed once
3. master.key is 32 bytes, mode 0600
4. bootstrap.json consumed on first start
5. `--force` re-initializes existing config
6. AWS CLI profile `spx` auto-configured | **DONE** | -| `spx admin join` | `--host` (required), `--node` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster
2. Missing host (error)
3. Missing node name (error) | **DONE** | +| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files
2. Root credentials printed once
3. master.key is 32 bytes, mode 0600
4. bootstrap.json consumed on first start
5. `--force` re-initializes existing config
6. AWS CLI profile `spx` auto-configured
7. Multi-node init generates join token and writes to `/join-token` | **DONE** | +| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster
2. Missing host (error)
3. Missing node name (error)
4. Missing token (error)
5. Invalid token (401)
6. Expired token (401) | **DONE** | ### Version diff --git a/docs/install-multi-node/README.md b/docs/install-multi-node/README.md index 7075dd268..1417d3324 100644 --- a/docs/install-multi-node/README.md +++ b/docs/install-multi-node/README.md @@ -106,12 +106,24 @@ sudo spx admin init \ Save the admin credentials — they will not be shown again. +The init output displays the join command including the token: + +``` +📡 Formation server started on 10.0.0.1:4432 + Waiting for 2 more node(s) to join... + Token expires in 30m0s + + Other nodes should run: + spx admin join --host 10.0.0.1:4432 --token spx_join_a8Bf3x9Kz2mN --node --bind +``` + **Server 2 — Join** (while init is running): ```bash sudo spx admin join \ --node node2 --bind $SPINIFEX_NODE2 --cluster-bind $SPINIFEX_NODE2 \ - --host $SPINIFEX_NODE1:4432 --region $AWS_REGION --az $AWS_AZ + --host $SPINIFEX_NODE1:4432 --token \ + --region $AWS_REGION --az $AWS_AZ ``` **Server 3 — Join** (while init is running): @@ -119,9 +131,13 @@ sudo spx admin join \ ```bash sudo spx admin join \ --node node3 --bind $SPINIFEX_NODE3 --cluster-bind $SPINIFEX_NODE3 \ - --host $SPINIFEX_NODE1:4432 --region $AWS_REGION --az $AWS_AZ + --host $SPINIFEX_NODE1:4432 --token \ + --region $AWS_REGION --az $AWS_AZ ``` +> **Note:** The join token expires 30 minutes after init by default. For larger deployments +> with slower provisioning, use `--token-ttl 2h` on the init command. + ## Step 5. Start Services On **all servers**: diff --git a/scripts/create-multi-node.sh b/scripts/create-multi-node.sh index ab0a45b45..c1ae2ca3a 100755 --- a/scripts/create-multi-node.sh +++ b/scripts/create-multi-node.sh @@ -39,7 +39,26 @@ echo "Starting init node (127.0.0.1) and joining nodes (127.0.0.2, 127.0.0.3)... INIT_PID=$! # Wait for formation server to start -sleep 2 +echo "Waiting for formation server..." +for i in $(seq 1 30); do + if curl -sk "https://127.0.0.1:4432/formation/health" > /dev/null 2>&1; then + echo "Formation server ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "ERROR: Formation server failed to start" + kill $INIT_PID 2>/dev/null || true + exit 1 + fi + sleep 1 +done + +# Read join token +JOIN_TOKEN=$(cat ~/node1/config/join-token) +if [ -z "$JOIN_TOKEN" ]; then + echo "ERROR: Join token file is empty" + exit 1 +fi # Join nodes concurrently ./bin/spx admin join \ @@ -47,6 +66,7 @@ sleep 2 --bind 127.0.0.2 \ --cluster-bind 127.0.0.2 \ --host 127.0.0.1:4432 \ + --token "$JOIN_TOKEN" \ --data-dir ~/node2/ \ --config-dir ~/node2/config/ \ --region ap-southeast-2 \ @@ -58,6 +78,7 @@ JOIN2_PID=$! --bind 127.0.0.3 \ --cluster-bind 127.0.0.3 \ --host 127.0.0.1:4432 \ + --token "$JOIN_TOKEN" \ --data-dir ~/node3/ \ --config-dir ~/node3/config/ \ --region ap-southeast-2 \ diff --git a/scripts/iac/configure-cluster.sh b/scripts/iac/configure-cluster.sh index 095e25451..48ee7d263 100755 --- a/scripts/iac/configure-cluster.sh +++ b/scripts/iac/configure-cluster.sh @@ -121,6 +121,14 @@ while ! curl -sk --connect-timeout 3 "https://$NODE1_IP:4432/formation/health" > done echo " Formation server ready" +# Read join token from init node +JOIN_TOKEN=$(remote "$NODE1_IP" "cat ~/spinifex/config/join-token") +if [ -z "$JOIN_TOKEN" ]; then + echo "Error: Failed to read join token from $NODE1_IP" + exit 1 +fi +echo " Join token loaded from node1" + # Nodes 2..N: join if [ "$NODE_COUNT" -gt 1 ]; then JOIN_PIDS=() @@ -134,6 +142,7 @@ if [ "$NODE_COUNT" -gt 1 ]; then --cluster-bind $ip \ --cluster-routes $NODE1_IP:4248 \ --host $NODE1_IP:4432 \ + --token $JOIN_TOKEN \ --data-dir ~/spinifex/ \ --config-dir ~/spinifex/config/ \ --region $REGION \ diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index 46a75b9d8..df84af1b5 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -329,7 +329,11 @@ if has_service "predastore"; then PREDASTORE_CMD="./bin/spx service predastore start" start_service "predastore" "$PREDASTORE_CMD" set_oom_score "predastore" "-500" - check_service "Predastore" "$SPINIFEX_PREDASTORE_HOST" "$SPINIFEX_PREDASTORE_PORT" + if is_multinode; then + echo " ⏭️ Skipping Predastore connectivity check (multi-node: needs quorum from peer nodes)" + else + check_service "Predastore" "$SPINIFEX_PREDASTORE_HOST" "$SPINIFEX_PREDASTORE_PORT" + fi else echo "2️⃣ Skipping Predastore (not a local service)" fi diff --git a/spinifex/formation/formation.go b/spinifex/formation/formation.go index 124820a14..e0a29acc8 100644 --- a/spinifex/formation/formation.go +++ b/spinifex/formation/formation.go @@ -2,7 +2,10 @@ package formation import ( "context" + crypto_rand "crypto/rand" + "crypto/subtle" "crypto/tls" + "encoding/base64" "encoding/json" "fmt" "log/slog" @@ -98,12 +101,25 @@ type FormationServer struct { caKey string masterKey string networkConfig *NetworkConfig + joinToken string + tokenExpiry time.Time done chan struct{} server *http.Server } +// GenerateJoinToken returns a token of the form "spx_join_<16 base64url chars>". +// Uses crypto/rand for 12 random bytes (96 bits entropy), base64 URL-safe encoded. +func GenerateJoinToken() (string, error) { + b := make([]byte, 12) + if _, err := crypto_rand.Read(b); err != nil { + return "", fmt.Errorf("generate join token: %w", err) + } + return "spx_join_" + base64.RawURLEncoding.EncodeToString(b), nil +} + // NewFormationServer creates a new formation server expecting the given number of nodes. -func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string, networkConfig *NetworkConfig) *FormationServer { +func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string, + networkConfig *NetworkConfig, joinToken string, tokenTTL time.Duration) *FormationServer { return &FormationServer{ expected: expected, nodes: make(map[string]NodeInfo), @@ -111,6 +127,8 @@ func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey st caCert: caCert, caKey: caKey, networkConfig: networkConfig, + joinToken: joinToken, + tokenExpiry: time.Now().Add(tokenTTL), done: make(chan struct{}), } } @@ -238,7 +256,44 @@ func (fs *FormationServer) Shutdown(ctx context.Context) error { return fs.server.Shutdown(ctx) } +// validateToken checks the Authorization header against the join token. +// Returns an error if the token is missing, invalid, or expired. +func (fs *FormationServer) validateToken(r *http.Request) error { + fs.mu.RLock() + defer fs.mu.RUnlock() + + if time.Now().After(fs.tokenExpiry) { + return fmt.Errorf("join token has expired") + } + + auth := r.Header.Get("Authorization") + if auth == "" { + return fmt.Errorf("missing Authorization header") + } + + const prefix = "Bearer " + if len(auth) < len(prefix) || auth[:len(prefix)] != prefix { + return fmt.Errorf("invalid Authorization header format") + } + token := auth[len(prefix):] + + if subtle.ConstantTimeCompare([]byte(token), []byte(fs.joinToken)) != 1 { + return fmt.Errorf("invalid join token") + } + + return nil +} + func (fs *FormationServer) handleJoin(w http.ResponseWriter, r *http.Request) { + if err := fs.validateToken(r); err != nil { + slog.Warn("Formation join rejected", "error", err, "remote_addr", r.RemoteAddr) + writeJSON(w, http.StatusUnauthorized, JoinResponse{ + Success: false, + Message: "unauthorized: " + err.Error(), + }) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 1*1024*1024) // 1 MB limit var req JoinRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -270,6 +325,12 @@ func (fs *FormationServer) handleJoin(w http.ResponseWriter, r *http.Request) { } func (fs *FormationServer) handleStatus(w http.ResponseWriter, r *http.Request) { + if err := fs.validateToken(r); err != nil { + slog.Warn("Formation status rejected", "error", err, "remote_addr", r.RemoteAddr) + http.Error(w, "unauthorized: "+err.Error(), http.StatusUnauthorized) + return + } + fs.mu.RLock() defer fs.mu.RUnlock() diff --git a/spinifex/formation/formation_test.go b/spinifex/formation/formation_test.go index 5357f4107..480d1b9f9 100644 --- a/spinifex/formation/formation_test.go +++ b/spinifex/formation/formation_test.go @@ -12,6 +12,9 @@ import ( "github.com/stretchr/testify/require" ) +const testToken = "spx_join_dGVzdHRva2VuMTIz" +const testTokenTTL = 1 * time.Hour + func testCreds() *SharedCredentials { return &SharedCredentials{ AccessKey: "AKIATEST1234567890AB", @@ -33,9 +36,53 @@ func testNode(name, ip string) NodeInfo { } } +func testServer(t *testing.T, fs *FormationServer) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("POST /formation/join", fs.handleJoin) + mux.HandleFunc("GET /formation/status", fs.handleStatus) + mux.HandleFunc("GET /formation/health", fs.handleHealth) + return httptest.NewServer(mux) +} + +func authPost(t *testing.T, url, token string, body []byte) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp +} + +func authGet(t *testing.T, url, token string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp +} + +func TestGenerateJoinToken(t *testing.T) { + t.Parallel() + token1, err := GenerateJoinToken() + require.NoError(t, err) + + assert.True(t, len(token1) > len("spx_join_")) + assert.Equal(t, "spx_join_", token1[:9]) + assert.Len(t, token1[9:], 16) + + token2, err := GenerateJoinToken() + require.NoError(t, err) + assert.NotEqual(t, token1, token2) +} + func TestNewFormationServer(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem", nil) + fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem", nil, testToken, testTokenTTL) assert.Equal(t, 3, fs.expected) assert.Empty(t, fs.nodes) @@ -44,7 +91,7 @@ func TestNewFormationServer(t *testing.T) { func TestRegisterNode(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "", nil) + fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL) err := fs.RegisterNode(testNode("node1", "10.0.0.1")) require.NoError(t, err) @@ -56,7 +103,7 @@ func TestRegisterNode(t *testing.T) { func TestRegisterDuplicateName(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "", nil) + fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) err := fs.RegisterNode(testNode("node1", "10.0.0.2")) @@ -66,7 +113,7 @@ func TestRegisterDuplicateName(t *testing.T) { func TestRegisterDuplicateIP(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "", nil) + fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) err := fs.RegisterNode(testNode("node2", "10.0.0.1")) @@ -76,7 +123,7 @@ func TestRegisterDuplicateIP(t *testing.T) { func TestIsComplete(t *testing.T) { t.Parallel() - fs := NewFormationServer(2, testCreds(), "", "", nil) + fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL) assert.False(t, fs.IsComplete()) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) @@ -87,7 +134,7 @@ func TestIsComplete(t *testing.T) { func TestWaitForCompletion(t *testing.T) { t.Parallel() - fs := NewFormationServer(2, testCreds(), "", "", nil) + fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL) go func() { time.Sleep(50 * time.Millisecond) @@ -102,7 +149,7 @@ func TestWaitForCompletion(t *testing.T) { func TestWaitForCompletionTimeout(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "", nil) + fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) @@ -111,18 +158,62 @@ func TestWaitForCompletionTimeout(t *testing.T) { assert.Contains(t, err.Error(), "timed out") } -func testServer(t *testing.T, fs *FormationServer) *httptest.Server { - t.Helper() - mux := http.NewServeMux() - mux.HandleFunc("POST /formation/join", fs.handleJoin) - mux.HandleFunc("GET /formation/status", fs.handleStatus) - mux.HandleFunc("GET /formation/health", fs.handleHealth) - return httptest.NewServer(mux) +func TestValidateToken_Valid(t *testing.T) { + t.Parallel() + fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("Authorization", "Bearer "+testToken) + assert.NoError(t, fs.validateToken(req)) } -func TestJoinEndpoint(t *testing.T) { +func TestValidateToken_Missing(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "", nil) + fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + err := fs.validateToken(req) + assert.Error(t, err) + assert.Contains(t, err.Error(), "missing Authorization header") +} + +func TestValidateToken_Invalid(t *testing.T) { + t.Parallel() + fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("Authorization", "Bearer wrong-token") + err := fs.validateToken(req) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid join token") +} + +func TestValidateToken_Expired(t *testing.T) { + t.Parallel() + fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, 1*time.Millisecond) + time.Sleep(5 * time.Millisecond) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("Authorization", "Bearer "+testToken) + err := fs.validateToken(req) + assert.Error(t, err) + assert.Contains(t, err.Error(), "expired") +} + +func TestValidateToken_BadFormat(t *testing.T) { + t.Parallel() + fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("Authorization", "Basic "+testToken) + err := fs.validateToken(req) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid Authorization header format") +} + +func TestJoinEndpoint_Unauthorized(t *testing.T) { + t.Parallel() + fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL) ts := testServer(t, fs) defer ts.Close() @@ -131,6 +222,19 @@ func TestJoinEndpoint(t *testing.T) { require.NoError(t, err) defer resp.Body.Close() + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) +} + +func TestJoinEndpoint_WithToken(t *testing.T) { + t.Parallel() + fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL) + ts := testServer(t, fs) + defer ts.Close() + + body, _ := json.Marshal(JoinRequest{NodeInfo: testNode("node1", "10.0.0.1")}) + resp := authPost(t, ts.URL+"/formation/join", testToken, body) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) var jr JoinResponse @@ -142,33 +246,43 @@ func TestJoinEndpoint(t *testing.T) { func TestJoinEndpointDuplicate(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "", nil) + fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL) ts := testServer(t, fs) defer ts.Close() body, _ := json.Marshal(JoinRequest{NodeInfo: testNode("node1", "10.0.0.1")}) - resp1, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body)) - require.NoError(t, err) + resp1 := authPost(t, ts.URL+"/formation/join", testToken, body) resp1.Body.Close() assert.Equal(t, http.StatusOK, resp1.StatusCode) - resp2, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body)) - require.NoError(t, err) + resp2 := authPost(t, ts.URL+"/formation/join", testToken, body) defer resp2.Body.Close() assert.Equal(t, http.StatusConflict, resp2.StatusCode) } +func TestStatusEndpoint_Unauthorized(t *testing.T) { + t.Parallel() + fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil, testToken, testTokenTTL) + ts := testServer(t, fs) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/formation/status") + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) +} + func TestStatusEndpointIncomplete(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil) + fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil, testToken, testTokenTTL) ts := testServer(t, fs) defer ts.Close() require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) - resp, err := http.Get(ts.URL + "/formation/status") - require.NoError(t, err) + resp := authGet(t, ts.URL+"/formation/status", testToken) defer resp.Body.Close() var sr StatusResponse @@ -184,15 +298,14 @@ func TestStatusEndpointIncomplete(t *testing.T) { func TestStatusEndpointComplete(t *testing.T) { t.Parallel() creds := testCreds() - fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data", nil) + fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data", nil, testToken, testTokenTTL) ts := testServer(t, fs) defer ts.Close() require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) require.NoError(t, fs.RegisterNode(testNode("node2", "10.0.0.2"))) - resp, err := http.Get(ts.URL + "/formation/status") - require.NoError(t, err) + resp := authGet(t, ts.URL+"/formation/status", testToken) defer resp.Body.Close() var sr StatusResponse @@ -209,14 +322,13 @@ func TestStatusEndpointComplete(t *testing.T) { func TestStatusEndpointMasterKey(t *testing.T) { t.Parallel() - fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key", nil) + fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key", nil, testToken, testTokenTTL) fs.SetMasterKey("dGVzdC1tYXN0ZXIta2V5LWJhc2U2NC1lbmNvZGVk") ts := testServer(t, fs) defer ts.Close() // Before completion: master key should not be exposed - resp, err := http.Get(ts.URL + "/formation/status") - require.NoError(t, err) + resp := authGet(t, ts.URL+"/formation/status", testToken) var sr StatusResponse require.NoError(t, json.NewDecoder(resp.Body).Decode(&sr)) resp.Body.Close() @@ -227,17 +339,16 @@ func TestStatusEndpointMasterKey(t *testing.T) { require.NoError(t, fs.RegisterNode(testNode("node2", "10.0.0.2"))) // After completion: master key should be present - resp, err = http.Get(ts.URL + "/formation/status") - require.NoError(t, err) + resp = authGet(t, ts.URL+"/formation/status", testToken) defer resp.Body.Close() require.NoError(t, json.NewDecoder(resp.Body).Decode(&sr)) assert.True(t, sr.Complete) assert.Equal(t, "dGVzdC1tYXN0ZXIta2V5LWJhc2U2NC1lbmNvZGVk", sr.MasterKey) } -func TestHealthEndpoint(t *testing.T) { +func TestHealthEndpoint_NoAuth(t *testing.T) { t.Parallel() - fs := NewFormationServer(1, testCreds(), "", "", nil) + fs := NewFormationServer(1, testCreds(), "", "", nil, testToken, testTokenTTL) ts := testServer(t, fs) defer ts.Close() @@ -289,22 +400,21 @@ func TestBuildPredastoreNodes(t *testing.T) { assert.Equal(t, "10.0.0.3", pnodes[2].Host) } -func TestFullFormationFlow(t *testing.T) { +func TestFullFormationFlow_WithToken(t *testing.T) { t.Parallel() creds := testCreds() - fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", nil) + fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", nil, testToken, testTokenTTL) ts := testServer(t, fs) defer ts.Close() - // 3 nodes join sequentially + // 3 nodes join sequentially with auth for i, n := range []struct{ name, ip string }{ {"node1", "10.0.0.1"}, {"node2", "10.0.0.2"}, {"node3", "10.0.0.3"}, } { body, _ := json.Marshal(JoinRequest{NodeInfo: testNode(n.name, n.ip)}) - resp, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body)) - require.NoError(t, err) + resp := authPost(t, ts.URL+"/formation/join", testToken, body) var jr JoinResponse require.NoError(t, json.NewDecoder(resp.Body).Decode(&jr)) @@ -314,9 +424,8 @@ func TestFullFormationFlow(t *testing.T) { assert.Equal(t, i+1, jr.Joined) } - // All nodes poll status — should be complete - resp, err := http.Get(ts.URL + "/formation/status") - require.NoError(t, err) + // All nodes poll status with auth — should be complete + resp := authGet(t, ts.URL+"/formation/status", testToken) defer resp.Body.Close() var sr StatusResponse diff --git a/tests/e2e/TEST_COVERAGE.md b/tests/e2e/TEST_COVERAGE.md index 9878be84a..506cfa365 100644 --- a/tests/e2e/TEST_COVERAGE.md +++ b/tests/e2e/TEST_COVERAGE.md @@ -306,11 +306,11 @@ Tests that EC2 resources are properly isolated between tenant accounts (Alpha, B - Simulated network IPs (no ramdisk — start-dev.sh uses disk-backed WAL/VB in CI) ### Phase 2: Cluster Initialization -- `spx admin init` (leader node1) +- `spx admin init` (leader node1) — generates join token, writes to `~/node1/config/join-token` - CA certificate trust -- Start node1 services -- `spx admin join` (node2, node3) -- Start node2 + node3 services +- Read join token from node1 config +- `spx admin join` with `--token` (node2, node3 — concurrent) +- Start all 3 node services concurrently ### Phase 3: Cluster Health Verification - Verify NATS cluster (3 nodes) @@ -562,7 +562,7 @@ Runs on a real 3-node libvirt cluster provisioned by OpenTofu (`scripts/tofu-clu 4. `git fetch` + checkout test branch on all nodes (all 3 repos) 5. `make build` on all nodes (parallel) 6. `setup-ovn.sh` — primary gets `--management`, secondaries connect to primary's OVN central -7. `spx admin init` on primary, `spx admin join` on secondaries (formation server handshake) +7. `spx admin init` on primary (generates join token), read token via SSH, `spx admin join --token` on secondaries 8. `start-dev.sh` on all nodes 9. Wait for `/health` (awsgw=ok) on all nodes 10. Install Spinifex CA certificate on all nodes diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index 98ac014c0..4613fbc6b 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -796,6 +796,16 @@ init_leader_node() { for i in $(seq 1 60); do if curl -sk "https://${NODE1_IP}:${CLUSTER_PORT}/formation/health" > /dev/null 2>&1; then echo " Formation server is ready (PID: $LEADER_INIT_PID)" + + # Read join token written by admin init + JOIN_TOKEN=$(cat "$HOME/node1/config/join-token") + if [ -z "$JOIN_TOKEN" ]; then + echo " ERROR: Join token file is empty" + return 1 + fi + echo " Join token loaded" + export JOIN_TOKEN + return 0 fi sleep 1 @@ -824,6 +834,7 @@ join_follower_node() { --cluster-bind "$node_ip" \ --cluster-routes "${NODE1_IP}:${NATS_CLUSTER_PORT}" \ --host "${NODE1_IP}:${CLUSTER_PORT}" \ + --token "$JOIN_TOKEN" \ --data-dir "$data_dir/" \ --config-dir "$data_dir/config/" \ --region ap-southeast-2 \ From 911b36a69622437ac6b93a24634c596fb7ca8945 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 16 Apr 2026 10:00:22 +1000 Subject: [PATCH 02/16] refactor: extract shared helpers from admin init/join (#142) --- cmd/spinifex/cmd/admin.go | 289 +++++++++++++++----------------------- 1 file changed, 112 insertions(+), 177 deletions(-) diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 872455236..f53d1dbb7 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -904,17 +904,10 @@ func runAdminInit(cmd *cobra.Command, args []string) { // Create config files from embedded templates fmt.Println("\n📝 Creating configuration files...") - // Create subdirectories - awsgwDir := filepath.Join(configDir, "awsgw") - predastoreDir := filepath.Join(configDir, "predastore") - natsDir := filepath.Join(configDir, "nats") - spinifexDir := filepath.Join(configDir, "spinifex") - - for _, dir := range []string{awsgwDir, predastoreDir, natsDir, spinifexDir} { - if err := os.MkdirAll(dir, 0700); err != nil { - fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err) - os.Exit(1) - } + dirs, err := createConfigSubdirs(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating config subdirectories: %v\n", err) + os.Exit(1) } portStr := strconv.Itoa(port) @@ -954,7 +947,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { os.Exit(1) } - predastorePath := filepath.Join(predastoreDir, "predastore.toml") + predastorePath := filepath.Join(dirs.Predastore, "predastore.toml") if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil { fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err) os.Exit(1) @@ -1032,38 +1025,12 @@ func runAdminInit(cmd *cobra.Command, args []string) { } // Generate config files - configs := []admin.ConfigFile{ - {Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate}, - {Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate}, - {Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate}, - } - // Skip template-based predastore.toml if multi-node was already generated - if predastoreNodesStr == "" { - configs = append(configs, admin.ConfigFile{ - Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate, - }) - } - - if err := admin.GenerateConfigFiles(configs, configSettings); err != nil { + if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, predastoreNodesStr != ""); err != nil { fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err) os.Exit(1) } - // Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system) - fmt.Println("\n🔧 Configuring AWS credentials...") - if err := admin.SetupAWSCredentials(bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, certPath, bindIP); err != nil { - fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err) - } else { - fmt.Println("✅ AWS credentials configured") - } - - admin.CreateServiceDirectories(spxRoot) - - // In production layout (running as root), set per-service ownership on - // directories and shared config files (root:spinifex with correct modes). - if os.Getuid() == 0 { - admin.SetServiceOwnership() - } + finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP) // Print success message fmt.Println("\n🎉 Spinifex initialization complete!") @@ -1196,31 +1163,25 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, fmt.Println("\n📝 Creating configuration files...") - // Create subdirectories - awsgwDir := filepath.Join(configDir, "awsgw") - predastoreDir := filepath.Join(configDir, "predastore") - natsDir := filepath.Join(configDir, "nats") - spinifexDir := filepath.Join(configDir, "spinifex") - - for _, dir := range []string{awsgwDir, predastoreDir, natsDir, spinifexDir} { - if err := os.MkdirAll(dir, 0700); err != nil { - fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err) - os.Exit(1) - } + dirs, err := createConfigSubdirs(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating config subdirectories: %v\n", err) + os.Exit(1) } portStr := strconv.Itoa(port) // Generate multi-node predastore config var predastoreNodeID int - if len(predastoreNodes) >= 3 { + hasPredastoreConfig := len(predastoreNodes) >= 3 + if hasPredastoreConfig { predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP) if err != nil { fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err) os.Exit(1) } - predastorePath := filepath.Join(predastoreDir, "predastore.toml") + predastorePath := filepath.Join(dirs.Predastore, "predastore.toml") if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil { fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err) os.Exit(1) @@ -1259,67 +1220,16 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, OVNSBAddr: "tcp:127.0.0.1:6642", } - // Apply cluster-wide network config to init node's config if networkConfig != nil { - configSettings.ExternalMode = networkConfig.ExternalMode - configSettings.ExternalDHCP = networkConfig.ExternalDHCP - configSettings.PoolName = networkConfig.PoolName - configSettings.PoolSource = networkConfig.PoolSource - configSettings.PoolStart = networkConfig.PoolStart - configSettings.PoolEnd = networkConfig.PoolEnd - configSettings.PoolGateway = networkConfig.PoolGateway - configSettings.PoolGatewayIP = networkConfig.PoolGatewayIP - configSettings.PoolPrefixLen = networkConfig.PoolPrefixLen - configSettings.PoolDNSServers = networkConfig.PoolDNSServers - - configSettings.BootstrapAccountId = networkConfig.BootstrapAccountId - configSettings.BootstrapVpcId = networkConfig.BootstrapVpcId - configSettings.BootstrapSubnetId = networkConfig.BootstrapSubnetId - configSettings.BootstrapIgwId = networkConfig.BootstrapIgwId - configSettings.BootstrapCidr = networkConfig.BootstrapCidr - configSettings.BootstrapSubnetCidr = networkConfig.BootstrapSubnetCidr - - // Auto-detect local network topology for per-node config - detected, err := admin.DetectNetwork() - if err == nil && detected.WAN != nil { - configSettings.ExternalIface = detected.WAN.Name - configSettings.WanBridge = detectedWanBridge(detected) - } + applyNetworkConfig(&configSettings, networkConfig) } - // Generate config files - configs := []admin.ConfigFile{ - {Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate}, - {Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate}, - {Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate}, - } - // Skip template-based predastore.toml if multi-node was generated - if len(predastoreNodes) < 3 { - configs = append(configs, admin.ConfigFile{ - Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate, - }) - } - - if err := admin.GenerateConfigFiles(configs, configSettings); err != nil { + if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil { fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err) os.Exit(1) } - // Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system) - fmt.Println("\n🔧 Configuring AWS credentials...") - if err := admin.SetupAWSCredentials(bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, certPath, bindIP); err != nil { - fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err) - } else { - fmt.Println("✅ AWS credentials configured") - } - - admin.CreateServiceDirectories(spxRoot) - - // In production layout (running as root), set per-service ownership on - // directories and shared config files (root:spinifex with correct modes). - if os.Getuid() == 0 { - admin.SetServiceOwnership() - } + finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP) // Keep formation server running briefly so joining nodes can fetch complete status fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...") @@ -1619,17 +1529,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) { fmt.Println("📝 Creating configuration files...") - // Create subdirectories - awsgwDir := filepath.Join(configDir, "awsgw") - predastoreDir := filepath.Join(configDir, "predastore") - natsDir := filepath.Join(configDir, "nats") - spinifexDir := filepath.Join(configDir, "spinifex") - - for _, dir := range []string{awsgwDir, predastoreDir, natsDir, spinifexDir} { - if err := os.MkdirAll(dir, 0700); err != nil { - fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err) - os.Exit(1) - } + dirs, err := createConfigSubdirs(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating config subdirectories: %v\n", err) + os.Exit(1) } portStr := strconv.Itoa(port) @@ -1645,7 +1548,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) { os.Exit(1) } - predastorePath := filepath.Join(predastoreDir, "predastore.toml") + predastorePath := filepath.Join(dirs.Predastore, "predastore.toml") if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil { fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err) os.Exit(1) @@ -1688,71 +1591,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) { OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP), } - // Apply cluster-wide network config from leader if statusResp.NetworkConfig != nil { - nc := statusResp.NetworkConfig - configSettings.ExternalMode = nc.ExternalMode - configSettings.ExternalDHCP = nc.ExternalDHCP - configSettings.PoolName = nc.PoolName - configSettings.PoolSource = nc.PoolSource - configSettings.PoolStart = nc.PoolStart - configSettings.PoolEnd = nc.PoolEnd - configSettings.PoolGateway = nc.PoolGateway - configSettings.PoolGatewayIP = nc.PoolGatewayIP - configSettings.PoolPrefixLen = nc.PoolPrefixLen - configSettings.PoolDNSServers = nc.PoolDNSServers - - configSettings.BootstrapAccountId = nc.BootstrapAccountId - configSettings.BootstrapVpcId = nc.BootstrapVpcId - configSettings.BootstrapSubnetId = nc.BootstrapSubnetId - configSettings.BootstrapIgwId = nc.BootstrapIgwId - configSettings.BootstrapCidr = nc.BootstrapCidr - configSettings.BootstrapSubnetCidr = nc.BootstrapSubnetCidr - - // Auto-detect local network topology for per-node config - if nc.ExternalMode != "" { - detected, err := admin.DetectNetwork() - if err == nil && detected.WAN != nil { - configSettings.ExternalIface = detected.WAN.Name - configSettings.WanBridge = detectedWanBridge(detected) - } - } + applyNetworkConfig(&configSettings, statusResp.NetworkConfig) } - // Generate config files - configs := []admin.ConfigFile{ - {Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate}, - {Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate}, - {Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate}, - } - // Skip template-based predastore.toml if multi-node was generated - if !hasPredastoreConfig { - configs = append(configs, admin.ConfigFile{ - Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate, - }) - } - - err = admin.GenerateConfigFiles(configs, configSettings) - if err != nil { + if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil { fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err) os.Exit(1) } - // Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system) - fmt.Println("\n🔧 Configuring AWS credentials...") - if err := admin.SetupAWSCredentials(creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, caCertPath, bindIP); err != nil { - fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err) - } else { - fmt.Println("✅ AWS credentials configured") - } - - admin.CreateServiceDirectories(dataDir) - - // In production layout (running as root), set per-service ownership on - // directories and shared config files (root:spinifex with correct modes). - if os.Getuid() == 0 { - admin.SetServiceOwnership() - } + finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP) // Print cluster summary fmt.Println("\n🎉 Node successfully joined cluster!") @@ -1986,6 +1834,93 @@ func runCertRenew(cmd *cobra.Command, _ []string) { fmt.Println("\n⚠️ Restart awsgw and daemon services to pick up the new certificate.") } +// configDirs holds the paths to config subdirectories created by createConfigSubdirs. +type configDirs struct { + AWSGW string + Predastore string + NATS string + Spinifex string +} + +// createConfigSubdirs creates the standard config subdirectories under configDir. +func createConfigSubdirs(configDir string) (configDirs, error) { + dirs := configDirs{ + AWSGW: filepath.Join(configDir, "awsgw"), + Predastore: filepath.Join(configDir, "predastore"), + NATS: filepath.Join(configDir, "nats"), + Spinifex: filepath.Join(configDir, "spinifex"), + } + for _, dir := range []string{dirs.AWSGW, dirs.Predastore, dirs.NATS, dirs.Spinifex} { + if err := os.MkdirAll(dir, 0700); err != nil { + return configDirs{}, fmt.Errorf("create directory %s: %w", dir, err) + } + } + return dirs, nil +} + +// generateAndWriteConfigs renders the standard config files (spinifex.toml, +// awsgw.toml, nats.conf, and optionally predastore.toml) from templates. +func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings admin.ConfigSettings, skipPredastore bool) error { + configs := []admin.ConfigFile{ + {Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate}, + {Name: filepath.Join(dirs.AWSGW, "awsgw.toml"), Path: filepath.Join(dirs.AWSGW, "awsgw.toml"), Template: awsgwTomlTemplate}, + {Name: filepath.Join(dirs.NATS, "nats.conf"), Path: filepath.Join(dirs.NATS, "nats.conf"), Template: natsConfTemplate}, + } + if !skipPredastore { + configs = append(configs, admin.ConfigFile{ + Name: filepath.Join(dirs.Predastore, "predastore.toml"), Path: filepath.Join(dirs.Predastore, "predastore.toml"), Template: predastoreTomlTemplate, + }) + } + return admin.GenerateConfigFiles(configs, settings) +} + +// finalizeNodeSetup configures AWS credentials, creates service directories, +// and sets ownership when running as root. +func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region, bindIP string) { + fmt.Println("\n🔧 Configuring AWS credentials...") + if err := admin.SetupAWSCredentials(adminAccessKey, adminSecretKey, region, certPath, bindIP); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err) + } else { + fmt.Println("✅ AWS credentials configured") + } + + admin.CreateServiceDirectories(dataDir) + + if os.Getuid() == 0 { + admin.SetServiceOwnership() + } +} + +// applyNetworkConfig copies cluster-wide network settings from a formation +// NetworkConfig into ConfigSettings and auto-detects the local WAN interface. +func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkConfig) { + settings.ExternalMode = nc.ExternalMode + settings.ExternalDHCP = nc.ExternalDHCP + settings.PoolName = nc.PoolName + settings.PoolSource = nc.PoolSource + settings.PoolStart = nc.PoolStart + settings.PoolEnd = nc.PoolEnd + settings.PoolGateway = nc.PoolGateway + settings.PoolGatewayIP = nc.PoolGatewayIP + settings.PoolPrefixLen = nc.PoolPrefixLen + settings.PoolDNSServers = nc.PoolDNSServers + + settings.BootstrapAccountId = nc.BootstrapAccountId + settings.BootstrapVpcId = nc.BootstrapVpcId + settings.BootstrapSubnetId = nc.BootstrapSubnetId + settings.BootstrapIgwId = nc.BootstrapIgwId + settings.BootstrapCidr = nc.BootstrapCidr + settings.BootstrapSubnetCidr = nc.BootstrapSubnetCidr + + if nc.ExternalMode != "" { + detected, err := admin.DetectNetwork() + if err == nil && detected.WAN != nil { + settings.ExternalIface = detected.WAN.Name + settings.WanBridge = detectedWanBridge(detected) + } + } +} + // writeBootstrapResult holds the admin credentials so callers can // write them to ~/.aws/credentials instead of the system credentials. type writeBootstrapResult struct { From 8e9aff3299bebd29df3fa0bf91c78af3d94c8bed Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 16 Apr 2026 10:15:10 +1000 Subject: [PATCH 03/16] chore: update old doc --- docs/INSTALL_ORIGINAL.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/INSTALL_ORIGINAL.md b/docs/INSTALL_ORIGINAL.md index 606592848..49db097ea 100644 --- a/docs/INSTALL_ORIGINAL.md +++ b/docs/INSTALL_ORIGINAL.md @@ -766,6 +766,20 @@ Start services on each node (in separate terminals or background): ./scripts/start-dev.sh ~/node3/ ``` +**Note:** NATS monitoring (port 8222) is bound to `127.0.0.1` by default for security. When running multiple nodes on the same machine, only the first node can bind this port — nodes 2 and 3 will fail to start NATS. To fix this, edit each node's `nats.conf` to either bind monitoring to the node's unique IP or disable it: + +```bash +# Option 1: Bind to node-specific IP +sed -i 's|^http: 127.0.0.1:8222|http: 127.0.0.2:8222|' ~/node2/config/nats/nats.conf +sed -i 's|^http: 127.0.0.1:8222|http: 127.0.0.3:8222|' ~/node3/config/nats/nats.conf + +# Option 2: Disable monitoring on nodes 2 and 3 +sed -i '/^http:/d' ~/node2/config/nats/nats.conf +sed -i '/^http:/d' ~/node3/config/nats/nats.conf +``` + +This only applies to simulated mode — real multi-server deployments have one node per machine with no port conflict. + ### Option B: Real Multi-Node (Physical Servers) For production or production-like deployments across multiple physical servers or VMs. From 29ba511b3865b8ddca750d652ec9e6f9b47877d0 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:31:28 +1000 Subject: [PATCH 04/16] feat: disable public nats monitoring (#143) --- cmd/spinifex/cmd/templates/nats.conf | 8 +-- spinifex/daemon/daemon_handlers.go | 2 +- .../migrate/003_nats_monitor_localhost.go | 51 ++++++++++++++++ spinifex/migrate/migrate_test.go | 61 +++++++++++++++++++ spinifex/services/nats/nats_test.go | 2 +- tests/e2e/lib/multinode-helpers.sh | 6 +- tests/e2e/run-multinode-e2e.sh | 6 +- tests/e2e/run-pseudo-multinode-e2e.sh | 6 ++ 8 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 spinifex/migrate/003_nats_monitor_localhost.go diff --git a/cmd/spinifex/cmd/templates/nats.conf b/cmd/spinifex/cmd/templates/nats.conf index 7baec86ae..267d4f4de 100644 --- a/cmd/spinifex/cmd/templates/nats.conf +++ b/cmd/spinifex/cmd/templates/nats.conf @@ -1,4 +1,4 @@ -# spinifex-config-version: 1 +# spinifex-config-version: 2 # NATS Server Configuration # Generated by Spinifex admin init @@ -49,9 +49,5 @@ max_payload: 1MB max_pending: 64MB # Monitoring -{{- if .BindIP }} -http: {{.BindIP}}:8222 -{{- else }} -http_port: 8222 -{{- end }} +http: 127.0.0.1:8222 diff --git a/spinifex/daemon/daemon_handlers.go b/spinifex/daemon/daemon_handlers.go index a0bfafcb9..558db5633 100644 --- a/spinifex/daemon/daemon_handlers.go +++ b/spinifex/daemon/daemon_handlers.go @@ -207,7 +207,7 @@ func (d *Daemon) queryNATSRole() string { if !d.config.HasService("nats") { return "" } - url := "http://" + net.JoinHostPort(d.daemonIP(), strconv.Itoa(natsMonitorPort)) + "/varz" + url := "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(natsMonitorPort)) + "/varz" return fetchNATSRole(url, roleHTTPClient) } diff --git a/spinifex/migrate/003_nats_monitor_localhost.go b/spinifex/migrate/003_nats_monitor_localhost.go new file mode 100644 index 000000000..a2c545653 --- /dev/null +++ b/spinifex/migrate/003_nats_monitor_localhost.go @@ -0,0 +1,51 @@ +package migrate + +import ( + "fmt" + "os" + "path/filepath" + "regexp" +) + +func init() { + DefaultRegistry.RegisterConfig("nats.conf", ConfigMigration{ + FromVersion: 1, + ToVersion: 2, + Description: "Bind NATS monitoring to localhost only", + Run: func(ctx ConfigContext) error { + path := filepath.Join(ctx.ConfigDir, "nats", "nats.conf") + content, err := os.ReadFile(path) + if err != nil { + return err + } + + // Match "http: :8222" or "http_port: 8222" + httpRe := regexp.MustCompile(`(?m)^http(?:_port)?:\s*.*8222\s*$`) + if !httpRe.Match(content) { + ctx.Logger.Info("nats.conf: no monitoring line found, skipping") + return nil + } + + updated := httpRe.ReplaceAll(content, []byte("http: 127.0.0.1:8222")) + + info, err := os.Stat(path) + if err != nil { + return err + } + if err := os.WriteFile(path, updated, info.Mode()); err != nil { + return err + } + + // Post-migration validation. + result, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("post-migration validation read failed: %w", err) + } + localhostRe := regexp.MustCompile(`(?m)^http:\s*127\.0\.0\.1:8222\s*$`) + if !localhostRe.Match(result) { + return fmt.Errorf("monitoring line not rewritten to localhost after migration") + } + return nil + }, + }) +} diff --git a/spinifex/migrate/migrate_test.go b/spinifex/migrate/migrate_test.go index cd470bad5..c9833a1a5 100644 --- a/spinifex/migrate/migrate_test.go +++ b/spinifex/migrate/migrate_test.go @@ -712,6 +712,67 @@ listen: 0.0.0.0:4222 assert.Equal(t, 2, v) } +// --- NATS monitoring localhost migration tests --- + +func TestNATSMonitorLocalhost(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + "BindIP 0.0.0.0", + "# spinifex-config-version: 1\n# NATS\nhttp: 0.0.0.0:8222\n", + "http: 127.0.0.1:8222", + }, + { + "BindIP specific IP", + "# spinifex-config-version: 1\n# NATS\nhttp: 10.0.0.1:8222\n", + "http: 127.0.0.1:8222", + }, + { + "no BindIP (http_port)", + "# spinifex-config-version: 1\n# NATS\nhttp_port: 8222\n", + "http: 127.0.0.1:8222", + }, + { + "already migrated (idempotent)", + "# spinifex-config-version: 1\n# NATS\nhttp: 127.0.0.1:8222\n", + "http: 127.0.0.1:8222", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + natsDir := filepath.Join(dir, "nats") + require.NoError(t, os.MkdirAll(natsDir, 0o755)) + confPath := filepath.Join(natsDir, "nats.conf") + require.NoError(t, os.WriteFile(confPath, []byte(tt.input), 0o640)) + + r := NewRegistry() + r.RegisterConfigTarget("nats.conf", "nats/nats.conf", &NATSConfVersionReader{}) + r.RegisterConfig("nats.conf", ConfigMigration{ + FromVersion: 1, + ToVersion: 2, + Description: "Bind NATS monitoring to localhost only", + Run: DefaultRegistry.configMigrations["nats.conf"][1].Run, + }) + + err := r.RunConfig("nats.conf", dir, dir) + require.NoError(t, err) + + data, err := os.ReadFile(confPath) + require.NoError(t, err) + assert.Contains(t, string(data), tt.expected) + // Verify no leftover non-localhost monitoring lines + assert.NotContains(t, string(data), "0.0.0.0:8222") + assert.NotContains(t, string(data), "10.0.0.1:8222") + assert.NotContains(t, string(data), "http_port:") + }) + } +} + func TestRunConfig_RejectsChainGap(t *testing.T) { dir := t.TempDir() natsDir := filepath.Join(dir, "nats") diff --git a/spinifex/services/nats/nats_test.go b/spinifex/services/nats/nats_test.go index c84e20969..23d0a0765 100644 --- a/spinifex/services/nats/nats_test.go +++ b/spinifex/services/nats/nats_test.go @@ -98,6 +98,6 @@ func TestRenderedConfig_HasMigrationVersionMarker(t *testing.T) { require.NoError(t, err) firstLine := strings.SplitN(string(raw), "\n", 2)[0] - assert.Equal(t, "# spinifex-config-version: 1", firstLine, + assert.Equal(t, "# spinifex-config-version: 2", firstLine, "nats.conf template must start with the current migration version marker") } diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index 4613fbc6b..7b5a32271 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -120,7 +120,7 @@ verify_nats_cluster() { # Check cluster info via monitoring endpoint on node1 local cluster_info - cluster_info=$(curl -s "http://${NODE1_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || { + cluster_info=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || { echo " ERROR: Cannot reach NATS monitoring endpoint" return 1 } @@ -645,8 +645,8 @@ verify_all_services_down() { all_down=false fi - # Check NATS - if curl -s --connect-timeout 2 "http://${node_ip}:${NATS_MONITOR_PORT}" > /dev/null 2>&1; then + # Check NATS (client port — monitoring is localhost-only) + if nc -z -w 2 "${node_ip}" ${NATS_CLIENT_PORT} 2>/dev/null; then echo " Node$i: NATS still responding" all_down=false fi diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index 17cb98bea..992567e59 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -333,7 +333,7 @@ echo "========================================" # NATS cluster: verify 2 unique peers from node1 echo "Checking NATS cluster..." -NATS_INFO=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || { +NATS_INFO=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || { echo " ERROR: Cannot reach NATS monitoring endpoint" exit 1 } @@ -809,7 +809,7 @@ else fi # Check NATS degraded state (should have 1 route instead of 2) -NATS_DEGRADED=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) +NATS_DEGRADED=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null) DEGRADED_PEERS=$(echo "$NATS_DEGRADED" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0") echo " NATS peers during failure: $DEGRADED_PEERS (expected: 1)" if [ "$DEGRADED_PEERS" -eq 1 ]; then @@ -849,7 +849,7 @@ echo " Waiting for NATS cluster to reform..." ATTEMPT=0 REFORMED=false while [ $ATTEMPT -lt 60 ]; do - NATS_RECOVER=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) + NATS_RECOVER=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null) RECOVER_PEERS=$(echo "$NATS_RECOVER" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0") if [ "$RECOVER_PEERS" -ge 2 ]; then echo " NATS cluster reformed ($RECOVER_PEERS peers)" diff --git a/tests/e2e/run-pseudo-multinode-e2e.sh b/tests/e2e/run-pseudo-multinode-e2e.sh index b8a22dd2c..1fad30130 100755 --- a/tests/e2e/run-pseudo-multinode-e2e.sh +++ b/tests/e2e/run-pseudo-multinode-e2e.sh @@ -134,6 +134,12 @@ wait $JOIN3_PID || { echo "ERROR: Node 3 join failed"; exit 1; } wait $LEADER_INIT_PID || { echo "ERROR: Leader init failed"; exit 1; } echo "Cluster formation complete — all configs generated" +# Disable monitoring on non-primary nodes to avoid port conflict on 127.0.0.1:8222 +for i in 2 3; do + sed -i '/^http:\s*127\.0\.0\.1:8222/d' "$HOME/node${i}/config/nats/nats.conf" + echo " Stripped NATS monitoring from node$i (avoids 127.0.0.1:8222 conflict)" +done + # Now start services (configs exist for all nodes) echo "" echo "Starting node services..." From ff9f9485d2f3c3e6017579204008fa680d02c2c2 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:46:12 +1000 Subject: [PATCH 05/16] feat: nats TLS (#144) --- .github/workflows/e2e.yml | 2 +- build/systemd/spinifex-awsgw.service | 2 +- build/systemd/spinifex-daemon.service | 2 +- build/systemd/spinifex-nats.service | 2 +- build/systemd/spinifex-predastore.service | 2 +- build/systemd/spinifex-viperblock.service | 2 +- build/systemd/spinifex-vpcd.service | 2 +- cmd/spinifex/cmd/admin.go | 4 +- cmd/spinifex/cmd/get.go | 2 +- cmd/spinifex/cmd/service.go | 2 + cmd/spinifex/cmd/templates/nats.conf | 15 +- .../cmd/templates/predastore-multinode.toml | 3 +- cmd/spinifex/cmd/templates/predastore.toml | 3 +- cmd/spinifex/cmd/templates/spinifex.toml | 3 +- scripts/check-coverage.sh | 7 +- scripts/diff-coverage.sh | 2 +- scripts/start-dev.sh | 38 +++-- spinifex/admin/admin.go | 4 +- spinifex/config/config.go | 7 +- spinifex/daemon/daemon.go | 7 +- spinifex/migrate/004_nats_tls.go | 148 ++++++++++++++++ spinifex/services/awsgw/awsgw.go | 11 +- spinifex/services/nats/nats_test.go | 26 ++- spinifex/services/viperblockd/viperblockd.go | 3 +- spinifex/services/vpcd/vpcd.go | 4 +- spinifex/utils/nats.go | 20 ++- spinifex/utils/nats_test.go | 158 +++++++++++++++++- tests/e2e/lib/multinode-helpers.sh | 5 +- 28 files changed, 434 insertions(+), 52 deletions(-) create mode 100644 spinifex/migrate/004_nats_tls.go diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 72a282416..b10f9a084 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -83,7 +83,7 @@ jobs: SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') SSH_KEY="${SSH_KEY/#\~/$HOME}" ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ - 'cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ./run-lb-e2e.sh' + "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh" - name: Teardown if: always() diff --git a/build/systemd/spinifex-awsgw.service b/build/systemd/spinifex-awsgw.service index fd61d424a..48ca3c640 100644 --- a/build/systemd/spinifex-awsgw.service +++ b/build/systemd/spinifex-awsgw.service @@ -22,7 +22,7 @@ Environment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key NoNewPrivileges=yes ProtectSystem=strict PrivateDevices=yes -ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key +ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem ReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex # Hardening diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index 4e5856956..c37d28224 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -23,7 +23,7 @@ PrivateDevices=no DevicePolicy=closed DeviceAllow=/dev/kvm rw DeviceAllow=/dev/net/tun rw -ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key +ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd # Hardening diff --git a/build/systemd/spinifex-nats.service b/build/systemd/spinifex-nats.service index f7995f42e..48036dc09 100644 --- a/build/systemd/spinifex-nats.service +++ b/build/systemd/spinifex-nats.service @@ -17,7 +17,7 @@ Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf NoNewPrivileges=yes ProtectSystem=strict PrivateDevices=yes -ReadOnlyPaths=/etc/spinifex/nats +ReadOnlyPaths=/etc/spinifex/nats /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem ReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex # Hardening diff --git a/build/systemd/spinifex-predastore.service b/build/systemd/spinifex-predastore.service index e42b95685..73614d31f 100644 --- a/build/systemd/spinifex-predastore.service +++ b/build/systemd/spinifex-predastore.service @@ -22,7 +22,7 @@ Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key NoNewPrivileges=yes ProtectSystem=strict PrivateDevices=yes -ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key +ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem ReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex # Hardening diff --git a/build/systemd/spinifex-viperblock.service b/build/systemd/spinifex-viperblock.service index c9c72e289..f06c0f3d1 100644 --- a/build/systemd/spinifex-viperblock.service +++ b/build/systemd/spinifex-viperblock.service @@ -21,7 +21,7 @@ Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/ NoNewPrivileges=yes ProtectSystem=strict PrivateDevices=yes -ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env +ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem ReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd # Hardening diff --git a/build/systemd/spinifex-vpcd.service b/build/systemd/spinifex-vpcd.service index 4e705762a..73f1b5cf7 100644 --- a/build/systemd/spinifex-vpcd.service +++ b/build/systemd/spinifex-vpcd.service @@ -19,7 +19,7 @@ Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/ # Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl) ProtectSystem=full PrivateDevices=yes -ReadOnlyPaths=/etc/spinifex/spinifex.toml +ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex # Hardening diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index f53d1dbb7..c59b70ad2 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -239,7 +239,7 @@ func init() { adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster") adminInitCmd.Flags().String("host", "", "Leader node to join (if not specified, tries multicast discovery)") adminInitCmd.Flags().Int("port", 4432, "Port to bind cluster services on") - adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node)") + adminInitCmd.Flags().String("bind", "127.0.0.1", "IP address to bind services to (e.g., 10.11.12.1 for multi-node)") adminInitCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)") adminInitCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)") adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.") @@ -864,7 +864,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { spxRoot = filepath.Clean(spxRoot) // Determine if this is a multi-node formation - isMultiNode := nodes >= 2 && bindIP != "0.0.0.0" + isMultiNode := nodes >= 2 && bindIP != "0.0.0.0" && bindIP != "127.0.0.1" if isMultiNode { // Build cluster-wide network config for propagation to joining nodes diff --git a/cmd/spinifex/cmd/get.go b/cmd/spinifex/cmd/get.go index 29c49cf46..dd8bb1e9e 100644 --- a/cmd/spinifex/cmd/get.go +++ b/cmd/spinifex/cmd/get.go @@ -90,7 +90,7 @@ func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) { } nodeConfig := cfg.Nodes[cfg.Node] - nc, err := utils.ConnectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token) + nc, err := utils.ConnectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token, nodeConfig.NATS.CACert) if err != nil { return nil, nil, fmt.Errorf("failed to connect to NATS: %w", err) } diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index f0503ea4a..269d602c0 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -288,6 +288,7 @@ var viperblockStartCmd = &cobra.Command{ service, err := service.New("viperblock", &viperblockd.Config{ NatsHost: nodeConfig.NATS.Host, NatsToken: nodeConfig.NATS.ACL.Token, + NatsCACert: nodeConfig.NATS.CACert, PluginPath: pluginPath, S3Host: nodeConfig.Predastore.Host, Bucket: nodeConfig.Predastore.Bucket, @@ -743,6 +744,7 @@ var vpcdStartCmd = &cobra.Command{ svc, err := service.New("vpcd", &vpcd.Config{ NatsHost: nodeConfig.NATS.Host, NatsToken: nodeConfig.NATS.ACL.Token, + NatsCACert: nodeConfig.NATS.CACert, OVNNBAddr: nodeConfig.VPCD.OVNNBAddr, OVNSBAddr: nodeConfig.VPCD.OVNSBAddr, BaseDir: nodeConfig.BaseDir, diff --git a/cmd/spinifex/cmd/templates/nats.conf b/cmd/spinifex/cmd/templates/nats.conf index 267d4f4de..e3ba95873 100644 --- a/cmd/spinifex/cmd/templates/nats.conf +++ b/cmd/spinifex/cmd/templates/nats.conf @@ -1,4 +1,4 @@ -# spinifex-config-version: 2 +# spinifex-config-version: 3 # NATS Server Configuration # Generated by Spinifex admin init @@ -9,12 +9,25 @@ listen: {{.BindIP}}:4222 listen: 0.0.0.0:4222 {{- end }} +tls { + cert_file: "{{.ConfigDir}}/server.pem" + key_file: "{{.ConfigDir}}/server.key" + ca_file: "{{.ConfigDir}}/ca.pem" +} + {{- if and .ClusterBindIP .ClusterRoutes }} cluster { name: {{ .ClusterName }} listen: {{.ClusterBindIP}}:4248 + tls { + cert_file: "{{.ConfigDir}}/server.pem" + key_file: "{{.ConfigDir}}/server.key" + ca_file: "{{.ConfigDir}}/ca.pem" + verify: true + } + routes = [ {{- range .ClusterRoutes }} "nats-route://{{$.NatsToken}}@{{ . }}", diff --git a/cmd/spinifex/cmd/templates/predastore-multinode.toml b/cmd/spinifex/cmd/templates/predastore-multinode.toml index f4f813245..a0b180ba7 100644 --- a/cmd/spinifex/cmd/templates/predastore-multinode.toml +++ b/cmd/spinifex/cmd/templates/predastore-multinode.toml @@ -1,6 +1,6 @@ # Predastore S3-compatible storage configuration file in TOML format. # Generated for multi-node cluster -version = "2" +version = "3" region = "{{.Region}}" host = "0.0.0.0" @@ -65,5 +65,6 @@ burst = 200 [iam] nats_url = "nats://{{.BindIP}}:4222" nats_token = "{{.NatsToken}}" +nats_ca_cert = "{{.ConfigDir}}/ca.pem" master_key_path = "{{.ConfigDir}}/master.key" access_keys_bucket = "spinifex-iam-access-keys" diff --git a/cmd/spinifex/cmd/templates/predastore.toml b/cmd/spinifex/cmd/templates/predastore.toml index e2202b74c..fc522d701 100644 --- a/cmd/spinifex/cmd/templates/predastore.toml +++ b/cmd/spinifex/cmd/templates/predastore.toml @@ -1,6 +1,6 @@ # Predastore S3-compatible storage configuration file in TOML format. # MULTI NODE used for testing -version = "2" +version = "3" region = "{{.Region}}" # Optionally configure port and host, or specify via daemon CLI options @@ -99,5 +99,6 @@ burst = 200 [iam] nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}}:4222" nats_token = "{{.NatsToken}}" +nats_ca_cert = "{{.ConfigDir}}/ca.pem" master_key_path = "{{.ConfigDir}}/master.key" access_keys_bucket = "spinifex-iam-access-keys" \ No newline at end of file diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index f839c6e31..432dd70f0 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -1,5 +1,5 @@ # Domain configuration file in TOML format. -version = "1.0" +version = "2" epoch = 1 node = "{{.Node}}" {{- if .ExternalMode}} @@ -69,6 +69,7 @@ config = "awsgw/awsgw.toml" [nodes.{{.Node}}.nats] host = "{{.BindIP}}:4222" +cacert = "{{.ConfigDir}}/ca.pem" [nodes.{{.Node}}.nats.acl] token = "{{.NatsToken}}" diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 0406c59c2..8a3d978ab 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -13,7 +13,12 @@ if [[ ! -s "$PROFILE" ]]; then exit 1 fi -TOTAL=$(go tool cover -func="$PROFILE" | tail -1 | awk '{print $NF}' | tr -d '%') +# Filter out packages excluded from coverage requirements (e.g. migrations) +FILTERED=$(mktemp) +trap 'rm -f "$FILTERED"' EXIT +awk 'NR==1 || !/\/migrate\//' "$PROFILE" > "$FILTERED" + +TOTAL=$(go tool cover -func="$FILTERED" | tail -1 | awk '{print $NF}' | tr -d '%') if [[ -z "$TOTAL" ]]; then echo "ERROR: No coverage data generated — tests may have failed to compile" diff --git a/scripts/diff-coverage.sh b/scripts/diff-coverage.sh index 84aa1d2c6..e12269dee 100755 --- a/scripts/diff-coverage.sh +++ b/scripts/diff-coverage.sh @@ -66,7 +66,7 @@ trap 'rm -rf "$TMPDIR"' EXIT # Only non-test .go files that were added or modified. # -M enables rename detection so file splits/moves only count truly changed lines. # -C enables copy detection for the same reason (file split = partial copy). -git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' | awk ' +git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' ':!spinifex/migrate/*' | awk ' /^\+\+\+ / { file = substr($2, 3) next diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index df84af1b5..114916b9a 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -271,19 +271,33 @@ if has_service "nats"; then # Wait for NATS JetStream to be ready before Predastore starts. # Predastore lazily opens IAM KV buckets — it starts with config-only auth # and activates IAM auth once the spinifex daemon creates the KV buckets. - echo "🔍 Waiting for NATS JetStream..." - NATS_JS_READY=false - for i in $(seq 1 15); do - # Send a NATS INFO probe — JetStream-enabled servers include "jetstream" in the INFO response - if echo "" | nc -w 1 "$NATS_CHECK_HOST" 4222 2>/dev/null | grep -q "jetstream"; then - echo " ✅ NATS JetStream is ready" - NATS_JS_READY=true - break + # Wait for NATS JetStream to be ready before Predastore starts. + # Predastore lazily opens IAM KV buckets — it starts with config-only auth + # and activates IAM auth once the spinifex daemon creates the KV buckets. + # Use the monitoring HTTP endpoint (plaintext nc probe doesn't work with TLS). + # Monitoring is only on the primary node; secondary nodes skip this check. + NATS_HAS_MONITORING=false + if [ -f "$CONFIG_DIR/nats/nats.conf" ] && grep -q '^http:' "$CONFIG_DIR/nats/nats.conf"; then + NATS_HAS_MONITORING=true + fi + + if [ "$NATS_HAS_MONITORING" = "true" ]; then + echo "🔍 Waiting for NATS JetStream..." + NATS_JS_READY=false + for i in $(seq 1 15); do + if curl -sf "http://127.0.0.1:8222/jsz" > /dev/null 2>&1; then + echo " ✅ NATS JetStream is ready" + NATS_JS_READY=true + break + fi + sleep 1 + done + if [ "$NATS_JS_READY" = "false" ]; then + echo " ⚠️ NATS JetStream not confirmed ready — Predastore may fall back to config-only auth" fi - sleep 1 - done - if [ "$NATS_JS_READY" = "false" ]; then - echo " ⚠️ NATS JetStream not confirmed ready — Predastore may fall back to config-only auth" + else + echo "🔍 NATS monitoring not configured — skipping JetStream readiness check" + sleep 2 fi else echo "1️⃣ Skipping NATS (not a local service)" diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 0752fef55..679c831a8 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -678,7 +678,7 @@ func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string, NotBefore: notBefore, NotAfter: notAfter, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, BasicConstraintsValid: true, DNSNames: dnsNames, IPAddresses: ipAddresses, @@ -740,7 +740,7 @@ func GenerateSelfSignedCert(certPath, keyPath string) error { NotBefore: notBefore, NotAfter: notAfter, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, BasicConstraintsValid: true, DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, diff --git a/spinifex/config/config.go b/spinifex/config/config.go index ca3e19fb4..92ef94ee6 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -117,9 +117,10 @@ type DaemonConfig struct { // NATSConfig holds the NATS configuration type NATSConfig struct { - Host string `json:"Host" mapstructure:"host"` - ACL NATSACL `json:"ACL" mapstructure:"acl"` - Sub NATSSub `json:"Sub" mapstructure:"sub"` + Host string `json:"Host" mapstructure:"host"` + CACert string `json:"CACert" mapstructure:"cacert"` + ACL NATSACL `json:"ACL" mapstructure:"acl"` + Sub NATSSub `json:"Sub" mapstructure:"sub"` } // NATSACL holds the NATS ACL configuration diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 0985c4c97..2d684ae05 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -851,7 +851,7 @@ func (d *Daemon) connectNATS() error { start := time.Now() for { - nc, err := utils.ConnectNATS(d.config.NATS.Host, d.config.NATS.ACL.Token) + nc, err := utils.ConnectNATS(d.config.NATS.Host, d.config.NATS.ACL.Token, d.config.NATS.CACert) if err == nil { d.natsConn = nc if time.Since(start) > time.Second { @@ -860,6 +860,11 @@ func (d *Daemon) connectNATS() error { return nil } + // TLS configuration errors are permanent — retrying will not help. + if strings.Contains(err.Error(), "CA cert") { + return fmt.Errorf("NATS TLS configuration error: %w", err) + } + elapsed := time.Since(start) if elapsed >= maxWait { return fmt.Errorf("NATS connect failed after %s: %w", elapsed.Round(time.Second), err) diff --git a/spinifex/migrate/004_nats_tls.go b/spinifex/migrate/004_nats_tls.go new file mode 100644 index 000000000..c2ad66c21 --- /dev/null +++ b/spinifex/migrate/004_nats_tls.go @@ -0,0 +1,148 @@ +package migrate + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +func init() { + // nats.conf: version 2 → 3 — add TLS blocks for client and cluster connections. + DefaultRegistry.RegisterConfig("nats.conf", ConfigMigration{ + FromVersion: 2, + ToVersion: 3, + Description: "Add TLS for client connections and mutual TLS for cluster routes", + Run: func(ctx ConfigContext) error { + path := filepath.Join(ctx.ConfigDir, "nats", "nats.conf") + content, err := os.ReadFile(path) + if err != nil { + return err + } + + text := string(content) + + // Skip if TLS is already configured. + if strings.Contains(text, "tls {") { + ctx.Logger.Info("nats.conf: TLS block already present, skipping") + return nil + } + + certFile := fmt.Sprintf("%s/server.pem", ctx.ConfigDir) + keyFile := fmt.Sprintf("%s/server.key", ctx.ConfigDir) + caFile := fmt.Sprintf("%s/ca.pem", ctx.ConfigDir) + + // 1. Inject client TLS block after the top-level listen line (port 4222 only, + // not the cluster listen on 4248). + clientTLS := fmt.Sprintf("\n\ntls {\n cert_file: \"%s\"\n key_file: \"%s\"\n ca_file: \"%s\"\n}", certFile, keyFile, caFile) + + listenRe := regexp.MustCompile(`(?m)^listen:\s*\S+:4222\s*$`) + if !listenRe.MatchString(text) { + return fmt.Errorf("nats.conf: client listen line (port 4222) not found") + } + loc := listenRe.FindStringIndex(text) + text = text[:loc[1]] + clientTLS + text[loc[1]:] + + // 2. Inject cluster TLS block inside the cluster {} block, after the listen line. + clusterTLS := fmt.Sprintf("\n\n tls {\n cert_file: \"%s\"\n key_file: \"%s\"\n ca_file: \"%s\"\n verify: true\n }", certFile, keyFile, caFile) + + // Match the cluster listen line and insert TLS block after it. + clusterListenRe := regexp.MustCompile(`(?m)^(\s*listen:\s*\S+:4248)\s*$`) + if clusterListenRe.MatchString(text) { + text = clusterListenRe.ReplaceAllStringFunc(text, func(match string) string { + return strings.TrimRight(match, "\n") + clusterTLS + }) + } + + info, err := os.Stat(path) + if err != nil { + return err + } + return os.WriteFile(path, []byte(text), info.Mode()) + }, + }) + + // spinifex.toml: version 1 → 2 — add cacert under [nodes.*.nats]. + DefaultRegistry.RegisterConfigTarget("spinifex.toml", "spinifex.toml", &TOMLVersionReader{}) + + DefaultRegistry.RegisterConfig("spinifex.toml", ConfigMigration{ + FromVersion: 1, + ToVersion: 2, + Description: "Add NATS CA cert path for TLS", + Run: func(ctx ConfigContext) error { + path := filepath.Join(ctx.ConfigDir, "spinifex.toml") + content, err := os.ReadFile(path) + if err != nil { + return err + } + + text := string(content) + + // Skip if cacert is already present. + if strings.Contains(text, "cacert") { + ctx.Logger.Info("spinifex.toml: cacert already present, skipping") + return nil + } + + // Rewrite 0.0.0.0 → 127.0.0.1 for the NATS host. 0.0.0.0 is a valid + // bind address but not a valid TLS connect address (certs have real IPs). + text = strings.ReplaceAll(text, `host = "0.0.0.0:4222"`, `host = "127.0.0.1:4222"`) + + // Insert cacert after each host line under [nodes.*.nats]. + // Pattern: host = "x.x.x.x:4222" followed by a newline. + caPath := fmt.Sprintf("%s/ca.pem", ctx.ConfigDir) + natsHostRe := regexp.MustCompile(`(?m)^(host\s*=\s*"[^"]+:4222")\s*$`) + if !natsHostRe.MatchString(text) { + return fmt.Errorf("spinifex.toml: NATS host line (host = \"...:4222\") not found, cannot insert cacert") + } + text = natsHostRe.ReplaceAllStringFunc(text, func(match string) string { + return strings.TrimRight(match, "\n") + fmt.Sprintf("\ncacert = \"%s\"", caPath) + }) + + info, err := os.Stat(path) + if err != nil { + return err + } + return os.WriteFile(path, []byte(text), info.Mode()) + }, + }) + + // predastore.toml: version 2 → 3 — add nats_ca_cert under [iam]. + DefaultRegistry.RegisterConfig("predastore.toml", ConfigMigration{ + FromVersion: 2, + ToVersion: 3, + Description: "Add NATS CA cert path for TLS", + Run: func(ctx ConfigContext) error { + path := filepath.Join(ctx.ConfigDir, "predastore", "predastore.toml") + content, err := os.ReadFile(path) + if err != nil { + return err + } + + text := string(content) + + // Skip if nats_ca_cert is already present. + if strings.Contains(text, "nats_ca_cert") { + ctx.Logger.Info("predastore.toml: nats_ca_cert already present, skipping") + return nil + } + + // Insert nats_ca_cert after the nats_token line. + caPath := fmt.Sprintf("%s/ca.pem", ctx.ConfigDir) + natsTokenRe := regexp.MustCompile(`(?m)^(nats_token\s*=\s*"[^"]*")\s*$`) + if !natsTokenRe.MatchString(text) { + return fmt.Errorf("predastore.toml: nats_token line not found, cannot insert nats_ca_cert") + } + text = natsTokenRe.ReplaceAllStringFunc(text, func(match string) string { + return strings.TrimRight(match, "\n") + fmt.Sprintf("\nnats_ca_cert = \"%s\"", caPath) + }) + + info, err := os.Stat(path) + if err != nil { + return err + } + return os.WriteFile(path, []byte(text), info.Mode()) + }, + }) +} diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go index c627fffdb..e46de75b9 100644 --- a/spinifex/services/awsgw/awsgw.go +++ b/spinifex/services/awsgw/awsgw.go @@ -102,7 +102,7 @@ func launchService(config *config.ClusterConfig) error { // Connect to NATS for service communication. On concurrent startup the // local NATS server may not be listening yet, so retry with backoff. - natsConn, err := connectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token) + natsConn, err := connectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token, nodeConfig.NATS.CACert) if err != nil { return err } @@ -205,13 +205,13 @@ func launchService(config *config.ClusterConfig) error { // connectNATS establishes a connection to NATS with retry/backoff. On concurrent // startup the local NATS server may not be listening yet. -func connectNATS(host, token string) (*nats.Conn, error) { +func connectNATS(host, token, caCertPath string) (*nats.Conn, error) { const maxWait = 5 * time.Minute retryDelay := 500 * time.Millisecond start := time.Now() for { - nc, err := utils.ConnectNATS(host, token) + nc, err := utils.ConnectNATS(host, token, caCertPath) if err == nil { if time.Since(start) > time.Second { slog.Info("NATS connection established", "elapsed", time.Since(start).Round(time.Second)) @@ -219,6 +219,11 @@ func connectNATS(host, token string) (*nats.Conn, error) { return nc, nil } + // TLS configuration errors are permanent — retrying will not help. + if strings.Contains(err.Error(), "CA cert") { + return nil, fmt.Errorf("NATS TLS configuration error: %w", err) + } + elapsed := time.Since(start) if elapsed >= maxWait { return nil, fmt.Errorf("NATS connect failed after %s: %w", elapsed.Round(time.Second), err) diff --git a/spinifex/services/nats/nats_test.go b/spinifex/services/nats/nats_test.go index 23d0a0765..f173fb322 100644 --- a/spinifex/services/nats/nats_test.go +++ b/spinifex/services/nats/nats_test.go @@ -9,6 +9,7 @@ import ( "text/template" "time" + "github.com/mulgadc/spinifex/spinifex/admin" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" @@ -34,6 +35,14 @@ func TestRenderedConfig_EnforcesAuth(t *testing.T) { tmpDir := t.TempDir() confPath := filepath.Join(tmpDir, "nats.conf") + // Generate test CA and server certs for TLS. + caCertPath := filepath.Join(tmpDir, "ca.pem") + caKeyPath := filepath.Join(tmpDir, "ca.key") + require.NoError(t, admin.GenerateCACert(caCertPath, caKeyPath)) + serverCertPath := filepath.Join(tmpDir, "server.pem") + serverKeyPath := filepath.Join(tmpDir, "server.key") + require.NoError(t, admin.GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, "127.0.0.1")) + // Read and render the production template. raw, err := os.ReadFile(templatePath(t)) require.NoError(t, err) @@ -50,6 +59,7 @@ func TestRenderedConfig_EnforcesAuth(t *testing.T) { "NatsToken": token, "DataDir": tmpDir, "LogDir": tmpDir, + "ConfigDir": tmpDir, }) f.Close() require.NoError(t, err) @@ -70,22 +80,30 @@ func TestRenderedConfig_EnforcesAuth(t *testing.T) { require.True(t, ns.ReadyForConnections(5*time.Second)) t.Cleanup(func() { ns.Shutdown() }) + // Build TLS option using the test CA cert for client connections. + tlsOpt := nats.RootCAs(caCertPath) + t.Run("no token rejected", func(t *testing.T) { - _, err := nats.Connect(ns.ClientURL(), nats.MaxReconnects(0)) + _, err := nats.Connect(ns.ClientURL(), tlsOpt, nats.MaxReconnects(0)) assert.Error(t, err, "unauthenticated connection should be rejected") }) t.Run("wrong token rejected", func(t *testing.T) { - _, err := nats.Connect(ns.ClientURL(), nats.Token("wrong-token"), nats.MaxReconnects(0)) + _, err := nats.Connect(ns.ClientURL(), tlsOpt, nats.Token("wrong-token"), nats.MaxReconnects(0)) assert.Error(t, err, "wrong token should be rejected") }) t.Run("correct token accepted", func(t *testing.T) { - nc, err := nats.Connect(ns.ClientURL(), nats.Token(token), nats.MaxReconnects(0)) + nc, err := nats.Connect(ns.ClientURL(), tlsOpt, nats.Token(token), nats.MaxReconnects(0)) require.NoError(t, err, "correct token should be accepted") defer nc.Close() assert.True(t, nc.IsConnected()) }) + + t.Run("plaintext rejected", func(t *testing.T) { + _, err := nats.Connect(ns.ClientURL(), nats.Token(token), nats.MaxReconnects(0)) + assert.Error(t, err, "plaintext connection should be rejected when TLS is enabled") + }) } // TestRenderedConfig_HasMigrationVersionMarker ensures the nats.conf template @@ -98,6 +116,6 @@ func TestRenderedConfig_HasMigrationVersionMarker(t *testing.T) { require.NoError(t, err) firstLine := strings.SplitN(string(raw), "\n", 2)[0] - assert.Equal(t, "# spinifex-config-version: 2", firstLine, + assert.Equal(t, "# spinifex-config-version: 3", firstLine, "nats.conf template must start with the current migration version marker") } diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index 2c0fe7955..cae94b3f7 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -39,6 +39,7 @@ type Config struct { Debug bool NatsHost string NatsToken string + NatsCACert string MountedVolumes []MountedVolume S3Host string Bucket string @@ -169,7 +170,7 @@ func (svc *Service) Reload() (err error) { func launchService(cfg *Config) (err error) { // Connect to NATS - nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken) + nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken, cfg.NatsCACert) if err != nil { slog.Error("Failed to connect to NATS", "err", err) return err diff --git a/spinifex/services/vpcd/vpcd.go b/spinifex/services/vpcd/vpcd.go index cb562f456..bbc95492d 100644 --- a/spinifex/services/vpcd/vpcd.go +++ b/spinifex/services/vpcd/vpcd.go @@ -44,6 +44,8 @@ type Config struct { NatsHost string // NatsToken is the NATS authentication token. NatsToken string + // NatsCACert is the path to the CA certificate for NATS TLS. + NatsCACert string // OVNNBAddr is the OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641"). OVNNBAddr string // OVNSBAddr is the OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642"), used for monitoring. @@ -272,7 +274,7 @@ func launchService(cfg *Config) error { slog.Info("OVN preflight passed (br-int exists, ovn-controller running)") // Connect to NATS - nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken) + nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken, cfg.NatsCACert) if err != nil { slog.Error("Failed to connect to NATS", "err", err) return err diff --git a/spinifex/utils/nats.go b/spinifex/utils/nats.go index 06f62057d..e4b7b84e2 100644 --- a/spinifex/utils/nats.go +++ b/spinifex/utils/nats.go @@ -1,10 +1,13 @@ package utils import ( + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" "log/slog" + "os" "time" "github.com/nats-io/nats.go" @@ -12,7 +15,8 @@ import ( // ConnectNATS establishes a connection to a NATS server with standard reconnect // handling and logging. If token is non-empty, token authentication is used. -func ConnectNATS(host, token string) (*nats.Conn, error) { +// If caCertPath is non-empty, TLS is enabled using the given CA certificate. +func ConnectNATS(host, token, caCertPath string) (*nats.Conn, error) { opts := []nats.Option{ nats.ReconnectWait(time.Second), nats.MaxReconnects(-1), @@ -28,6 +32,20 @@ func ConnectNATS(host, token string) (*nats.Conn, error) { opts = append(opts, nats.Token(token)) } + if caCertPath != "" { + caCert, err := os.ReadFile(caCertPath) + if err != nil { + return nil, fmt.Errorf("read CA cert %s: %w", caCertPath, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to parse CA cert from %s", caCertPath) + } + opts = append(opts, nats.Secure(&tls.Config{ + RootCAs: pool, + })) + } + nc, err := nats.Connect(host, opts...) if err != nil { return nil, fmt.Errorf("NATS connect failed: %w", err) diff --git a/spinifex/utils/nats_test.go b/spinifex/utils/nats_test.go index 4b861462d..b382ee6ac 100644 --- a/spinifex/utils/nats_test.go +++ b/spinifex/utils/nats_test.go @@ -1,7 +1,18 @@ package utils import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" "testing" "time" @@ -21,7 +32,7 @@ func startTestNATSServer(t *testing.T) *server.Server { func TestConnectNATS_Success(t *testing.T) { ns := startTestNATSServer(t) - nc, err := ConnectNATS(ns.ClientURL(), "") + nc, err := ConnectNATS(ns.ClientURL(), "", "") require.NoError(t, err) defer nc.Close() @@ -44,7 +55,7 @@ func TestConnectNATS_WithToken(t *testing.T) { t.Cleanup(func() { ns.Shutdown() }) // With correct token — should succeed - nc, err := ConnectNATS(ns.ClientURL(), "test-token-123") + nc, err := ConnectNATS(ns.ClientURL(), "test-token-123", "") require.NoError(t, err) defer nc.Close() assert.True(t, nc.IsConnected()) @@ -65,17 +76,156 @@ func TestConnectNATS_WrongToken(t *testing.T) { require.True(t, ns.ReadyForConnections(5*time.Second)) t.Cleanup(func() { ns.Shutdown() }) - _, err = ConnectNATS(ns.ClientURL(), "wrong-token") + _, err = ConnectNATS(ns.ClientURL(), "wrong-token", "") assert.Error(t, err) assert.Contains(t, err.Error(), "NATS connect failed") } func TestConnectNATS_BadAddress(t *testing.T) { - _, err := ConnectNATS("nats://127.0.0.1:1", "") + _, err := ConnectNATS("nats://127.0.0.1:1", "", "") assert.Error(t, err) assert.Contains(t, err.Error(), "NATS connect failed") } +func TestConnectNATS_MissingCACert(t *testing.T) { + _, err := ConnectNATS("nats://127.0.0.1:4222", "", "/nonexistent/ca.pem") + require.Error(t, err) + assert.Contains(t, err.Error(), "read CA cert") + assert.Contains(t, err.Error(), "/nonexistent/ca.pem") +} + +func TestConnectNATS_MalformedCACert(t *testing.T) { + tmp := t.TempDir() + badCert := filepath.Join(tmp, "bad-ca.pem") + require.NoError(t, os.WriteFile(badCert, []byte("not a PEM certificate"), 0o644)) + + _, err := ConnectNATS("nats://127.0.0.1:4222", "", badCert) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse CA cert") +} + +// generateTestCA creates an ephemeral CA cert+key and writes PEM files to dir. +func generateTestCA(t *testing.T, dir, name string) (certPath, keyPath string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + + certPath = filepath.Join(dir, name+".pem") + keyPath = filepath.Join(dir, name+".key") + require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600)) + return certPath, keyPath +} + +// generateTestServerCert creates a server cert signed by the given CA. +func generateTestServerCert(t *testing.T, dir, caCertPath, caKeyPath string) (certPath, keyPath string) { + t.Helper() + caCertPEM, err := os.ReadFile(caCertPath) + require.NoError(t, err) + block, _ := pem.Decode(caCertPEM) + caCert, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + caKeyPEM, err := os.ReadFile(caKeyPath) + require.NoError(t, err) + keyBlock, _ := pem.Decode(caKeyPEM) + caKey, err := x509.ParseECPrivateKey(keyBlock.Bytes) + require.NoError(t, err) + + serverKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &serverKey.PublicKey, caKey) + require.NoError(t, err) + + certPath = filepath.Join(dir, "server.pem") + keyPath = filepath.Join(dir, "server.key") + require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)) + keyDER, err := x509.MarshalECPrivateKey(serverKey) + require.NoError(t, err) + require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600)) + return certPath, keyPath +} + +// startTLSNATSServer starts a NATS server with TLS using the given cert files. +func startTLSNATSServer(t *testing.T, serverCertPath, serverKeyPath, caCertPath string) *server.Server { + t.Helper() + cert, err := tls.LoadX509KeyPair(serverCertPath, serverKeyPath) + require.NoError(t, err) + caPEM, err := os.ReadFile(caCertPath) + require.NoError(t, err) + pool := x509.NewCertPool() + require.True(t, pool.AppendCertsFromPEM(caPEM)) + + tlsCfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientCAs: pool, + } + + opts := &server.Options{ + Host: "127.0.0.1", + Port: -1, + NoLog: true, + NoSigs: true, + TLSConfig: tlsCfg, + } + + ns, err := server.NewServer(opts) + require.NoError(t, err) + go ns.Start() + require.True(t, ns.ReadyForConnections(5*time.Second)) + t.Cleanup(func() { ns.Shutdown() }) + return ns +} + +func TestConnectNATS_TLSSuccess(t *testing.T) { + tmp := t.TempDir() + caCertPath, caKeyPath := generateTestCA(t, tmp, "ca") + serverCertPath, serverKeyPath := generateTestServerCert(t, tmp, caCertPath, caKeyPath) + + ns := startTLSNATSServer(t, serverCertPath, serverKeyPath, caCertPath) + + nc, err := ConnectNATS(ns.ClientURL(), "", caCertPath) + require.NoError(t, err) + defer nc.Close() + assert.True(t, nc.IsConnected()) +} + +func TestConnectNATS_WrongCA(t *testing.T) { + tmp := t.TempDir() + caCertPath, caKeyPath := generateTestCA(t, tmp, "ca") + serverCertPath, serverKeyPath := generateTestServerCert(t, tmp, caCertPath, caKeyPath) + wrongCACertPath, _ := generateTestCA(t, tmp, "wrong-ca") + + ns := startTLSNATSServer(t, serverCertPath, serverKeyPath, caCertPath) + + _, err := ConnectNATS(ns.ClientURL(), "", wrongCACertPath) + assert.Error(t, err, "connection with wrong CA should fail") +} + func TestNATSRequest_Success(t *testing.T) { ns := startTestNATSServer(t) diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index 7b5a32271..c69a2acc2 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -779,10 +779,9 @@ init_leader_node() { # shellcheck disable=SC2086 ./bin/spx admin init \ --node node1 \ + --nodes 3 \ --bind "${NODE1_IP}" \ --cluster-bind "${NODE1_IP}" \ - --cluster-routes "${NODE1_IP}:${NATS_CLUSTER_PORT}" \ - --predastore-nodes "${NODE1_IP},${NODE2_IP},${NODE3_IP}" \ --port ${CLUSTER_PORT} \ --region ap-southeast-2 \ --az ap-southeast-2a \ @@ -827,12 +826,10 @@ join_follower_node() { # Remove old node directory rm -rf "$data_dir/" - # Route to node1 (seed node) - other nodes discovered via NATS gossip ./bin/spx admin join \ --node "node$node_num" \ --bind "$node_ip" \ --cluster-bind "$node_ip" \ - --cluster-routes "${NODE1_IP}:${NATS_CLUSTER_PORT}" \ --host "${NODE1_IP}:${CLUSTER_PORT}" \ --token "$JOIN_TOKEN" \ --data-dir "$data_dir/" \ From 39865497837a102c9c9eeb33e513b9bb9a150d35 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:51:46 +1000 Subject: [PATCH 06/16] fix: log failed attempts at warn level for compliance --- spinifex/gateway/auth.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spinifex/gateway/auth.go b/spinifex/gateway/auth.go index 5a8172335..8c329b52b 100644 --- a/spinifex/gateway/auth.go +++ b/spinifex/gateway/auth.go @@ -106,7 +106,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler { ak, err := gw.IAMService.LookupAccessKey(accessKey) if err != nil { if strings.Contains(err.Error(), awserrors.ErrorIAMNoSuchEntity) { - slog.Debug("Access key not found", "accessKeyID", accessKey) + slog.Warn("Auth failure: access key not found", "accessKeyID", accessKey, "sourceIP", clientIP) gw.RateLimiter.RecordFailure(clientIP) gw.writeSigV4Error(w, r, awserrors.ErrorInvalidClientTokenId) return @@ -116,7 +116,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler { return } if ak.Status != handlers_iam.AccessKeyStatusActive { - slog.Debug("Access key inactive", "accessKeyID", accessKey) + slog.Warn("Auth failure: access key inactive", "accessKeyID", accessKey, "sourceIP", clientIP) gw.RateLimiter.RecordFailure(clientIP) gw.writeSigV4Error(w, r, awserrors.ErrorInvalidClientTokenId) return @@ -172,8 +172,9 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler { // Compare signatures using constant-time comparison to prevent timing attacks if subtle.ConstantTimeCompare([]byte(expectedSignature), []byte(providedSignature)) != 1 { - slog.Debug("Signature mismatch", - "accessKeyId", accessKey, + slog.Warn("Auth failure: signature mismatch", + "accessKeyID", accessKey, + "sourceIP", clientIP, ) gw.RateLimiter.RecordFailure(clientIP) gw.writeSigV4Error(w, r, awserrors.ErrorSignatureDoesNotMatch) From 6f02f6bc2201d0dfd443a76e5e2be6437481cd6f Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:45:21 +1000 Subject: [PATCH 07/16] feat(vpcd): log=true on default-deny ACLs (#145) --- spinifex/services/vpcd/mock_ovn.go | 19 ++++++-- spinifex/services/vpcd/nbdb/model.go | 4 ++ spinifex/services/vpcd/ovn.go | 16 +++++- spinifex/services/vpcd/security.go | 55 ++++++++++++++++++--- spinifex/services/vpcd/security_test.go | 65 +++++++++++++++++++++---- 5 files changed, 134 insertions(+), 25 deletions(-) diff --git a/spinifex/services/vpcd/mock_ovn.go b/spinifex/services/vpcd/mock_ovn.go index 24cbb1ca1..01bf67096 100644 --- a/spinifex/services/vpcd/mock_ovn.go +++ b/spinifex/services/vpcd/mock_ovn.go @@ -532,7 +532,7 @@ func (m *MockOVNClient) SetPortGroupPorts(_ context.Context, name string, ports // ACLs -func (m *MockOVNClient) AddACL(_ context.Context, portGroupName string, direction string, priority int, match string, action string) error { +func (m *MockOVNClient) AddACL(_ context.Context, portGroupName string, spec ACLSpec) error { m.mu.Lock() defer m.mu.Unlock() pg, exists := m.portGroups[portGroupName] @@ -541,10 +541,19 @@ func (m *MockOVNClient) AddACL(_ context.Context, portGroupName string, directio } acl := &nbdb.ACL{ UUID: utils.GenerateResourceID("acl"), - Direction: direction, - Priority: priority, - Match: match, - Action: action, + Direction: spec.Direction, + Priority: spec.Priority, + Match: spec.Match, + Action: spec.Action, + Log: spec.Log, + } + if spec.Name != "" { + name := spec.Name + acl.Name = &name + } + if spec.Severity != "" { + severity := spec.Severity + acl.Severity = &severity } m.acls[acl.UUID] = acl pg.ACLs = append(pg.ACLs, acl.UUID) diff --git a/spinifex/services/vpcd/nbdb/model.go b/spinifex/services/vpcd/nbdb/model.go index a98eade4d..811df1ef3 100644 --- a/spinifex/services/vpcd/nbdb/model.go +++ b/spinifex/services/vpcd/nbdb/model.go @@ -104,10 +104,14 @@ type PortGroup struct { // ACL represents an OVN ACL rule attached to a port group or logical switch. type ACL struct { UUID string `ovsdb:"_uuid"` + Name *string `ovsdb:"name"` Direction string `ovsdb:"direction"` // "to-lport" or "from-lport" Priority int `ovsdb:"priority"` Match string `ovsdb:"match"` Action string `ovsdb:"action"` // "allow-related", "drop", "allow", "reject" + Log bool `ovsdb:"log"` + Severity *string `ovsdb:"severity"` // "alert", "warning", "notice", "info", "debug" + Meter *string `ovsdb:"meter"` ExternalIDs map[string]string `ovsdb:"external_ids"` } diff --git a/spinifex/services/vpcd/ovn.go b/spinifex/services/vpcd/ovn.go index 18332845e..a8ada215f 100644 --- a/spinifex/services/vpcd/ovn.go +++ b/spinifex/services/vpcd/ovn.go @@ -34,6 +34,18 @@ func (c *LiveOVNClient) transactOps(ctx context.Context, ops []ovsdb.Operation) return err } +// ACLSpec describes an OVN ACL rule for attachment to a port group. +// Name, Severity, and Meter are optional — set Severity when Log is true. +type ACLSpec struct { + Direction string // "to-lport" or "from-lport" + Priority int + Match string + Action string // "allow-related", "drop", "allow", "reject" + Name string + Log bool + Severity string // "alert", "warning", "notice", "info", "debug" +} + // OVNClient defines the interface for interacting with the OVN Northbound Database. // This abstraction allows for mock implementations in tests. type OVNClient interface { @@ -88,7 +100,7 @@ type OVNClient interface { SetPortGroupPorts(ctx context.Context, name string, ports []string) error // ACLs (attached to port groups) - AddACL(ctx context.Context, portGroupName string, direction string, priority int, match string, action string) error + AddACL(ctx context.Context, portGroupName string, spec ACLSpec) error ClearACLs(ctx context.Context, portGroupName string) error // Gateway Chassis (HA scheduling for gateway router ports) @@ -821,7 +833,7 @@ func (c *LiveOVNClient) SetPortGroupPorts(_ context.Context, _ string, _ []strin return fmt.Errorf("SetPortGroupPorts: not yet implemented for live OVN client") } -func (c *LiveOVNClient) AddACL(_ context.Context, _ string, _ string, _ int, _ string, _ string) error { +func (c *LiveOVNClient) AddACL(_ context.Context, _ string, _ ACLSpec) error { return fmt.Errorf("AddACL: not yet implemented for live OVN client") } diff --git a/spinifex/services/vpcd/security.go b/spinifex/services/vpcd/security.go index a8cd05ca2..c19f74e2e 100644 --- a/spinifex/services/vpcd/security.go +++ b/spinifex/services/vpcd/security.go @@ -16,6 +16,12 @@ const ( TopicUpdateSG = "vpc.update-sg" ) +// denyACLSeverity is the syslog severity OVN uses when a packet matches a +// default-deny ACL. "info" is loud enough to be captured by a syslog forwarder +// yet quiet enough to avoid paging on every dropped scan packet. Operators can +// promote it at their log collector if they want higher-priority alerts. +const denyACLSeverity = "info" + // SGEvent carries security group state from the handler to vpcd. type SGEvent struct { GroupId string `json:"group_id"` @@ -48,11 +54,13 @@ func (h *TopologyHandler) handleCreateSG(msg *nats.Msg) { return } - // Add default deny ACL (priority 900) — drop all traffic not explicitly allowed - if err := h.ovn.AddACL(ctx, pgName, "to-lport", 900, fmt.Sprintf("outport == @%s && ip4", pgName), "drop"); err != nil { + // Add default deny ACLs (priority 900) — drop all traffic not explicitly + // allowed. Logging is enabled so boundary communications are observable via + // syslog (CMMC SC.L1-3.13.1). + if err := h.ovn.AddACL(ctx, pgName, denyIngressACL(pgName)); err != nil { slog.Warn("vpcd: failed to add default deny ingress ACL", "pg", pgName, "err", err) } - if err := h.ovn.AddACL(ctx, pgName, "from-lport", 900, fmt.Sprintf("inport == @%s && ip4", pgName), "drop"); err != nil { + if err := h.ovn.AddACL(ctx, pgName, denyEgressACL(pgName)); err != nil { slog.Warn("vpcd: failed to add default deny egress ACL", "pg", pgName, "err", err) } @@ -127,11 +135,11 @@ func (h *TopologyHandler) handleUpdateSG(msg *nats.Msg) { slog.Warn("vpcd: failed to clear ACLs for update", "pg", pgName, "err", err) } - // Re-add default deny ACLs (priority 900) - if err := h.ovn.AddACL(ctx, pgName, "to-lport", 900, fmt.Sprintf("outport == @%s && ip4", pgName), "drop"); err != nil { + // Re-add default deny ACLs (priority 900) with logging enabled. + if err := h.ovn.AddACL(ctx, pgName, denyIngressACL(pgName)); err != nil { slog.Warn("vpcd: failed to re-add default deny ingress ACL", "pg", pgName, "err", err) } - if err := h.ovn.AddACL(ctx, pgName, "from-lport", 900, fmt.Sprintf("inport == @%s && ip4", pgName), "drop"); err != nil { + if err := h.ovn.AddACL(ctx, pgName, denyEgressACL(pgName)); err != nil { slog.Warn("vpcd: failed to re-add default deny egress ACL", "pg", pgName, "err", err) } @@ -150,18 +158,49 @@ func (h *TopologyHandler) handleUpdateSG(msg *nats.Msg) { // addRuleACLs adds OVN ACLs for a set of ingress and egress rules at priority 1000 // (higher than the default deny at 900, so allow rules take precedence). +// Allow rules are not logged — accept logging on a private network is +// high-volume and low-signal. Only denies carry Log=true. func (h *TopologyHandler) addRuleACLs(ctx context.Context, pgName string, ingress, egress []SGRuleForACL) { for _, rule := range ingress { match := BuildIngressACLMatch(pgName, rule) - if err := h.ovn.AddACL(ctx, pgName, "to-lport", 1000, match, "allow-related"); err != nil { + spec := ACLSpec{Direction: "to-lport", Priority: 1000, Match: match, Action: "allow-related"} + if err := h.ovn.AddACL(ctx, pgName, spec); err != nil { slog.Warn("vpcd: failed to add ingress ACL", "pg", pgName, "match", match, "err", err) } } for _, rule := range egress { match := BuildEgressACLMatch(pgName, rule) - if err := h.ovn.AddACL(ctx, pgName, "from-lport", 1000, match, "allow-related"); err != nil { + spec := ACLSpec{Direction: "from-lport", Priority: 1000, Match: match, Action: "allow-related"} + if err := h.ovn.AddACL(ctx, pgName, spec); err != nil { slog.Warn("vpcd: failed to add egress ACL", "pg", pgName, "match", match, "err", err) } } } + +// denyIngressACL builds the default-deny ingress ACL for a port group with +// logging enabled (CMMC SC.L1-3.13.1 boundary monitoring). +func denyIngressACL(pgName string) ACLSpec { + return ACLSpec{ + Direction: "to-lport", + Priority: 900, + Match: fmt.Sprintf("outport == @%s && ip4", pgName), + Action: "drop", + Name: pgName + "-deny-ingress", + Log: true, + Severity: denyACLSeverity, + } +} + +// denyEgressACL is the egress counterpart to denyIngressACL. +func denyEgressACL(pgName string) ACLSpec { + return ACLSpec{ + Direction: "from-lport", + Priority: 900, + Match: fmt.Sprintf("inport == @%s && ip4", pgName), + Action: "drop", + Name: pgName + "-deny-egress", + Log: true, + Severity: denyACLSeverity, + } +} diff --git a/spinifex/services/vpcd/security_test.go b/spinifex/services/vpcd/security_test.go index 3417782c5..a953c33d0 100644 --- a/spinifex/services/vpcd/security_test.go +++ b/spinifex/services/vpcd/security_test.go @@ -40,10 +40,44 @@ func TestSecurity_CreatePortGroup(t *testing.T) { assert.NotNil(t, pg) // Verify default deny ACLs were created (2 deny ACLs at priority 900) + // and that each has logging enabled per CMMC SC.L1-3.13.1. + type aclSnapshot struct { + action string + log bool + severity string + name string + } + var snapshots []aclSnapshot mock.mu.Lock() aclCount := len(pg.ACLs) + for _, aclUUID := range pg.ACLs { + a := mock.acls[aclUUID] + if a == nil { + continue + } + snap := aclSnapshot{action: a.Action, log: a.Log} + if a.Severity != nil { + snap.severity = *a.Severity + } + if a.Name != nil { + snap.name = *a.Name + } + snapshots = append(snapshots, snap) + } mock.mu.Unlock() + assert.Equal(t, 2, aclCount, "should have 2 default deny ACLs (ingress + egress)") + denyCount := 0 + for _, s := range snapshots { + if s.action != "drop" { + continue + } + denyCount++ + assert.True(t, s.log, "default deny ACL must have log=true for boundary monitoring") + assert.Equal(t, "info", s.severity, "default deny ACL must use info severity") + assert.Contains(t, s.name, "sg_abc123-deny-", "default deny ACL must have a name for syslog correlation") + } + assert.Equal(t, 2, denyCount, "should have 2 drop ACLs") } func TestSecurity_DeletePortGroup(t *testing.T) { @@ -123,30 +157,41 @@ func TestSecurity_UpdateSGAddRules(t *testing.T) { assertSuccess(t, resp, "update SG with ingress rules") // Verify ACLs were created: 2 default deny + 2 ingress allow = 4 + type aclSnapshot struct { + action string + match string + log bool + } + var snapshots []aclSnapshot mock.mu.Lock() pg := mock.portGroups["sg_upd1"] aclCount := len(pg.ACLs) - - // Collect match strings from the ACLs - var matches []string for _, aclUUID := range pg.ACLs { - acl := mock.acls[aclUUID] - if acl != nil { - matches = append(matches, acl.Match) + a := mock.acls[aclUUID] + if a == nil { + continue } + snapshots = append(snapshots, aclSnapshot{action: a.Action, match: a.Match, log: a.Log}) } mock.mu.Unlock() assert.Equal(t, 4, aclCount, "should have 4 ACLs (2 deny + 2 ingress allow)") - // Check that at least one match contains tcp.dst == 22 + // Check that at least one match contains tcp.dst == 22. Also verify + // logging policy: denies logged, allows not logged (CMMC SC.L1-3.13.1). foundSSH := false foundHTTPS := false - for _, m := range matches { - if containsAll(m, "tcp.dst == 22", "ip4.src == 10.0.0.0/8") { + for _, s := range snapshots { + switch s.action { + case "drop": + assert.True(t, s.log, "deny ACL must be logged") + case "allow-related": + assert.False(t, s.log, "allow ACL must not be logged (high volume, low signal)") + } + if containsAll(s.match, "tcp.dst == 22", "ip4.src == 10.0.0.0/8") { foundSSH = true } - if containsAll(m, "tcp.dst == 443") { + if containsAll(s.match, "tcp.dst == 443") { foundHTTPS = true } } From 084e93cfd0a0dc69308878b8243992384863f096 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:00:27 +1000 Subject: [PATCH 08/16] feat: validate image checksums (#147) --- cmd/spinifex/cmd/admin.go | 42 +++-- cmd/spinifex/cmd/admin_test.go | 34 +++++ docs/COMMANDS.md | 33 +++- spinifex/utils/images.go | 236 +++++++++++++++++++++++++++++ spinifex/utils/utils.go | 6 - spinifex/utils/utils_test.go | 269 +++++++++++++++++++++++++++++++++ 6 files changed, 604 insertions(+), 16 deletions(-) create mode 100644 cmd/spinifex/cmd/admin_test.go diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index c59b70ad2..739596558 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -286,6 +286,7 @@ func init() { imagesImportCmd.Flags().String("platform", "Linux/UNIX", "Specified platform (e.g Linux/UNIX, Windows)") imagesImportCmd.Flags().StringSlice("tag", nil, "Tag to apply to the imported AMI as key=value (repeatable; e.g. --tag spinifex:managed-by=elbv2)") imagesImportCmd.Flags().Bool("force", false, "Force command execution (overwrites existing files)") + imagesImportCmd.Flags().Bool("skip-verify", false, "Skip catalog-image checksum verification (INSECURE; operator assumes integrity responsibility)") } func runimagesImportCmd(cmd *cobra.Command, args []string) { @@ -297,6 +298,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { cfgFile, _ := cmd.Flags().GetString("config") forceCmd, _ := cmd.Flags().GetBool("force") + skipVerify, _ := cmd.Flags().GetBool("skip-verify") ostmpDir, _ := cmd.Flags().GetString("tmp-dir") // Use default config path @@ -399,16 +401,12 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { os.Exit(1) } - fmt.Printf("✅ Created config directory: %s\n", imagePath) - // Next, if the file is selected to download, fetch it, extract disk image, and save to path if imageName != "" && localFile == "" { // Download the file to the image path filename := path.Base(image.URL) imageFile = fmt.Sprintf("%s/%s", imagePath, filename) - fmt.Printf("Downloading image %s to %s\n", image.URL, imageFile) - // If image path exists, skip if admin.FileExists(imageFile) && !forceCmd { fmt.Printf("Image file already exists, skipping download, use --force to overwrite: %s\n", imageFile) @@ -421,8 +419,23 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { } } - // Update image file path for later extraction - //imagePath = imageFilePath + // Verify before extract: the catalog digest is of the artifact as it + // sits on the mirror (.tar.xz/.img/.raw). Also catches a poisoned + // cache on the re-run path — failure leaves the file on disk so the + // operator can inspect; recover with --force. + if skipVerify { + fmt.Fprintf(os.Stderr, "⚠️ --skip-verify set: checksum verification skipped for %s\n", imageName) + } else { + if image.Checksum == "" || image.ChecksumType == "" { + fmt.Fprintf(os.Stderr, "Catalog entry %q is missing Checksum/ChecksumType; refusing import.\n", imageName) + os.Exit(1) + } + if err := utils.VerifyImageChecksum(imageFile, image.Checksum, image.ChecksumType); err != nil { + printChecksumError(os.Stderr, imageFile, imageName, image, err) + os.Exit(1) + } + fmt.Printf("✅ Verified image checksum (%s)\n", image.ChecksumType) + } } // Next, validate if the image is raw, tar, gz, xv, etc. We need to upload the raw image @@ -434,9 +447,6 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { } extractedImagePath, err := utils.ExtractDiskImageFromFile(imageFile, imagePath) - - fmt.Println("Extracted image to:", extractedImagePath) - if err != nil { fmt.Fprintf(os.Stderr, "Could not extract image: %v\n", err) os.Exit(1) @@ -2134,3 +2144,17 @@ func installCACertificate(caPemPath string) { fmt.Printf("✅ CA certificate installed to system trust store\n") } + +// printChecksumError writes the failure, the source URL (printed for every +// error kind so 404/non-HTTPS/size-cap failures tell the operator which URL +// to investigate), and the exact --force recovery command. The cached file +// is left in place: an implicit auto-delete would mutate state inside +// "verify", and a tampered artifact is forensically useful intact. +func printChecksumError(w io.Writer, imageFile, imageName string, image utils.Images, err error) { + fmt.Fprintf(w, "Image integrity verification failed: %v\n", err) + fmt.Fprintf(w, " file: %s\n", imageFile) + fmt.Fprintf(w, " source: %s\n", image.Checksum) + fmt.Fprintln(w) + fmt.Fprintln(w, "The cached file was left in place. To re-download and retry:") + fmt.Fprintf(w, " spx admin images import --name %s --force\n", imageName) +} diff --git a/cmd/spinifex/cmd/admin_test.go b/cmd/spinifex/cmd/admin_test.go new file mode 100644 index 000000000..4fdb92cac --- /dev/null +++ b/cmd/spinifex/cmd/admin_test.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "bytes" + "errors" + "fmt" + "testing" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/stretchr/testify/assert" +) + +// Regression guard: source URL must print for every error kind, not just +// mismatch, so a 404/non-HTTPS/size-cap failure still tells the operator +// which URL to investigate. +func TestPrintChecksumError(t *testing.T) { + image := utils.Images{Checksum: "https://example.com/SUMS", ChecksumType: "sha512"} + const imageFile = "/var/lib/img.tar.xz" + const imageName = "debian-12-x86_64" + + errs := []error{ + fmt.Errorf("%w: expected abc got def", utils.ErrChecksumMismatch), + fmt.Errorf("%w: 404", utils.ErrChecksumFetchFailed), + errors.New("open /x: no such file"), + } + for _, e := range errs { + var buf bytes.Buffer + printChecksumError(&buf, imageFile, imageName, image, e) + out := buf.String() + assert.Contains(t, out, image.Checksum, "source URL must print for: %v", e) + assert.Contains(t, out, imageFile) + assert.Contains(t, out, "spx admin images import --name "+imageName+" --force") + } +} diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 90410eeb0..cca285a4b 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -89,9 +89,40 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status | |---------|-------|---------------|-------------|------------|--------| -| `spx admin images import` | `--name` | Cluster must be running, image file must exist locally | Imports a local disk image (QCOW2) as an AMI → uploads to Predastore S3 → registers as available AMI | 1. Import valid QCOW2 image
2. Image appears in `describe-images` | **DONE** | +| `spx admin images import` | `--name`, `--file`, `--force`, `--skip-verify` | Cluster must be running; either `--name` (catalog download) or `--file` (operator-supplied media) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, and verify the SHA-256/SHA-512 digest before extraction. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip verification — operator-supplied media is outside Spinifex's trust boundary, and the skip is logged at INFO for audit. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. | 1. Import valid catalog image (verifies checksum)
2. Tampered cache hit fails with `ErrChecksumMismatch`
3. `--force` recovers after a mismatch
4. `--file` import skips verification
5. `--skip-verify` bypasses checksum with WARN | **DONE** | | `spx admin images list` | — | None | Lists available OS images that can be imported or downloaded | 1. List available images | **DONE** | +#### Image integrity verification (CMMC SI.L1-3.14.2) + +Catalog imports (`spx admin images import --name `) verify the image +against the catalog-declared SHA-256/SHA-512 digest before extraction. The sums +file is fetched from the catalog `Checksum` URL over HTTPS only (cross-scheme +redirects refused), and verification runs on both fresh downloads and cache +hits so a poisoned cache is caught on the next import. + +On mismatch the import exits non-zero, the cached file is left on disk for +inspection, and the printed guidance is `spx admin images import --name +--force` to re-download. + +`--file` imports skip verification by design: operator-supplied media is +outside Spinifex's trust boundary and the operator is responsible for +integrity (e.g. `sha256sum` against a trusted upstream digest before import). +The skip is recorded as an INFO `slog` event with `reason=local-file-import` +so a CMMC assessor can audit the decision from journald. + +`--skip-verify` bypasses the checksum step for catalog imports. The command +still downloads via the catalog URL but does not compare the image digest +against the sums file. Intended for narrow cases such as debugging upstream +mirror issues or running against a transiently-broken `latest/` path; the +skip is logged at WARN with `reason=skip-verify-flag` and printed to stderr +so operators and assessors see it. Prefer `--file` with an out-of-band +verified image over `--skip-verify` whenever possible. + +**Limitation:** verification confirms the image matches the digest the mirror +served. A mirror compromise that swaps both image and sums file is not +detected; closing that gap requires GPG signature verification of the sums +file, deferred to a later phase. + ## AWS Commands ### EC2 - Instance Management diff --git a/spinifex/utils/images.go b/spinifex/utils/images.go index 6134ed3cc..79a0e7676 100644 --- a/spinifex/utils/images.go +++ b/spinifex/utils/images.go @@ -3,11 +3,23 @@ package utils import ( "bufio" "bytes" + "context" + "crypto/sha256" + "crypto/sha512" + "crypto/subtle" + "crypto/tls" + "crypto/x509" + "encoding/hex" "errors" "fmt" + "hash" "io" + "log/slog" + "net/http" + "net/url" "os" "os/exec" + "os/signal" "path/filepath" "runtime" "strings" @@ -18,6 +30,230 @@ import ( var ErrQCOWDetected = errors.New("qcow format detected") +const sumsFileMaxSize = 1 * 1024 * 1024 // 1 MB, matches formation.go JoinRequest cap. + +var ( + ErrChecksumMismatch = errors.New("image checksum mismatch") + ErrChecksumNotFound = errors.New("checksum entry for image filename not found") + ErrUnsupportedChecksumType = errors.New("unsupported checksum type") + ErrChecksumFetchFailed = errors.New("checksum fetch failed") +) + +// checksumExtraRootCAs is a test-only hook so unit tests can stand up an +// httptest.NewTLSServer (which uses a self-signed cert) without relaxing the +// HTTPS enforcement. Production builds leave this nil and the verification +// client uses the system trust store. +var checksumExtraRootCAs *x509.CertPool + +// checksumFetchTimeout is a var (not const) so tests can shrink it to exercise +// the context-deadline path without waiting 30s. +var checksumFetchTimeout = 30 * time.Second + +// VerifyImageChecksum fetches the sums file at checksumURL, locates the entry +// for imagePath's basename, hashes imagePath with the algorithm named by +// checksumType ("sha256" or "sha512"), and compares digests. +// +// Fails closed: every error path returns without accepting the image. A +// non-HTTPS scheme, non-2xx status, transport error, response over 1 MB, or +// cross-scheme redirect all wrap ErrChecksumFetchFailed. Unknown algorithm +// wraps ErrUnsupportedChecksumType. Missing filename entry wraps +// ErrChecksumNotFound. Digest mismatch wraps ErrChecksumMismatch and the +// wrapped error's %v includes expected and actual hex. +func VerifyImageChecksum(imagePath, checksumURL, checksumType string) error { + hasher, err := newHasher(checksumType) + if err != nil { + return err + } + + expected, err := fetchExpectedDigest(checksumURL, filepath.Base(imagePath)) + if err != nil { + slog.Error("image checksum fetch failed", "source", checksumURL, "err", err) + return err + } + + // Distinct error for a catalog/sums-file algorithm mismatch — without this + // it'd surface as "tampering" via ConstantTimeCompare's length-0 return. + if len(expected) != hasher.Size()*2 { + return fmt.Errorf("%w: digest length %d from sums file does not match %s output length %d", + ErrChecksumFetchFailed, len(expected), checksumType, hasher.Size()*2) + } + + actual, err := hashImageFile(imagePath, hasher) + if err != nil { + return fmt.Errorf("hash image file: %w", err) + } + + if subtle.ConstantTimeCompare([]byte(expected), []byte(actual)) != 1 { + slog.Error("image checksum mismatch", + "image", imagePath, + "algorithm", checksumType, + "expected", expected, + "actual", actual, + "source", checksumURL, + ) + return fmt.Errorf("%w: expected %s got %s", ErrChecksumMismatch, expected, actual) + } + + return nil +} + +func newHasher(checksumType string) (hash.Hash, error) { + switch strings.ToLower(checksumType) { + case "sha256": + return sha256.New(), nil + case "sha512": + return sha512.New(), nil + default: + return nil, fmt.Errorf("%w: %q", ErrUnsupportedChecksumType, checksumType) + } +} + +// fetchExpectedDigest downloads the sums file and returns the hex digest for +// the given filename. Enforces HTTPS on initial URL and every redirect hop, +// caps redirects at 10, and truncates response at sumsFileMaxSize+1 so the +// size check is unambiguous. +func fetchExpectedDigest(checksumURL, filename string) (string, error) { + parsed, err := url.Parse(checksumURL) + if err != nil { + return "", fmt.Errorf("%w: parse url: %v", ErrChecksumFetchFailed, err) + } + if parsed.Scheme != "https" { + return "", fmt.Errorf("%w: non-https checksum url scheme %q", ErrChecksumFetchFailed, parsed.Scheme) + } + + ctx, cancel := context.WithTimeout(context.Background(), checksumFetchTimeout) + defer cancel() + intCh := make(chan os.Signal, 1) + signal.Notify(intCh, os.Interrupt) + defer signal.Stop(intCh) + go func() { + select { + case <-intCh: + cancel() + case <-ctx.Done(): + } + }() + + transport := &http.Transport{} + if checksumExtraRootCAs != nil { + transport.TLSClientConfig = &tls.Config{RootCAs: checksumExtraRootCAs} + } + client := &http.Client{ + Timeout: checksumFetchTimeout, + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("stopped after 10 redirects") + } + if req.URL.Scheme != "https" { + return fmt.Errorf("refusing non-https redirect to %s", req.URL.Redacted()) + } + return nil + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, checksumURL, nil) + if err != nil { + return "", fmt.Errorf("%w: build request: %v", ErrChecksumFetchFailed, err) + } + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("%w: %v", ErrChecksumFetchFailed, err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return "", fmt.Errorf("%w: unexpected status %s", ErrChecksumFetchFailed, resp.Status) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, sumsFileMaxSize+1)) + if err != nil { + return "", fmt.Errorf("%w: read body: %v", ErrChecksumFetchFailed, err) + } + if len(body) > sumsFileMaxSize { + return "", fmt.Errorf("%w: sums file exceeds %d byte limit", ErrChecksumFetchFailed, sumsFileMaxSize) + } + + digest, err := parseSumsFile(body, filename) + if err != nil { + return "", err + } + return digest, nil +} + +// parseSumsFile scans a coreutils-style sums file and returns the hex digest +// matching filename. Filename match is case-sensitive: upstream Debian/Ubuntu/ +// Alpine sums are consistently lowercase and a divergence is a real signal. +// +// Accepts three on-the-wire shapes: " " (text mode), " *" +// (binary mode), and a bare single-token "" line (single-file .sha512 from +// Alpine, which does not carry a filename). +func parseSumsFile(body []byte, filename string) (string, error) { + var bareDigest string + bareCount := 0 + scanner := bufio.NewScanner(bytes.NewReader(body)) + scanner.Buffer(make([]byte, 0, 64*1024), sumsFileMaxSize+1) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + switch len(fields) { + case 1: + bareDigest = fields[0] + bareCount++ + case 2: + name := strings.TrimPrefix(fields[1], "*") + if name == filename { + return fields[0], nil + } + default: + // More than two fields: not a format we recognise. Skip rather than + // reject outright — signed sums files occasionally have trailing + // commentary we want to tolerate. + } + } + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("%w: scan sums file: %v", ErrChecksumFetchFailed, err) + } + + // Alpine single-file .sha512 fallback: exactly one bare-digest line in the + // whole file. Any more and we refuse, because we can't tell which line is + // the right one without a filename. + if bareCount == 1 { + return bareDigest, nil + } + + return "", fmt.Errorf("%w: %s", ErrChecksumNotFound, filename) +} + +// hashImageFile streams the image through hasher and returns the digest as +// lowercase hex. Zero-byte files are rejected outright so a truncated +// download surfaces as an explicit error rather than an opaque "checksum +// mismatch". +func hashImageFile(imagePath string, hasher hash.Hash) (string, error) { + f, err := os.Open(imagePath) + if err != nil { + return "", err + } + defer f.Close() + + stat, err := f.Stat() + if err != nil { + return "", err + } + if stat.Size() == 0 { + return "", fmt.Errorf("image file %s is empty (likely a truncated or failed download)", imagePath) + } + + if _, err := io.Copy(hasher, f); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + type Images struct { Name string `json:"name"` Description string `json:"description"` diff --git a/spinifex/utils/utils.go b/spinifex/utils/utils.go index de05fa6b9..8683efb51 100644 --- a/spinifex/utils/utils.go +++ b/spinifex/utils/utils.go @@ -249,9 +249,7 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout WithTotal(int(cl)). Start() - var written int64 reader := io.TeeReader(resp.Body, progressWriter(func(n int) { - written += int64(n) // Update progress bar with the number of bytes read in this chunk bar.Add(n) })) @@ -261,8 +259,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout if err != nil { return fmt.Errorf("copy error: %v", err) } - - pterm.Printf("Saved %s (%s)\n", filename, humanBytes(SafeInt64ToUint64(written))) return err } else { // Unknown size: spinner that shows bytes downloaded @@ -280,8 +276,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout if err != nil { return fmt.Errorf("copy error: %v", err) } - - pterm.Printf("Saved %s (%s)\n", filename, humanBytes(SafeInt64ToUint64(written))) } return nil diff --git a/spinifex/utils/utils_test.go b/spinifex/utils/utils_test.go index 30cfcf89a..9230a7774 100644 --- a/spinifex/utils/utils_test.go +++ b/spinifex/utils/utils_test.go @@ -1,12 +1,20 @@ package utils import ( + "crypto/sha256" + "crypto/sha512" + "crypto/x509" + "encoding/hex" "fmt" + "net/http" + "net/http/httptest" "os" "os/exec" + "path/filepath" "runtime" "strings" "sync" + "sync/atomic" "syscall" "testing" "time" @@ -1005,3 +1013,264 @@ func TestHashMAC(t *testing.T) { // Different input → different output assert.NotEqual(t, mac, HashMAC("02:00:00", "other-id")) } + +// writeTempImage writes content to a file named name under t.TempDir() and +// returns the absolute path. +func writeTempImage(t *testing.T, name string, content []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + require.NoError(t, os.WriteFile(path, content, 0o600)) + return path +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func sha512Hex(b []byte) string { + sum := sha512.Sum512(b) + return hex.EncodeToString(sum[:]) +} + +// newSumsServer stands up a TLS test server returning body for any request, +// configures the package-level trust hook, and registers cleanup. Returns the +// server URL. +func newSumsServer(t *testing.T, body string) string { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + trustTestServer(t, srv) + return srv.URL + "/SUMS" +} + +// trustTestServer adds srv's certificate to the package-level trust pool so +// VerifyImageChecksum's HTTPS-only client accepts it. Restores the previous +// pool on test cleanup. +func trustTestServer(t *testing.T, srv *httptest.Server) { + t.Helper() + prev := checksumExtraRootCAs + pool := x509.NewCertPool() + if prev != nil { + pool = prev.Clone() + } + pool.AddCert(srv.Certificate()) + checksumExtraRootCAs = pool + t.Cleanup(func() { checksumExtraRootCAs = prev }) +} + +func TestVerifyImageChecksum(t *testing.T) { + const debianName = "debian-12-generic-amd64.tar.xz" + const ubuntuName = "noble-server-cloudimg-amd64.img" + const alpineName = "alb-alpine-3.21.6-x86_64.raw" + + debianBytes := []byte("debian-image-bytes-fixture") + ubuntuBytes := []byte("ubuntu-image-bytes-fixture") + alpineBytes := []byte("alpine-image-bytes-fixture") + + t.Run("debian sha512 multi-line match", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + body := fmt.Sprintf("%s %s\n%s other-file\n", + sha512Hex(debianBytes), debianName, sha512Hex([]byte("unrelated"))) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + assert.NoError(t, err) + }) + + t.Run("ubuntu sha256 binary-mode asterisk match", func(t *testing.T) { + img := writeTempImage(t, ubuntuName, ubuntuBytes) + body := fmt.Sprintf("%s *%s\n", sha256Hex(ubuntuBytes), ubuntuName) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha256") + assert.NoError(t, err) + }) + + t.Run("alpine single-line named match", func(t *testing.T) { + img := writeTempImage(t, alpineName, alpineBytes) + body := fmt.Sprintf("%s %s\n", sha512Hex(alpineBytes), alpineName) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + assert.NoError(t, err) + }) + + t.Run("alpine bare-digest fallback", func(t *testing.T) { + img := writeTempImage(t, alpineName, alpineBytes) + body := sha512Hex(alpineBytes) + "\n" + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + assert.NoError(t, err) + }) + + t.Run("case-sensitive filename mismatch", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + // Capitalised "Debian-..." in sums file: legitimate divergence from upstream + // convention, treated as not-found. + body := fmt.Sprintf("%s Debian-12-generic-amd64.tar.xz\n", sha512Hex(debianBytes)) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + assert.ErrorIs(t, err, ErrChecksumNotFound) + }) + + t.Run("comment lines skipped", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + body := fmt.Sprintf("# Hash file generated by upstream\n# PGP-signed below\n%s %s\n", + sha512Hex(debianBytes), debianName) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + assert.NoError(t, err) + }) + + t.Run("digest mismatch", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + wrong := sha512Hex([]byte("a different blob entirely")) + body := fmt.Sprintf("%s %s\n", wrong, debianName) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumMismatch) + // Both expected and actual digests should be in the rendered error so + // operators can compare without scraping slog. + assert.Contains(t, err.Error(), wrong) + assert.Contains(t, err.Error(), sha512Hex(debianBytes)) + }) + + t.Run("filename absent from sums file", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + body := fmt.Sprintf("%s some-other-file.tar.xz\n%s yet-another.iso\n", + sha512Hex([]byte("a")), sha512Hex([]byte("b"))) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + assert.ErrorIs(t, err, ErrChecksumNotFound) + }) + + t.Run("unsupported checksum type rejected before any fetch", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + // Server URL is gibberish — function must fail before reaching it. + err := VerifyImageChecksum(img, "https://example.invalid/SUMS", "md5") + assert.ErrorIs(t, err, ErrUnsupportedChecksumType) + }) + + t.Run("http 404 wraps fetch failure", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + trustTestServer(t, srv) + err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512") + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + }) + + t.Run("non-https initial url rejected before any request", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + // Hit a port nothing listens on — if HTTPS check ever regresses we'd + // still get an error, but we want ErrChecksumFetchFailed wrapping the + // scheme refusal specifically. + err := VerifyImageChecksum(img, "http://127.0.0.1:1/SUMS", "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + assert.Contains(t, err.Error(), "non-https") + }) + + t.Run("non-https redirect refused", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + // Plain-HTTP target the TLS server will try to redirect us to. + plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "%s %s\n", sha512Hex(debianBytes), debianName) + })) + t.Cleanup(plain.Close) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, plain.URL+"/SUMS", http.StatusFound) + })) + t.Cleanup(srv.Close) + trustTestServer(t, srv) + err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + }) + + t.Run("redirect cap exceeded", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + var hits atomic.Int32 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + http.Redirect(w, r, r.URL.Path, http.StatusFound) + })) + t.Cleanup(srv.Close) + trustTestServer(t, srv) + err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + assert.GreaterOrEqual(t, int(hits.Load()), 10) + }) + + t.Run("sums file size cap exceeded", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + // Body of 2 MB of unrelated entries — over the 1 MB cap. + var b strings.Builder + junk := strings.Repeat("a", 128) + for b.Len() <= sumsFileMaxSize+1 { + fmt.Fprintf(&b, "%s %s.bin\n", sha512Hex([]byte(junk)), junk) + } + url := newSumsServer(t, b.String()) + err := VerifyImageChecksum(img, url, "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + assert.Contains(t, err.Error(), "exceeds") + }) + + t.Run("context deadline exceeded wraps fetch failure", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + t.Cleanup(srv.Close) + trustTestServer(t, srv) + + prev := checksumFetchTimeout + checksumFetchTimeout = 100 * time.Millisecond + t.Cleanup(func() { checksumFetchTimeout = prev }) + + err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + }) + + t.Run("digest length mismatch rejected before hashing", func(t *testing.T) { + // sha256 digest served, caller declares sha512: algorithm disagreement, + // not tampering. Must be distinct from ErrChecksumMismatch. + img := writeTempImage(t, debianName, debianBytes) + body := fmt.Sprintf("%s %s\n", sha256Hex(debianBytes), debianName) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + assert.NotErrorIs(t, err, ErrChecksumMismatch) + assert.Contains(t, err.Error(), "digest length") + }) + + t.Run("empty image file rejected with clear error", func(t *testing.T) { + img := writeTempImage(t, debianName, nil) + body := fmt.Sprintf("%s %s\n", sha512Hex(nil), debianName) + url := newSumsServer(t, body) + err := VerifyImageChecksum(img, url, "sha512") + require.Error(t, err) + assert.NotErrorIs(t, err, ErrChecksumMismatch) + assert.Contains(t, err.Error(), "empty") + }) + + t.Run("transport error wrapped", func(t *testing.T) { + img := writeTempImage(t, debianName, debianBytes) + // Stand up a TLS server, trust it, then close immediately so any + // subsequent request fails at the transport layer. + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + trustTestServer(t, srv) + url := srv.URL + "/SUMS" + srv.Close() + err := VerifyImageChecksum(img, url, "sha512") + require.Error(t, err) + assert.ErrorIs(t, err, ErrChecksumFetchFailed) + }) +} From 883fac980cd0bf507447a9f248257d81046fbc86 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 10:32:49 +1000 Subject: [PATCH 09/16] fix: remove unused /join /config endpoints (#148) --- spinifex/daemon/daemon.go | 181 +-------------------------------- spinifex/daemon/daemon_test.go | 56 ---------- spinifex/types/cluster.go | 23 ----- 3 files changed, 3 insertions(+), 257 deletions(-) diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 2d684ae05..d6c0f8547 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -56,7 +56,6 @@ import ( "github.com/mulgadc/spinifex/spinifex/vm" "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats.go" - "github.com/pelletier/go-toml/v2" ) type BlockDeviceMapping struct { @@ -550,8 +549,8 @@ func (d *Daemon) Start() error { return fmt.Errorf("failed to connect to NATS: %w", err) } - // ClusterManager must start before JetStream init so other nodes can join - // via /join endpoint and help form the NATS cluster. + // ClusterManager must start before JetStream init so peers can reach + // /health during bootstrap. if err := d.ClusterManager(); err != nil { return fmt.Errorf("failed to start cluster manager: %w", err) } @@ -1388,27 +1387,6 @@ func (d *Daemon) computeConfigHash() (string, error) { return hex.EncodeToString(hash[:]), nil } -// saveClusterConfig writes the cluster config to disk in TOML format -func (d *Daemon) saveClusterConfig() error { - if d.configPath == "" { - return fmt.Errorf("config path not set") - } - - // Marshal to TOML - configTOML, err := toml.Marshal(d.clusterConfig) - if err != nil { - return fmt.Errorf("failed to marshal config to TOML: %w", err) - } - - // Write to config file - if err := os.WriteFile(d.configPath, configTOML, 0600); err != nil { - return fmt.Errorf("failed to write config: %w", err) - } - - slog.Info("Cluster config saved", "path", d.configPath, "epoch", d.clusterConfig.Epoch) - return nil -} - // ClusterManager starts the HTTP cluster management server func (d *Daemon) ClusterManager() error { // Get daemon host from config @@ -1476,160 +1454,6 @@ func (d *Daemon) ClusterManager() error { } }) - // Join endpoint - accepts new nodes joining the cluster - r.Post("/join", func(w http.ResponseWriter, r *http.Request) { - var req types.NodeJoinRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusBadRequest) - if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{ - Success: false, - Message: "invalid request body", - }); err != nil { - slog.Error("Failed to encode join error response", "error", err) - } - return - } - - slog.Info("Node join request received", "node", req.Node, "region", req.Region, "az", req.AZ) - - // Validate request - if req.Node == "" || req.Region == "" || req.AZ == "" { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusBadRequest) - if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{ - Success: false, - Message: "node, region, and az are required", - }); err != nil { - slog.Error("Failed to encode join validation response", "error", err) - } - return - } - - // Check if node already exists - if _, exists := d.clusterConfig.Nodes[req.Node]; exists { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusConflict) - if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{ - Success: false, - Message: fmt.Sprintf("node %s already exists in cluster", req.Node), - }); err != nil { - slog.Error("Failed to encode join conflict response", "error", err) - } - return - } - - // Add new node to cluster config - d.mu.Lock() - newNodeConfig := config.Config{ - Node: req.Node, - Region: req.Region, - AZ: req.AZ, - DataDir: req.DataDir, - Daemon: config.DaemonConfig{ - Host: req.DaemonHost, - }, - // Copy shared config from current node - NATS: d.config.NATS, - Predastore: d.config.Predastore, - AWSGW: d.config.AWSGW, - BaseDir: req.DataDir, - } - - d.clusterConfig.Nodes[req.Node] = newNodeConfig - d.clusterConfig.Epoch++ // Increment epoch for version tracking - d.mu.Unlock() - - // Save updated config - if err := d.saveClusterConfig(); err != nil { - slog.Error("Failed to save cluster config", "error", err) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusInternalServerError) - if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{ - Success: false, - Message: "failed to save cluster config", - }); err != nil { - slog.Error("Failed to encode join save-error response", "error", err) - } - return - } - - configHash, _ := d.computeConfigHash() - - slog.Info("Node joined cluster", "node", req.Node, "epoch", d.clusterConfig.Epoch) - - // Update JetStream KV replicas to match new cluster size - // This may fail if the new node's NATS server isn't running yet - that's OK, - // replicas can be updated later when the cluster is fully formed - if d.jsManager != nil { - newReplicaCount := len(d.clusterConfig.Nodes) - if err := d.jsManager.UpdateReplicas(newReplicaCount); err != nil { - slog.Warn("Failed to update JetStream replicas (new node NATS may not be ready yet)", "targetReplicas", newReplicaCount, "error", err) - } - } - - // Send only shared cluster data (exclude node-specific top-level fields) - sharedData := &types.SharedClusterData{ - Epoch: d.clusterConfig.Epoch, - Version: d.clusterConfig.Version, - Nodes: d.clusterConfig.Nodes, - } - - // Read CA certificate and key to share with joining node for per-node cert generation - configDir := filepath.Dir(d.configPath) - caCertPath := filepath.Join(configDir, "ca.pem") - caKeyPath := filepath.Join(configDir, "ca.key") - - var caCert, caKey string - if caCertPEM, err := os.ReadFile(caCertPath); err == nil { - caCert = string(caCertPEM) - } else { - slog.Warn("Could not read CA cert for join response", "error", err) - } - - if caKeyPEM, err := os.ReadFile(caKeyPath); err == nil { - caKey = string(caKeyPEM) - } else { - slog.Warn("Could not read CA key for join response", "error", err) - } - - // Read predastore.toml to share with joining node (for multi-node predastore) - var predastoreConfig string - predastorePath := filepath.Join(configDir, "predastore", "predastore.toml") - if content, err := os.ReadFile(predastorePath); err == nil { - predastoreConfig = string(content) - } else { - slog.Warn("Could not read predastore.toml for join response", "path", predastorePath, "error", err) - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{ - Success: true, - Message: fmt.Sprintf("node %s successfully joined cluster", req.Node), - SharedData: sharedData, - ConfigHash: configHash, - JoiningNode: req.Node, - CACert: caCert, - CAKey: caKey, - PredastoreConfig: predastoreConfig, - }); err != nil { - slog.Error("Failed to encode join success response", "error", err) - } - }) - - // Get cluster config endpoint - r.Get("/config", func(w http.ResponseWriter, r *http.Request) { - configHash, _ := d.computeConfigHash() - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(map[string]any{ - "config": d.clusterConfig, - "config_hash": configHash, - }); err != nil { - slog.Error("Failed to encode config response", "error", err) - } - }) - // Load TLS certificate (C-5: serve over HTTPS instead of plaintext HTTP) // Resolve relative cert paths against config directory (cert lives alongside spinifex.toml). // For binary installs, systemd sets absolute paths via env vars; for dev, the config @@ -1655,6 +1479,7 @@ func (d *Daemon) ClusterManager() error { tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS12, } d.clusterServer = &http.Server{ diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index ef338313a..34724fe45 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -34,7 +34,6 @@ import ( "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/vm" "github.com/nats-io/nats.go" - "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -3087,61 +3086,6 @@ func TestComputeConfigHash_ExcludesNodeField(t *testing.T) { assert.Equal(t, h1, h2, "changing top-level Node should not affect config hash") } -// --- saveClusterConfig --- - -func TestSaveClusterConfig_WritesToDisk(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "spinifex.toml") - - d := &Daemon{ - configPath: path, - clusterConfig: &config.ClusterConfig{ - Epoch: 5, - Version: "2.0", - Node: "test-node", - Nodes: map[string]config.Config{ - "test-node": {Region: "us-west-2"}, - }, - }, - } - - err := d.saveClusterConfig() - require.NoError(t, err) - - data, err := os.ReadFile(path) - require.NoError(t, err) - - var loaded config.ClusterConfig - require.NoError(t, toml.Unmarshal(data, &loaded)) - assert.Equal(t, uint64(5), loaded.Epoch) - assert.Equal(t, "2.0", loaded.Version) - assert.Equal(t, "us-west-2", loaded.Nodes["test-node"].Region) - - // Verify permissions - info, _ := os.Stat(path) - assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) -} - -func TestSaveClusterConfig_ErrorOnEmptyPath(t *testing.T) { - d := &Daemon{ - configPath: "", - clusterConfig: &config.ClusterConfig{}, - } - err := d.saveClusterConfig() - assert.Error(t, err) - assert.Contains(t, err.Error(), "config path not set") -} - -func TestSaveClusterConfig_ErrorOnInvalidPath(t *testing.T) { - d := &Daemon{ - configPath: "/nonexistent/dir/spinifex.toml", - clusterConfig: &config.ClusterConfig{}, - } - err := d.saveClusterConfig() - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to write config") -} - // --- instanceTypeVCPUs / instanceTypeMemoryMiB nil safety --- func TestInstanceTypeVCPUs_NilSafety(t *testing.T) { diff --git a/spinifex/types/cluster.go b/spinifex/types/cluster.go index b616d5404..76f27210e 100644 --- a/spinifex/types/cluster.go +++ b/spinifex/types/cluster.go @@ -2,16 +2,6 @@ package types import "github.com/mulgadc/spinifex/spinifex/config" -// Cluster manager API types - -type NodeJoinRequest struct { - Node string `json:"node"` - Region string `json:"region"` - AZ string `json:"az"` - DataDir string `json:"data_dir"` - DaemonHost string `json:"daemon_host"` // Host:port where this node's daemon is accessible -} - // SharedClusterData contains only the shared cluster information (no node-specific top-level fields) type SharedClusterData struct { Epoch uint64 `json:"epoch" toml:"epoch"` @@ -19,19 +9,6 @@ type SharedClusterData struct { Nodes map[string]config.Config `json:"nodes" toml:"nodes"` } -type NodeJoinResponse struct { - Success bool `json:"success"` - Message string `json:"message"` - SharedData *SharedClusterData `json:"shared_data,omitempty"` - ConfigHash string `json:"config_hash,omitempty"` - JoiningNode string `json:"joining_node,omitempty"` // The node name that is joining - // CA distribution for per-node certificate generation - CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate - CAKey string `json:"ca_key,omitempty"` // PEM-encoded CA private key - // Predastore config distribution for multi-node clusters - PredastoreConfig string `json:"predastore_config,omitempty"` // Full predastore.toml content -} - type NodeHealthResponse struct { Node string `json:"node"` Status string `json:"status"` From 6f58a3f1900bdffe95eb18a3755044ea13f3ef04 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:17:16 +1000 Subject: [PATCH 10/16] feat(docs): malware protection guide (#149) --- docs/INSTALL_ORIGINAL.md | 1289 ----------------- .../creating-accounts/README.md | 0 .../iam-users-and-policies/README.md | 0 .../spinifex-admin-cli/README.md | 0 .../launching-instances/README.md | 0 docs/{ => compute}/placement-groups/README.md | 0 docs/{ => compute}/vpc-networking/README.md | 18 +- .../install-airgapped/README.md | 0 .../install-multi-node/README.md | 0 .../install-source/README.md | 0 docs/{ => getting-started}/install/README.md | 0 .../setting-up-your-cluster/README.md | 0 docs/{ => getting-started}/update/README.md | 0 docs/{ => migration}/hybrid-sync/README.md | 0 .../moving-aws-workload/README.md | 0 docs/security/malware-protection/README.md | 314 ++++ .../bastion-private-subnet/README.md | 0 .../bastion-private-subnet/main.tf | 0 .../nginx-alb/README.md | 0 .../nginx-alb/main.tf | 0 .../nginx-webserver/README.md | 0 .../nginx-webserver/main.tf | 0 .../s3-webapp/README.md | 0 .../s3-webapp/main.tf | 0 .../s3-webapp/terraform.tfvars.example | 0 25 files changed, 323 insertions(+), 1298 deletions(-) delete mode 100644 docs/INSTALL_ORIGINAL.md rename docs/{ => administration}/creating-accounts/README.md (100%) rename docs/{ => administration}/iam-users-and-policies/README.md (100%) rename docs/{ => administration}/spinifex-admin-cli/README.md (100%) rename docs/{ => compute}/launching-instances/README.md (100%) rename docs/{ => compute}/placement-groups/README.md (100%) rename docs/{ => compute}/vpc-networking/README.md (96%) rename docs/{ => getting-started}/install-airgapped/README.md (100%) rename docs/{ => getting-started}/install-multi-node/README.md (100%) rename docs/{ => getting-started}/install-source/README.md (100%) rename docs/{ => getting-started}/install/README.md (100%) rename docs/{ => getting-started}/setting-up-your-cluster/README.md (100%) rename docs/{ => getting-started}/update/README.md (100%) rename docs/{ => migration}/hybrid-sync/README.md (100%) rename docs/{ => migration}/moving-aws-workload/README.md (100%) create mode 100644 docs/security/malware-protection/README.md rename docs/{terraform => terraform-workbooks}/bastion-private-subnet/README.md (100%) rename docs/{terraform => terraform-workbooks}/bastion-private-subnet/main.tf (100%) rename docs/{terraform => terraform-workbooks}/nginx-alb/README.md (100%) rename docs/{terraform => terraform-workbooks}/nginx-alb/main.tf (100%) rename docs/{terraform => terraform-workbooks}/nginx-webserver/README.md (100%) rename docs/{terraform => terraform-workbooks}/nginx-webserver/main.tf (100%) rename docs/{terraform => terraform-workbooks}/s3-webapp/README.md (100%) rename docs/{terraform => terraform-workbooks}/s3-webapp/main.tf (100%) rename docs/{terraform => terraform-workbooks}/s3-webapp/terraform.tfvars.example (100%) diff --git a/docs/INSTALL_ORIGINAL.md b/docs/INSTALL_ORIGINAL.md deleted file mode 100644 index 49db097ea..000000000 --- a/docs/INSTALL_ORIGINAL.md +++ /dev/null @@ -1,1289 +0,0 @@ -# Spinifex Installation - -Notes for development environment installation. - -## Dependencies - -The currently supported operating systems that are validated and tested include: - -- Ubuntu 22.04 -- Ubuntu 24.04 -- Ubuntu 25.10 -- Debian 12.13 - -For the development preview, please use one of the supported versions above. - -## Download - -Create the base directory for the Spinifex development environment. - -```bash -mkdir -p ~/Development/mulga/ -cd ~/Development/mulga/ -git clone https://github.com/mulgadc/spinifex.git -``` - -### Quick Install - -To bootstrap all dependencies of Spinifex in one step (QEMU, Go, AWS CLI, OVN/OVS): - -```bash -sudo make -C spinifex quickinstall -``` - -Ensure Go is available in your `PATH` if not previously installed: - -```bash -export PATH=$PATH:/usr/local/go/bin/ -``` - -### Manual Install - -Alternatively, run the following steps to manually set up the required dependencies. - -```bash -sudo add-apt-repository universe -sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file ovn-central ovn-host openvswitch-switch -``` - -**Note:** OVN and Open vSwitch are required for VPC networking (virtual switches, routers, DHCP, Geneve overlay). The packages are installed above, but the setup and configuration step (`setup-ovn.sh`) is covered later — see the [Setup OVN](#setup-ovn) section below. - -Ensure the Go toolkit is installed for version 1.26.2 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/). - -Confirm Go is correctly installed, and set in your $PATH. - -```bash -go version -``` - -Spinifex provides AWS API/SDK layer functionality, which requires the AWS CLI tool to be installed to interface with the system. - -```bash -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -unzip awscliv2.zip -sudo ./aws/install -``` - -Confirm awscli version > 2.0 is installed. - -[https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) - -```bash -aws --version -``` - -## Build - -Setup the dev environment and package dependencies on [viperblock](https://github.com/mulgadc/viperblock/) and [predastore](https://github.com/mulgadc/predastore/). - -```bash -cd spinifex -./scripts/clone-deps.sh # Clone viperblock + predastore repositories -./scripts/dev-setup.sh # Setup complete development environment -``` - -Once complete, confirm `./bin/spx` exists and is executable. - -# Single Node Installation - -For rapid development and testing, `spx` can be installed locally as a single node instance. Follow the instructions below for a complete working environment. For multi node installation, skip to [Multi-Node Installation](#multi-node-installation) - -## Setup OVN - -OVN provides the virtual networking layer for VPC instances. This step configures OVS bridges, starts the OVN controller, and enables Geneve tunnel support. For a single-node install, run the setup script with `--management` to start all OVN services locally: - -```bash -./scripts/setup-ovn.sh --management -``` - -This creates the `br-int` integration bridge, starts `ovn-controller`, starts the OVN central databases (NB DB + SB DB), configures Geneve tunnel endpoints, enables IP forwarding, and creates a sudoers rule so the Spinifex daemon can manage tap devices and OVS ports without running as root. Spinifex will not start without this step. - -## Init - -When running Spinifex for the first time, run the init function to create the default directories for data, config files and layout required. - -Example single node installation to get started, this will create a new region `ap-southeast-2` and availability zone `ap-southeast-2a` on your local instance: - -```bash -./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1 -``` - -## Trust CA Certificate (Required) - -Spinifex generates a local Certificate Authority (CA) during initialization. For AWS CLI and other tools to trust Spinifex services over HTTPS, you must add the CA certificate to your system's trust store. - -```bash -sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt -sudo update-ca-certificates -``` - -Verify the certificate was added: - -```bash -ls -la /usr/local/share/ca-certificates/spinifex-ca.crt -``` - -## Launch Services - -Start the core services for development. - -```bash -./scripts/start-dev.sh -``` - -Set the AWS profile to use the default admin account created during init: - -```bash -export AWS_PROFILE=spinifex -``` - -All subsequent `aws` CLI commands will use this account. - -## Create SSH Key - -For first install, create or import an existing key pair which can be used to launch EC2 instances. - -### Import Existing Key - -Import an existing key pair, replace `~/.ssh/id_rsa.pub` with your specified key. - -```bash -aws ec2 import-key-pair --key-name "spinifex-key" --public-key-material fileb://~/.ssh/id_rsa.pub -``` - -If no key exists, generate one using `ssh-keygen` and repeat the command above. - -```bash -ssh-keygen -t rsa -``` - -### Create New Key - -Alternatively, create a new key pair using the AWS CLI tool and store the JSON output of the AWS SDK using the `jq` command. - -```bash -aws ec2 create-key-pair \ - --key-name spinifex-key \ -| jq -r '.KeyMaterial | rtrimstr("\n")' > ~/.ssh/spinifex-key -``` - -Update permissions for the key for SSH to accept reading the file. - -```bash -chmod 600 ~/.ssh/spinifex-key -``` - -Next, generate a public key from the specified private key pair. - -```bash -ssh-keygen -y -f ~/.ssh/spinifex-key > ~/.ssh/spinifex-key.pub -``` - -### Verify New Key - -```bash -aws ec2 describe-key-pairs -``` - -```json -{ - "KeyPairs": [ - { - "KeyPairId": "key-3caa0a34f53d12a3", - "KeyType": "ed25519", - "Tags": [], - "CreateTime": "2025-10-28T13:39:23.458000+00:00", - "KeyName": "spinifex-key", - "KeyFingerprint": "SHA256:/g/A5OkeZeSydz9WUErXYVdCt00b0VbfN6RLn2YVFAY" - } - ] -} -``` - -## Create AMI Template - -The Spinifex CLI tool offers 2 ways of importing an image. You can use automatic image importing, and select from our range of images. Alternatively, provide your own image source file (e.g image.tar.gz, image.gz, image.tar.xz) and the tool will automatically extract and upload the raw OS image as an AMI (after validating the image contains UEFI/BIOS boot capability). - -**Note:** When downloading OS images, use platforms that support the `cloud-init` feature to automatically bootstrap when using the Spinifex EC2 functionality to access SSH and networking services. - -### Automatic Image Import - -Discover available images to automatically download and install. This will pull the images from the distro official mirror and simplify the process to bootstrap a Spinifex installation with AMIs for common operating systems. - -```bash -./bin/spx admin images list -``` - -```bash -NAME | DISTRO | VERSION | ARCH | BOOT -debian-12-arm64 | debian | 12 | arm64 | bios -debian-12-x86_64 | debian | 12 | x86_64 | bios -ubuntu-24.04-arm64 | ubuntu | 24.04 | arm64 | bios -ubuntu-24.04-x86_64 | ubuntu | 24.04 | x86_64 | bios -``` - -Next, choose the image you would like to import as an AMI. Choose the appropriate `x86_64` or `arm64` architecture depending on your platform. - -```bash -./bin/spx admin images import --name debian-12-arm64 -``` - -### Manual Image Import - -Using this method you can import any OS disk image. For example, download the Debian 12 image from the repository [https://cloud.debian.org/images/cloud/bookworm/latest/](https://cloud.debian.org/images/cloud/bookworm/latest/). - -Download the image: - -```bash -wget https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.tar.xz -O ~/debian-12-genericcloud-arm64.tar.xz -``` - -Import as an AMI to the backend store: - -```bash -./bin/spx admin images import --file ~/debian-12-genericcloud-arm64.tar.xz --arch arm64 --distro debian --version 12 -``` - -### Verify Image Import - -Export the AMI. - -```bash -export SPINIFEX_AMI="ami-XXX" -``` - -Next, verify available disk images to confirm the import was successful. - -```bash -aws ec2 describe-images --image-ids $SPINIFEX_AMI -``` - -## Create VPC and Subnet - -Every EC2 instance runs inside a VPC (Virtual Private Cloud) with an isolated virtual network. Spinifex creates a default VPC automatically during initialization, but you can also create your own. - -### Create a VPC - -```bash -aws ec2 create-vpc --cidr-block 10.200.0.0/16 -``` - -```json -{ - "Vpc": { - "VpcId": "vpc-1035bd70d9bc10b06", - "CidrBlock": "10.200.0.0/16", - "State": "available", - "IsDefault": false - } -} -``` - -Export the VPC ID: - -```bash -export SPINIFEX_VPC="vpc-XXX" -``` - -### Create a Subnet - -Create a subnet within the VPC. All instances launched into this subnet will receive a private IP address from its CIDR range via DHCP. - -```bash -aws ec2 create-subnet --vpc-id $SPINIFEX_VPC --cidr-block 10.200.1.0/24 -``` - -```json -{ - "Subnet": { - "SubnetId": "subnet-6e7f829e3a4b1c5d0", - "VpcId": "vpc-1035bd70d9bc10b06", - "CidrBlock": "10.200.1.0/24", - "AvailableIpAddressCount": 249, - "State": "available" - } -} -``` - -Export the Subnet ID — this is required when launching instances: - -```bash -export SPINIFEX_SUBNET="subnet-XXX" -``` - -### Verify VPC Networking - -Confirm the VPC and subnet were created: - -```bash -aws ec2 describe-vpcs -aws ec2 describe-subnets -``` - -Behind the scenes, Spinifex's `vpcd` service translates these into OVN topology: the VPC becomes a logical router, the subnet becomes a logical switch with DHCP options, and each instance launched into the subnet gets an OVN port with automatic IP assignment. - -## Run Instance - -Once Spinifex is successfully installed and bootstrapped with a system AMI, SSH keys, and a VPC subnet, proceed to run an instance. - -### Query Instance Types - -Depending on the host platform Spinifex is installed, different compute instances will be available. - -Query available instance types and choose an instance type available on your host: - -```bash -aws ec2 describe-instance-types -``` - -```json -{ - "InstanceTypes": [ - { - "InstanceType": "t3.small", - "CurrentGeneration": true, - "SupportedRootDeviceTypes": [ - "ebs" - ], - "SupportedVirtualizationTypes": [ - "hvm" - ], - "Hypervisor": "kvm", - "ProcessorInfo": { - "SupportedArchitectures": [ - "x86_64" - ] - }, - "VCpuInfo": { - "DefaultVCpus": 2 - }, - "MemoryInfo": { - "SizeInMiB": 2048 - }, - "BurstablePerformanceSupported": false - }, - ] -} -``` - -Export the instance-type: - -```sh -export SPINIFEX_INSTANCE="t3.small" -``` - -### Run Instance - -Next, launch a new instance. The `--subnet-id` places the instance into your VPC subnet, where it will receive a private IP address via DHCP. The `spinifex-key` is the SSH key specified in the previous stage. - -```bash -aws ec2 run-instances \ - --image-id $SPINIFEX_AMI \ - --instance-type $SPINIFEX_INSTANCE \ - --key-name spinifex-key \ - --subnet-id $SPINIFEX_SUBNET \ - --count 1 -``` - -A sample response is below from the `RunInstance` request. Note the `InstanceId` and `PrivateIpAddress` — the instance has been assigned an IP from the subnet's CIDR range: - -```json -{ - "ReservationId": "r-f101157331e261a68", - "OwnerId": "123456789012", - "Instances": [ - { - "InstanceId": "i-36765eb0c6609e4d2", - "ImageId": "ami-15c3dcfe607460f15", - "State": { - "Code": 0, - "Name": "pending" - }, - "PrivateIpAddress": "10.200.1.5", - "SubnetId": "subnet-6e7f829e3a4b1c5d0", - "VpcId": "vpc-1035bd70d9bc10b06", - "KeyName": "spinifex-key", - "InstanceType": "t3.micro", - "LaunchTime": "2025-11-12T13:07:47.548000+00:00", - "NetworkInterfaces": [ - { - "NetworkInterfaceId": "eni-a1b2c3d4e5f6g7h8i", - "PrivateIpAddress": "10.200.1.5", - "MacAddress": "02:00:00:1a:2b:3c", - "SubnetId": "subnet-6e7f829e3a4b1c5d0", - "VpcId": "vpc-1035bd70d9bc10b06", - "Status": "in-use", - "Attachment": { - "DeviceIndex": 0, - "Status": "attached" - } - } - ] - } - ] -} -``` - -Export the instance ID. - -```bash -export INSTANCE_ID="i-XXX" -``` - -### Verify Instance - -Next, validate the running instance is ready. - -```bash -aws ec2 describe-instances --instance-ids $INSTANCE_ID -``` - -Confirm the `State.Name` attribute is set as `running` and the instance has a `PrivateIpAddress` assigned from the subnet. - -```json -{ - "Reservations": [ - { - "ReservationId": "r-f101157331e261a68", - "OwnerId": "123456789012", - "Instances": [ - { - "InstanceId": "i-36765eb0c6609e4d2", - "ImageId": "ami-15c3dcfe607460f15", - "State": { - "Code": 16, - "Name": "running" - }, - "PrivateIpAddress": "10.200.1.5", - "SubnetId": "subnet-6e7f829e3a4b1c5d0", - "VpcId": "vpc-1035bd70d9bc10b06", - "KeyName": "spinifex-key", - "InstanceType": "t3.micro", - "LaunchTime": "2025-11-12T13:07:47.548000+00:00" - } - ] - } - ] -} -``` - -## SSH Connection (development) - -For a Spinifex development environment (toggled off for production), a local SSH port forwarding will be active to connect directly to the instance, regardless of the VPC and network settings. - -Determine the SSH port allocated. - -```bash -ps auxw | grep $INSTANCE_ID -``` - -```bash -qemu-system-x86_64 -daemonize -pidfile /run/user/1000/i-36765eb0c6609e4d.pid -qmp unix:/run/user/1000/qmp-i-36765eb0c6609e4d2.sock,server,nowait -enable-kvm -M ubuntu -serial pty -cpu host -smp 2 -m 1024 -drive file=nbd://127.0.0.1:42653,format=raw,if=none,media=disk,id=os -drive file=nbd://127.0.0.1:44499,format=raw,if=virtio,media=cdrom,id=cloudinit -device virtio-blk-pci,drive=os,bootindex=1 -device virtio-rng-pci -device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp:127.0.0.1:33683-:22 -``` - -Note the `hostfwd=tcp:127.0.0.1:33683-:22`, in this case the local port `33683` will connect to the new instance. - -```bash -ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p 33683 - -... - -Linux spinifex-vm-36765eb0 6.1.0-40-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.153-1 (2025-09-20) x86_64 -ec2-user@spinifex-vm-36765eb0:~$ -``` - -Congratulations! Your first AMI image is imported, a new EC2 instance launched, and successfully connected via SSH for the configured SSH key, using the OS `cloud-init` procedure. - -## Managing Instances - -### Stop Instance - -To stop a running instance gracefully: - -```bash -aws ec2 stop-instances --instance-ids $INSTANCE_ID -``` - -```json -{ - "StoppingInstances": [ - { - "InstanceId": "i-36765eb0c6609e4d2", - "CurrentState": { - "Code": 64, - "Name": "stopping" - }, - "PreviousState": { - "Code": 16, - "Name": "running" - } - } - ] -} -``` - -Next, confirm the instance has stopped as requested. - -```bash -aws ec2 describe-instances --instance-ids $INSTANCE_ID -``` - -```json -{ - "Reservations": [ - { - "ReservationId": "r-f101157331e261a68", - "OwnerId": "123456789012", - "Instances": [ - { - "InstanceId": "i-36765eb0c6609e4d2", - "ImageId": "ami-15c3dcfe607460f15", - "State": { - "Code": 80, - "Name": "stopped" - }, - "KeyName": "spinifex-key", - "InstanceType": "t3.micro", - "LaunchTime": "2025-11-12T13:07:47.548000+00:00" - } - ] - } - ] -} -``` - -### Start Instance - -To start a previously stopped instance: - -```bash -aws ec2 start-instances --instance-ids $INSTANCE_ID -``` - -```json -{ - "StartingInstances": [ - { - "InstanceId": "i-36765eb0c6609e4d2", - "CurrentState": { - "Code": 0, - "Name": "pending" - }, - "PreviousState": { - "Code": 80, - "Name": "stopped" - } - } - ] -} -``` - -Next, validate the instance is running as expected. - -```bash -aws ec2 describe-instances --instance-ids "i-36765eb0c6609e4d2" -``` - -```json -{ - "Reservations": [ - { - "ReservationId": "r-f101157331e261a68", - "OwnerId": "123456789012", - "Instances": [ - { - "InstanceId": "i-36765eb0c6609e4d2", - "ImageId": "ami-15c3dcfe607460f15", - "State": { - "Code": 16, - "Name": "running" - }, - "KeyName": "spinifex-key", - "InstanceType": "t3.micro", - "LaunchTime": "2025-11-12T13:07:47.548000+00:00" - } - ] - } - ] -} -``` - -### Terminate Instance - -To terminate an instance, which will first stop the instance, and on success, remove the EBS volumes and permanently remove the instance data. - -```bash -aws ec2 terminate-instances --instance-ids $INSTANCE_ID -``` - -```json -{ - "TerminatingInstances": [ - { - "InstanceId": "i-36765eb0c6609e4d2", - "CurrentState": { - "Code": 16, - "Name": "running" - }, - "PreviousState": { - "Code": 16, - "Name": "running" - } - } - ] -} -``` - -Next, validate the instance is removed, this may take a few minutes depending on the instance volume size. - -```bash -aws ec2 describe-instances --instance-ids $INSTANCE_ID -``` - -On success no data will be returned, since the instance is no longer available. - -## Additional Accounts (Optional) - -The default `spx` account is created automatically during init. To create additional isolated accounts with their own resources (instances, volumes, VPCs, key pairs): - -```bash -./bin/spx admin account create --name myteam -``` - -This provisions an IAM account with an admin user, generates access keys, and configures a local AWS CLI profile named `spinifex-myteam`. Resources created under one account are isolated from other accounts. - -```bash -export AWS_PROFILE=spinifex-myteam -``` - -## UI Management Panel - -The Spinifex Platform also offers a web interface for managing spx services. Simply have the spinifex server running and go to [https://localhost:3000](https://localhost:3000) in your browser to continue. - -**Note:** If you are not viewing the website on the same machine that is running the spinifex server, you will need to go to [https://localhost:9999](https://localhost:9999) and [https://localhost:8443](https://localhost:8443) and accept the certificates. If you have added the CA to your local machine this is not needed. - -# Multi-Node Installation - -Spinifex is designed for distributed, multi-server deployments. A Spinifex cluster operates as a fully distributed infrastructure region — similar to an AWS Region — where multiple nodes work together to provide high availability, data durability, and fault tolerance across the platform. - -When deploying a multi-node cluster, you define a **region** (e.g., `ap-southeast-2`, `us-east-1`) and **availability zones** to organize your infrastructure. Services are distributed across nodes: NATS provides a clustered message bus for request routing and JetStream replication, Predastore uses Raft consensus with Reed-Solomon erasure coding (RS 2+1) for durable S3-compatible object storage, and the Spinifex daemon on each node can independently serve AWS API requests. The result is a platform where compute, storage, and API services are replicated — an AMI stored on one node is available from any node, an EC2 instance launched via any gateway is visible cluster-wide, and the loss of a single node does not compromise data or availability. - -Cluster formation is automatic. When initializing with `--nodes 3`, the init node starts a formation server and waits for other nodes to join. Once all nodes have registered, each node receives the full cluster topology — credentials, CA certificates, NATS routes, Predastore peer lists — and generates its own configuration files. No manual configuration synchronization is required. - -## Choose a Deployment Mode - -| | Simulated (Option A) | Real Multi-Server (Option B) | -|---|---|---| -| **Servers** | 1 machine, loopback IPs | 3+ physical/virtual servers | -| **Use case** | Development & testing | Production & staging | -| **Network** | `127.0.0.1/2/3` (no setup) | Real LAN IPs | -| **Data dirs** | `~/node1/`, `~/node2/`, `~/node3/` | `~/spinifex/` on each server | - -Both options use `SPINIFEX_NODE1`, `SPINIFEX_NODE2`, and `SPINIFEX_NODE3` environment variables. Set them in the option you choose, then all subsequent sections (verification, usage, shutdown) reference them. - -### Option A: Simulated Multi-Node (Single Server) - -For development and testing, a 3-node cluster can be simulated on a single machine using loopback addresses. Linux handles the full `127.0.0.0/8` range natively, so `127.0.0.1`, `127.0.0.2`, and `127.0.0.3` all work without any network configuration. Each node uses a separate data directory to isolate configuration, logs, and state. - -#### 1. Build and Set Variables - -```bash -make build - -export SPINIFEX_NODE1=127.0.0.1 -export SPINIFEX_NODE2=127.0.0.2 -export SPINIFEX_NODE3=127.0.0.3 -``` - -#### 2. Setup OVN - -For simulated mode on a single machine, run the setup script once with `--management`: - -```bash -./scripts/setup-ovn.sh --management -``` - -This starts OVN central services and configures the local compute node. All three simulated nodes share the same OVS/OVN instance on the host. - -#### 3. Form the Cluster - -The formation process requires running init and join commands concurrently. The init node blocks until all expected nodes have joined. - -**Terminal 1 — Initialize the leader node:** - -```bash -./bin/spx admin init \ - --node node1 \ - --nodes 3 \ - --bind $SPINIFEX_NODE1 \ - --cluster-bind $SPINIFEX_NODE1 \ - --port 4432 \ - --spinifex-dir ~/node1/ \ - --config-dir ~/node1/config/ \ - --region ap-southeast-2 \ - --az ap-southeast-2a -``` - -This starts a formation server on `$SPINIFEX_NODE1:4432` and waits for 2 more nodes to join. - -**Terminal 2 — Join node 2 (while init is running):** - -```bash -./bin/spx admin join \ - --node node2 \ - --bind $SPINIFEX_NODE2 \ - --cluster-bind $SPINIFEX_NODE2 \ - --host $SPINIFEX_NODE1:4432 \ - --data-dir ~/node2/ \ - --config-dir ~/node2/config/ \ - --region ap-southeast-2 \ - --az ap-southeast-2a -``` - -**Terminal 3 — Join node 3 (while init is running):** - -```bash -./bin/spx admin join \ - --node node3 \ - --bind $SPINIFEX_NODE3 \ - --cluster-bind $SPINIFEX_NODE3 \ - --host $SPINIFEX_NODE1:4432 \ - --data-dir ~/node3/ \ - --config-dir ~/node3/config/ \ - --region ap-southeast-2 \ - --az ap-southeast-2a -``` - -All three processes will exit with a cluster summary once formation is complete. - -#### 4. Start Services - -Start services on each node (in separate terminals or background): - -```bash -./scripts/start-dev.sh ~/node1/ -./scripts/start-dev.sh ~/node2/ -./scripts/start-dev.sh ~/node3/ -``` - -**Note:** NATS monitoring (port 8222) is bound to `127.0.0.1` by default for security. When running multiple nodes on the same machine, only the first node can bind this port — nodes 2 and 3 will fail to start NATS. To fix this, edit each node's `nats.conf` to either bind monitoring to the node's unique IP or disable it: - -```bash -# Option 1: Bind to node-specific IP -sed -i 's|^http: 127.0.0.1:8222|http: 127.0.0.2:8222|' ~/node2/config/nats/nats.conf -sed -i 's|^http: 127.0.0.1:8222|http: 127.0.0.3:8222|' ~/node3/config/nats/nats.conf - -# Option 2: Disable monitoring on nodes 2 and 3 -sed -i '/^http:/d' ~/node2/config/nats/nats.conf -sed -i '/^http:/d' ~/node3/config/nats/nats.conf -``` - -This only applies to simulated mode — real multi-server deployments have one node per machine with no port conflict. - -### Option B: Real Multi-Node (Physical Servers) - -For production or production-like deployments across multiple physical servers or VMs. - -#### Prerequisites - -On **each server**: - -1. Install Spinifex dependencies (see [Dependencies](#dependencies) above) -2. Clone the repository and build: - -```bash -mkdir -p ~/Development/mulga/ -cd ~/Development/mulga/ -git clone https://github.com/mulgadc/spinifex.git -cd spinifex -./scripts/clone-deps.sh -make build -``` - -3. Identify the bind IP for each server: - -```bash -ip addr show | grep "inet " - inet 127.0.0.1/8 scope host lo - inet 192.168.1.10/24 brd 192.168.1.255 scope global eth0 - inet 10.0.0.10/24 brd 10.0.0.255 scope global eth1 -``` - -With a **single NIC**, use the one real IP (e.g. `192.168.1.10`) for everything. With **two NICs**, the primary interface (e.g. `eth0`) is your management IP for `--bind` and `--cluster-bind`, and the secondary (e.g. `eth1`) is for `--encap-ip` (Geneve overlay) — see [Dual NIC Configuration](#dual-nic-configuration). - -#### 1. Set Variables - -Export the IPs for your three servers (replace with your actual IPs) on each server: - -```bash -export SPINIFEX_NODE1=192.168.1.10 -export SPINIFEX_NODE2=192.168.1.11 -export SPINIFEX_NODE3=192.168.1.12 -``` - -#### 2. Setup OVN - -OVN must be configured on every server before forming the cluster. Server 1 runs OVN central (the NB and SB databases), so it must be set up first. Servers 2 and 3 connect to server 1's OVN databases as compute nodes. - -**Note:** The examples below use a single NIC where `--encap-ip` is the same as the node's bind IP. If your servers have a dedicated overlay NIC, see [Dual NIC Configuration](#dual-nic-configuration) for how to separate management and Geneve tunnel traffic. - -**Server 1 — OVN central + compute (run first):** - -```bash -cd ~/Development/mulga/spinifex -./scripts/setup-ovn.sh --management --encap-ip=$SPINIFEX_NODE1 -``` - -Verify OVN central is ready before proceeding to other servers: - -```bash -sudo ovn-sbctl show -``` - -**Server 2 — Compute node (after server 1 is ready):** - -```bash -cd ~/Development/mulga/spinifex -./scripts/setup-ovn.sh --ovn-remote=tcp:$SPINIFEX_NODE1:6642 --encap-ip=$SPINIFEX_NODE2 -``` - -**Server 3 — Compute node (after server 1 is ready):** - -```bash -cd ~/Development/mulga/spinifex -./scripts/setup-ovn.sh --ovn-remote=tcp:$SPINIFEX_NODE1:6642 --encap-ip=$SPINIFEX_NODE3 -``` - -Verify OVN is running on all servers: - -```bash -sudo systemctl is-active ovn-controller -``` - -On server 1, confirm all chassis have registered: - -```bash -sudo ovn-sbctl show -``` - -You should see 3 chassis entries, one per server, each with a Geneve encap IP. - -If your servers have a dedicated data/overlay NIC separate from the management NIC, use the data NIC's IP for `--encap-ip` instead — see [Dual NIC Configuration](#dual-nic-configuration) below. - -#### 3. Form the Cluster - -**Server 1 — Initialize the cluster:** - -```bash -cd ~/Development/mulga/spinifex - -./bin/spx admin init \ - --node node1 \ - --nodes 3 \ - --bind $SPINIFEX_NODE1 \ - --cluster-bind $SPINIFEX_NODE1 \ - --port 4432 \ - --spinifex-dir ~/spinifex/ \ - --config-dir ~/spinifex/config/ \ - --region us-east-1 \ - --az us-east-1a -``` - -**Server 2 — Join the cluster (while init is running):** - -```bash -cd ~/Development/mulga/spinifex - -./bin/spx admin join \ - --node node2 \ - --bind $SPINIFEX_NODE2 \ - --cluster-bind $SPINIFEX_NODE2 \ - --host $SPINIFEX_NODE1:4432 \ - --data-dir ~/spinifex/ \ - --config-dir ~/spinifex/config/ \ - --region us-east-1 \ - --az us-east-1a -``` - -**Server 3 — Join the cluster (while init is running):** - -```bash -cd ~/Development/mulga/spinifex - -./bin/spx admin join \ - --node node3 \ - --bind $SPINIFEX_NODE3 \ - --cluster-bind $SPINIFEX_NODE3 \ - --host $SPINIFEX_NODE1:4432 \ - --data-dir ~/spinifex/ \ - --config-dir ~/spinifex/config/ \ - --region us-east-1 \ - --az us-east-1a -``` - -All three processes will exit with a cluster summary once formation is complete. - -#### 4. Start Services - -After formation completes, start services on **all servers**: - -```bash -./scripts/start-dev.sh -``` - -## Verify the Cluster - -From any node, view cluster status and resource capacity: - -```bash -./bin/spx get nodes -# NAME STATUS IP REGION AZ UPTIME VMs SERVICES -# node1 Ready $SPINIFEX_NODE1 ... ... 2m 0 nats,predastore,viperblock,daemon,... -# node2 Ready $SPINIFEX_NODE2 ... ... 2m 0 nats,predastore,viperblock,daemon -# node3 Ready $SPINIFEX_NODE3 ... ... 2m 0 nats,predastore,viperblock,daemon - -./bin/spx top nodes -# NAME CPU (used/total) MEM (used/total) VMs -# node1 0/16 0.0Gi/30.6Gi 0 -# node2 0/16 0.0Gi/30.6Gi 0 -# node3 0/16 0.0Gi/30.6Gi 0 -``` - -All nodes should show `Ready` status. Use `./bin/spx get vms` to see running instances across the cluster. - -Check individual daemon health endpoints: - -```bash -curl -sk https://$SPINIFEX_NODE1:4432/health -curl -sk https://$SPINIFEX_NODE2:4432/health -curl -sk https://$SPINIFEX_NODE3:4432/health -``` - -Check NATS cluster routing (from any node): - -```bash -grep -i "route\|cluster" ~/node1/logs/nats.log # simulated -grep -i "route\|cluster" ~/spinifex/logs/nats.log # real multi-server -``` - -Check Predastore Raft consensus: - -```bash -grep -i "leader\|election" ~/node1/logs/predastore.log # simulated -grep -i "leader\|election" ~/spinifex/logs/predastore.log # real multi-server -``` - -## Trust CA Certificate - -The CA certificate is generated by the init node and distributed to all joining nodes during formation. On **each node** (or the single machine for simulated mode), trust the CA: - -```bash -sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt -sudo update-ca-certificates -``` - -For simulated mode, the config directory is under each node's data directory — copy from any node since the CA is the same: - -```bash -sudo cp ~/node1/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt -sudo update-ca-certificates -``` - -Set the AWS profile to use the default admin account created during init: - -```bash -export AWS_PROFILE=spinifex -``` - -## Using the Cluster - -Connect to any node's AWS Gateway — all nodes serve the same cluster state: - -```bash -AWS_ENDPOINT_URL=https://$SPINIFEX_NODE1:9999 aws ec2 describe-instances -AWS_ENDPOINT_URL=https://$SPINIFEX_NODE2:9999 aws ec2 describe-instances -``` - -### Create SSH Key - -Create or import an SSH key pair from **any node** — the key is stored in Predastore and available cluster-wide: - -```bash -aws ec2 import-key-pair --key-name "spinifex-key" --public-key-material fileb://~/.ssh/id_rsa.pub -``` - -Or generate a new key pair: - -```bash -aws ec2 create-key-pair \ - --key-name spinifex-key \ -| jq -r '.KeyMaterial | rtrimstr("\n")' > ~/.ssh/spinifex-key -chmod 600 ~/.ssh/spinifex-key -``` - -**Important:** If you want to ssh into instances, you will need to copy the private key to every node in the cluster. Instance SSH port forwarding is only accessible from the node running the VM, so you need the key available on all nodes: - -```bash -# From the node where the key was created, copy to other nodes: -scp ~/.ssh/spinifex-key tf-user@$SPINIFEX_NODE2:~/.ssh/spinifex-key -scp ~/.ssh/spinifex-key tf-user@$SPINIFEX_NODE3:~/.ssh/spinifex-key -``` - -### Create AMI Template - -Import an OS image from **any node**. The AMI is stored in Predastore and available cluster-wide: - -```bash -./bin/spx admin images list -./bin/spx admin images import --name debian-12-x86_64 -``` - -Verify and export the AMI ID: - -```bash -aws ec2 describe-images -export SPINIFEX_AMI="ami-XXX" -``` - -See the [single-node AMI section](#create-ami-template) for more options (manual import, custom images). - -### VPC Networking - -Create a VPC and subnet for your cluster. This only needs to be done once — the VPC is shared across all nodes. - -```bash -aws ec2 create-vpc --cidr-block 10.200.0.0/16 -export SPINIFEX_VPC="vpc-XXX" - -aws ec2 create-subnet --vpc-id $SPINIFEX_VPC --cidr-block 10.200.1.0/24 -export SPINIFEX_SUBNET="subnet-XXX" -``` - -Verify the OVN topology was created (from server 1): - -```bash -sudo ovn-nbctl lr-list # One logical router per VPC -sudo ovn-nbctl ls-list # One logical switch per subnet -``` - -### Launching Instances Across Nodes - -Use `--count` to launch multiple instances in a single request. Spinifex distributes them across nodes in the cluster based on available capacity — instances land on different physical hosts but share the same VPC subnet, and can route traffic to each other over OVN's Geneve overlay: - -```bash -aws ec2 run-instances \ - --image-id $SPINIFEX_AMI \ - --instance-type $SPINIFEX_INSTANCE \ - --key-name spinifex-key \ - --subnet-id $SPINIFEX_SUBNET \ - --count 3 -``` - -This launches 3 instances distributed across the cluster. Each receives a unique private IP from the subnet (e.g. `10.200.1.5`, `10.200.1.6`, `10.200.1.7`). Instances on different physical hosts communicate transparently through Geneve tunnels — no additional configuration required. - -Verify all instances are running and have received their private IPs: - -```bash -aws ec2 describe-instances -``` - -Check which node each instance landed on: - -```bash -./bin/spx get vms -``` - -### Verifying Cross-Host Connectivity - -SSH into an instance. The dev SSH port forwarding binds to the node running the VM, so you must SSH from that node (or use its IP). Find the forwarded port: - -```bash -# On the node running the instance: -ps auxw | grep hostfwd -# Look for: hostfwd=tcp::-:22 -``` - -```bash -ssh -i ~/.ssh/spinifex-key ec2-user@ -p -``` - -From inside the VM, ping an instance running on a **different physical node**: - -```bash -ping 10.200.1.6 -``` - -If the ping succeeds across hosts, the OVN Geneve overlay is working correctly — traffic is being encapsulated and routed between physical servers transparently. - -Check logs on each node for debugging. Log locations depend on your deployment mode — `~/node{1,2,3}/logs/` for simulated, `~/spinifex/logs/` for real multi-server. - -## Advanced Configuration - -### Service Co-location - -Every node that runs EC2 instances (compute) **must** also run the viperblock service. EBS volumes are mounted locally via Unix domain sockets (NBD) — the daemon publishes to a node-specific NATS topic (`ebs.{nodeName}.mount`) which is handled by the viperblock instance on the same server. Nodes without viperblock cannot run EC2 instances. - -In the default deployment, every node runs all services (NATS, Predastore, Viperblock, Daemon, Gateway), which satisfies this requirement. - -### Network Configuration - -Minimum **1 NIC** required. **2 NICs recommended** for production: - -| NIC | Purpose | Traffic | -|-----|---------|---------| -| **NIC 1 — Management** | Cluster coordination, admin access | NATS cluster routes, Predastore Raft, daemon health, SSH, AWS Gateway | -| **NIC 2 — Overlay** | VPC networking between hosts | Geneve tunnels (UDP 6081), OVN datapath, inter-host VM traffic | - -With a single NIC, all traffic shares one interface. This works but is not recommended for production — Geneve tunnel traffic competes with cluster coordination traffic. - -### Dual NIC Configuration - -When using two NICs, each flag controls which NIC carries which traffic: - -| Flag | NIC | Used by | Example | -|------|-----|---------|---------| -| `--bind` | Management | Formation server, daemon health, AWS gateway | `192.168.1.10` | -| `--cluster-bind` | Management | NATS cluster routes, Predastore Raft consensus | `192.168.1.10` | -| `--encap-ip` | Overlay | Geneve tunnels (VM-to-VM traffic across hosts) | `10.0.0.10` | - -If `--cluster-bind` is not specified, it defaults to `--bind`. For single NIC deployments, all three use the same IP. - -Example with management NIC `192.168.1.0/24` and overlay NIC `10.0.0.0/24`. The order is the same as the single NIC flow — setup OVN on all servers first, then form the cluster. - -**Step 1 — Setup OVN on all servers:** - -```bash -# Server 1 (OVN central + compute): -./scripts/setup-ovn.sh --management --encap-ip=10.0.0.10 - -# Server 2 (compute, after server 1 is ready): -./scripts/setup-ovn.sh --ovn-remote=tcp:192.168.1.10:6642 --encap-ip=10.0.0.11 - -# Server 3 (compute, after server 1 is ready): -./scripts/setup-ovn.sh --ovn-remote=tcp:192.168.1.10:6642 --encap-ip=10.0.0.12 -``` - -**Step 2 — Form the cluster (same as single NIC, but with `--cluster-bind`):** - -Server 1: - -```bash -./bin/spx admin init \ - --node node1 \ - --nodes 3 \ - --bind 192.168.1.10 \ - --cluster-bind 192.168.1.10 \ - --port 4432 \ - --spinifex-dir ~/spinifex/ \ - --config-dir ~/spinifex/config/ \ - --region us-east-1 \ - --az us-east-1a -``` - -Server 2 (while init is running): - -```bash -./bin/spx admin join \ - --node node2 \ - --bind 192.168.1.11 \ - --cluster-bind 192.168.1.11 \ - --host 192.168.1.10:4432 \ - --data-dir ~/spinifex/ \ - --config-dir ~/spinifex/config/ \ - --region us-east-1 \ - --az us-east-1a -``` - -Server 3 (while init is running): - -```bash -./bin/spx admin join \ - --node node3 \ - --bind 192.168.1.12 \ - --cluster-bind 192.168.1.12 \ - --host 192.168.1.10:4432 \ - --data-dir ~/spinifex/ \ - --config-dir ~/spinifex/config/ \ - --region us-east-1 \ - --az us-east-1a -``` - -### IP Aliases (Simulated Mode) - -For a more production-like simulation using dedicated IP addresses on a network interface: - -```bash -sudo ip addr add 10.11.12.1/24 dev eth0 -sudo ip addr add 10.11.12.2/24 dev eth0 -sudo ip addr add 10.11.12.3/24 dev eth0 -``` - -Replace `eth0` with your network interface name (e.g., `enp0s3`, `ens33`). Then export these as your `SPINIFEX_NODE` variables instead of the loopback addresses. - -To remove the aliases: - -```bash -sudo ip addr del 10.11.12.1/24 dev eth0 -sudo ip addr del 10.11.12.2/24 dev eth0 -sudo ip addr del 10.11.12.3/24 dev eth0 -``` - -## Shutdown - -When running in the multi-server cluster mode a graceful shutdown is required to synchronize and coordinate nodes. - -On any node, issue the following command to shutdown nodes cluster wide: - -```sh -./bin/spx admin cluster shutdown -``` - -Example output: -```bash -Starting coordinated cluster shutdown (3 nodes) -Phases: gate -> drain -> storage -> persist -> infra -Timeout per phase: 2m0s - -[GATE] Sending to 3 node(s)... - [GATE] node1: stopped awsgw, spinifex-ui - [GATE] node3: stopped awsgw, spinifex-ui - [GATE] node2: stopped awsgw, spinifex-ui -[GATE] Complete (3/3 nodes, 2.005s) - -[DRAIN] Sending to 3 node(s)... - [DRAIN] node1: 0/0 VMs remaining - [DRAIN] node1: 0/0 VMs remaining - [DRAIN] node2: 0/0 VMs remaining - [DRAIN] node2: 0/0 VMs remaining - [DRAIN] node3: 0/0 VMs remaining - [DRAIN] node3: 0/0 VMs remaining - [DRAIN] node1: OK - [DRAIN] node3: OK - [DRAIN] node2: OK -[DRAIN] Complete (3/3 nodes, 5ms) - -[STORAGE] Sending to 3 node(s)... - [STORAGE] node1: stopped viperblock - [STORAGE] node3: stopped viperblock - [STORAGE] node2: stopped viperblock -[STORAGE] Complete (3/3 nodes, 1.09s) - -[PERSIST] Sending to 3 node(s)... - [PERSIST] node1: stopped predastore - [PERSIST] node3: stopped predastore - [PERSIST] node2: stopped predastore -[PERSIST] Complete (3/3 nodes, 1.003s) - -[INFRA] Sending final shutdown to all nodes... -[INFRA] Complete -Cluster shutdown complete (6.104s) - -``` - -If EC2 instances are running, the stop process will gracefully terminate them, unmount attached EBS volumes (via NBD), and flush the write-ahead-log (WAL) to the S3 server (Predastore). This may take a few minutes depending on instance volume sizes. - -To fully remove a simulated cluster's data: - -```bash -rm -rf ~/node1/ ~/node2/ ~/node3/ -``` diff --git a/docs/creating-accounts/README.md b/docs/administration/creating-accounts/README.md similarity index 100% rename from docs/creating-accounts/README.md rename to docs/administration/creating-accounts/README.md diff --git a/docs/iam-users-and-policies/README.md b/docs/administration/iam-users-and-policies/README.md similarity index 100% rename from docs/iam-users-and-policies/README.md rename to docs/administration/iam-users-and-policies/README.md diff --git a/docs/spinifex-admin-cli/README.md b/docs/administration/spinifex-admin-cli/README.md similarity index 100% rename from docs/spinifex-admin-cli/README.md rename to docs/administration/spinifex-admin-cli/README.md diff --git a/docs/launching-instances/README.md b/docs/compute/launching-instances/README.md similarity index 100% rename from docs/launching-instances/README.md rename to docs/compute/launching-instances/README.md diff --git a/docs/placement-groups/README.md b/docs/compute/placement-groups/README.md similarity index 100% rename from docs/placement-groups/README.md rename to docs/compute/placement-groups/README.md diff --git a/docs/vpc-networking/README.md b/docs/compute/vpc-networking/README.md similarity index 96% rename from docs/vpc-networking/README.md rename to docs/compute/vpc-networking/README.md index 18e233b00..a6315076f 100644 --- a/docs/vpc-networking/README.md +++ b/docs/compute/vpc-networking/README.md @@ -39,7 +39,7 @@ Spinifex maps AWS VPC concepts directly to OVN constructs: ## Network Path

- VPC logical topology — WAN, br-wan, VPC logical router, subnets, ENIs + VPC logical topology — WAN, br-wan, VPC logical router, subnets, ENIs

Cross-host traffic uses **Geneve tunnels** (UDP 6081) over the management/overlay @@ -58,7 +58,7 @@ same VPC (even across subnets and hosts via the overlay). They cannot reach the internet or be reached from the WAN.

- Private subnet — instance hits router, no default route, packet dropped + Private subnet — instance hits router, no default route, packet dropped

If the VPC has an IGW attached, private subnet instances CAN reach the internet @@ -71,7 +71,7 @@ Instances get both a private IP and a public IP. The public IP is a 1:1 NAT managed by OVN — the instance OS only sees its private IP.

- Public subnet — outbound SNAT and inbound DNAT between private and public IPs + Public subnet — outbound SNAT and inbound DNAT between private and public IPs

**Requirements for a public subnet:** @@ -162,7 +162,7 @@ Regardless of whether IPs come from a static range or DHCP, the OVN behavior is identical:

- Two independent DHCP conversations — host-to-router and VM-to-OVN + Two independent DHCP conversations — host-to-router and VM-to-OVN

### Choosing Static vs DHCP @@ -252,7 +252,7 @@ The connection between them is logical, not physical — OVN's `localnet` port t maps the logical external switch to `br-wan` via `ovn-bridge-mappings`.

- Data path — VM TAP through br-int, OVN pipeline, localnet, br-wan, WAN + Data path — VM TAP through br-int, OVN pipeline, localnet, br-wan, WAN

## Bridge Modes @@ -267,7 +267,7 @@ The WAN NIC is added directly to `br-wan` as an OVS port. OVS sees all traffic on the wire — any MAC, any protocol. No filtering, no workarounds.

- Direct bridge mode — dedicated WAN NIC added directly to br-wan + Direct bridge mode — dedicated WAN NIC added directly to br-wan

**Use when:** The host has a NIC dedicated to external/WAN traffic that is NOT @@ -293,7 +293,7 @@ macvlan is added to br-wan. The host keeps its IP on the original NIC — SSH stays up.

- Macvlan mode — WAN NIC shared between host traffic and macvlan child on br-wan + Macvlan mode — WAN NIC shared between host traffic and macvlan child on br-wan

**Use when:** The WAN NIC is the same NIC used for SSH. Single-NIC homelabs, @@ -397,7 +397,7 @@ ip link show spx-ext-eth0 ## Network Flow Diagram

- Bare-metal host — br-int overlay, br-wan WAN bridge with macvlan, OVN NAT pipeline + Bare-metal host — br-int overlay, br-wan WAN bridge with macvlan, OVN NAT pipeline

## Configuration Reference @@ -408,7 +408,7 @@ levels: cluster-wide mode, IP pool definitions, and per-node NIC settings. ## Configuration Levels

- spinifex.toml configuration layers — cluster mode, IP pools, per-node NIC + spinifex.toml configuration layers — cluster mode, IP pools, per-node NIC

## Cluster-Wide: external_mode diff --git a/docs/install-airgapped/README.md b/docs/getting-started/install-airgapped/README.md similarity index 100% rename from docs/install-airgapped/README.md rename to docs/getting-started/install-airgapped/README.md diff --git a/docs/install-multi-node/README.md b/docs/getting-started/install-multi-node/README.md similarity index 100% rename from docs/install-multi-node/README.md rename to docs/getting-started/install-multi-node/README.md diff --git a/docs/install-source/README.md b/docs/getting-started/install-source/README.md similarity index 100% rename from docs/install-source/README.md rename to docs/getting-started/install-source/README.md diff --git a/docs/install/README.md b/docs/getting-started/install/README.md similarity index 100% rename from docs/install/README.md rename to docs/getting-started/install/README.md diff --git a/docs/setting-up-your-cluster/README.md b/docs/getting-started/setting-up-your-cluster/README.md similarity index 100% rename from docs/setting-up-your-cluster/README.md rename to docs/getting-started/setting-up-your-cluster/README.md diff --git a/docs/update/README.md b/docs/getting-started/update/README.md similarity index 100% rename from docs/update/README.md rename to docs/getting-started/update/README.md diff --git a/docs/hybrid-sync/README.md b/docs/migration/hybrid-sync/README.md similarity index 100% rename from docs/hybrid-sync/README.md rename to docs/migration/hybrid-sync/README.md diff --git a/docs/moving-aws-workload/README.md b/docs/migration/moving-aws-workload/README.md similarity index 100% rename from docs/moving-aws-workload/README.md rename to docs/migration/moving-aws-workload/README.md diff --git a/docs/security/malware-protection/README.md b/docs/security/malware-protection/README.md new file mode 100644 index 000000000..b8bc2b911 --- /dev/null +++ b/docs/security/malware-protection/README.md @@ -0,0 +1,314 @@ +--- +title: "Malware Protection" +description: "Operator guide for host-based malware protection, file integrity monitoring, and AV scanning on Spinifex nodes" +category: "Security" +sections: + - overview +tags: + - security + - compliance + - cmmc + - malware + - antivirus + - file integrity +resources: + - title: "Spinifex Repository" + url: "https://github.com/mulgadc/spinifex" + - title: "NIST SP 800-171 Rev 3" + url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final" + - title: "CMMC Level 1 Self-Assessment Guide v2.0" + url: "https://dodcio.defense.gov/CMMC/Documentation" + - title: "ClamAV Documentation" + url: "https://docs.clamav.net" + - title: "AIDE — Advanced Intrusion Detection Environment" + url: "https://aide.github.io" + - title: "Wazuh Documentation" + url: "https://documentation.wazuh.com" +--- + +# Malware Protection + +> Operator guide for host-based malware protection, file integrity monitoring, and AV scanning on Spinifex nodes + +## Table of Contents + +- [Overview](#overview) +- [CMMC Practices Covered](#cmmc-practices-covered) +- [Approach](#approach) +- [1. Designated Locations](#1-designated-locations-sil1-3142-a) +- [2. Update Requirements](#2-update-requirements-sil1-3144) +- [3. Scan Schedule](#3-scan-schedule-sil1-3145) +- [4. Reference Configurations](#4-reference-configurations) +- [5. Operator Checklist](#5-operator-checklist) +- [6. Related Documents](#6-related-documents) + +--- + +## Overview + +**Audience:** Operators deploying Spinifex into environments subject to CMMC Level 1 (or organisations that otherwise require host-based malware protection). + +**Scope:** The Spinifex nodes themselves — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd` and the OVN control plane. Guest VMs launched on the platform are the responsibility of the workload owner and are out of scope. + +## CMMC Practices Covered + +This guide captures the operator-side controls required to meet three CMMC Level 1 practices that Spinifex itself cannot enforce — they depend on host-based tooling chosen and operated by the deployment team. + +| Practice | Title | Objective | +|----------|-------|-----------| +| SI.L1-3.14.2 | Malicious Code Protection | [a] Designated locations for malicious code protection are identified. [b] Protection from malicious code at designated locations is provided. | +| SI.L1-3.14.4 | Update Malicious Code Protection | [a] Malicious code protection mechanisms are updated when new releases are available. | +| SI.L1-3.14.5 | System & File Scanning | [a] Periodic scans of the information system are defined. [b] Periodic scans of the information system are performed. [c] Real-time scans of files from external sources are defined and performed as files are downloaded, opened, or executed. | + +## Approach + +Spinifex does not bundle or mandate a specific anti-malware product. Operators will typically have an existing endpoint security standard (CrowdStrike Falcon, Microsoft Defender for Linux, Wazuh, ClamAV, Sophos, ESET, etc.). This guide: + +1. Names the Spinifex components and filesystem paths that must be included in scan and integrity-monitoring coverage (the "designated locations"). +2. Specifies update and scan cadence required to meet SI.L1-3.14.4 and SI.L1-3.14.5. +3. Provides reference configurations for open-source stacks (ClamAV for AV, AIDE for file integrity, Wazuh for centralised monitoring) as a known-good baseline. Operators running a commercial EDR should map these locations and cadences into their existing product. + +## 1. Designated Locations (SI.L1-3.14.2 [a]) + +All Spinifex nodes must include the following in their malware protection scope. Paths assume the production install layout (`/etc/spinifex` exists); development layouts under `$HOME/spinifex` are not supported for compliance deployments. + +### 1.1 Executable binaries + +| Path | Contents | +|------|----------| +| `/usr/local/bin/spx` | Spinifex CLI and service entry point (`spx service start`). | +| `/usr/local/bin/predastore` | Predastore S3 server binary. | +| `/usr/local/bin/viperblock` | Viperblock NBD block storage binary. | +| `/usr/local/bin/nats-server` | Bundled NATS server. | +| `/usr/local/bin/ovn-*`, `/usr/local/bin/ovs-*` | OVN / OVS control-plane and datapath binaries. | +| `/usr/local/share/ca-certificates/spinifex-ca.crt` | Cluster CA cert installed into the system trust store. | + +### 1.2 Configuration and secrets + +| Path | Contents | +|------|----------| +| `/etc/spinifex/` | Cluster config directory: `spinifex.toml`, `predastore.toml`, `nats.conf`. | +| `/etc/spinifex/ca.pem`, `/etc/spinifex/ca.key` | Cluster root CA certificate and private key (leader only). | +| `/etc/spinifex/node.crt`, `/etc/spinifex/node.key` | Per-node TLS certificate and key. | +| `/etc/spinifex/master.key` | Master encryption key that wraps per-volume DEKs and IAM secrets. **Highest-value artifact on the node.** | +| `/etc/spinifex/systemd.env` | Service environment file (shared by all `spinifex-*.service` units). | + +### 1.3 Service units and privilege configuration + +| Path | Contents | +|------|----------| +| `/etc/systemd/system/spinifex-*.service` | Unit files for daemon, awsgw, nats, predastore, viperblock, vpcd, spinifex-ui. | +| `/etc/sudoers`, `/etc/sudoers.d/` | Any drop-ins granting privileges to service accounts. | +| `/etc/ssh/sshd_config`, `/etc/ssh/sshd_config.d/` | Host SSH daemon configuration. | + +### 1.4 Runtime and data locations + +| Path | Purpose | Notes | +|------|---------|-------| +| `/var/lib/spinifex/spinifex/` | Daemon state, local bead/catalogue metadata. | Include in scan scope; exclude large, high-churn subtrees from real-time AV if performance requires (document the exclusion). | +| `/var/lib/spinifex/predastore/` | Local Predastore S3 backing data (chunks, metadata). | See exclusion note below. | +| `/var/lib/spinifex/viperblock/` | Local Viperblock WAL and chunk data for EBS volumes. | See exclusion note below. | +| `/var/lib/spinifex/nats/` | NATS JetStream data. | See exclusion note below. | +| `/var/lib/spinifex/awsgw/` | AWS gateway state. | Include in scan scope. | +| `/var/lib/spinifex/vpcd/` | VPC daemon state. | Include in scan scope. | +| `/run/spinifex/`, `/run/spinifex/nbd/` | Runtime sockets and PID files. | Integrity monitoring not meaningful (transient); exclude from AIDE. | +| `/var/log/spinifex/` | Service logs. | Scan for malicious content indicators; do not block on writes. | + +**Performance exclusions.** Viperblock WAL, Predastore chunks, and NATS JetStream files are large, high-churn, block-level storage that contains customer VM data. Real-time on-access AV scanning of these paths is both a significant performance cost and a confused-deputy problem (the daemon, not the operator, is the writer). Recommendation: cover these paths with **periodic scheduled scans** (SI.L1-3.14.5 [b]) but exclude them from real-time on-access hooks. Record the exclusion and its justification in the system security plan. + +### 1.5 External file ingest — real-time scan required (SI.L1-3.14.5 [c]) + +The following are "files from external sources" within the meaning of 3.14.5 [c] and **must** be scanned at ingest, before Spinifex consumes them: + +| Ingest point | What lands here | How it reaches the node | +|--------------|-----------------|-------------------------| +| VM image catalogue downloads | Debian / Ubuntu / Alpine / Rocky cloud images pulled by `processImage()` in `utils/images.go`. | Spinifex downloads over HTTPS and verifies the catalogue SHA checksum. Operators layer AV on top: scan the downloaded image file before it is registered. | +| `ImportImage` / user-uploaded AMIs | Operator- or tenant-supplied VMDK/VHD/RAW images. | Uploaded to Predastore and registered via the EC2 API. | +| Cloud-init user-data | Base64 payloads attached to instances. | Passed via `RunInstances`; written to the CIDATA seed ISO before boot. | +| ISO / media attached to instances | Arbitrary ISO images attached as CD-ROM to VMs. | Uploaded to Predastore. | + +Spinifex does not currently invoke an AV scanner on these ingest paths in-process. Until that integration exists, operators must either (a) use an on-access AV hook (ClamAV `clamonacc`, EDR file-write monitor) that sees Predastore writes, or (b) run a pre-ingest workflow that scans artifacts before they are uploaded to Spinifex. + +## 2. Update Requirements (SI.L1-3.14.4) + +Malicious code protection mechanisms must be updated when new releases are available. For signature-based AV this means signature feeds; for EDR products it means agent version pins. Apply to every node. + +| Component | Cadence | Mechanism | +|-----------|---------|-----------| +| AV signatures (ClamAV `freshclam`, commercial EDR cloud feeds) | At least every 4 hours; real-time if the vendor supports it. | Systemd timer for `freshclam` (see §4.1); EDR agents typically self-update. | +| AV engine / agent version | Within 7 days of vendor release for non-critical; within 48 hours for vendor-marked critical. | OS package manager or vendor deployment pipeline; align with the flaw-remediation SLAs in `docs/security/flaw-remediation-policy.md`. | +| File-integrity baseline (AIDE) | Re-baseline on every Spinifex upgrade or config change. | Re-run `aide --init` as part of the upgrade runbook; commit the resulting hash to the change record. | + +Evidence: the system security plan must record (a) which product is deployed, (b) how updates are delivered, and (c) a sample of update-success log entries from the last 30 days. + +## 3. Scan Schedule (SI.L1-3.14.5) + +### 3.1 Periodic scans (objectives [a], [b]) + +| Target | Frequency | Type | +|--------|-----------|------| +| Designated locations §1.1–§1.3 (binaries, config, secrets, unit files) | Daily | Full AV scan + AIDE integrity check. | +| Runtime data under `/var/lib/spinifex/*` (§1.4) | Weekly | Full AV scan. | +| Whole root filesystem | Monthly | Full AV scan. | + +### 3.2 Real-time scans (objective [c]) + +| Trigger | Action | +|---------|--------| +| Write to the external-ingest paths in §1.5 | AV scan before the file is considered available. A positive detection must fail the ingest and emit a structured log line to `/var/log/spinifex/` and the operator's SIEM. | +| Execution of a binary in `/usr/local/bin/` or `/usr/local/share/` | On-access AV check (ClamAV `clamonacc` or EDR equivalent). | + +### 3.3 Logging + +All scan runs — scheduled and real-time — must produce a log entry that includes scanner name, version, signature version, target, outcome, and duration. Forward to the same log collection used for Spinifex service logs (see `docs/security/network-connections.md` for permitted external endpoints). + +## 4. Reference Configurations + +The configurations below are reference baselines. They are known to meet the CMMC L1 objectives on a standard Debian 12 Spinifex node; they are not the only valid implementation. Operators running a commercial EDR should treat §1 and §3 as the contract and configure their chosen product to match. + +### 4.1 ClamAV (signature-based AV) + +Install: + +```bash +apt-get install -y clamav clamav-daemon clamav-freshclam clamonacc +``` + +Signature updates — drop-in `/etc/systemd/system/clamav-freshclam.timer.d/spinifex.conf`: + +```ini +[Timer] +OnCalendar=*-*-* 00,04,08,12,16,20:00:00 +Persistent=true +``` + +Enable and start: + +```bash +systemctl enable --now clamav-freshclam.timer clamav-daemon.service clamav-clamonacc.service +``` + +On-access scanning — `/etc/clamav/clamd.conf` additions: + +``` +OnAccessIncludePath /usr/local/bin +OnAccessIncludePath /etc/spinifex +OnAccessIncludePath /etc/systemd/system +OnAccessPreventionAction clamd-block +OnAccessExcludeUname root +OnAccessExcludePath /var/lib/spinifex/viperblock +OnAccessExcludePath /var/lib/spinifex/predastore +OnAccessExcludePath /var/lib/spinifex/nats +``` + +Scheduled scans — `/etc/systemd/system/spinifex-clamav-scan.service`: + +```ini +[Unit] +Description=Spinifex scheduled AV scan +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/clamdscan --fdpass --multiscan \ + /usr/local/bin /etc/spinifex /etc/systemd/system \ + /var/lib/spinifex/awsgw /var/lib/spinifex/vpcd \ + /var/lib/spinifex/spinifex /var/log/spinifex +StandardOutput=append:/var/log/spinifex/clamav-scheduled.log +StandardError=append:/var/log/spinifex/clamav-scheduled.log +``` + +Paired timer `spinifex-clamav-scan.timer`: + +```ini +[Unit] +Description=Daily Spinifex AV scan + +[Timer] +OnCalendar=*-*-* 03:15:00 +Persistent=true + +[Install] +WantedBy=timers.target +``` + +For the weekly `/var/lib/spinifex/*` data scan and the monthly root scan, add two additional timer/service pairs with `OnCalendar=weekly` / `OnCalendar=monthly` and the appropriate scan targets. + +### 4.2 AIDE (file integrity monitoring) + +Install: + +```bash +apt-get install -y aide aide-common +``` + +Policy — `/etc/aide/aide.conf.d/90_spinifex`: + +``` +# Binaries — any change is suspicious +/usr/local/bin/spx f+p+u+g+s+m+c+sha256 +/usr/local/bin/predastore f+p+u+g+s+m+c+sha256 +/usr/local/bin/viperblock f+p+u+g+s+m+c+sha256 +/usr/local/bin/nats-server f+p+u+g+s+m+c+sha256 + +# Config and secrets +/etc/spinifex f+p+u+g+s+m+c+sha256 +/etc/systemd/system/spinifex-.*\.service$ f+p+u+g+s+m+c+sha256 + +# CA trust store +/usr/local/share/ca-certificates/spinifex-ca.crt f+p+u+g+s+m+c+sha256 + +# Exclude high-churn runtime data (covered by scheduled AV instead) +!/var/lib/spinifex/viperblock +!/var/lib/spinifex/predastore +!/var/lib/spinifex/nats +!/var/log/spinifex +!/run/spinifex +``` + +Baseline: + +```bash +aideinit +mv /var/lib/aide/aide.db.new /var/lib/aide/aide.db +``` + +Daily check — a systemd timer invoking `aide --check`. Non-zero exit must alert. Re-baseline after every Spinifex upgrade (`make reinstall`, package upgrade, config change) and record the new hash. + +### 4.3 Wazuh (centralised AV + FIM + alerting) + +For multi-node clusters, a Wazuh agent on each Spinifex node feeding a central Wazuh manager is the simplest way to discharge 3.14.2 / 3.14.4 / 3.14.5 with consolidated evidence. + +Key agent configuration (`/var/ossec/etc/ossec.conf` fragments): + +```xml + + 43200 + /etc/spinifex + /usr/local/bin + /etc/systemd/system + /var/lib/spinifex/viperblock + /var/lib/spinifex/predastore + /var/lib/spinifex/nats + /run/spinifex + + + + 3600 + +``` + +Pair with the Wazuh ClamAV integration (or a commercial AV module) for signature-based detection, and use the Wazuh manager's rule engine to alert on signature updates failing (SI.L1-3.14.4 evidence) and on scheduled-scan failures (SI.L1-3.14.5 evidence). + +## 5. Operator Checklist + +Use this list to confirm a node meets the three CMMC practices before the node is admitted to a production cluster: + +- [ ] Host AV product installed, running, and reporting to a central console or log sink. +- [ ] Signature update timer/job running; last successful update within the cadence in §2. +- [ ] AV and file-integrity policy covers every path in §1.1–§1.4, with documented exclusions for §1.4 high-churn paths. +- [ ] External-ingest paths in §1.5 have real-time scanning in place (either at the Predastore write layer or in the operator's upload workflow). +- [ ] Daily, weekly, and monthly scheduled scans defined per §3.1 and visible in recent run history. +- [ ] Scan and update logs forwarded to the same SIEM / log collector used for Spinifex service logs. +- [ ] AIDE (or equivalent FIM) baseline exists; re-baseline step is part of the Spinifex upgrade runbook. +- [ ] System security plan documents the deployed product, update mechanism, scan cadence, and exclusions. diff --git a/docs/terraform/bastion-private-subnet/README.md b/docs/terraform-workbooks/bastion-private-subnet/README.md similarity index 100% rename from docs/terraform/bastion-private-subnet/README.md rename to docs/terraform-workbooks/bastion-private-subnet/README.md diff --git a/docs/terraform/bastion-private-subnet/main.tf b/docs/terraform-workbooks/bastion-private-subnet/main.tf similarity index 100% rename from docs/terraform/bastion-private-subnet/main.tf rename to docs/terraform-workbooks/bastion-private-subnet/main.tf diff --git a/docs/terraform/nginx-alb/README.md b/docs/terraform-workbooks/nginx-alb/README.md similarity index 100% rename from docs/terraform/nginx-alb/README.md rename to docs/terraform-workbooks/nginx-alb/README.md diff --git a/docs/terraform/nginx-alb/main.tf b/docs/terraform-workbooks/nginx-alb/main.tf similarity index 100% rename from docs/terraform/nginx-alb/main.tf rename to docs/terraform-workbooks/nginx-alb/main.tf diff --git a/docs/terraform/nginx-webserver/README.md b/docs/terraform-workbooks/nginx-webserver/README.md similarity index 100% rename from docs/terraform/nginx-webserver/README.md rename to docs/terraform-workbooks/nginx-webserver/README.md diff --git a/docs/terraform/nginx-webserver/main.tf b/docs/terraform-workbooks/nginx-webserver/main.tf similarity index 100% rename from docs/terraform/nginx-webserver/main.tf rename to docs/terraform-workbooks/nginx-webserver/main.tf diff --git a/docs/terraform/s3-webapp/README.md b/docs/terraform-workbooks/s3-webapp/README.md similarity index 100% rename from docs/terraform/s3-webapp/README.md rename to docs/terraform-workbooks/s3-webapp/README.md diff --git a/docs/terraform/s3-webapp/main.tf b/docs/terraform-workbooks/s3-webapp/main.tf similarity index 100% rename from docs/terraform/s3-webapp/main.tf rename to docs/terraform-workbooks/s3-webapp/main.tf diff --git a/docs/terraform/s3-webapp/terraform.tfvars.example b/docs/terraform-workbooks/s3-webapp/terraform.tfvars.example similarity index 100% rename from docs/terraform/s3-webapp/terraform.tfvars.example rename to docs/terraform-workbooks/s3-webapp/terraform.tfvars.example From 1a41731ef5292f1cd019a035f2f9923707c300c9 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:59:10 +1000 Subject: [PATCH 11/16] chore: tweak malware-protection doc --- docs/security/malware-protection/README.md | 79 ++++++++++------------ 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/docs/security/malware-protection/README.md b/docs/security/malware-protection/README.md index b8bc2b911..d21c58dca 100644 --- a/docs/security/malware-protection/README.md +++ b/docs/security/malware-protection/README.md @@ -1,6 +1,6 @@ --- title: "Malware Protection" -description: "Operator guide for host-based malware protection, file integrity monitoring, and AV scanning on Spinifex nodes" +description: "Operator guide for host-based malware protection and file integrity monitoring on Spinifex nodes" category: "Security" sections: - overview @@ -12,23 +12,19 @@ tags: - antivirus - file integrity resources: - - title: "Spinifex Repository" - url: "https://github.com/mulgadc/spinifex" - title: "NIST SP 800-171 Rev 3" url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final" - title: "CMMC Level 1 Self-Assessment Guide v2.0" url: "https://dodcio.defense.gov/CMMC/Documentation" - title: "ClamAV Documentation" url: "https://docs.clamav.net" - - title: "AIDE — Advanced Intrusion Detection Environment" + - title: "AIDE" url: "https://aide.github.io" - - title: "Wazuh Documentation" - url: "https://documentation.wazuh.com" --- # Malware Protection -> Operator guide for host-based malware protection, file integrity monitoring, and AV scanning on Spinifex nodes +> Operator guide for host-based malware protection and file integrity monitoring on Spinifex nodes ## Table of Contents @@ -40,7 +36,6 @@ resources: - [3. Scan Schedule](#3-scan-schedule-sil1-3145) - [4. Reference Configurations](#4-reference-configurations) - [5. Operator Checklist](#5-operator-checklist) -- [6. Related Documents](#6-related-documents) --- @@ -48,7 +43,7 @@ resources: **Audience:** Operators deploying Spinifex into environments subject to CMMC Level 1 (or organisations that otherwise require host-based malware protection). -**Scope:** The Spinifex nodes themselves — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd` and the OVN control plane. Guest VMs launched on the platform are the responsibility of the workload owner and are out of scope. +**Scope:** The Spinifex nodes themselves — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd`, `spinifex-ui` and the OVN control plane. Guest VMs launched on the platform are the responsibility of the workload owner and are out of scope. ## CMMC Practices Covered @@ -72,57 +67,55 @@ Spinifex does not bundle or mandate a specific anti-malware product. Operators w All Spinifex nodes must include the following in their malware protection scope. Paths assume the production install layout (`/etc/spinifex` exists); development layouts under `$HOME/spinifex` are not supported for compliance deployments. -### 1.1 Executable binaries +### 1.1 Executable Binaries | Path | Contents | |------|----------| -| `/usr/local/bin/spx` | Spinifex CLI and service entry point (`spx service start`). | -| `/usr/local/bin/predastore` | Predastore S3 server binary. | -| `/usr/local/bin/viperblock` | Viperblock NBD block storage binary. | -| `/usr/local/bin/nats-server` | Bundled NATS server. | -| `/usr/local/bin/ovn-*`, `/usr/local/bin/ovs-*` | OVN / OVS control-plane and datapath binaries. | -| `/usr/local/share/ca-certificates/spinifex-ca.crt` | Cluster CA cert installed into the system trust store. | +| `/usr/local/bin/spx` | Spinifex CLI and service entry point. All `spinifex-*.service` units invoke `spx service start`, so every service binary is reached through this one path. | +| `/usr/bin/ovn-*`, `/usr/bin/ovs-*` | OVN / OVS control-plane and datapath binaries (installed via `apt` from the `ovn-central`, `ovn-host`, `openvswitch-switch` packages). | +| `/usr/lib/-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` | Viperblock NBD plugin loaded by `nbdkit` to expose EBS volumes as block devices. The exact path comes from `nbdkit --dump-config` (`plugindir=`); on Debian 12 amd64 it is `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/`. | +| `/usr/local/share/ca-certificates/spinifex-ca.crt` | Cluster CA cert installed into the system trust store | -### 1.2 Configuration and secrets +### 1.2 Configuration and Secrets | Path | Contents | |------|----------| | `/etc/spinifex/` | Cluster config directory: `spinifex.toml`, `predastore.toml`, `nats.conf`. | | `/etc/spinifex/ca.pem`, `/etc/spinifex/ca.key` | Cluster root CA certificate and private key (leader only). | -| `/etc/spinifex/node.crt`, `/etc/spinifex/node.key` | Per-node TLS certificate and key. | +| `/etc/spinifex/server.pem`, `/etc/spinifex/server.key` | Per-node TLS certificate and key. | | `/etc/spinifex/master.key` | Master encryption key that wraps per-volume DEKs and IAM secrets. **Highest-value artifact on the node.** | | `/etc/spinifex/systemd.env` | Service environment file (shared by all `spinifex-*.service` units). | -### 1.3 Service units and privilege configuration +### 1.3 Service Units and Privilege Configuration | Path | Contents | |------|----------| -| `/etc/systemd/system/spinifex-*.service` | Unit files for daemon, awsgw, nats, predastore, viperblock, vpcd, spinifex-ui. | -| `/etc/sudoers`, `/etc/sudoers.d/` | Any drop-ins granting privileges to service accounts. | +| `/etc/systemd/system/spinifex-*.service` | Unit files for daemon, awsgw, nats, predastore, viperblock, vpcd, ui. | +| `/etc/sudoers`, `/etc/sudoers.d/` | Any drop-ins granting privileges to service accounts (OVN/OVS management, NBD mounts, etc.). | | `/etc/ssh/sshd_config`, `/etc/ssh/sshd_config.d/` | Host SSH daemon configuration. | -### 1.4 Runtime and data locations +### 1.4 Runtime and Data Locations | Path | Purpose | Notes | |------|---------|-------| -| `/var/lib/spinifex/spinifex/` | Daemon state, local bead/catalogue metadata. | Include in scan scope; exclude large, high-churn subtrees from real-time AV if performance requires (document the exclusion). | +| `/var/lib/spinifex/spinifex/` | Daemon state and local metadata. | Include in scan scope. | +| `/var/lib/spinifex/awsgw/` | AWS gateway state. | Include in scan scope. | +| `/var/lib/spinifex/vpcd/` | VPC daemon state. | Include in scan scope. | | `/var/lib/spinifex/predastore/` | Local Predastore S3 backing data (chunks, metadata). | See exclusion note below. | | `/var/lib/spinifex/viperblock/` | Local Viperblock WAL and chunk data for EBS volumes. | See exclusion note below. | | `/var/lib/spinifex/nats/` | NATS JetStream data. | See exclusion note below. | -| `/var/lib/spinifex/awsgw/` | AWS gateway state. | Include in scan scope. | -| `/var/lib/spinifex/vpcd/` | VPC daemon state. | Include in scan scope. | | `/run/spinifex/`, `/run/spinifex/nbd/` | Runtime sockets and PID files. | Integrity monitoring not meaningful (transient); exclude from AIDE. | | `/var/log/spinifex/` | Service logs. | Scan for malicious content indicators; do not block on writes. | -**Performance exclusions.** Viperblock WAL, Predastore chunks, and NATS JetStream files are large, high-churn, block-level storage that contains customer VM data. Real-time on-access AV scanning of these paths is both a significant performance cost and a confused-deputy problem (the daemon, not the operator, is the writer). Recommendation: cover these paths with **periodic scheduled scans** (SI.L1-3.14.5 [b]) but exclude them from real-time on-access hooks. Record the exclusion and its justification in the system security plan. +**Performance exclusions.** Viperblock WAL, Predastore chunks, and NATS JetStream files are large, high-churn, block-level storage that contains customer VM data. Real-time on-access AV scanning of these paths is both a significant performance cost and a confused-deputy problem (the daemon, not the operator, is the writer). Cover these paths with **periodic scheduled scans** (SI.L1-3.14.5 [b]) but exclude them from real-time on-access hooks. Record the exclusion and its justification in the system security plan. -### 1.5 External file ingest — real-time scan required (SI.L1-3.14.5 [c]) +### 1.5 External File Ingest — Real-time Scan Required (SI.L1-3.14.5 [c]) The following are "files from external sources" within the meaning of 3.14.5 [c] and **must** be scanned at ingest, before Spinifex consumes them: | Ingest point | What lands here | How it reaches the node | |--------------|-----------------|-------------------------| -| VM image catalogue downloads | Debian / Ubuntu / Alpine / Rocky cloud images pulled by `processImage()` in `utils/images.go`. | Spinifex downloads over HTTPS and verifies the catalogue SHA checksum. Operators layer AV on top: scan the downloaded image file before it is registered. | +| VM image catalogue downloads | Debian / Ubuntu / Alpine / Rocky cloud images extracted during `spx admin images import`. | Spinifex downloads over HTTPS and verifies the catalogue SHA checksum. Operators layer AV on top: scan the downloaded image file before it is registered. | | `ImportImage` / user-uploaded AMIs | Operator- or tenant-supplied VMDK/VHD/RAW images. | Uploaded to Predastore and registered via the EC2 API. | | Cloud-init user-data | Base64 payloads attached to instances. | Passed via `RunInstances`; written to the CIDATA seed ISO before boot. | | ISO / media attached to instances | Arbitrary ISO images attached as CD-ROM to VMs. | Uploaded to Predastore. | @@ -136,14 +129,14 @@ Malicious code protection mechanisms must be updated when new releases are avail | Component | Cadence | Mechanism | |-----------|---------|-----------| | AV signatures (ClamAV `freshclam`, commercial EDR cloud feeds) | At least every 4 hours; real-time if the vendor supports it. | Systemd timer for `freshclam` (see §4.1); EDR agents typically self-update. | -| AV engine / agent version | Within 7 days of vendor release for non-critical; within 48 hours for vendor-marked critical. | OS package manager or vendor deployment pipeline; align with the flaw-remediation SLAs in `docs/security/flaw-remediation-policy.md`. | +| AV engine / agent version | Within 7 days of vendor release for non-critical; within 48 hours for vendor-marked critical. | OS package manager or vendor deployment pipeline; align with the flaw-remediation SLAs. | | File-integrity baseline (AIDE) | Re-baseline on every Spinifex upgrade or config change. | Re-run `aide --init` as part of the upgrade runbook; commit the resulting hash to the change record. | -Evidence: the system security plan must record (a) which product is deployed, (b) how updates are delivered, and (c) a sample of update-success log entries from the last 30 days. +**Evidence:** the system security plan must record (a) which product is deployed, (b) how updates are delivered, and (c) a sample of update-success log entries from the last 30 days. ## 3. Scan Schedule (SI.L1-3.14.5) -### 3.1 Periodic scans (objectives [a], [b]) +### 3.1 Periodic Scans (objectives [a], [b]) | Target | Frequency | Type | |--------|-----------|------| @@ -151,16 +144,16 @@ Evidence: the system security plan must record (a) which product is deployed, (b | Runtime data under `/var/lib/spinifex/*` (§1.4) | Weekly | Full AV scan. | | Whole root filesystem | Monthly | Full AV scan. | -### 3.2 Real-time scans (objective [c]) +### 3.2 Real-time Scans (objective [c]) | Trigger | Action | |---------|--------| | Write to the external-ingest paths in §1.5 | AV scan before the file is considered available. A positive detection must fail the ingest and emit a structured log line to `/var/log/spinifex/` and the operator's SIEM. | -| Execution of a binary in `/usr/local/bin/` or `/usr/local/share/` | On-access AV check (ClamAV `clamonacc` or EDR equivalent). | +| Execution of a binary in `/usr/local/bin/`, `/usr/bin/ovn-*`, `/usr/bin/ovs-*`, or load of `nbdkit-viperblock-plugin.so` from the nbdkit plugin dir | On-access AV check (ClamAV `clamonacc` or EDR equivalent). | ### 3.3 Logging -All scan runs — scheduled and real-time — must produce a log entry that includes scanner name, version, signature version, target, outcome, and duration. Forward to the same log collection used for Spinifex service logs (see `docs/security/network-connections.md` for permitted external endpoints). +All scan runs — scheduled and real-time — must produce a log entry that includes scanner name, version, signature version, target, outcome, and duration. Forward to the same log collection used for Spinifex service logs. ## 4. Reference Configurations @@ -192,10 +185,10 @@ On-access scanning — `/etc/clamav/clamd.conf` additions: ``` OnAccessIncludePath /usr/local/bin +OnAccessIncludePath /usr/lib/x86_64-linux-gnu/nbdkit/plugins OnAccessIncludePath /etc/spinifex OnAccessIncludePath /etc/systemd/system OnAccessPreventionAction clamd-block -OnAccessExcludeUname root OnAccessExcludePath /var/lib/spinifex/viperblock OnAccessExcludePath /var/lib/spinifex/predastore OnAccessExcludePath /var/lib/spinifex/nats @@ -211,7 +204,8 @@ After=network-online.target [Service] Type=oneshot ExecStart=/usr/bin/clamdscan --fdpass --multiscan \ - /usr/local/bin /etc/spinifex /etc/systemd/system \ + /usr/local/bin /usr/lib/x86_64-linux-gnu/nbdkit/plugins \ + /etc/spinifex /etc/systemd/system \ /var/lib/spinifex/awsgw /var/lib/spinifex/vpcd \ /var/lib/spinifex/spinifex /var/log/spinifex StandardOutput=append:/var/log/spinifex/clamav-scheduled.log @@ -246,17 +240,16 @@ Policy — `/etc/aide/aide.conf.d/90_spinifex`: ``` # Binaries — any change is suspicious -/usr/local/bin/spx f+p+u+g+s+m+c+sha256 -/usr/local/bin/predastore f+p+u+g+s+m+c+sha256 -/usr/local/bin/viperblock f+p+u+g+s+m+c+sha256 -/usr/local/bin/nats-server f+p+u+g+s+m+c+sha256 +/usr/local/bin/spx f+p+u+g+s+m+c+sha256 +# Viperblock NBD plugin — adjust for the node (see `nbdkit --dump-config`) +/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so f+p+u+g+s+m+c+sha256 # Config and secrets -/etc/spinifex f+p+u+g+s+m+c+sha256 -/etc/systemd/system/spinifex-.*\.service$ f+p+u+g+s+m+c+sha256 +/etc/spinifex f+p+u+g+s+m+c+sha256 +/etc/systemd/system/spinifex-.*\.service$ f+p+u+g+s+m+c+sha256 # CA trust store -/usr/local/share/ca-certificates/spinifex-ca.crt f+p+u+g+s+m+c+sha256 +/usr/local/share/ca-certificates/spinifex-ca.crt f+p+u+g+s+m+c+sha256 # Exclude high-churn runtime data (covered by scheduled AV instead) !/var/lib/spinifex/viperblock From 8d6f4df52601b2088f4a766ede635a30d016d95e Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:13:44 +1000 Subject: [PATCH 12/16] fix: setting up cluster image --- {docs => .github/assets}/spinifex-ui.jpg | Bin README.md | 4 ++-- .../setting-up-your-cluster/README.md | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) rename {docs => .github/assets}/spinifex-ui.jpg (100%) diff --git a/docs/spinifex-ui.jpg b/.github/assets/spinifex-ui.jpg similarity index 100% rename from docs/spinifex-ui.jpg rename to .github/assets/spinifex-ui.jpg diff --git a/README.md b/README.md index ee5188565..7e86479cb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- Go + Go License Docs CI @@ -130,7 +130,7 @@ Each component can be developed independently. See component-specific documentat Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.

- Spinifex web console — dashboard view + Spinifex web console — dashboard view

The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up: diff --git a/docs/getting-started/setting-up-your-cluster/README.md b/docs/getting-started/setting-up-your-cluster/README.md index 57f05998b..f82a623a5 100644 --- a/docs/getting-started/setting-up-your-cluster/README.md +++ b/docs/getting-started/setting-up-your-cluster/README.md @@ -272,7 +272,9 @@ spx get vms Spinifex ships with a built-in web console — an alternative to the AWS CLI, SDKs, and Terraform, analogous to the AWS Management Console. Every action in the UI is a standard AWS SigV4 API call, so the same IAM policies and audit behaviour apply. -![Spinifex Web UI](../spinifex-ui.jpg) +

+ Spinifex Web UI +

### Open the Console From e46b42793887a05ebe8bf340c05f5e6ab2621206 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:44:48 +1000 Subject: [PATCH 13/16] feat: network-connections docs (#150) --- .../install-multi-node/README.md | 2 +- docs/getting-started/install-source/README.md | 5 +- docs/security/network-connections/README.md | 177 ++++++++++++++++++ 3 files changed, 179 insertions(+), 5 deletions(-) create mode 100644 docs/security/network-connections/README.md diff --git a/docs/getting-started/install-multi-node/README.md b/docs/getting-started/install-multi-node/README.md index 1417d3324..a08ec6d60 100644 --- a/docs/getting-started/install-multi-node/README.md +++ b/docs/getting-started/install-multi-node/README.md @@ -59,7 +59,7 @@ export AWS_AZ=us-east-1a ## Step 3. Setup OVN Networking -Server 1 runs OVN central and must be set up first. If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC) or `--macvlan --wan-iface=eth0` (single-NIC). +Server 1 runs OVN central and must be set up first. If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC). **Server 1:** diff --git a/docs/getting-started/install-source/README.md b/docs/getting-started/install-source/README.md index c3fd077e4..2cefe1a20 100644 --- a/docs/getting-started/install-source/README.md +++ b/docs/getting-started/install-source/README.md @@ -65,14 +65,11 @@ If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-det ./scripts/setup-ovn.sh --management ``` -If your WAN is a physical NIC, choose one: +If your WAN is a physical NIC: ```bash # Dedicated WAN NIC (not your SSH connection): ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1 - -# Single-NIC host (SSH-safe macvlan): -./scripts/setup-ovn.sh --management --macvlan --wan-iface=enp0s3 ``` ## Step 4. Development initialisation diff --git a/docs/security/network-connections/README.md b/docs/security/network-connections/README.md new file mode 100644 index 000000000..724cdcbde --- /dev/null +++ b/docs/security/network-connections/README.md @@ -0,0 +1,177 @@ +--- +title: "External Connection Inventory" +description: "Operator inventory of inbound listeners and outbound connections on Spinifex nodes, for CMMC AC.L1-3.1.20 compliance" +category: "Security" +sections: + - overview +tags: + - security + - compliance + - cmmc + - network + - connections + - boundary +resources: + - title: "NIST SP 800-171 Rev 3" + url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final" + - title: "CMMC Level 1 Self-Assessment Guide v2.0" + url: "https://dodcio.defense.gov/CMMC/Documentation" + - title: "NIST SP 800-41 Rev 1 — Guidelines on Firewalls and Firewall Policy" + url: "https://csrc.nist.gov/pubs/sp/800/41/r1/final" +--- + +# External Connection Inventory + +> Operator inventory of inbound listeners and outbound connections on Spinifex nodes, for CMMC AC.L1-3.1.20 compliance + +## Table of Contents + +- [Overview](#overview) +- [CMMC Practices Covered](#cmmc-practices-covered) +- [Approach](#approach) +- [1. Inbound Listeners](#1-inbound-listeners) +- [2. Outbound Connections](#2-outbound-connections) +- [3. Cross-Node (Internal) Connections](#3-cross-node-internal-connections) +- [4. Limiting Controls](#4-limiting-controls) +- [5. Configuration Surface](#5-configuration-surface) +- [6. Operator Checklist](#6-operator-checklist) + +--- + +## Overview + +**Audience:** Operators deploying Spinifex into environments subject to CMMC Level 1, or any site that requires a documented inventory of system connections. + +**Scope:** Network connections originated by or terminated at the Spinifex nodes — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd`, `spinifex-ui`, and the OVN control plane. Guest VM traffic is the workload owner's responsibility and out of scope. + +**Boundary definition.** For the purposes of this document: + +- **External** means outside the Spinifex cluster's trusted network perimeter — the public internet, the operator's corporate network, tenant users of the AWS API, and guest VMs. +- **Internal** means between Spinifex nodes inside the cluster subnet(s) defined in `spinifex.toml`. + +AC.L1-3.1.20 applies specifically to **external** connections. Internal cluster connections are documented here as well so operators can build an accurate firewall policy. + +## CMMC Practices Covered + +This guide addresses AC.L1-3.1.20. The related boundary-protection practice SC.L1-3.13.1 is covered by OVN ACL and security-group enforcement in `vpcd` and is documented separately. + +| Practice | Title | Objective | +|----------|-------|-----------| +| AC.L1-3.1.20 | External Connections | [a] Connections to external systems are identified. [b] The use of external systems is identified. [c] Connections to external systems are verified. [d] The use of external systems is verified. [e] Connections to external systems are controlled/limited. [f] The use of external systems is controlled/limited. | + +## Approach + +Spinifex has a small, enumerable set of network surfaces: + +1. **Inbound listeners** — the TCP/UDP ports each node binds. These are the attack surface exposed to whoever can reach the node. +2. **Outbound connections** — the destinations the node's services reach out to. Today this is a short list: peer Spinifex nodes, OS image mirrors, and install telemetry. +3. **Cross-node connections** — inter-node control- and data-plane traffic inside the cluster subnet. + +The inventory in §1–§2 satisfies objectives [a]/[b]. The **Auth / Verification** columns throughout satisfy [c]/[d]. §4 and §5 satisfy [e]/[f]. The default install meets [c]–[f] for every listed connection; the operator's remaining work is to record the inventory in the system security plan, apply host/network firewall rules per §4, and audit §5 on a recurring schedule. + +## 1. Inbound Listeners + +"Scope" classifies intended reach: + +- **External** — reachable by tenant/operator networks. Authenticated and TLS-protected. +- **Cluster** — reachable only from peer Spinifex nodes. Operator must restrict via host or network firewall. +- **Localhost** — bound to `127.0.0.1`; not reachable off-node. + +| Port | Service | Protocol | Scope | Purpose | Auth / Verification | +|------|---------|----------|-------|---------|--------------------| +| 9999 | spinifex-awsgw | HTTPS | External | AWS-compatible API (EC2, S3, ELBv2, IAM) — customer endpoint | AWS SigV4 + TLS (cluster CA) | +| 3000 | spinifex-ui | HTTPS | External | Operator web dashboard | Session cookie + TLS | +| 22 | OpenSSH | SSH | External | Operator administration | Key-based auth (operator-managed) | +| 4432 | Formation server | HTTPS | External (bootstrap only) | Cluster join coordination; active only while a join token is valid. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ | +| 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) | +| 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) | +| 8443 | spinifex-predastore | HTTPS | Cluster | S3-compatible object storage (AMIs, snapshots, user objects) | AWS SigV4 + TLS | +| 6660–6662 | predastore (Raft) | TCP | Cluster | Metadata consensus (3 nodes) | Cluster network only | +| 9991–9993 | predastore (data shards) | TCP | Cluster | Erasure-coded data shard transport | Cluster network only | +| 6641 | OVN Northbound DB | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned | +| 6642 | OVN Southbound DB | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned | +| 8222 | spinifex-nats (monitoring) | HTTP | Localhost | `varz`/`subsz` metrics consumed by the daemon | Loopback only | +| socket / dynamic TCP | nbdkit (Viperblock) | NBD | Host-local / cluster | Block device transport for guest EBS volumes | Unix socket by default; TCP only in remote/DPU mode | + +¹ **Formation port lifecycle.** 4432 opens during `spx admin init` / `spx admin join` while a bootstrap token is outstanding and closes once the cluster is formed (token TTL default 30 min, `--token-ttl`). The server presents an ephemeral self-signed cert that pre-dates trust bootstrap, so the joining node dials with `InsecureSkipVerify` — the only production dial that does. Authenticity rests on the operator supplying the leader address out-of-band plus possession of the bearer token. Document in the security plan so reviewers do not flag 4432 as a persistent open port. + +**Development-only listeners.** When `dev_networking=true`, QEMU opens arbitrary host TCP ports for SSH port-forwarding into guest VMs. Production installs (the `/etc/spinifex` layout) do not enable this; it must not appear on compliance nodes. + +## 2. Outbound Connections + +Spinifex nodes initiate a small, fixed set of outbound connections. + +**To external destinations:** + +| Destination | Purpose | Protocol | Verification | +|-------------|---------|----------|--------------| +| `https://cdimage.debian.org/cdimage/cloud/bookworm/latest/` | Debian 12 cloud image download (`spx admin images import`) | HTTPS | TLS (system trust store) + SHA256/SHA512 checksum against published manifest | +| `https://cloud-images.ubuntu.com/noble/current/` | Ubuntu 24.04 LTS cloud image download | HTTPS | TLS + checksum verification | +| `https://d2yp8ipz5jfqcw.cloudfront.net` | Alpine image for managed HAProxy load-balancer | HTTPS | TLS + checksum verification | +| `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. Payload: machine ID, event type, region/AZ, node name, node count, bind IP, Spinifex version, OS/arch. | HTTPS | TLS (system trust store). Opt out via `--no-telemetry` or `SPX_NO_TELEMETRY=1`. | + +**To peer nodes (cluster-internal):** NATS federation (4248), Predastore S3 (8443), OVN NB/SB (6641/6642) — see §3 for encryption and verification of each. The daemon also probes local Predastore Raft status at `:6660/status` (TLS, cluster CA) and local NATS monitoring at `127.0.0.1:8222/varz` (loopback HTTP). + +**Update checks and metadata.** Spinifex does not check for updates and does not consume a cloud metadata service (`169.254.169.254` is served *by* the cluster to guest VMs). Node software updates come from the operator's OS package channel. The install-telemetry endpoint above is the only vendor-operated destination contacted by a node; closed-egress deployments should disable it and record the opt-out in the security plan. + +**Air-gapped deployments.** The three image URLs are the only destinations needed for the standard image catalogue. Mirror them locally and use `spx admin images import --file` with pre-staged files. Telemetry must also be disabled. See [Air-gapped install](https://docs.mulgadc.com/docs/install-airgapped). + +## 3. Cross-Node (Internal) Connections + +Control- and data-plane traffic between Spinifex nodes, for completeness and firewall planning: + +| Connection | Port(s) | Encryption / Auth | Notes | +|-----------|---------|-------------------|-------| +| NATS cluster routes | 4248 | Mutual TLS + cluster token | Full mesh between NATS servers | +| Predastore S3 | 8443 | TLS + AWS SigV4 | Cross-node object reads/writes | +| Predastore Raft | 6660–6662 | Cluster network only | Metadata consensus | +| Predastore shards | 9991–9993 | Cluster network only | Erasure-coded data shards | +| OVN NB/SB | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane | +| OVN tunnels (Geneve) | UDP 6081 | None | Tenant traffic overlay between chassis, inside the cluster subnet | + +Nodes **must** sit on a network segment that is not routed to tenant/guest VMs or to the internet. Predastore Raft/shards and OVN DBs are cluster-internal and must not be reachable from anywhere else. + +## 4. Limiting Controls + +Default external surface is three listeners — **9999** (AWS API), **3000** (UI), **22** (SSH) — plus **4432** transiently during bootstrap. Every other listener is cluster-only and the operator must enforce this with a host firewall (`nftables`/`iptables`/`firewalld`) or an upstream network ACL. Minimal `nftables` reference: + +``` +# External: from anywhere +tcp dport { 22, 9999, 3000 } accept + +# Cluster-only: replace 10.0.1.0/24 with your cluster CIDR +ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641, 6642, 6660-6662, 9991-9993 } accept +ip saddr 10.0.1.0/24 udp dport 6081 accept + +# Default deny +tcp dport 0-65535 drop +udp dport 0-65535 drop +``` + +Port 4432 must be closed outside the bootstrap window; `spx admin join` opens it transiently. Outbound egress can be limited to the image-catalogue hostnames in §2 plus the operator's OS package repositories; on air-gapped nodes, block all outbound HTTPS and use `spx admin images import --file`. + +## 5. Configuration Surface + +Every listener and outbound destination is controlled by one of these files. Changes require a service restart. + +| File | Keys | Controls | +|------|------|----------| +| `/etc/spinifex/spinifex.toml` | `nodes..{awsgw,nats,predastore,daemon}.host`, `nodes..vpcd.ovn_{nb,sb}_addr`, `nodes..daemon.dev_networking` | Per-service bind addresses/ports; dev-mode QEMU port forwarding. | +| `/etc/spinifex/nats.conf` | `listen`, `cluster.listen`, `cluster.routes`, `http`, `tls`, `cluster.authorization` | NATS client/cluster/monitoring listeners, peer routes, TLS, cluster token. | +| `/etc/spinifex/predastore.toml` | `host`, `port`, `[[db]].port`, `[[nodes]].port`, `tls.*` | Predastore S3 listener, Raft ports, shard ports, TLS certs. | +| OVN packages (`ovn-central`, `ovn-host`) | `ovn-nb-db`, `ovn-sb-db` (via `ovs-vsctl set open_vswitch …`) | OVN DB bind addresses. | +| Spinifex UI service | Built-in defaults: `host = "0.0.0.0"`, `port = 3000`. No `spinifex.toml` block today. | UI listener. | +| `spx admin init` / `spx admin join` | `--port`, `--token-ttl`, `--no-telemetry` (or `SPX_NO_TELEMETRY=1`) | Formation port, token TTL, telemetry opt-out. | +| `utils/images.go` `AvailableImages` | Image catalogue URLs | Outbound HTTPS destinations for image downloads. | + +## 6. Operator Checklist + +- Inventory recorded in the system security plan — inbound (§1), outbound (§2), cross-node (§3) — matches what is observed on the node (`ss -tlnp`, `ss -unlp`). +- Host firewall enforces the external/cluster/localhost split in §4: external surface limited to 9999, 3000, 22 (and 4432 only during bootstrap). +- Cluster subnet is isolated from tenant guest VM networks and from the public internet. +- Formation port 4432 is closed on nodes not actively running a bootstrap token. +- Outbound HTTPS restricted to the §2 image-catalogue hosts, or replaced with air-gapped import. +- Install telemetry (`install.mulgadc.com`) is either permitted and recorded in the security plan, or disabled via `SPX_NO_TELEMETRY=1` / `--no-telemetry`. +- OVN NB/SB (6641/6642) exposure limited to the cluster subnet pending the L2 TLS work. +- SSH (22) configured to operator-managed keys only; password auth disabled in `sshd_config`. +- Periodic review (at least annually, and after any topology change) confirms this inventory still matches the deployed configuration. From e131caf9ecc552dd177c8eb8ee2c5761143e1da6 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:41:58 +1000 Subject: [PATCH 14/16] fix: doc links --- docs/security/malware-protection/README.md | 26 ++++++++++----------- docs/security/network-connections/README.md | 24 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/security/malware-protection/README.md b/docs/security/malware-protection/README.md index d21c58dca..8415f99dd 100644 --- a/docs/security/malware-protection/README.md +++ b/docs/security/malware-protection/README.md @@ -128,7 +128,7 @@ Malicious code protection mechanisms must be updated when new releases are avail | Component | Cadence | Mechanism | |-----------|---------|-----------| -| AV signatures (ClamAV `freshclam`, commercial EDR cloud feeds) | At least every 4 hours; real-time if the vendor supports it. | Systemd timer for `freshclam` (see §4.1); EDR agents typically self-update. | +| AV signatures (ClamAV `freshclam`, commercial EDR cloud feeds) | At least every 4 hours; real-time if the vendor supports it. | Systemd timer for `freshclam` (see [§4.1](#41-clamav-signature-based-av)); EDR agents typically self-update. | | AV engine / agent version | Within 7 days of vendor release for non-critical; within 48 hours for vendor-marked critical. | OS package manager or vendor deployment pipeline; align with the flaw-remediation SLAs. | | File-integrity baseline (AIDE) | Re-baseline on every Spinifex upgrade or config change. | Re-run `aide --init` as part of the upgrade runbook; commit the resulting hash to the change record. | @@ -140,15 +140,15 @@ Malicious code protection mechanisms must be updated when new releases are avail | Target | Frequency | Type | |--------|-----------|------| -| Designated locations §1.1–§1.3 (binaries, config, secrets, unit files) | Daily | Full AV scan + AIDE integrity check. | -| Runtime data under `/var/lib/spinifex/*` (§1.4) | Weekly | Full AV scan. | +| Designated locations [§1.1–§1.3](#11-executable-binaries) (binaries, config, secrets, unit files) | Daily | Full AV scan + AIDE integrity check. | +| Runtime data under `/var/lib/spinifex/*` ([§1.4](#14-runtime-and-data-locations)) | Weekly | Full AV scan. | | Whole root filesystem | Monthly | Full AV scan. | ### 3.2 Real-time Scans (objective [c]) | Trigger | Action | |---------|--------| -| Write to the external-ingest paths in §1.5 | AV scan before the file is considered available. A positive detection must fail the ingest and emit a structured log line to `/var/log/spinifex/` and the operator's SIEM. | +| Write to the external-ingest paths in [§1.5](#1-designated-locations-sil1-3142-a) | AV scan before the file is considered available. A positive detection must fail the ingest and emit a structured log line to `/var/log/spinifex/` and the operator's SIEM. | | Execution of a binary in `/usr/local/bin/`, `/usr/bin/ovn-*`, `/usr/bin/ovs-*`, or load of `nbdkit-viperblock-plugin.so` from the nbdkit plugin dir | On-access AV check (ClamAV `clamonacc` or EDR equivalent). | ### 3.3 Logging @@ -157,7 +157,7 @@ All scan runs — scheduled and real-time — must produce a log entry that incl ## 4. Reference Configurations -The configurations below are reference baselines. They are known to meet the CMMC L1 objectives on a standard Debian 12 Spinifex node; they are not the only valid implementation. Operators running a commercial EDR should treat §1 and §3 as the contract and configure their chosen product to match. +The configurations below are reference baselines. They are known to meet the CMMC L1 objectives on a standard Debian 12 Spinifex node; they are not the only valid implementation. Operators running a commercial EDR should treat [§1](#1-designated-locations-sil1-3142-a) and [§3](#3-scan-schedule-sil1-3145) as the contract and configure their chosen product to match. ### 4.1 ClamAV (signature-based AV) @@ -297,11 +297,11 @@ Pair with the Wazuh ClamAV integration (or a commercial AV module) for signature Use this list to confirm a node meets the three CMMC practices before the node is admitted to a production cluster: -- [ ] Host AV product installed, running, and reporting to a central console or log sink. -- [ ] Signature update timer/job running; last successful update within the cadence in §2. -- [ ] AV and file-integrity policy covers every path in §1.1–§1.4, with documented exclusions for §1.4 high-churn paths. -- [ ] External-ingest paths in §1.5 have real-time scanning in place (either at the Predastore write layer or in the operator's upload workflow). -- [ ] Daily, weekly, and monthly scheduled scans defined per §3.1 and visible in recent run history. -- [ ] Scan and update logs forwarded to the same SIEM / log collector used for Spinifex service logs. -- [ ] AIDE (or equivalent FIM) baseline exists; re-baseline step is part of the Spinifex upgrade runbook. -- [ ] System security plan documents the deployed product, update mechanism, scan cadence, and exclusions. +- Host AV product installed, running, and reporting to a central console or log sink. +- Signature update timer/job running; last successful update within the cadence in [§2](#2-update-requirements-sil1-3144). +- AV and file-integrity policy covers every path in [§1.1–§1.4](#11-executable-binaries), with documented exclusions for [§1.4](#14-runtime-and-data-locations) high-churn paths. +- External-ingest paths in [§1.5](#1-designated-locations-sil1-3142-a) have real-time scanning in place (either at the Predastore write layer or in the operator's upload workflow). +- Daily, weekly, and monthly scheduled scans defined per [§3.1](#31-periodic-scans-objectives-a-b) and visible in recent run history. +- Scan and update logs forwarded to the same SIEM / log collector used for Spinifex service logs. +- AIDE (or equivalent FIM) baseline exists; re-baseline step is part of the Spinifex upgrade runbook. +- System security plan documents the deployed product, update mechanism, scan cadence, and exclusions. diff --git a/docs/security/network-connections/README.md b/docs/security/network-connections/README.md index 724cdcbde..445593448 100644 --- a/docs/security/network-connections/README.md +++ b/docs/security/network-connections/README.md @@ -1,6 +1,6 @@ --- title: "External Connection Inventory" -description: "Operator inventory of inbound listeners and outbound connections on Spinifex nodes, for CMMC AC.L1-3.1.20 compliance" +description: "Operator inventory of inbound listeners and outbound connections on Spinifex nodes" category: "Security" sections: - overview @@ -22,7 +22,7 @@ resources: # External Connection Inventory -> Operator inventory of inbound listeners and outbound connections on Spinifex nodes, for CMMC AC.L1-3.1.20 compliance +> Operator inventory of inbound listeners and outbound connections on Spinifex nodes ## Table of Contents @@ -67,7 +67,7 @@ Spinifex has a small, enumerable set of network surfaces: 2. **Outbound connections** — the destinations the node's services reach out to. Today this is a short list: peer Spinifex nodes, OS image mirrors, and install telemetry. 3. **Cross-node connections** — inter-node control- and data-plane traffic inside the cluster subnet. -The inventory in §1–§2 satisfies objectives [a]/[b]. The **Auth / Verification** columns throughout satisfy [c]/[d]. §4 and §5 satisfy [e]/[f]. The default install meets [c]–[f] for every listed connection; the operator's remaining work is to record the inventory in the system security plan, apply host/network firewall rules per §4, and audit §5 on a recurring schedule. +The inventory in [§1](#1-inbound-listeners)–[§2](#2-outbound-connections) satisfies objectives [a]/[b]. The **Auth / Verification** columns throughout satisfy [c]/[d]. [§4](#4-limiting-controls) and [§5](#5-configuration-surface) satisfy [e]/[f]. The default install meets [c]–[f] for every listed connection; the operator's remaining work is to record the inventory in the system security plan, apply host/network firewall rules per [§4](#4-limiting-controls), and audit [§5](#5-configuration-surface) on a recurring schedule. ## 1. Inbound Listeners @@ -105,12 +105,12 @@ Spinifex nodes initiate a small, fixed set of outbound connections. | Destination | Purpose | Protocol | Verification | |-------------|---------|----------|--------------| -| `https://cdimage.debian.org/cdimage/cloud/bookworm/latest/` | Debian 12 cloud image download (`spx admin images import`) | HTTPS | TLS (system trust store) + SHA256/SHA512 checksum against published manifest | -| `https://cloud-images.ubuntu.com/noble/current/` | Ubuntu 24.04 LTS cloud image download | HTTPS | TLS + checksum verification | +| `https://cdimage.debian.org/cdimage/cloud/bookworm/latest` | Debian 12 cloud image download | HTTPS | TLS + checksum verification | +| `https://cloud-images.ubuntu.com/noble/current` | Ubuntu 24.04 LTS cloud image download | HTTPS | TLS + checksum verification | | `https://d2yp8ipz5jfqcw.cloudfront.net` | Alpine image for managed HAProxy load-balancer | HTTPS | TLS + checksum verification | -| `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. Payload: machine ID, event type, region/AZ, node name, node count, bind IP, Spinifex version, OS/arch. | HTTPS | TLS (system trust store). Opt out via `--no-telemetry` or `SPX_NO_TELEMETRY=1`. | +| `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. | HTTPS | TLS | -**To peer nodes (cluster-internal):** NATS federation (4248), Predastore S3 (8443), OVN NB/SB (6641/6642) — see §3 for encryption and verification of each. The daemon also probes local Predastore Raft status at `:6660/status` (TLS, cluster CA) and local NATS monitoring at `127.0.0.1:8222/varz` (loopback HTTP). +**To peer nodes (cluster-internal):** NATS federation (4248), Predastore S3 (8443), OVN NB/SB (6641/6642) — see [§3](#3-cross-node-internal-connections) for encryption and verification of each. The daemon also probes local Predastore Raft status at `:6660/status` (TLS, cluster CA) and local NATS monitoring at `127.0.0.1:8222/varz` (loopback HTTP). **Update checks and metadata.** Spinifex does not check for updates and does not consume a cloud metadata service (`169.254.169.254` is served *by* the cluster to guest VMs). Node software updates come from the operator's OS package channel. The install-telemetry endpoint above is the only vendor-operated destination contacted by a node; closed-egress deployments should disable it and record the opt-out in the security plan. @@ -118,7 +118,7 @@ Spinifex nodes initiate a small, fixed set of outbound connections. ## 3. Cross-Node (Internal) Connections -Control- and data-plane traffic between Spinifex nodes, for completeness and firewall planning: +Control-plane and data-plane traffic between Spinifex nodes, for completeness and firewall planning: | Connection | Port(s) | Encryption / Auth | Notes | |-----------|---------|-------------------|-------| @@ -148,7 +148,7 @@ tcp dport 0-65535 drop udp dport 0-65535 drop ``` -Port 4432 must be closed outside the bootstrap window; `spx admin join` opens it transiently. Outbound egress can be limited to the image-catalogue hostnames in §2 plus the operator's OS package repositories; on air-gapped nodes, block all outbound HTTPS and use `spx admin images import --file`. +Port 4432 must be closed outside the bootstrap window; `spx admin join` opens it transiently. Outbound egress can be limited to the image-catalogue hostnames in [§2](#2-outbound-connections) plus the operator's OS package repositories; on air-gapped nodes, block all outbound HTTPS and use `spx admin images import --file`. ## 5. Configuration Surface @@ -166,11 +166,11 @@ Every listener and outbound destination is controlled by one of these files. Cha ## 6. Operator Checklist -- Inventory recorded in the system security plan — inbound (§1), outbound (§2), cross-node (§3) — matches what is observed on the node (`ss -tlnp`, `ss -unlp`). -- Host firewall enforces the external/cluster/localhost split in §4: external surface limited to 9999, 3000, 22 (and 4432 only during bootstrap). +- Inventory recorded in the system security plan — inbound ([§1](#1-inbound-listeners)), outbound ([§2](#2-outbound-connections)), cross-node ([§3](#3-cross-node-internal-connections)) — matches what is observed on the node (`ss -tlnp`, `ss -unlp`). +- Host firewall enforces the external/cluster/localhost split in [§4](#4-limiting-controls): external surface limited to 9999, 3000, 22 (and 4432 only during bootstrap). - Cluster subnet is isolated from tenant guest VM networks and from the public internet. - Formation port 4432 is closed on nodes not actively running a bootstrap token. -- Outbound HTTPS restricted to the §2 image-catalogue hosts, or replaced with air-gapped import. +- Outbound HTTPS restricted to the [§2](#2-outbound-connections) image-catalogue hosts, or replaced with air-gapped import. - Install telemetry (`install.mulgadc.com`) is either permitted and recorded in the security plan, or disabled via `SPX_NO_TELEMETRY=1` / `--no-telemetry`. - OVN NB/SB (6641/6642) exposure limited to the cluster subnet pending the L2 TLS work. - SSH (22) configured to operator-managed keys only; password auth disabled in `sshd_config`. From 14efe4412593b317bddf3171df45cda2f28cf7f0 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:15:44 +1000 Subject: [PATCH 15/16] feat: flaw remediation policy doc (#151) --- .../flaw-remediation-policy/README.md | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 docs/security/flaw-remediation-policy/README.md diff --git a/docs/security/flaw-remediation-policy/README.md b/docs/security/flaw-remediation-policy/README.md new file mode 100644 index 000000000..927eb8287 --- /dev/null +++ b/docs/security/flaw-remediation-policy/README.md @@ -0,0 +1,153 @@ +--- +title: "Flaw Remediation Policy" +description: "CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex" +category: "Security" +sections: + - overview +tags: + - security + - compliance + - cmmc + - vulnerabilities + - cvss + - patching +resources: + - title: "NIST SP 800-171 Rev 3" + url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final" + - title: "CMMC Level 1 Self-Assessment Guide v2.0" + url: "https://dodcio.defense.gov/CMMC/Documentation" + - title: "FIRST CVSS v3.1 Specification" + url: "https://www.first.org/cvss/v3-1/specification-document" + - title: "Go govulncheck" + url: "https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck" + - title: "GitHub Dependabot" + url: "https://docs.github.com/en/code-security/dependabot" +--- + +# Flaw Remediation Policy + +> CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex + +## Table of Contents + +- [Overview](#overview) +- [CMMC Practices Covered](#cmmc-practices-covered) +- [Approach](#approach) +- [1. Identification](#1-identification) +- [2. Severity](#2-severity) +- [3. Remediation SLAs](#3-remediation-slas) +- [4. Reporting Workflow](#4-reporting-workflow) +- [5. Operator Responsibilities](#5-operator-responsibilities) +- [6. Evidence](#6-evidence) +- [7. Operator Checklist](#7-operator-checklist) + +--- + +## Overview + +**Audience:** Spinifex maintainers who publish releases, and operators deploying Spinifex into CMMC Level 1 environments. + +**Scope:** Flaws in Spinifex's own code and its direct dependencies (Go modules, UI npm packages, GitHub Actions). Operating system, kernel, OVN/OVS, and hypervisor CVEs are the operator's responsibility — see [§5](#5-operator-responsibilities). + +## CMMC Practices Covered + +| Practice | Title | Objective | +|----------|-------|-----------| +| SI.L1-3.14.1 | Flaw Remediation | [a] Time to identify flaws is specified. [b] Flaws identified in time. [c] Time to report flaws is specified. [d] Flaws reported in time. [e] Time to correct flaws is specified. [f] Flaws corrected in time. | + +## Approach + +Spinifex ships a short dependency chain and identification is already automated: Dependabot raises security PRs the moment GitHub links a CVE to a shipped dependency, `govulncheck` runs on every PR, and GitHub Security Advisories watch all three `mulgadc/*` repositories. + +Sections map to the three scored objectives: [§1](#1-identification) names the identification sources (objectives [a]/[b]); [§3](#3-remediation-slas) and [§4](#4-reporting-workflow) set the report and correct timeframes (objectives [c]–[f]). [§5](#5-operator-responsibilities) delineates layers the operator owns, and [§6](#6-evidence) covers the records needed to demonstrate compliance at assessment. + +## 1. Identification + +| Source | Frequency | Coverage | +|--------|-----------|----------| +| Dependabot security updates | Immediate — raised as soon as GitHub links an advisory to a dependency | Go modules, GitHub Actions, UI npm packages | +| Dependabot version updates | Weekly | Routine version bumps (non-security) | +| `govulncheck` (CI `lint_and_security` job) | Every PR and push to `main` | Go stdlib + module CVEs reachable in the binary | +| GitHub Security Advisories | Continuous | `mulgadc/spinifex`, `mulgadc/predastore`, `mulgadc/viperblock` | +| Third-party reports (sensitive) | Ad hoc | [GitHub private vulnerability reporting](https://github.com/mulgadc/spinifex/security/advisories/new) | +| Third-party reports (non-sensitive) | Ad hoc | Public GitHub issue, labelled `security` | +| Internal discovery | Ad hoc | Review, audit, red team, incident response | + +Dependabot security PRs are merged immediately once CI passes — for dependency CVEs this collapses identify, report, and correct into a single action. The separate weekly cadence applies only to non-security version bumps. + +Reporters: use private vulnerability reporting for anything plausibly Critical or High; a public issue is fine for hardening suggestions and non-exploitable bugs. A flaw is **identified** once it appears in any channel above with enough detail to score CVSS. + +## 2. Severity + +CVSS v3.1 base score from the upstream advisory; otherwise scored by maintainers and the vector recorded on the tracking issue. + +| Severity | CVSS v3.1 | Examples | +|----------|-----------|----------| +| Critical | 9.0 – 10.0 | Unauthenticated RCE reachable from tenant networks; `awsgw` auth bypass; master-key disclosure | +| High | 7.0 – 8.9 | Authenticated RCE; intra-service privilege escalation; cluster-internal auth bypass; memory disclosure exposing credentials. | +| Medium | 4.0 – 6.9 | Authenticated DoS; non-secret information disclosure; limited-blast-radius logic errors | +| Low | 0.1 – 3.9 | Heavily mitigated issues — local-only, non-default config, negligible impact | + +**Severity modifiers.** Bump one tier **up** if any of the following apply: the flaw is under active exploitation, reaches the master encryption key or cluster CA private key, or allows cross-tenant data access. Bump one tier **down** if the flaw is only reachable by a cluster-internal process already holding credentials equivalent to the attack's outcome. + +## 3. Remediation SLAs + +Clocks start at the moment of identification ([§1](#1-identification-sources)). All three objectives — identify [a], report [c], correct [e] — are time-bound. + +| Severity | Identify | Report | Correct | +|----------|----------|--------|---------| +| Critical | 24 hours | 48 hours, with patch | 48 hours | +| High | 48 hours | 7 days, with patch | 7 days | +| Medium | 7 days | Next release changelog | 30 days | +| Low | 30 days | Next release changelog | 90 days | + +**Identify** - triaged into a tracking issue, CVSS scored, affected components and versions determined, severity label applied. + +**Report** - GitHub Security Advisory is published (for Critical/High) or the fix is described in the release notes (for Medium/Low). Operators are the audience; the report must let them determine whether their deployment is affected. + +**Correct** - tagged Spinifex release containing the fix is available for operators to pull. For dependency CVEs, this is a release that bumps the vulnerable module to a fixed version. For first-party code, it is a release containing the patch. + +If an upstream embargo applies, the correct-SLA pauses until disclosure; the identify-SLA does not. Record the embargo on the tracking issue. + +## 4. Reporting Workflow + +1. **Intake** — External findings arrive via GitHub (private for sensitive, public issue otherwise). Maintainers mirror each finding into the internal tracker with CVSS score, affected components, and advisory link. Private disclosures stay private until the advisory is drafted. +2. **Triage** — Validate severity within the identify-SLA, confirm reachability in Spinifex, and either close as not-applicable (with justification) or proceed. +3. **Fix** — Patch lands on `main` via the normal PR + E2E workflow. Commit message references the CVE / GHSA ID. +4. **Release** — Tagged release cut; release notes state affected versions, CVSS, and upgrade guidance. Critical/High get a published GHSA. +5. **Notify** — GitHub release/advisory notifications reach subscribed operators automatically. Out-of-band email notification is triggered only for Critical. + +Not-applicable findings (unreachable, already patched, scanner false positive) are still tracked to closure so every identification has an audit trail. + +## 5. Operator Responsibilities + +Operators own remediation for layers Spinifex does not ship: + +| Layer | Responsibility | +|-------|----------------| +| Spinifex releases | Apply released patches within the correct-SLA above. | +| Host OS + kernel | Track distribution advisories (Debian DSA, Ubuntu USN) at matching cadence. | +| OVN / OVS | Patched via the operator's package channel alongside the OS. | +| QEMU / KVM / libvirt | As above — hypervisor CVEs affect tenant isolation. | +| Host AV / EDR agent | See https://docs.mulgadc.com/docs/security/malware-protection §2. | + +Operators should subscribe to the `mulgadc/spinifex` release feed (Watch → Custom → Releases + Security Advisories) so Critical/High advisories arrive via GitHub's notification channel rather than polling. + +## 6. Evidence and Record Keeping + +For CMMC assessment, retain the following for at least 12 months: + +- Tracking record for every identified flaw (GitHub issue, GHSA, or internal tracker ID) with identify / triage / close timestamps. +- Dependabot history and `govulncheck` CI logs — export before GitHub Actions retention expires if shorter. +- Release notes and GHSAs for each corrected flaw. +- Operator patch-apply records (change tickets, config-management runs) showing releases deployed within SLA. + +These demonstrate objectives [b], [d], [f] — "within the specified time" — in practice. + +## 7. Operator Checklist + +- System security plan references this policy as the identify / report / correct timeframes for Spinifex. +- Change-management records patch-apply timestamps for every Spinifex release. +- Operator subscribes to `mulgadc/spinifex`, `mulgadc/predastore`, `mulgadc/viperblock` releases and advisories. +- OS / kernel / hypervisor patching cadence documented and at least as strict as [§3](#3-remediation-slas). +- Annual review confirms at least one patch cycle completed within SLA in the prior 12 months, with evidence per [§6](#6-evidence). From 4f991095ee1323a43295889f5ed363f396cc664c Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:28:43 +1000 Subject: [PATCH 16/16] feat: physical security guide --- .../physical-security-guide/README.md | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 docs/security/physical-security-guide/README.md diff --git a/docs/security/physical-security-guide/README.md b/docs/security/physical-security-guide/README.md new file mode 100644 index 000000000..e016d9035 --- /dev/null +++ b/docs/security/physical-security-guide/README.md @@ -0,0 +1,205 @@ +--- +title: "Physical Security Operator Guide" +description: "Operator guide for physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes" +category: "Security" +sections: + - overview +tags: + - security + - compliance + - cmmc + - physical + - facilities + - access control +resources: + - title: "NIST SP 800-171 Rev 3" + url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final" + - title: "CMMC Level 1 Self-Assessment Guide v2.0" + url: "https://dodcio.defense.gov/CMMC/Documentation" + - title: "NIST SP 800-53 Rev 5 — PE Family" + url: "https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final" + - title: "NIST SP 800-116 Rev 1 — PIV for Physical Access" + url: "https://csrc.nist.gov/pubs/sp/800/116/r1/final" +--- + +# Physical Security Operator Guide + +> Operator guide for physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes + +## Table of Contents + +- [Overview](#overview) +- [CMMC Practices Covered](#cmmc-practices-covered) +- [Approach](#approach) +- [1. Protected Assets](#1-protected-assets) +- [2. Limit Physical Access (PE.L1-3.10.1)](#2-limit-physical-access-pel1-3101) +- [3. Escort and Monitor Visitors (PE.L1-3.10.3)](#3-escort-and-monitor-visitors-pel1-3103) +- [4. Physical Access Logs (PE.L1-3.10.4)](#4-physical-access-logs-pel1-3104) +- [5. Manage Physical Access Devices (PE.L1-3.10.5)](#5-manage-physical-access-devices-pel1-3105) +- [6. Evidence and Record Keeping](#6-evidence-and-record-keeping) +- [7. Operator Checklist](#7-operator-checklist) + +--- + +## Overview + +**Audience:** Operators deploying Spinifex into environments subject to CMMC Level 1, or any site that requires documented physical protection of compute and storage infrastructure. + +**Scope:** The physical environment housing Spinifex nodes — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd`, `spinifex-ui`, and the OVN control plane — together with the network equipment, cabling, console/KVM access paths, and backup media that support them. Tenant workloads and operator endpoints (laptops, jump hosts) are out of scope. + +## CMMC Practices Covered + +PE.L1 objectives are discharged by the operator's facility, not by Spinifex. This guide names the assets, cadences, and evidence required. + +| Practice | Title | Objective | +|----------|-------|-----------| +| PE.L1-3.10.1 | Limit Physical Access | [a] Authorized individuals allowed physical access are identified. [b] Physical access to organizational systems, equipment, and operating environments is limited to authorized individuals. | +| PE.L1-3.10.3 | Escort Visitors | [a] Visitors are escorted. [b] Visitor activity is monitored. | +| PE.L1-3.10.4 | Physical Access Logs | [a] Audit logs of physical access are maintained. | +| PE.L1-3.10.5 | Manage Physical Access Devices | [a] Physical access devices are identified. [b] Physical access devices are controlled. [c] Physical access devices are managed. | + +## Approach + +Spinifex does not mandate a specific access-control product. Operators typically deploy into a facility that already has badge readers, CCTV, and a visitor-management system (Kisi, HID, Lenel, Genetec, Envoy, Traction Guest, paper logs, etc.). Manual procedures (locked cabinet, paper sign-in, tracked key list) are acceptable for small sites provided the evidence trail exists; multi-rack sites should feed an electronic access-control system into the same SIEM used for Spinifex service logs. + +## 1. Protected Assets + +The physical protection boundary must enclose every item in this table. An asset is "protected" when access to it requires passing through a controlled barrier (locked room, cage, or cabinet). + +| Asset | Why it must be inside the boundary | +|-------|-----------------------------------| +| Spinifex node chassis | Host console, BMC, and disks hold `/etc/spinifex/master.key`, cluster CA key, per-node TLS keys, and all tenant volume data. | +| Network switches and routers serving the cluster subnet | Physical access permits traffic capture, port mirroring, and control-plane tampering. | +| Structured cabling (top-of-rack to host, host to storage) | Passive taps are trivial on unprotected cabling. | +| Console / KVM / serial aggregators | Bypass host authentication; reach GRUB, single-user mode, BMC. | +| Backup media (tapes, removable disks, off-site copies) | Carry the same data as primary storage; covered by MP.L1-3.8.3 for disposal. | +| Facility power and HVAC cutoffs serving the rack | Unauthorised de-energise is a denial-of-service and a risk to in-flight writes. | + +Nodes deployed in an unstaffed remote or edge location must be installed in a locked enclosure with tamper-evident seals and, where feasible, a sensor (door contact, accelerometer) feeding the central monitoring system. Record the enclosure location, seal serial, and sensor channel in the asset register. + +## 2. Limit Physical Access (PE.L1-3.10.1) + +### 2.1 Authorized Individuals ([a]) + +Maintain a written access list naming every individual authorized to enter each protected space. For each entry record: + +- Full name and employing organization. +- Role justifying access (e.g. "Spinifex operator", "facilities", "vendor: OEM field service"). +- Scope — which protected spaces, and whether escorted or unescorted. +- Start date, scheduled review date, and end date when access is removed. +- Approver (named individual, not a role mailbox). + +Review the list at least quarterly and immediately on personnel change (role change, departure, contractor end-of-engagement). Revocations must be effective in the access-control system promptly per operator policy. + +### 2.2 Enforcement ([b]) + +Access to protected spaces must require authentication at the barrier — a badge, PIN, key, biometric, or combination. Unaccompanied access by individuals not on the list in [§2.1](#21-authorized-individuals-a) must not be possible. Specifically: + +- Doors and cages: electronic access control (badge/PIN) with door-forced and door-held alarms wired to the monitoring system. Mechanical-only locks are acceptable for lab/edge deployments provided key distribution is tracked under [§5](#5-manage-physical-access-devices-pel1-3105). +- Racks and cabinets holding Spinifex nodes: locked at all times when unattended. Key or combination distribution tracked as an access device under [§5](#5-manage-physical-access-devices-pel1-3105). +- Remote/edge enclosures: locked with tamper-evident seal; seal integrity checked on every site visit and recorded in the visit log. + +Shared credentials (one badge used by several people, a rack key left in the cage) are not acceptable. + +## 3. Escort and Monitor Visitors (PE.L1-3.10.3) + +A visitor is any individual entering a protected space who is not on the authorized-access list in [§2.1](#21-authorized-individuals-a). This includes vendor field engineers, auditors, janitorial staff, and employees of the operator who do not hold Spinifex access. + +### 3.1 Escort ([a]) + +- Every visitor must be signed in by a named authorized escort before entering the protected space. +- The escort must remain physically present with the visitor for the duration of the visit. Passing a visitor between escorts is permitted; leaving a visitor unaccompanied is not. +- Work inside a rack or at a node console requires one escort per visitor. + +### 3.2 Monitoring ([b]) + +Visitor activity must be monitored by at least one of: + +- Continuous physical presence of the escort in line-of-sight of the visitor. This is the minimum. +- CCTV coverage of the protected space with recordings retained for at least 30 days. Cameras must cover rack fronts and rears, door entries, and any console/KVM positions. +- For vendor maintenance involving system access (BMC, console, disk swap): a second operator witness on-site, or a recorded screen share if the work is performed from the console. Record the witness or session recording ID in the visitor log. + +CCTV is strongly recommended for any facility hosting more than a single rack. It also serves as corroborating evidence for [§4](#4-physical-access-logs-pel1-3104) logs and is useful for incident response. + +## 4. Physical Access Logs (PE.L1-3.10.4) + +### 4.1 What to Log + +Every entry into a protected space must produce a log entry capturing: + +| Field | Source | +|-------|--------| +| Identity (name, badge ID, or visitor record ID) | Access control system or sign-in sheet. | +| Timestamp in and timestamp out | Access control system; manual for paper logs. | +| Protected space entered | Door/cage ID. | +| Purpose | Free-text; required for visitors, recommended for authorized staff. | +| Escort (for visitors) | Named individual from the authorized list. | +| Associated change or ticket ID | When the visit is tied to a Spinifex change (upgrade, disk swap, node rebuild). | + +### 4.2 Retention and Review + +- **Retention:** at least 12 months. CCTV recordings associated with the same visit, where used to discharge [§3.2](#32-monitoring-b), should be retained for the same period. +- **Review cadence:** monthly spot-check of 10% of entries, quarterly full review against the authorized-access list. Discrepancies (unknown badge ID, visitor with no matching sign-in, escort named who was off-site) must be investigated and the finding recorded. +- **Alerting:** out-of-hours entries, repeated failed reads at a single door, and door-forced / door-held events must fire an alert to the on-call operator. Treat these as incidents until triaged. + +### 4.3 Integration with Spinifex Logs + +Forward physical-access events to the same SIEM or log collector used for Spinifex service logs (see [Malware Protection §3](https://docs.mulgadc.com/docs/security/malware-protection#3-scan-schedule-sil1-3145)) so a `master.key` access attempt can be correlated with a physical entry into the rack. + +## 5. Manage Physical Access Devices (PE.L1-3.10.5) + +Physical access devices are badges, key cards, PINs, mechanical keys, rack keys, safe combinations, and tamper seals. USB tokens or HSMs holding the cluster CA key or master encryption key also belong in the register. + +### 5.1 Identification ([a]) + +Maintain a device register with one entry per issued device. Minimum fields: + +| Field | Notes | +|-------|-------| +| Device ID | Badge serial, key stamp, seal serial, HSM serial. | +| Device type | Badge / mechanical key / PIN / combination / seal / hardware token. | +| Scope | Which barriers it opens, or which asset it protects. | +| Holder | Named individual. Shared holders are not acceptable for badges or PINs. | +| Issued date, issued by | Audit trail for [c]. | +| Returned or revoked date, reason | Populated on personnel change. | + +### 5.2 Control ([b]) + +- Issue only to individuals named in the [§2.1](#21-authorized-individuals-a) access list. Scope of the device must not exceed that individual's authorized scope. +- Badges and PINs must be unique per holder. Mechanical keys that must be shared (e.g. a single rack key) are permitted only when distribution is tracked by check-out/check-in against the register. +- Lost or compromised devices trigger immediate revocation: badges deactivated in the access-control system, PINs changed, affected mechanical locks re-keyed within 30 days, tamper seals re-applied on the next site visit. Record the event and remediation in the register. +- Terminated or reassigned personnel surrender all devices on last day. Mark returned in the register. + +### 5.3 Management ([c]) + +- **Inventory review:** at least quarterly, reconcile the device register against (a) the access-control system's badge database, (b) the physical key count, and (c) the authorized-access list in [§2.1](#21-authorized-individuals-a). Discrepancies must be resolved before the review is marked complete. +- **Rotation / re-keying:** mechanical locks re-keyed on loss of any key, and at least every five years. Default combinations (safe, cabinet, BMC default passwords at install) must be changed before the device enters service. +- **Spares and master keys:** held in a secured location (locked cabinet or safe inside a protected space), accessible only to named individuals, with any access itself logged under [§4](#4-physical-access-logs-pel1-3104). + +## 6. Evidence and Record Keeping + +For CMMC assessment, retain the following for at least 12 months (longer where facility or contract policy requires): + +- **Authorized access list** ([§2.1](#21-authorized-individuals-a)) with review dates and approver names. +- **Access-control system configuration** showing which badges are permitted at which readers, exported or screenshotted at each quarterly review. +- **Physical access logs** ([§4](#4-physical-access-logs-pel1-3104)) and, where applicable, CCTV retention policy and sample recordings. +- **Visitor logs** with escort names for every visit. +- **Device register** ([§5.1](#51-identification-a)) with issue, return, and revocation history. +- **Incident records** for door-forced/door-held events, lost devices, failed seal checks, and any access-review discrepancies, with remediation outcomes. +- **Annual review attestation** from the facility or security owner confirming the controls above operated for the prior 12 months. + +## 7. Operator Checklist + +Use this list to confirm a site meets the four CMMC practices before admitting Spinifex nodes to a production cluster: + +- Authorized-access list exists, names every protected space, is reviewed at least quarterly, and has an approver for every entry. +- Every protected space enforces authentication at the barrier; shared credentials are not in use for badged/PIN'd spaces. +- Remote/edge enclosures are locked, sealed, and listed in the asset register with seal serials recorded. +- Visitor procedure is documented: sign-in, escort (one-per-visitor at equipment), sign-out. +- CCTV or equivalent monitoring covers the rack fronts, rears, and console positions, with 30-day minimum retention. +- Physical access logs are produced for every entry, retained 12 months, and reviewed on the cadence in [§4.2](#42-retention-and-review). +- Door-forced, door-held, and out-of-hours events alert the on-call operator. +- Physical access events forwarded to the SIEM used for Spinifex service logs (see [Malware Protection §3](https://docs.mulgadc.com/docs/security/malware-protection#3-scan-schedule-sil1-3145)). +- Device register reconciles quarterly against the access-control system and physical key count; discrepancies are closed before sign-off. +- Lost-device and termination procedures trigger immediate revocation per [§5.2](#52-control-b). +- System security plan references this guide and records the facility, access-control product, CCTV retention, and log-forwarding destination.