Merge branch 'main' into feat/iso-installer

.github/workflows/e2e.yml

@@ -83,7 +83,7 @@ jobs:
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            'cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh  && ./run-lb-e2e.sh'
+            "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
 
       - name: Teardown
         if: always()

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev/"><img src="https://img.shields.io/badge/Go-1.22+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.22+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
   <a href="https://github.com/mulgadc/spinifex/actions"><img src="https://img.shields.io/badge/CI-e2e-58a6ff?style=flat-square" alt="CI"></a>
@@ -130,7 +130,7 @@ Each component can be developed independently. See component-specific documentat
 Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src="docs/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
 </p>
 
 The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:

build/systemd/spinifex-awsgw.service

@@ -23,7 +23,7 @@ Environment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
-ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex
 
 # Hardening

build/systemd/spinifex-daemon.service

@@ -23,7 +23,7 @@ PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
-ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
 
 # Hardening

build/systemd/spinifex-nats.service

@@ -17,7 +17,7 @@ Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
-ReadOnlyPaths=/etc/spinifex/nats
+ReadOnlyPaths=/etc/spinifex/nats /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex
 
 # Hardening

build/systemd/spinifex-predastore.service

@@ -22,7 +22,7 @@ Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
-ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex
 
 # Hardening

build/systemd/spinifex-viperblock.service

@@ -21,7 +21,7 @@ Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
 NoNewPrivileges=yes
 ProtectSystem=strict
 PrivateDevices=yes
-ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd
 
 # Hardening

build/systemd/spinifex-vpcd.service

@@ -19,7 +19,7 @@ Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
 # Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl)
 ProtectSystem=full
 PrivateDevices=yes
-ReadOnlyPaths=/etc/spinifex/spinifex.toml
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
 ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
 
 # Hardening

cmd/spinifex/cmd/admin.go

@@ -23,6 +23,7 @@ import (
 	_ "embed"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -252,11 +253,12 @@ func init() {
 	adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster")
 	adminInitCmd.Flags().String("host", "", "Leader node to join (if not specified, tries multicast discovery)")
 	adminInitCmd.Flags().Int("port", 4432, "Port to bind cluster services on")
-	adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node)")
+	adminInitCmd.Flags().String("bind", "127.0.0.1", "IP address to bind services to (e.g., 10.11.12.1 for multi-node)")
 	adminInitCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
 	adminInitCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
 	adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
 	adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
+	adminInitCmd.Flags().String("token-ttl", "30m", "Join token validity duration (e.g. 30m, 1h, 2h)")
 	adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name")
 	adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)")
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")
@@ -281,10 +283,12 @@ func init() {
 	adminJoinCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.2 for multi-node on single host)")
 	adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
 	adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
+	adminJoinCmd.Flags().String("token", "", "Join token from the init node (required)")
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
 	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.MarkFlagRequired("node")
 	adminJoinCmd.MarkFlagRequired("host")
+	adminJoinCmd.MarkFlagRequired("token")
 
 	imagesImportCmd.Flags().String("tmp-dir", os.TempDir(), "Temporary directory for image import processing")
 
@@ -296,6 +300,7 @@ func init() {
 	imagesImportCmd.Flags().String("platform", "Linux/UNIX", "Specified platform (e.g Linux/UNIX, Windows)")
 	imagesImportCmd.Flags().StringSlice("tag", nil, "Tag to apply to the imported AMI as key=value (repeatable; e.g. --tag spinifex:managed-by=elbv2)")
 	imagesImportCmd.Flags().Bool("force", false, "Force command execution (overwrites existing files)")
+	imagesImportCmd.Flags().Bool("skip-verify", false, "Skip catalog-image checksum verification (INSECURE; operator assumes integrity responsibility)")
 }
 
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
@@ -307,6 +312,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 
 	cfgFile, _ := cmd.Flags().GetString("config")
 	forceCmd, _ := cmd.Flags().GetBool("force")
+	skipVerify, _ := cmd.Flags().GetBool("skip-verify")
 	ostmpDir, _ := cmd.Flags().GetString("tmp-dir")
 
 	// Use default config path
@@ -409,16 +415,12 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	fmt.Printf("✅ Created config directory: %s\n", imagePath)
-
 	// Next, if the file is selected to download, fetch it, extract disk image, and save to path
 	if imageName != "" && localFile == "" {
 		// Download the file to the image path
 		filename := path.Base(image.URL)
 		imageFile = fmt.Sprintf("%s/%s", imagePath, filename)
 
-		fmt.Printf("Downloading image %s to %s\n", image.URL, imageFile)
-
 		// If image path exists, skip
 		if admin.FileExists(imageFile) && !forceCmd {
 			fmt.Printf("Image file already exists, skipping download, use --force to overwrite: %s\n", imageFile)
@@ -431,8 +433,23 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 			}
 		}
 
-		// Update image file path for later extraction
-		//imagePath = imageFilePath
+		// Verify before extract: the catalog digest is of the artifact as it
+		// sits on the mirror (.tar.xz/.img/.raw). Also catches a poisoned
+		// cache on the re-run path — failure leaves the file on disk so the
+		// operator can inspect; recover with --force.
+		if skipVerify {
+			fmt.Fprintf(os.Stderr, "⚠️  --skip-verify set: checksum verification skipped for %s\n", imageName)
+		} else {
+			if image.Checksum == "" || image.ChecksumType == "" {
+				fmt.Fprintf(os.Stderr, "Catalog entry %q is missing Checksum/ChecksumType; refusing import.\n", imageName)
+				os.Exit(1)
+			}
+			if err := utils.VerifyImageChecksum(imageFile, image.Checksum, image.ChecksumType); err != nil {
+				printChecksumError(os.Stderr, imageFile, imageName, image, err)
+				os.Exit(1)
+			}
+			fmt.Printf("✅ Verified image checksum (%s)\n", image.ChecksumType)
+		}
 	}
 
 	// Next, validate if the image is raw, tar, gz, xv, etc. We need to upload the raw image
@@ -444,9 +461,6 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	}
 
 	extractedImagePath, err := utils.ExtractDiskImageFromFile(imageFile, imagePath)
-
-	fmt.Println("Extracted image to:", extractedImagePath)
-
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Could not extract image: %v\n", err)
 		os.Exit(1)
@@ -615,6 +629,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 	predastoreNodesStr, _ := cmd.Flags().GetString("predastore-nodes")
 	formationTimeoutStr, _ := cmd.Flags().GetString("formation-timeout")
+	tokenTTLStr, _ := cmd.Flags().GetString("token-ttl")
 	clusterName, _ := cmd.Flags().GetString("cluster-name")
 	services, _ := cmd.Flags().GetStringSlice("services")
 
@@ -873,7 +888,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	spxRoot = filepath.Clean(spxRoot)
 
 	// Determine if this is a multi-node formation
-	isMultiNode := nodes >= 2 && bindIP != "0.0.0.0"
+	isMultiNode := nodes >= 2 && bindIP != "0.0.0.0" && bindIP != "127.0.0.1"
 
 	if isMultiNode {
 		// Build cluster-wide network config for propagation to joining nodes
@@ -904,7 +919,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName,
 			configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind,
-			port, nodes, formationTimeoutStr, services, networkConfig)
+			port, nodes, formationTimeoutStr, tokenTTLStr, services, networkConfig)
 		return
 	}
 
@@ -913,17 +928,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// Create config files from embedded templates
 	fmt.Println("\n📝 Creating configuration files...")
 
-	// Create subdirectories
-	awsgwDir := filepath.Join(configDir, "awsgw")
-	predastoreDir := filepath.Join(configDir, "predastore")
-	natsDir := filepath.Join(configDir, "nats")
-	spinifexDir := filepath.Join(configDir, "spinifex")
-
-	for _, dir := range []string{awsgwDir, predastoreDir, natsDir, spinifexDir} {
-		if err := os.MkdirAll(dir, 0700); err != nil {
-			fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
-			os.Exit(1)
-		}
+	dirs, err := createConfigSubdirs(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating config subdirectories: %v\n", err)
+		os.Exit(1)
 	}
 
 	portStr := strconv.Itoa(port)
@@ -963,7 +971,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
-		predastorePath := filepath.Join(predastoreDir, "predastore.toml")
+		predastorePath := filepath.Join(dirs.Predastore, "predastore.toml")
 		if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil {
 			fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
 			os.Exit(1)
@@ -1041,38 +1049,12 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 
 	// Generate config files
-	configs := []admin.ConfigFile{
-		{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
-		{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
-		{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
-	}
-	// Skip template-based predastore.toml if multi-node was already generated
-	if predastoreNodesStr == "" {
-		configs = append(configs, admin.ConfigFile{
-			Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
-		})
-	}
-
-	if err := admin.GenerateConfigFiles(configs, configSettings); err != nil {
+	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, predastoreNodesStr != ""); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
 		os.Exit(1)
 	}
 
-	// Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system)
-	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, certPath, bindIP); err != nil {
-		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
-	} else {
-		fmt.Println("✅ AWS credentials configured")
-	}
-
-	admin.CreateServiceDirectories(spxRoot)
-
-	// In production layout (running as root), set per-service ownership on
-	// directories and shared config files (root:spinifex with correct modes).
-	if os.Getuid() == 0 {
-		admin.SetServiceOwnership()
-	}
+	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP)
 
 	// Print success message
 	fmt.Println("\n🎉 Spinifex initialization complete!")
@@ -1088,13 +1070,29 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 // then generates configs with complete cluster topology.
 func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName,
 	configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind string,
-	port, expectedNodes int, formationTimeoutStr string, services []string, networkConfig *formation.NetworkConfig) {
+	port, expectedNodes int, formationTimeoutStr, tokenTTLStr string, services []string, networkConfig *formation.NetworkConfig) {
 	formationTimeout, err := time.ParseDuration(formationTimeoutStr)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err)
 		os.Exit(1)
 	}
 
+	tokenTTL, err := time.ParseDuration(tokenTTLStr)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: Invalid --token-ttl: %v\n", err)
+		os.Exit(1)
+	}
+	if tokenTTL < formationTimeout+1*time.Minute {
+		fmt.Fprintf(os.Stderr, "❌ Error: --token-ttl (%s) must be >= --formation-timeout + 1m (%s)\n", tokenTTL, formationTimeout+1*time.Minute)
+		os.Exit(1)
+	}
+
+	joinToken, err := formation.GenerateJoinToken()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating join token: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Generate IAM master key for the cluster
 	masterKey, err := handlers_iam.GenerateMasterKey()
 	if err != nil {
@@ -1133,7 +1131,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		AdminSecretKey: bootstrapResult.AdminSecretKey,
 	}
 
-	fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig)
+	fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig, joinToken, tokenTTL)
 
 	// Include master key in formation server for distribution to joining nodes
 	fs.SetMasterKey(base64.StdEncoding.EncodeToString(masterKey))
@@ -1152,6 +1150,13 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
+	// Write join token to file for automated workflows
+	tokenPath := filepath.Join(configDir, "join-token")
+	if err := os.WriteFile(tokenPath, []byte(joinToken), 0600); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error writing join token: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Start formation server
 	formationAddr := fmt.Sprintf("%s:%d", bindIP, port)
 	if err := fs.Start(formationAddr); err != nil {
@@ -1161,12 +1166,15 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	fmt.Printf("\n📡 Formation server started on %s\n", formationAddr)
 	fmt.Printf("   Waiting for %d more node(s) to join...\n", expectedNodes-1)
-	fmt.Printf("   Other nodes should run: spx admin join --host %s --node <name> --bind <ip>\n\n", formationAddr)
+	fmt.Printf("   Token expires in %s\n\n", tokenTTL)
+	fmt.Printf("   Other nodes should run:\n")
+	fmt.Printf("   spx admin join --host %s --token %s --node <name> --bind <ip>\n\n", formationAddr, joinToken)
 
 	// Wait for all nodes to register
 	if err := fs.WaitForCompletion(formationTimeout); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ %v\n", err)
 		fs.Shutdown(context.Background())
+		os.Remove(tokenPath)
 		os.Exit(1)
 	}
 
@@ -1179,31 +1187,25 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	fmt.Println("\n📝 Creating configuration files...")
 
-	// Create subdirectories
-	awsgwDir := filepath.Join(configDir, "awsgw")
-	predastoreDir := filepath.Join(configDir, "predastore")
-	natsDir := filepath.Join(configDir, "nats")
-	spinifexDir := filepath.Join(configDir, "spinifex")
-
-	for _, dir := range []string{awsgwDir, predastoreDir, natsDir, spinifexDir} {
-		if err := os.MkdirAll(dir, 0700); err != nil {
-			fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
-			os.Exit(1)
-		}
+	dirs, err := createConfigSubdirs(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating config subdirectories: %v\n", err)
+		os.Exit(1)
 	}
 
 	portStr := strconv.Itoa(port)
 
 	// Generate multi-node predastore config
 	var predastoreNodeID int
-	if len(predastoreNodes) >= 3 {
+	hasPredastoreConfig := len(predastoreNodes) >= 3
+	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
 		}
 
-		predastorePath := filepath.Join(predastoreDir, "predastore.toml")
+		predastorePath := filepath.Join(dirs.Predastore, "predastore.toml")
 		if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil {
 			fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
 			os.Exit(1)
@@ -1242,67 +1244,16 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		OVNSBAddr: "tcp:127.0.0.1:6642",
 	}
 
-	// Apply cluster-wide network config to init node's config
 	if networkConfig != nil {
-		configSettings.ExternalMode = networkConfig.ExternalMode
-		configSettings.ExternalDHCP = networkConfig.ExternalDHCP
-		configSettings.PoolName = networkConfig.PoolName
-		configSettings.PoolSource = networkConfig.PoolSource
-		configSettings.PoolStart = networkConfig.PoolStart
-		configSettings.PoolEnd = networkConfig.PoolEnd
-		configSettings.PoolGateway = networkConfig.PoolGateway
-		configSettings.PoolGatewayIP = networkConfig.PoolGatewayIP
-		configSettings.PoolPrefixLen = networkConfig.PoolPrefixLen
-		configSettings.PoolDNSServers = networkConfig.PoolDNSServers
-
-		configSettings.BootstrapAccountId = networkConfig.BootstrapAccountId
-		configSettings.BootstrapVpcId = networkConfig.BootstrapVpcId
-		configSettings.BootstrapSubnetId = networkConfig.BootstrapSubnetId
-		configSettings.BootstrapIgwId = networkConfig.BootstrapIgwId
-		configSettings.BootstrapCidr = networkConfig.BootstrapCidr
-		configSettings.BootstrapSubnetCidr = networkConfig.BootstrapSubnetCidr
-
-		// Auto-detect local network topology for per-node config
-		detected, err := admin.DetectNetwork()
-		if err == nil && detected.WAN != nil {
-			configSettings.ExternalIface = detected.WAN.Name
-			configSettings.WanBridge = detectedWanBridge(detected)
-		}
+		applyNetworkConfig(&configSettings, networkConfig)
 	}
 
-	// Generate config files
-	configs := []admin.ConfigFile{
-		{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
-		{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
-		{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
-	}
-	// Skip template-based predastore.toml if multi-node was generated
-	if len(predastoreNodes) < 3 {
-		configs = append(configs, admin.ConfigFile{
-			Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
-		})
-	}
-
-	if err := admin.GenerateConfigFiles(configs, configSettings); err != nil {
+	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
 		os.Exit(1)
 	}
 
-	// Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system)
-	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, certPath, bindIP); err != nil {
-		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
-	} else {
-		fmt.Println("✅ AWS credentials configured")
-	}
-
-	admin.CreateServiceDirectories(spxRoot)
-
-	// In production layout (running as root), set per-service ownership on
-	// directories and shared config files (root:spinifex with correct modes).
-	if os.Getuid() == 0 {
-		admin.SetServiceOwnership()
-	}
+	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP)
 
 	// Keep formation server running briefly so joining nodes can fetch complete status
 	fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...")
@@ -1310,6 +1261,9 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	// Shutdown formation server
 	fs.Shutdown(context.Background())
+	if err := os.Remove(tokenPath); err != nil && !errors.Is(err, os.ErrNotExist) {
+		slog.Warn("Failed to remove join token file", "path", tokenPath, "error", err)
+	}
 
 	// Print cluster summary
 	fmt.Println("\n🎉 Cluster formation complete!")
@@ -1334,6 +1288,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	node, _ := cmd.Flags().GetString("node")
 	leaderHost, _ := cmd.Flags().GetString("host")
+	joinToken, _ := cmd.Flags().GetString("token")
 	region, _ := cmd.Flags().GetString("region")
 	az, _ := cmd.Flags().GetString("az")
 	dataDir, _ := cmd.Flags().GetString("data-dir")
@@ -1416,7 +1371,15 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 
 	joinURL := fmt.Sprintf("https://%s/formation/join", leaderHost)
-	resp, err := client.Post(joinURL, "application/json", bytes.NewBuffer(reqBody))
+	req, err := http.NewRequest(http.MethodPost, joinURL, bytes.NewBuffer(reqBody))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error creating join request: %v\n", err)
+		os.Exit(1)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+joinToken)
+
+	resp, err := client.Do(req)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err)
 		fmt.Fprintf(os.Stderr, "Make sure the leader node has run 'spx admin init' and is accessible at %s\n", leaderHost)
@@ -1448,7 +1411,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	var statusResp formation.StatusResponse
 
 	for {
-		sResp, err := client.Get(statusURL)
+		statusReq, err := http.NewRequest(http.MethodGet, statusURL, nil)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error creating status request: %v\n", err)
+			os.Exit(1)
+		}
+		statusReq.Header.Set("Authorization", "Bearer "+joinToken)
+
+		sResp, err := client.Do(statusReq)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "❌ Error polling formation status: %v\n", err)
 			os.Exit(1)
@@ -1461,8 +1431,17 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
+		if sResp.StatusCode == http.StatusUnauthorized {
+			fmt.Fprintf(os.Stderr, "❌ Error: join token rejected by formation server (expired or invalid)\n")
+			os.Exit(1)
+		}
+		if sResp.StatusCode != http.StatusOK {
+			fmt.Fprintf(os.Stderr, "❌ Error: unexpected status %d from formation server\n", sResp.StatusCode)
+			os.Exit(1)
+		}
+
 		if err := json.Unmarshal(sBody, &statusResp); err != nil {
-			fmt.Fprintf(os.Stderr, "Error parsing status response: %v\n", err)
+			fmt.Fprintf(os.Stderr, "❌ Error parsing status response: %v\n", err)
 			os.Exit(1)
 		}
 
@@ -1574,17 +1553,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	fmt.Println("📝 Creating configuration files...")
 
-	// Create subdirectories
-	awsgwDir := filepath.Join(configDir, "awsgw")
-	predastoreDir := filepath.Join(configDir, "predastore")
-	natsDir := filepath.Join(configDir, "nats")
-	spinifexDir := filepath.Join(configDir, "spinifex")
-
-	for _, dir := range []string{awsgwDir, predastoreDir, natsDir, spinifexDir} {
-		if err := os.MkdirAll(dir, 0700); err != nil {
-			fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
-			os.Exit(1)
-		}
+	dirs, err := createConfigSubdirs(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating config subdirectories: %v\n", err)
+		os.Exit(1)
 	}
 
 	portStr := strconv.Itoa(port)
@@ -1600,7 +1572,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
-		predastorePath := filepath.Join(predastoreDir, "predastore.toml")
+		predastorePath := filepath.Join(dirs.Predastore, "predastore.toml")
 		if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil {
 			fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
 			os.Exit(1)
@@ -1643,71 +1615,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
 	}
 
-	// Apply cluster-wide network config from leader
 	if statusResp.NetworkConfig != nil {
-		nc := statusResp.NetworkConfig
-		configSettings.ExternalMode = nc.ExternalMode
-		configSettings.ExternalDHCP = nc.ExternalDHCP
-		configSettings.PoolName = nc.PoolName
-		configSettings.PoolSource = nc.PoolSource
-		configSettings.PoolStart = nc.PoolStart
-		configSettings.PoolEnd = nc.PoolEnd
-		configSettings.PoolGateway = nc.PoolGateway
-		configSettings.PoolGatewayIP = nc.PoolGatewayIP
-		configSettings.PoolPrefixLen = nc.PoolPrefixLen
-		configSettings.PoolDNSServers = nc.PoolDNSServers
-
-		configSettings.BootstrapAccountId = nc.BootstrapAccountId
-		configSettings.BootstrapVpcId = nc.BootstrapVpcId
-		configSettings.BootstrapSubnetId = nc.BootstrapSubnetId
-		configSettings.BootstrapIgwId = nc.BootstrapIgwId
-		configSettings.BootstrapCidr = nc.BootstrapCidr
-		configSettings.BootstrapSubnetCidr = nc.BootstrapSubnetCidr
-
-		// Auto-detect local network topology for per-node config
-		if nc.ExternalMode != "" {
-			detected, err := admin.DetectNetwork()
-			if err == nil && detected.WAN != nil {
-				configSettings.ExternalIface = detected.WAN.Name
-				configSettings.WanBridge = detectedWanBridge(detected)
-			}
-		}
+		applyNetworkConfig(&configSettings, statusResp.NetworkConfig)
 	}
 
-	// Generate config files
-	configs := []admin.ConfigFile{
-		{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
-		{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
-		{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
-	}
-	// Skip template-based predastore.toml if multi-node was generated
-	if !hasPredastoreConfig {
-		configs = append(configs, admin.ConfigFile{
-			Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
-		})
-	}
-
-	err = admin.GenerateConfigFiles(configs, configSettings)
-	if err != nil {
+	if err := generateAndWriteConfigs(dirs, spinifexTomlPath, configSettings, hasPredastoreConfig); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
 		os.Exit(1)
 	}
 
-	// Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system)
-	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, caCertPath, bindIP); err != nil {
-		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
-	} else {
-		fmt.Println("✅ AWS credentials configured")
-	}
-
-	admin.CreateServiceDirectories(dataDir)
-
-	// In production layout (running as root), set per-service ownership on
-	// directories and shared config files (root:spinifex with correct modes).
-	if os.Getuid() == 0 {
-		admin.SetServiceOwnership()
-	}
+	finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP)
 
 	// Print cluster summary
 	fmt.Println("\n🎉 Node successfully joined cluster!")
@@ -1941,6 +1858,93 @@ func runCertRenew(cmd *cobra.Command, _ []string) {
 	fmt.Println("\n⚠️  Restart awsgw and daemon services to pick up the new certificate.")
 }
 
+// configDirs holds the paths to config subdirectories created by createConfigSubdirs.
+type configDirs struct {
+	AWSGW      string
+	Predastore string
+	NATS       string
+	Spinifex   string
+}
+
+// createConfigSubdirs creates the standard config subdirectories under configDir.
+func createConfigSubdirs(configDir string) (configDirs, error) {
+	dirs := configDirs{
+		AWSGW:      filepath.Join(configDir, "awsgw"),
+		Predastore: filepath.Join(configDir, "predastore"),
+		NATS:       filepath.Join(configDir, "nats"),
+		Spinifex:   filepath.Join(configDir, "spinifex"),
+	}
+	for _, dir := range []string{dirs.AWSGW, dirs.Predastore, dirs.NATS, dirs.Spinifex} {
+		if err := os.MkdirAll(dir, 0700); err != nil {
+			return configDirs{}, fmt.Errorf("create directory %s: %w", dir, err)
+		}
+	}
+	return dirs, nil
+}
+
+// generateAndWriteConfigs renders the standard config files (spinifex.toml,
+// awsgw.toml, nats.conf, and optionally predastore.toml) from templates.
+func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings admin.ConfigSettings, skipPredastore bool) error {
+	configs := []admin.ConfigFile{
+		{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
+		{Name: filepath.Join(dirs.AWSGW, "awsgw.toml"), Path: filepath.Join(dirs.AWSGW, "awsgw.toml"), Template: awsgwTomlTemplate},
+		{Name: filepath.Join(dirs.NATS, "nats.conf"), Path: filepath.Join(dirs.NATS, "nats.conf"), Template: natsConfTemplate},
+	}
+	if !skipPredastore {
+		configs = append(configs, admin.ConfigFile{
+			Name: filepath.Join(dirs.Predastore, "predastore.toml"), Path: filepath.Join(dirs.Predastore, "predastore.toml"), Template: predastoreTomlTemplate,
+		})
+	}
+	return admin.GenerateConfigFiles(configs, settings)
+}
+
+// finalizeNodeSetup configures AWS credentials, creates service directories,
+// and sets ownership when running as root.
+func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region, bindIP string) {
+	fmt.Println("\n🔧 Configuring AWS credentials...")
+	if err := admin.SetupAWSCredentials(adminAccessKey, adminSecretKey, region, certPath, bindIP); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
+	} else {
+		fmt.Println("✅ AWS credentials configured")
+	}
+
+	admin.CreateServiceDirectories(dataDir)
+
+	if os.Getuid() == 0 {
+		admin.SetServiceOwnership()
+	}
+}
+
+// applyNetworkConfig copies cluster-wide network settings from a formation
+// NetworkConfig into ConfigSettings and auto-detects the local WAN interface.
+func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkConfig) {
+	settings.ExternalMode = nc.ExternalMode
+	settings.ExternalDHCP = nc.ExternalDHCP
+	settings.PoolName = nc.PoolName
+	settings.PoolSource = nc.PoolSource
+	settings.PoolStart = nc.PoolStart
+	settings.PoolEnd = nc.PoolEnd
+	settings.PoolGateway = nc.PoolGateway
+	settings.PoolGatewayIP = nc.PoolGatewayIP
+	settings.PoolPrefixLen = nc.PoolPrefixLen
+	settings.PoolDNSServers = nc.PoolDNSServers
+
+	settings.BootstrapAccountId = nc.BootstrapAccountId
+	settings.BootstrapVpcId = nc.BootstrapVpcId
+	settings.BootstrapSubnetId = nc.BootstrapSubnetId
+	settings.BootstrapIgwId = nc.BootstrapIgwId
+	settings.BootstrapCidr = nc.BootstrapCidr
+	settings.BootstrapSubnetCidr = nc.BootstrapSubnetCidr
+
+	if nc.ExternalMode != "" {
+		detected, err := admin.DetectNetwork()
+		if err == nil && detected.WAN != nil {
+			settings.ExternalIface = detected.WAN.Name
+			settings.WanBridge = detectedWanBridge(detected)
+		}
+	}
+}
+
 // writeBootstrapResult holds the admin credentials so callers can
 // write them to ~/.aws/credentials instead of the system credentials.
 type writeBootstrapResult struct {
@@ -2337,3 +2341,17 @@ func resolveIfaceIP(iface string) string {
 	}
 	return ""
 }
+
+// printChecksumError writes the failure, the source URL (printed for every
+// error kind so 404/non-HTTPS/size-cap failures tell the operator which URL
+// to investigate), and the exact --force recovery command. The cached file
+// is left in place: an implicit auto-delete would mutate state inside
+// "verify", and a tampered artifact is forensically useful intact.
+func printChecksumError(w io.Writer, imageFile, imageName string, image utils.Images, err error) {
+	fmt.Fprintf(w, "Image integrity verification failed: %v\n", err)
+	fmt.Fprintf(w, "  file:     %s\n", imageFile)
+	fmt.Fprintf(w, "  source:   %s\n", image.Checksum)
+	fmt.Fprintln(w)
+	fmt.Fprintln(w, "The cached file was left in place. To re-download and retry:")
+	fmt.Fprintf(w, "  spx admin images import --name %s --force\n", imageName)
+}

cmd/spinifex/cmd/admin_test.go

@@ -0,0 +1,34 @@
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/stretchr/testify/assert"
+)
+
+// Regression guard: source URL must print for every error kind, not just
+// mismatch, so a 404/non-HTTPS/size-cap failure still tells the operator
+// which URL to investigate.
+func TestPrintChecksumError(t *testing.T) {
+	image := utils.Images{Checksum: "https://example.com/SUMS", ChecksumType: "sha512"}
+	const imageFile = "/var/lib/img.tar.xz"
+	const imageName = "debian-12-x86_64"
+
+	errs := []error{
+		fmt.Errorf("%w: expected abc got def", utils.ErrChecksumMismatch),
+		fmt.Errorf("%w: 404", utils.ErrChecksumFetchFailed),
+		errors.New("open /x: no such file"),
+	}
+	for _, e := range errs {
+		var buf bytes.Buffer
+		printChecksumError(&buf, imageFile, imageName, image, e)
+		out := buf.String()
+		assert.Contains(t, out, image.Checksum, "source URL must print for: %v", e)
+		assert.Contains(t, out, imageFile)
+		assert.Contains(t, out, "spx admin images import --name "+imageName+" --force")
+	}
+}

cmd/spinifex/cmd/get.go

@@ -90,7 +90,7 @@ func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) {
 	}
 
 	nodeConfig := cfg.Nodes[cfg.Node]
-	nc, err := utils.ConnectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token)
+	nc, err := utils.ConnectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token, nodeConfig.NATS.CACert)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to connect to NATS: %w", err)
 	}

cmd/spinifex/cmd/service.go

@@ -288,6 +288,7 @@ var viperblockStartCmd = &cobra.Command{
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:   nodeConfig.NATS.Host,
 			NatsToken:  nodeConfig.NATS.ACL.Token,
+			NatsCACert: nodeConfig.NATS.CACert,
 			PluginPath: pluginPath,
 			S3Host:     nodeConfig.Predastore.Host,
 			Bucket:     nodeConfig.Predastore.Bucket,
@@ -743,6 +744,7 @@ var vpcdStartCmd = &cobra.Command{
 		svc, err := service.New("vpcd", &vpcd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
+			NatsCACert:        nodeConfig.NATS.CACert,
 			OVNNBAddr:         nodeConfig.VPCD.OVNNBAddr,
 			OVNSBAddr:         nodeConfig.VPCD.OVNSBAddr,
 			BaseDir:           nodeConfig.BaseDir,

cmd/spinifex/cmd/templates/nats.conf

@@ -1,4 +1,4 @@
-# spinifex-config-version: 1
+# spinifex-config-version: 3
 # NATS Server Configuration
 # Generated by Spinifex admin init
 
@@ -9,12 +9,25 @@ listen: {{.BindIP}}:4222
 listen: 0.0.0.0:4222
 {{- end }}
 
+tls {
+  cert_file: "{{.ConfigDir}}/server.pem"
+  key_file:  "{{.ConfigDir}}/server.key"
+  ca_file:   "{{.ConfigDir}}/ca.pem"
+}
+
 {{- if and .ClusterBindIP .ClusterRoutes }}
 
 cluster {
   name: {{ .ClusterName }}
   listen: {{.ClusterBindIP}}:4248
 
+  tls {
+    cert_file: "{{.ConfigDir}}/server.pem"
+    key_file:  "{{.ConfigDir}}/server.key"
+    ca_file:   "{{.ConfigDir}}/ca.pem"
+    verify:    true
+  }
+
   routes = [
 {{- range .ClusterRoutes }}
     "nats-route://{{$.NatsToken}}@{{ . }}",
@@ -49,9 +62,5 @@ max_payload: 1MB
 max_pending: 64MB
 
 # Monitoring
-{{- if .BindIP }}
-http: {{.BindIP}}:8222
-{{- else }}
-http_port: 8222
-{{- end }}
+http: 127.0.0.1:8222
 

cmd/spinifex/cmd/templates/predastore-multinode.toml

@@ -1,6 +1,6 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # Generated for multi-node cluster
-version = "2"
+version = "3"
 region = "{{.Region}}"
 
 host = "0.0.0.0"
@@ -65,5 +65,6 @@ burst = 200
 [iam]
 nats_url = "nats://{{.BindIP}}:4222"
 nats_token = "{{.NatsToken}}"
+nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
 access_keys_bucket = "spinifex-iam-access-keys"

cmd/spinifex/cmd/templates/predastore.toml

@@ -1,6 +1,6 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # MULTI NODE used for testing
-version = "2"
+version = "3"
 region = "{{.Region}}"
 
 # Optionally configure port and host, or specify via daemon CLI options
@@ -99,5 +99,6 @@ burst = 200
 [iam]
 nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}}:4222"
 nats_token = "{{.NatsToken}}"
+nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
 access_keys_bucket = "spinifex-iam-access-keys"

cmd/spinifex/cmd/templates/spinifex.toml

@@ -1,5 +1,5 @@
 # Domain configuration file in TOML format.
-version = "1.0"
+version = "2"
 epoch = 1
 node = "{{.Node}}"
 {{- if .ExternalMode}}
@@ -69,6 +69,7 @@ config = "awsgw/awsgw.toml"
 
 [nodes.{{.Node}}.nats]
 host = "{{.BindIP}}:4222"
+cacert = "{{.ConfigDir}}/ca.pem"
 
 [nodes.{{.Node}}.nats.acl]
 token = "{{.NatsToken}}"

docs/COMMANDS.md

@@ -51,8 +51,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------|---------------|-------------|------------|--------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files<br>2. Root credentials printed once<br>3. master.key is 32 bytes, mode 0600<br>4. bootstrap.json consumed on first start<br>5. `--force` re-initializes existing config<br>6. AWS CLI profile `spx` auto-configured | **DONE** |
-| `spx admin join` | `--host` (required), `--node` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster<br>2. Missing host (error)<br>3. Missing node name (error) | **DONE** |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files<br>2. Root credentials printed once<br>3. master.key is 32 bytes, mode 0600<br>4. bootstrap.json consumed on first start<br>5. `--force` re-initializes existing config<br>6. AWS CLI profile `spx` auto-configured<br>7. Multi-node init generates join token and writes to `<config-dir>/join-token` | **DONE** |
+| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster<br>2. Missing host (error)<br>3. Missing node name (error)<br>4. Missing token (error)<br>5. Invalid token (401)<br>6. Expired token (401) | **DONE** |
 
 ### Version
 
@@ -89,9 +89,40 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------|---------------|-------------|------------|--------|
-| `spx admin images import` | `--name` | Cluster must be running, image file must exist locally | Imports a local disk image (QCOW2) as an AMI → uploads to Predastore S3 → registers as available AMI | 1. Import valid QCOW2 image<br>2. Image appears in `describe-images` | **DONE** |
+| `spx admin images import` | `--name`, `--file`, `--force`, `--skip-verify` | Cluster must be running; either `--name` (catalog download) or `--file` (operator-supplied media) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, and verify the SHA-256/SHA-512 digest before extraction. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip verification — operator-supplied media is outside Spinifex's trust boundary, and the skip is logged at INFO for audit. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. | 1. Import valid catalog image (verifies checksum)<br>2. Tampered cache hit fails with `ErrChecksumMismatch`<br>3. `--force` recovers after a mismatch<br>4. `--file` import skips verification<br>5. `--skip-verify` bypasses checksum with WARN | **DONE** |
 | `spx admin images list` | — | None | Lists available OS images that can be imported or downloaded | 1. List available images | **DONE** |
 
+#### Image integrity verification (CMMC SI.L1-3.14.2)
+
+Catalog imports (`spx admin images import --name <name>`) verify the image
+against the catalog-declared SHA-256/SHA-512 digest before extraction. The sums
+file is fetched from the catalog `Checksum` URL over HTTPS only (cross-scheme
+redirects refused), and verification runs on both fresh downloads and cache
+hits so a poisoned cache is caught on the next import.
+
+On mismatch the import exits non-zero, the cached file is left on disk for
+inspection, and the printed guidance is `spx admin images import --name <name>
+--force` to re-download.
+
+`--file` imports skip verification by design: operator-supplied media is
+outside Spinifex's trust boundary and the operator is responsible for
+integrity (e.g. `sha256sum` against a trusted upstream digest before import).
+The skip is recorded as an INFO `slog` event with `reason=local-file-import`
+so a CMMC assessor can audit the decision from journald.
+
+`--skip-verify` bypasses the checksum step for catalog imports. The command
+still downloads via the catalog URL but does not compare the image digest
+against the sums file. Intended for narrow cases such as debugging upstream
+mirror issues or running against a transiently-broken `latest/` path; the
+skip is logged at WARN with `reason=skip-verify-flag` and printed to stderr
+so operators and assessors see it. Prefer `--file` with an out-of-band
+verified image over `--skip-verify` whenever possible.
+
+**Limitation:** verification confirms the image matches the digest the mirror
+served. A mirror compromise that swaps both image and sums file is not
+detected; closing that gap requires GPG signature verification of the sums
+file, deferred to a later phase.
+
 ## AWS Commands
 
 ### EC2 - Instance Management

docs/compute/vpc-networking/README.md

@@ -39,7 +39,7 @@ Spinifex maps AWS VPC concepts directly to OVN constructs:
 ## Network Path
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-network-path.svg" alt="VPC logical topology — WAN, br-wan, VPC logical router, subnets, ENIs" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-network-path.svg" alt="VPC logical topology — WAN, br-wan, VPC logical router, subnets, ENIs" width="900">
 </p>
 
 Cross-host traffic uses **Geneve tunnels** (UDP 6081) over the management/overlay
@@ -58,7 +58,7 @@ same VPC (even across subnets and hosts via the overlay). They cannot reach the
 internet or be reached from the WAN.
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-private-subnet-flow.svg" alt="Private subnet — instance hits router, no default route, packet dropped" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-private-subnet-flow.svg" alt="Private subnet — instance hits router, no default route, packet dropped" width="900">
 </p>
 
 If the VPC has an IGW attached, private subnet instances CAN reach the internet
@@ -71,7 +71,7 @@ Instances get both a private IP and a public IP. The public IP is a 1:1 NAT
 managed by OVN — the instance OS only sees its private IP.
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-public-subnet-flow.svg" alt="Public subnet — outbound SNAT and inbound DNAT between private and public IPs" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-public-subnet-flow.svg" alt="Public subnet — outbound SNAT and inbound DNAT between private and public IPs" width="900">
 </p>
 
 **Requirements for a public subnet:**
@@ -162,7 +162,7 @@ Regardless of whether IPs come from a static range or DHCP, the OVN behavior
 is identical:
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-dhcp-conversations.svg" alt="Two independent DHCP conversations — host-to-router and VM-to-OVN" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-dhcp-conversations.svg" alt="Two independent DHCP conversations — host-to-router and VM-to-OVN" width="900">
 </p>
 
 ### Choosing Static vs DHCP
@@ -252,7 +252,7 @@ The connection between them is logical, not physical — OVN's `localnet` port t
 maps the logical external switch to `br-wan` via `ovn-bridge-mappings`.
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-datapath-localnet.svg" alt="Data path — VM TAP through br-int, OVN pipeline, localnet, br-wan, WAN" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-datapath-localnet.svg" alt="Data path — VM TAP through br-int, OVN pipeline, localnet, br-wan, WAN" width="900">
 </p>
 
 ## Bridge Modes
@@ -267,7 +267,7 @@ The WAN NIC is added directly to `br-wan` as an OVS port. OVS sees all
 traffic on the wire — any MAC, any protocol. No filtering, no workarounds.
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-direct-bridge.svg" alt="Direct bridge mode — dedicated WAN NIC added directly to br-wan" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-direct-bridge.svg" alt="Direct bridge mode — dedicated WAN NIC added directly to br-wan" width="900">
 </p>
 
 **Use when:** The host has a NIC dedicated to external/WAN traffic that is NOT
@@ -293,7 +293,7 @@ macvlan is added to br-wan. The host keeps its IP on the original NIC —
 SSH stays up.
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-macvlan.svg" alt="Macvlan mode — WAN NIC shared between host traffic and macvlan child on br-wan" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-macvlan.svg" alt="Macvlan mode — WAN NIC shared between host traffic and macvlan child on br-wan" width="900">
 </p>
 
 **Use when:** The WAN NIC is the same NIC used for SSH. Single-NIC homelabs,
@@ -397,7 +397,7 @@ ip link show spx-ext-eth0
 ## Network Flow Diagram
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-host-flow.svg" alt="Bare-metal host — br-int overlay, br-wan WAN bridge with macvlan, OVN NAT pipeline" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-host-flow.svg" alt="Bare-metal host — br-int overlay, br-wan WAN bridge with macvlan, OVN NAT pipeline" width="900">
 </p>
 
 ## Configuration Reference
@@ -408,7 +408,7 @@ levels: cluster-wide mode, IP pool definitions, and per-node NIC settings.
 ## Configuration Levels
 
 <p align="center">
-  <img src="../../.github/assets/diagrams/vpc-config-layers.svg" alt="spinifex.toml configuration layers — cluster mode, IP pools, per-node NIC" width="900">
+  <img src="../../../.github/assets/diagrams/vpc-config-layers.svg" alt="spinifex.toml configuration layers — cluster mode, IP pools, per-node NIC" width="900">
 </p>
 
 ## Cluster-Wide: external_mode

docs/getting-started/install-multi-node/README.md

@@ -59,7 +59,7 @@ export AWS_AZ=us-east-1a
 
 ## Step 3. Setup OVN Networking
 
-Server 1 runs OVN central and must be set up first. If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC) or `--macvlan --wan-iface=eth0` (single-NIC).
+Server 1 runs OVN central and must be set up first. If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC).
 
 **Server 1:**
 
@@ -106,12 +106,24 @@ sudo spx admin init \
 
 Save the admin credentials — they will not be shown again.
 
+The init output displays the join command including the token:
+
+```
+📡 Formation server started on 10.0.0.1:4432
+   Waiting for 2 more node(s) to join...
+   Token expires in 30m0s
+
+   Other nodes should run:
+   spx admin join --host 10.0.0.1:4432 --token spx_join_a8Bf3x9Kz2mN --node <name> --bind <ip>
+```
+
 **Server 2 — Join** (while init is running):
 
 ```bash
 sudo spx admin join \
   --node node2 --bind $SPINIFEX_NODE2 --cluster-bind $SPINIFEX_NODE2 \
-  --host $SPINIFEX_NODE1:4432 --region $AWS_REGION --az $AWS_AZ
+  --host $SPINIFEX_NODE1:4432 --token <token-from-init-output> \
+  --region $AWS_REGION --az $AWS_AZ
 ```
 
 **Server 3 — Join** (while init is running):
@@ -119,9 +131,13 @@ sudo spx admin join \
 ```bash
 sudo spx admin join \
   --node node3 --bind $SPINIFEX_NODE3 --cluster-bind $SPINIFEX_NODE3 \
-  --host $SPINIFEX_NODE1:4432 --region $AWS_REGION --az $AWS_AZ
+  --host $SPINIFEX_NODE1:4432 --token <token-from-init-output> \
+  --region $AWS_REGION --az $AWS_AZ
 ```
 
+> **Note:** The join token expires 30 minutes after init by default. For larger deployments
+> with slower provisioning, use `--token-ttl 2h` on the init command.
+
 ## Step 5. Start Services
 
 On **all servers**:

docs/getting-started/install-source/README.md

@@ -65,14 +65,11 @@ If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-det
 ./scripts/setup-ovn.sh --management
 ```
 
-If your WAN is a physical NIC, choose one:
+If your WAN is a physical NIC:
 
 ```bash
 # Dedicated WAN NIC (not your SSH connection):
 ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1
-
-# Single-NIC host (SSH-safe macvlan):
-./scripts/setup-ovn.sh --management --macvlan --wan-iface=enp0s3
 ```
 
 ## Step 4. Development initialisation

docs/getting-started/setting-up-your-cluster/README.md

@@ -272,7 +272,9 @@ spx get vms
 
 Spinifex ships with a built-in web console — an alternative to the AWS CLI, SDKs, and Terraform, analogous to the AWS Management Console. Every action in the UI is a standard AWS SigV4 API call, so the same IAM policies and audit behaviour apply.
 
-![Spinifex Web UI](../spinifex-ui.jpg)
+<p align="center">
+  <img src="../../../.github/assets/spinifex-ui.jpg" alt="Spinifex Web UI" width="900">
+</p>
 
 ### Open the Console
 

docs/security/flaw-remediation-policy/README.md

@@ -0,0 +1,153 @@
+---
+title: "Flaw Remediation Policy"
+description: "CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex"
+category: "Security"
+sections:
+  - overview
+tags:
+  - security
+  - compliance
+  - cmmc
+  - vulnerabilities
+  - cvss
+  - patching
+resources:
+  - title: "NIST SP 800-171 Rev 3"
+    url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final"
+  - title: "CMMC Level 1 Self-Assessment Guide v2.0"
+    url: "https://dodcio.defense.gov/CMMC/Documentation"
+  - title: "FIRST CVSS v3.1 Specification"
+    url: "https://www.first.org/cvss/v3-1/specification-document"
+  - title: "Go govulncheck"
+    url: "https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck"
+  - title: "GitHub Dependabot"
+    url: "https://docs.github.com/en/code-security/dependabot"
+---
+
+# Flaw Remediation Policy
+
+> CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex
+
+## Table of Contents
+
+- [Overview](#overview)
+- [CMMC Practices Covered](#cmmc-practices-covered)
+- [Approach](#approach)
+- [1. Identification](#1-identification)
+- [2. Severity](#2-severity)
+- [3. Remediation SLAs](#3-remediation-slas)
+- [4. Reporting Workflow](#4-reporting-workflow)
+- [5. Operator Responsibilities](#5-operator-responsibilities)
+- [6. Evidence](#6-evidence)
+- [7. Operator Checklist](#7-operator-checklist)
+
+---
+
+## Overview
+
+**Audience:** Spinifex maintainers who publish releases, and operators deploying Spinifex into CMMC Level 1 environments.
+
+**Scope:** Flaws in Spinifex's own code and its direct dependencies (Go modules, UI npm packages, GitHub Actions). Operating system, kernel, OVN/OVS, and hypervisor CVEs are the operator's responsibility — see [§5](#5-operator-responsibilities).
+
+## CMMC Practices Covered
+
+| Practice | Title | Objective |
+|----------|-------|-----------|
+| SI.L1-3.14.1 | Flaw Remediation | [a] Time to identify flaws is specified. [b] Flaws identified in time. [c] Time to report flaws is specified. [d] Flaws reported in time. [e] Time to correct flaws is specified. [f] Flaws corrected in time. |
+
+## Approach
+
+Spinifex ships a short dependency chain and identification is already automated: Dependabot raises security PRs the moment GitHub links a CVE to a shipped dependency, `govulncheck` runs on every PR, and GitHub Security Advisories watch all three `mulgadc/*` repositories.
+
+Sections map to the three scored objectives: [§1](#1-identification) names the identification sources (objectives [a]/[b]); [§3](#3-remediation-slas) and [§4](#4-reporting-workflow) set the report and correct timeframes (objectives [c]–[f]). [§5](#5-operator-responsibilities) delineates layers the operator owns, and [§6](#6-evidence) covers the records needed to demonstrate compliance at assessment.
+
+## 1. Identification
+
+| Source | Frequency | Coverage |
+|--------|-----------|----------|
+| Dependabot security updates | Immediate — raised as soon as GitHub links an advisory to a dependency | Go modules, GitHub Actions, UI npm packages |
+| Dependabot version updates | Weekly | Routine version bumps (non-security) |
+| `govulncheck` (CI `lint_and_security` job) | Every PR and push to `main` | Go stdlib + module CVEs reachable in the binary |
+| GitHub Security Advisories | Continuous | `mulgadc/spinifex`, `mulgadc/predastore`, `mulgadc/viperblock` |
+| Third-party reports (sensitive) | Ad hoc | [GitHub private vulnerability reporting](https://github.com/mulgadc/spinifex/security/advisories/new) |
+| Third-party reports (non-sensitive) | Ad hoc | Public GitHub issue, labelled `security` |
+| Internal discovery | Ad hoc | Review, audit, red team, incident response |
+
+Dependabot security PRs are merged immediately once CI passes — for dependency CVEs this collapses identify, report, and correct into a single action. The separate weekly cadence applies only to non-security version bumps.
+
+Reporters: use private vulnerability reporting for anything plausibly Critical or High; a public issue is fine for hardening suggestions and non-exploitable bugs. A flaw is **identified** once it appears in any channel above with enough detail to score CVSS.
+
+## 2. Severity
+
+CVSS v3.1 base score from the upstream advisory; otherwise scored by maintainers and the vector recorded on the tracking issue.
+
+| Severity | CVSS v3.1 | Examples |
+|----------|-----------|----------|
+| Critical | 9.0 – 10.0 | Unauthenticated RCE reachable from tenant networks; `awsgw` auth bypass; master-key disclosure |
+| High | 7.0 – 8.9 | Authenticated RCE; intra-service privilege escalation; cluster-internal auth bypass; memory disclosure exposing credentials. |
+| Medium | 4.0 – 6.9 | Authenticated DoS; non-secret information disclosure; limited-blast-radius logic errors |
+| Low | 0.1 – 3.9 | Heavily mitigated issues — local-only, non-default config, negligible impact |
+
+**Severity modifiers.** Bump one tier **up** if any of the following apply: the flaw is under active exploitation, reaches the master encryption key or cluster CA private key, or allows cross-tenant data access. Bump one tier **down** if the flaw is only reachable by a cluster-internal process already holding credentials equivalent to the attack's outcome.
+
+## 3. Remediation SLAs
+
+Clocks start at the moment of identification ([§1](#1-identification-sources)). All three objectives — identify [a], report [c], correct [e] — are time-bound.
+
+| Severity | Identify | Report | Correct |
+|----------|----------|--------|---------|
+| Critical | 24 hours | 48 hours, with patch | 48 hours |
+| High     | 48 hours | 7 days, with patch | 7 days |
+| Medium   | 7 days   | Next release changelog | 30 days |
+| Low      | 30 days  | Next release changelog | 90 days |
+
+**Identify** - triaged into a tracking issue, CVSS scored, affected components and versions determined, severity label applied.
+
+**Report** - GitHub Security Advisory is published (for Critical/High) or the fix is described in the release notes (for Medium/Low). Operators are the audience; the report must let them determine whether their deployment is affected.
+
+**Correct** - tagged Spinifex release containing the fix is available for operators to pull. For dependency CVEs, this is a release that bumps the vulnerable module to a fixed version. For first-party code, it is a release containing the patch.
+
+If an upstream embargo applies, the correct-SLA pauses until disclosure; the identify-SLA does not. Record the embargo on the tracking issue.
+
+## 4. Reporting Workflow
+
+1. **Intake** — External findings arrive via GitHub (private for sensitive, public issue otherwise). Maintainers mirror each finding into the internal tracker with CVSS score, affected components, and advisory link. Private disclosures stay private until the advisory is drafted.
+2. **Triage** — Validate severity within the identify-SLA, confirm reachability in Spinifex, and either close as not-applicable (with justification) or proceed.
+3. **Fix** — Patch lands on `main` via the normal PR + E2E workflow. Commit message references the CVE / GHSA ID.
+4. **Release** — Tagged release cut; release notes state affected versions, CVSS, and upgrade guidance. Critical/High get a published GHSA.
+5. **Notify** — GitHub release/advisory notifications reach subscribed operators automatically. Out-of-band email notification is triggered only for Critical.
+
+Not-applicable findings (unreachable, already patched, scanner false positive) are still tracked to closure so every identification has an audit trail.
+
+## 5. Operator Responsibilities
+
+Operators own remediation for layers Spinifex does not ship:
+
+| Layer | Responsibility |
+|-------|----------------|
+| Spinifex releases | Apply released patches within the correct-SLA above. |
+| Host OS + kernel | Track distribution advisories (Debian DSA, Ubuntu USN) at matching cadence. |
+| OVN / OVS | Patched via the operator's package channel alongside the OS. |
+| QEMU / KVM / libvirt | As above — hypervisor CVEs affect tenant isolation. |
+| Host AV / EDR agent | See https://docs.mulgadc.com/docs/security/malware-protection §2. |
+
+Operators should subscribe to the `mulgadc/spinifex` release feed (Watch → Custom → Releases + Security Advisories) so Critical/High advisories arrive via GitHub's notification channel rather than polling.
+
+## 6. Evidence and Record Keeping
+
+For CMMC assessment, retain the following for at least 12 months:
+
+- Tracking record for every identified flaw (GitHub issue, GHSA, or internal tracker ID) with identify / triage / close timestamps.
+- Dependabot history and `govulncheck` CI logs — export before GitHub Actions retention expires if shorter.
+- Release notes and GHSAs for each corrected flaw.
+- Operator patch-apply records (change tickets, config-management runs) showing releases deployed within SLA.
+
+These demonstrate objectives [b], [d], [f] — "within the specified time" — in practice.
+
+## 7. Operator Checklist
+
+- System security plan references this policy as the identify / report / correct timeframes for Spinifex.
+- Change-management records patch-apply timestamps for every Spinifex release.
+- Operator subscribes to `mulgadc/spinifex`, `mulgadc/predastore`, `mulgadc/viperblock` releases and advisories.
+- OS / kernel / hypervisor patching cadence documented and at least as strict as [§3](#3-remediation-slas).
+- Annual review confirms at least one patch cycle completed within SLA in the prior 12 months, with evidence per [§6](#6-evidence).

docs/security/malware-protection/README.md

@@ -0,0 +1,307 @@
+---
+title: "Malware Protection"
+description: "Operator guide for host-based malware protection and file integrity monitoring on Spinifex nodes"
+category: "Security"
+sections:
+  - overview
+tags:
+  - security
+  - compliance
+  - cmmc
+  - malware
+  - antivirus
+  - file integrity
+resources:
+  - title: "NIST SP 800-171 Rev 3"
+    url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final"
+  - title: "CMMC Level 1 Self-Assessment Guide v2.0"
+    url: "https://dodcio.defense.gov/CMMC/Documentation"
+  - title: "ClamAV Documentation"
+    url: "https://docs.clamav.net"
+  - title: "AIDE"
+    url: "https://aide.github.io"
+---
+
+# Malware Protection
+
+> Operator guide for host-based malware protection and file integrity monitoring on Spinifex nodes
+
+## Table of Contents
+
+- [Overview](#overview)
+- [CMMC Practices Covered](#cmmc-practices-covered)
+- [Approach](#approach)
+- [1. Designated Locations](#1-designated-locations-sil1-3142-a)
+- [2. Update Requirements](#2-update-requirements-sil1-3144)
+- [3. Scan Schedule](#3-scan-schedule-sil1-3145)
+- [4. Reference Configurations](#4-reference-configurations)
+- [5. Operator Checklist](#5-operator-checklist)
+
+---
+
+## Overview
+
+**Audience:** Operators deploying Spinifex into environments subject to CMMC Level 1 (or organisations that otherwise require host-based malware protection).
+
+**Scope:** The Spinifex nodes themselves — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd`, `spinifex-ui` and the OVN control plane. Guest VMs launched on the platform are the responsibility of the workload owner and are out of scope.
+
+## CMMC Practices Covered
+
+This guide captures the operator-side controls required to meet three CMMC Level 1 practices that Spinifex itself cannot enforce — they depend on host-based tooling chosen and operated by the deployment team.
+
+| Practice | Title | Objective |
+|----------|-------|-----------|
+| SI.L1-3.14.2 | Malicious Code Protection | [a] Designated locations for malicious code protection are identified. [b] Protection from malicious code at designated locations is provided. |
+| SI.L1-3.14.4 | Update Malicious Code Protection | [a] Malicious code protection mechanisms are updated when new releases are available. |
+| SI.L1-3.14.5 | System & File Scanning | [a] Periodic scans of the information system are defined. [b] Periodic scans of the information system are performed. [c] Real-time scans of files from external sources are defined and performed as files are downloaded, opened, or executed. |
+
+## Approach
+
+Spinifex does not bundle or mandate a specific anti-malware product. Operators will typically have an existing endpoint security standard (CrowdStrike Falcon, Microsoft Defender for Linux, Wazuh, ClamAV, Sophos, ESET, etc.). This guide:
+
+1. Names the Spinifex components and filesystem paths that must be included in scan and integrity-monitoring coverage (the "designated locations").
+2. Specifies update and scan cadence required to meet SI.L1-3.14.4 and SI.L1-3.14.5.
+3. Provides reference configurations for open-source stacks (ClamAV for AV, AIDE for file integrity, Wazuh for centralised monitoring) as a known-good baseline. Operators running a commercial EDR should map these locations and cadences into their existing product.
+
+## 1. Designated Locations (SI.L1-3.14.2 [a])
+
+All Spinifex nodes must include the following in their malware protection scope. Paths assume the production install layout (`/etc/spinifex` exists); development layouts under `$HOME/spinifex` are not supported for compliance deployments.
+
+### 1.1 Executable Binaries
+
+| Path | Contents |
+|------|----------|
+| `/usr/local/bin/spx` | Spinifex CLI and service entry point. All `spinifex-*.service` units invoke `spx service <name> start`, so every service binary is reached through this one path. |
+| `/usr/bin/ovn-*`, `/usr/bin/ovs-*` | OVN / OVS control-plane and datapath binaries (installed via `apt` from the `ovn-central`, `ovn-host`, `openvswitch-switch` packages). |
+| `/usr/lib/<arch>-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` | Viperblock NBD plugin loaded by `nbdkit` to expose EBS volumes as block devices. The exact path comes from `nbdkit --dump-config` (`plugindir=`); on Debian 12 amd64 it is `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/`. |
+| `/usr/local/share/ca-certificates/spinifex-ca.crt` | Cluster CA cert installed into the system trust store |
+
+### 1.2 Configuration and Secrets
+
+| Path | Contents |
+|------|----------|
+| `/etc/spinifex/` | Cluster config directory: `spinifex.toml`, `predastore.toml`, `nats.conf`. |
+| `/etc/spinifex/ca.pem`, `/etc/spinifex/ca.key` | Cluster root CA certificate and private key (leader only). |
+| `/etc/spinifex/server.pem`, `/etc/spinifex/server.key` | Per-node TLS certificate and key. |
+| `/etc/spinifex/master.key` | Master encryption key that wraps per-volume DEKs and IAM secrets. **Highest-value artifact on the node.** |
+| `/etc/spinifex/systemd.env` | Service environment file (shared by all `spinifex-*.service` units). |
+
+### 1.3 Service Units and Privilege Configuration
+
+| Path | Contents |
+|------|----------|
+| `/etc/systemd/system/spinifex-*.service` | Unit files for daemon, awsgw, nats, predastore, viperblock, vpcd, ui. |
+| `/etc/sudoers`, `/etc/sudoers.d/` | Any drop-ins granting privileges to service accounts (OVN/OVS management, NBD mounts, etc.). |
+| `/etc/ssh/sshd_config`, `/etc/ssh/sshd_config.d/` | Host SSH daemon configuration. |
+
+### 1.4 Runtime and Data Locations
+
+| Path | Purpose | Notes |
+|------|---------|-------|
+| `/var/lib/spinifex/spinifex/` | Daemon state and local metadata. | Include in scan scope. |
+| `/var/lib/spinifex/awsgw/` | AWS gateway state. | Include in scan scope. |
+| `/var/lib/spinifex/vpcd/` | VPC daemon state. | Include in scan scope. |
+| `/var/lib/spinifex/predastore/` | Local Predastore S3 backing data (chunks, metadata). | See exclusion note below. |
+| `/var/lib/spinifex/viperblock/` | Local Viperblock WAL and chunk data for EBS volumes. | See exclusion note below. |
+| `/var/lib/spinifex/nats/` | NATS JetStream data. | See exclusion note below. |
+| `/run/spinifex/`, `/run/spinifex/nbd/` | Runtime sockets and PID files. | Integrity monitoring not meaningful (transient); exclude from AIDE. |
+| `/var/log/spinifex/` | Service logs. | Scan for malicious content indicators; do not block on writes. |
+
+**Performance exclusions.** Viperblock WAL, Predastore chunks, and NATS JetStream files are large, high-churn, block-level storage that contains customer VM data. Real-time on-access AV scanning of these paths is both a significant performance cost and a confused-deputy problem (the daemon, not the operator, is the writer). Cover these paths with **periodic scheduled scans** (SI.L1-3.14.5 [b]) but exclude them from real-time on-access hooks. Record the exclusion and its justification in the system security plan.
+
+### 1.5 External File Ingest — Real-time Scan Required (SI.L1-3.14.5 [c])
+
+The following are "files from external sources" within the meaning of 3.14.5 [c] and **must** be scanned at ingest, before Spinifex consumes them:
+
+| Ingest point | What lands here | How it reaches the node |
+|--------------|-----------------|-------------------------|
+| VM image catalogue downloads | Debian / Ubuntu / Alpine / Rocky cloud images extracted during `spx admin images import`. | Spinifex downloads over HTTPS and verifies the catalogue SHA checksum. Operators layer AV on top: scan the downloaded image file before it is registered. |
+| `ImportImage` / user-uploaded AMIs | Operator- or tenant-supplied VMDK/VHD/RAW images. | Uploaded to Predastore and registered via the EC2 API. |
+| Cloud-init user-data | Base64 payloads attached to instances. | Passed via `RunInstances`; written to the CIDATA seed ISO before boot. |
+| ISO / media attached to instances | Arbitrary ISO images attached as CD-ROM to VMs. | Uploaded to Predastore. |
+
+Spinifex does not currently invoke an AV scanner on these ingest paths in-process. Until that integration exists, operators must either (a) use an on-access AV hook (ClamAV `clamonacc`, EDR file-write monitor) that sees Predastore writes, or (b) run a pre-ingest workflow that scans artifacts before they are uploaded to Spinifex.
+
+## 2. Update Requirements (SI.L1-3.14.4)
+
+Malicious code protection mechanisms must be updated when new releases are available. For signature-based AV this means signature feeds; for EDR products it means agent version pins. Apply to every node.
+
+| Component | Cadence | Mechanism |
+|-----------|---------|-----------|
+| AV signatures (ClamAV `freshclam`, commercial EDR cloud feeds) | At least every 4 hours; real-time if the vendor supports it. | Systemd timer for `freshclam` (see [§4.1](#41-clamav-signature-based-av)); EDR agents typically self-update. |
+| AV engine / agent version | Within 7 days of vendor release for non-critical; within 48 hours for vendor-marked critical. | OS package manager or vendor deployment pipeline; align with the flaw-remediation SLAs. |
+| File-integrity baseline (AIDE) | Re-baseline on every Spinifex upgrade or config change. | Re-run `aide --init` as part of the upgrade runbook; commit the resulting hash to the change record. |
+
+**Evidence:** the system security plan must record (a) which product is deployed, (b) how updates are delivered, and (c) a sample of update-success log entries from the last 30 days.
+
+## 3. Scan Schedule (SI.L1-3.14.5)
+
+### 3.1 Periodic Scans (objectives [a], [b])
+
+| Target | Frequency | Type |
+|--------|-----------|------|
+| Designated locations [§1.1–§1.3](#11-executable-binaries) (binaries, config, secrets, unit files) | Daily | Full AV scan + AIDE integrity check. |
+| Runtime data under `/var/lib/spinifex/*` ([§1.4](#14-runtime-and-data-locations)) | Weekly | Full AV scan. |
+| Whole root filesystem | Monthly | Full AV scan. |
+
+### 3.2 Real-time Scans (objective [c])
+
+| Trigger | Action |
+|---------|--------|
+| Write to the external-ingest paths in [§1.5](#1-designated-locations-sil1-3142-a) | AV scan before the file is considered available. A positive detection must fail the ingest and emit a structured log line to `/var/log/spinifex/` and the operator's SIEM. |
+| Execution of a binary in `/usr/local/bin/`, `/usr/bin/ovn-*`, `/usr/bin/ovs-*`, or load of `nbdkit-viperblock-plugin.so` from the nbdkit plugin dir | On-access AV check (ClamAV `clamonacc` or EDR equivalent). |
+
+### 3.3 Logging
+
+All scan runs — scheduled and real-time — must produce a log entry that includes scanner name, version, signature version, target, outcome, and duration. Forward to the same log collection used for Spinifex service logs.
+
+## 4. Reference Configurations
+
+The configurations below are reference baselines. They are known to meet the CMMC L1 objectives on a standard Debian 12 Spinifex node; they are not the only valid implementation. Operators running a commercial EDR should treat [§1](#1-designated-locations-sil1-3142-a) and [§3](#3-scan-schedule-sil1-3145) as the contract and configure their chosen product to match.
+
+### 4.1 ClamAV (signature-based AV)
+
+Install:
+
+```bash
+apt-get install -y clamav clamav-daemon clamav-freshclam clamonacc
+```
+
+Signature updates — drop-in `/etc/systemd/system/clamav-freshclam.timer.d/spinifex.conf`:
+
+```ini
+[Timer]
+OnCalendar=*-*-* 00,04,08,12,16,20:00:00
+Persistent=true
+```
+
+Enable and start:
+
+```bash
+systemctl enable --now clamav-freshclam.timer clamav-daemon.service clamav-clamonacc.service
+```
+
+On-access scanning — `/etc/clamav/clamd.conf` additions:
+
+```
+OnAccessIncludePath /usr/local/bin
+OnAccessIncludePath /usr/lib/x86_64-linux-gnu/nbdkit/plugins
+OnAccessIncludePath /etc/spinifex
+OnAccessIncludePath /etc/systemd/system
+OnAccessPreventionAction clamd-block
+OnAccessExcludePath /var/lib/spinifex/viperblock
+OnAccessExcludePath /var/lib/spinifex/predastore
+OnAccessExcludePath /var/lib/spinifex/nats
+```
+
+Scheduled scans — `/etc/systemd/system/spinifex-clamav-scan.service`:
+
+```ini
+[Unit]
+Description=Spinifex scheduled AV scan
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/clamdscan --fdpass --multiscan \
+    /usr/local/bin /usr/lib/x86_64-linux-gnu/nbdkit/plugins \
+    /etc/spinifex /etc/systemd/system \
+    /var/lib/spinifex/awsgw /var/lib/spinifex/vpcd \
+    /var/lib/spinifex/spinifex /var/log/spinifex
+StandardOutput=append:/var/log/spinifex/clamav-scheduled.log
+StandardError=append:/var/log/spinifex/clamav-scheduled.log
+```
+
+Paired timer `spinifex-clamav-scan.timer`:
+
+```ini
+[Unit]
+Description=Daily Spinifex AV scan
+
+[Timer]
+OnCalendar=*-*-* 03:15:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+```
+
+For the weekly `/var/lib/spinifex/*` data scan and the monthly root scan, add two additional timer/service pairs with `OnCalendar=weekly` / `OnCalendar=monthly` and the appropriate scan targets.
+
+### 4.2 AIDE (file integrity monitoring)
+
+Install:
+
+```bash
+apt-get install -y aide aide-common
+```
+
+Policy — `/etc/aide/aide.conf.d/90_spinifex`:
+
+```
+# Binaries — any change is suspicious
+/usr/local/bin/spx                                f+p+u+g+s+m+c+sha256
+# Viperblock NBD plugin — adjust <arch> for the node (see `nbdkit --dump-config`)
+/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so  f+p+u+g+s+m+c+sha256
+
+# Config and secrets
+/etc/spinifex                                     f+p+u+g+s+m+c+sha256
+/etc/systemd/system/spinifex-.*\.service$         f+p+u+g+s+m+c+sha256
+
+# CA trust store
+/usr/local/share/ca-certificates/spinifex-ca.crt  f+p+u+g+s+m+c+sha256
+
+# Exclude high-churn runtime data (covered by scheduled AV instead)
+!/var/lib/spinifex/viperblock
+!/var/lib/spinifex/predastore
+!/var/lib/spinifex/nats
+!/var/log/spinifex
+!/run/spinifex
+```
+
+Baseline:
+
+```bash
+aideinit
+mv /var/lib/aide/aide.db.new /var/lib/aide/aide.db
+```
+
+Daily check — a systemd timer invoking `aide --check`. Non-zero exit must alert. Re-baseline after every Spinifex upgrade (`make reinstall`, package upgrade, config change) and record the new hash.
+
+### 4.3 Wazuh (centralised AV + FIM + alerting)
+
+For multi-node clusters, a Wazuh agent on each Spinifex node feeding a central Wazuh manager is the simplest way to discharge 3.14.2 / 3.14.4 / 3.14.5 with consolidated evidence.
+
+Key agent configuration (`/var/ossec/etc/ossec.conf` fragments):
+
+```xml
+<syscheck>
+  <frequency>43200</frequency>   <!-- 12h FIM baseline -->
+  <directories check_all="yes" realtime="yes">/etc/spinifex</directories>
+  <directories check_all="yes" realtime="yes">/usr/local/bin</directories>
+  <directories check_all="yes" realtime="yes">/etc/systemd/system</directories>
+  <ignore>/var/lib/spinifex/viperblock</ignore>
+  <ignore>/var/lib/spinifex/predastore</ignore>
+  <ignore>/var/lib/spinifex/nats</ignore>
+  <ignore>/run/spinifex</ignore>
+</syscheck>
+
+<rootcheck>
+  <frequency>3600</frequency>
+</rootcheck>
+```
+
+Pair with the Wazuh ClamAV integration (or a commercial AV module) for signature-based detection, and use the Wazuh manager's rule engine to alert on signature updates failing (SI.L1-3.14.4 evidence) and on scheduled-scan failures (SI.L1-3.14.5 evidence).
+
+## 5. Operator Checklist
+
+Use this list to confirm a node meets the three CMMC practices before the node is admitted to a production cluster:
+
+- Host AV product installed, running, and reporting to a central console or log sink.
+- Signature update timer/job running; last successful update within the cadence in [§2](#2-update-requirements-sil1-3144).
+- AV and file-integrity policy covers every path in [§1.1–§1.4](#11-executable-binaries), with documented exclusions for [§1.4](#14-runtime-and-data-locations) high-churn paths.
+- External-ingest paths in [§1.5](#1-designated-locations-sil1-3142-a) have real-time scanning in place (either at the Predastore write layer or in the operator's upload workflow).
+- Daily, weekly, and monthly scheduled scans defined per [§3.1](#31-periodic-scans-objectives-a-b) and visible in recent run history.
+- Scan and update logs forwarded to the same SIEM / log collector used for Spinifex service logs.
+- AIDE (or equivalent FIM) baseline exists; re-baseline step is part of the Spinifex upgrade runbook.
+- System security plan documents the deployed product, update mechanism, scan cadence, and exclusions.

docs/security/network-connections/README.md

@@ -0,0 +1,177 @@
+---
+title: "External Connection Inventory"
+description: "Operator inventory of inbound listeners and outbound connections on Spinifex nodes"
+category: "Security"
+sections:
+  - overview
+tags:
+  - security
+  - compliance
+  - cmmc
+  - network
+  - connections
+  - boundary
+resources:
+  - title: "NIST SP 800-171 Rev 3"
+    url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final"
+  - title: "CMMC Level 1 Self-Assessment Guide v2.0"
+    url: "https://dodcio.defense.gov/CMMC/Documentation"
+  - title: "NIST SP 800-41 Rev 1 — Guidelines on Firewalls and Firewall Policy"
+    url: "https://csrc.nist.gov/pubs/sp/800/41/r1/final"
+---
+
+# External Connection Inventory
+
+> Operator inventory of inbound listeners and outbound connections on Spinifex nodes
+
+## Table of Contents
+
+- [Overview](#overview)
+- [CMMC Practices Covered](#cmmc-practices-covered)
+- [Approach](#approach)
+- [1. Inbound Listeners](#1-inbound-listeners)
+- [2. Outbound Connections](#2-outbound-connections)
+- [3. Cross-Node (Internal) Connections](#3-cross-node-internal-connections)
+- [4. Limiting Controls](#4-limiting-controls)
+- [5. Configuration Surface](#5-configuration-surface)
+- [6. Operator Checklist](#6-operator-checklist)
+
+---
+
+## Overview
+
+**Audience:** Operators deploying Spinifex into environments subject to CMMC Level 1, or any site that requires a documented inventory of system connections.
+
+**Scope:** Network connections originated by or terminated at the Spinifex nodes — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd`, `spinifex-ui`, and the OVN control plane. Guest VM traffic is the workload owner's responsibility and out of scope.
+
+**Boundary definition.** For the purposes of this document:
+
+- **External** means outside the Spinifex cluster's trusted network perimeter — the public internet, the operator's corporate network, tenant users of the AWS API, and guest VMs.
+- **Internal** means between Spinifex nodes inside the cluster subnet(s) defined in `spinifex.toml`.
+
+AC.L1-3.1.20 applies specifically to **external** connections. Internal cluster connections are documented here as well so operators can build an accurate firewall policy.
+
+## CMMC Practices Covered
+
+This guide addresses AC.L1-3.1.20. The related boundary-protection practice SC.L1-3.13.1 is covered by OVN ACL and security-group enforcement in `vpcd` and is documented separately.
+
+| Practice | Title | Objective |
+|----------|-------|-----------|
+| AC.L1-3.1.20 | External Connections | [a] Connections to external systems are identified. [b] The use of external systems is identified. [c] Connections to external systems are verified. [d] The use of external systems is verified. [e] Connections to external systems are controlled/limited. [f] The use of external systems is controlled/limited. |
+
+## Approach
+
+Spinifex has a small, enumerable set of network surfaces:
+
+1. **Inbound listeners** — the TCP/UDP ports each node binds. These are the attack surface exposed to whoever can reach the node.
+2. **Outbound connections** — the destinations the node's services reach out to. Today this is a short list: peer Spinifex nodes, OS image mirrors, and install telemetry.
+3. **Cross-node connections** — inter-node control- and data-plane traffic inside the cluster subnet.
+
+The inventory in [§1](#1-inbound-listeners)–[§2](#2-outbound-connections) satisfies objectives [a]/[b]. The **Auth / Verification** columns throughout satisfy [c]/[d]. [§4](#4-limiting-controls) and [§5](#5-configuration-surface) satisfy [e]/[f]. The default install meets [c]–[f] for every listed connection; the operator's remaining work is to record the inventory in the system security plan, apply host/network firewall rules per [§4](#4-limiting-controls), and audit [§5](#5-configuration-surface) on a recurring schedule.
+
+## 1. Inbound Listeners
+
+"Scope" classifies intended reach:
+
+- **External** — reachable by tenant/operator networks. Authenticated and TLS-protected.
+- **Cluster** — reachable only from peer Spinifex nodes. Operator must restrict via host or network firewall.
+- **Localhost** — bound to `127.0.0.1`; not reachable off-node.
+
+| Port | Service | Protocol | Scope | Purpose | Auth / Verification |
+|------|---------|----------|-------|---------|--------------------|
+| 9999 | spinifex-awsgw | HTTPS | External | AWS-compatible API (EC2, S3, ELBv2, IAM) — customer endpoint | AWS SigV4 + TLS (cluster CA) |
+| 3000 | spinifex-ui | HTTPS | External | Operator web dashboard | Session cookie + TLS |
+| 22 | OpenSSH | SSH | External | Operator administration | Key-based auth (operator-managed) |
+| 4432 | Formation server | HTTPS | External (bootstrap only) | Cluster join coordination; active only while a join token is valid. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
+| 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
+| 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
+| 8443 | spinifex-predastore | HTTPS | Cluster | S3-compatible object storage (AMIs, snapshots, user objects) | AWS SigV4 + TLS |
+| 6660–6662 | predastore (Raft) | TCP | Cluster | Metadata consensus (3 nodes) | Cluster network only |
+| 9991–9993 | predastore (data shards) | TCP | Cluster | Erasure-coded data shard transport | Cluster network only |
+| 6641 | OVN Northbound DB | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned |
+| 6642 | OVN Southbound DB | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned |
+| 8222 | spinifex-nats (monitoring) | HTTP | Localhost | `varz`/`subsz` metrics consumed by the daemon | Loopback only |
+| socket / dynamic TCP | nbdkit (Viperblock) | NBD | Host-local / cluster | Block device transport for guest EBS volumes | Unix socket by default; TCP only in remote/DPU mode |
+
+¹ **Formation port lifecycle.** 4432 opens during `spx admin init` / `spx admin join` while a bootstrap token is outstanding and closes once the cluster is formed (token TTL default 30 min, `--token-ttl`). The server presents an ephemeral self-signed cert that pre-dates trust bootstrap, so the joining node dials with `InsecureSkipVerify` — the only production dial that does. Authenticity rests on the operator supplying the leader address out-of-band plus possession of the bearer token. Document in the security plan so reviewers do not flag 4432 as a persistent open port.
+
+**Development-only listeners.** When `dev_networking=true`, QEMU opens arbitrary host TCP ports for SSH port-forwarding into guest VMs. Production installs (the `/etc/spinifex` layout) do not enable this; it must not appear on compliance nodes.
+
+## 2. Outbound Connections
+
+Spinifex nodes initiate a small, fixed set of outbound connections.
+
+**To external destinations:**
+
+| Destination | Purpose | Protocol | Verification |
+|-------------|---------|----------|--------------|
+| `https://cdimage.debian.org/cdimage/cloud/bookworm/latest` | Debian 12 cloud image download | HTTPS | TLS + checksum verification |
+| `https://cloud-images.ubuntu.com/noble/current` | Ubuntu 24.04 LTS cloud image download | HTTPS | TLS + checksum verification |
+| `https://d2yp8ipz5jfqcw.cloudfront.net` | Alpine image for managed HAProxy load-balancer | HTTPS | TLS + checksum verification |
+| `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. | HTTPS | TLS |
+
+**To peer nodes (cluster-internal):** NATS federation (4248), Predastore S3 (8443), OVN NB/SB (6641/6642) — see [§3](#3-cross-node-internal-connections) for encryption and verification of each. The daemon also probes local Predastore Raft status at `<bind IP>:6660/status` (TLS, cluster CA) and local NATS monitoring at `127.0.0.1:8222/varz` (loopback HTTP).
+
+**Update checks and metadata.** Spinifex does not check for updates and does not consume a cloud metadata service (`169.254.169.254` is served *by* the cluster to guest VMs). Node software updates come from the operator's OS package channel. The install-telemetry endpoint above is the only vendor-operated destination contacted by a node; closed-egress deployments should disable it and record the opt-out in the security plan.
+
+**Air-gapped deployments.** The three image URLs are the only destinations needed for the standard image catalogue. Mirror them locally and use `spx admin images import --file` with pre-staged files. Telemetry must also be disabled. See [Air-gapped install](https://docs.mulgadc.com/docs/install-airgapped).
+
+## 3. Cross-Node (Internal) Connections
+
+Control-plane and data-plane traffic between Spinifex nodes, for completeness and firewall planning:
+
+| Connection | Port(s) | Encryption / Auth | Notes |
+|-----------|---------|-------------------|-------|
+| NATS cluster routes | 4248 | Mutual TLS + cluster token | Full mesh between NATS servers |
+| Predastore S3 | 8443 | TLS + AWS SigV4 | Cross-node object reads/writes |
+| Predastore Raft | 6660–6662 | Cluster network only | Metadata consensus |
+| Predastore shards | 9991–9993 | Cluster network only | Erasure-coded data shards |
+| OVN NB/SB | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane |
+| OVN tunnels (Geneve) | UDP 6081 | None | Tenant traffic overlay between chassis, inside the cluster subnet |
+
+Nodes **must** sit on a network segment that is not routed to tenant/guest VMs or to the internet. Predastore Raft/shards and OVN DBs are cluster-internal and must not be reachable from anywhere else.
+
+## 4. Limiting Controls
+
+Default external surface is three listeners — **9999** (AWS API), **3000** (UI), **22** (SSH) — plus **4432** transiently during bootstrap. Every other listener is cluster-only and the operator must enforce this with a host firewall (`nftables`/`iptables`/`firewalld`) or an upstream network ACL. Minimal `nftables` reference:
+
+```
+# External: from anywhere
+tcp dport { 22, 9999, 3000 } accept
+
+# Cluster-only: replace 10.0.1.0/24 with your cluster CIDR
+ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641, 6642, 6660-6662, 9991-9993 } accept
+ip saddr 10.0.1.0/24 udp dport 6081 accept
+
+# Default deny
+tcp dport 0-65535 drop
+udp dport 0-65535 drop
+```
+
+Port 4432 must be closed outside the bootstrap window; `spx admin join` opens it transiently. Outbound egress can be limited to the image-catalogue hostnames in [§2](#2-outbound-connections) plus the operator's OS package repositories; on air-gapped nodes, block all outbound HTTPS and use `spx admin images import --file`.
+
+## 5. Configuration Surface
+
+Every listener and outbound destination is controlled by one of these files. Changes require a service restart.
+
+| File | Keys | Controls |
+|------|------|----------|
+| `/etc/spinifex/spinifex.toml` | `nodes.<node>.{awsgw,nats,predastore,daemon}.host`, `nodes.<node>.vpcd.ovn_{nb,sb}_addr`, `nodes.<node>.daemon.dev_networking` | Per-service bind addresses/ports; dev-mode QEMU port forwarding. |
+| `/etc/spinifex/nats.conf` | `listen`, `cluster.listen`, `cluster.routes`, `http`, `tls`, `cluster.authorization` | NATS client/cluster/monitoring listeners, peer routes, TLS, cluster token. |
+| `/etc/spinifex/predastore.toml` | `host`, `port`, `[[db]].port`, `[[nodes]].port`, `tls.*` | Predastore S3 listener, Raft ports, shard ports, TLS certs. |
+| OVN packages (`ovn-central`, `ovn-host`) | `ovn-nb-db`, `ovn-sb-db` (via `ovs-vsctl set open_vswitch …`) | OVN DB bind addresses. |
+| Spinifex UI service | Built-in defaults: `host = "0.0.0.0"`, `port = 3000`. No `spinifex.toml` block today. | UI listener. |
+| `spx admin init` / `spx admin join` | `--port`, `--token-ttl`, `--no-telemetry` (or `SPX_NO_TELEMETRY=1`) | Formation port, token TTL, telemetry opt-out. |
+| `utils/images.go` `AvailableImages` | Image catalogue URLs | Outbound HTTPS destinations for image downloads. |
+
+## 6. Operator Checklist
+
+- Inventory recorded in the system security plan — inbound ([§1](#1-inbound-listeners)), outbound ([§2](#2-outbound-connections)), cross-node ([§3](#3-cross-node-internal-connections)) — matches what is observed on the node (`ss -tlnp`, `ss -unlp`).
+- Host firewall enforces the external/cluster/localhost split in [§4](#4-limiting-controls): external surface limited to 9999, 3000, 22 (and 4432 only during bootstrap).
+- Cluster subnet is isolated from tenant guest VM networks and from the public internet.
+- Formation port 4432 is closed on nodes not actively running a bootstrap token.
+- Outbound HTTPS restricted to the [§2](#2-outbound-connections) image-catalogue hosts, or replaced with air-gapped import.
+- Install telemetry (`install.mulgadc.com`) is either permitted and recorded in the security plan, or disabled via `SPX_NO_TELEMETRY=1` / `--no-telemetry`.
+- OVN NB/SB (6641/6642) exposure limited to the cluster subnet pending the L2 TLS work.
+- SSH (22) configured to operator-managed keys only; password auth disabled in `sshd_config`.
+- Periodic review (at least annually, and after any topology change) confirms this inventory still matches the deployed configuration.

docs/security/physical-security-guide/README.md

@@ -0,0 +1,205 @@
+---
+title: "Physical Security Operator Guide"
+description: "Operator guide for physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes"
+category: "Security"
+sections:
+  - overview
+tags:
+  - security
+  - compliance
+  - cmmc
+  - physical
+  - facilities
+  - access control
+resources:
+  - title: "NIST SP 800-171 Rev 3"
+    url: "https://csrc.nist.gov/pubs/sp/800/171/r3/final"
+  - title: "CMMC Level 1 Self-Assessment Guide v2.0"
+    url: "https://dodcio.defense.gov/CMMC/Documentation"
+  - title: "NIST SP 800-53 Rev 5 — PE Family"
+    url: "https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final"
+  - title: "NIST SP 800-116 Rev 1 — PIV for Physical Access"
+    url: "https://csrc.nist.gov/pubs/sp/800/116/r1/final"
+---
+
+# Physical Security Operator Guide
+
+> Operator guide for physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes
+
+## Table of Contents
+
+- [Overview](#overview)
+- [CMMC Practices Covered](#cmmc-practices-covered)
+- [Approach](#approach)
+- [1. Protected Assets](#1-protected-assets)
+- [2. Limit Physical Access (PE.L1-3.10.1)](#2-limit-physical-access-pel1-3101)
+- [3. Escort and Monitor Visitors (PE.L1-3.10.3)](#3-escort-and-monitor-visitors-pel1-3103)
+- [4. Physical Access Logs (PE.L1-3.10.4)](#4-physical-access-logs-pel1-3104)
+- [5. Manage Physical Access Devices (PE.L1-3.10.5)](#5-manage-physical-access-devices-pel1-3105)
+- [6. Evidence and Record Keeping](#6-evidence-and-record-keeping)
+- [7. Operator Checklist](#7-operator-checklist)
+
+---
+
+## Overview
+
+**Audience:** Operators deploying Spinifex into environments subject to CMMC Level 1, or any site that requires documented physical protection of compute and storage infrastructure.
+
+**Scope:** The physical environment housing Spinifex nodes — the Linux hosts running `spinifex-daemon`, `spinifex-awsgw`, `spinifex-nats`, `spinifex-predastore`, `spinifex-viperblock`, `spinifex-vpcd`, `spinifex-ui`, and the OVN control plane — together with the network equipment, cabling, console/KVM access paths, and backup media that support them. Tenant workloads and operator endpoints (laptops, jump hosts) are out of scope.
+
+## CMMC Practices Covered
+
+PE.L1 objectives are discharged by the operator's facility, not by Spinifex. This guide names the assets, cadences, and evidence required.
+
+| Practice | Title | Objective |
+|----------|-------|-----------|
+| PE.L1-3.10.1 | Limit Physical Access | [a] Authorized individuals allowed physical access are identified. [b] Physical access to organizational systems, equipment, and operating environments is limited to authorized individuals. |
+| PE.L1-3.10.3 | Escort Visitors | [a] Visitors are escorted. [b] Visitor activity is monitored. |
+| PE.L1-3.10.4 | Physical Access Logs | [a] Audit logs of physical access are maintained. |
+| PE.L1-3.10.5 | Manage Physical Access Devices | [a] Physical access devices are identified. [b] Physical access devices are controlled. [c] Physical access devices are managed. |
+
+## Approach
+
+Spinifex does not mandate a specific access-control product. Operators typically deploy into a facility that already has badge readers, CCTV, and a visitor-management system (Kisi, HID, Lenel, Genetec, Envoy, Traction Guest, paper logs, etc.). Manual procedures (locked cabinet, paper sign-in, tracked key list) are acceptable for small sites provided the evidence trail exists; multi-rack sites should feed an electronic access-control system into the same SIEM used for Spinifex service logs.
+
+## 1. Protected Assets
+
+The physical protection boundary must enclose every item in this table. An asset is "protected" when access to it requires passing through a controlled barrier (locked room, cage, or cabinet).
+
+| Asset | Why it must be inside the boundary |
+|-------|-----------------------------------|
+| Spinifex node chassis | Host console, BMC, and disks hold `/etc/spinifex/master.key`, cluster CA key, per-node TLS keys, and all tenant volume data. |
+| Network switches and routers serving the cluster subnet | Physical access permits traffic capture, port mirroring, and control-plane tampering. |
+| Structured cabling (top-of-rack to host, host to storage) | Passive taps are trivial on unprotected cabling. |
+| Console / KVM / serial aggregators | Bypass host authentication; reach GRUB, single-user mode, BMC. |
+| Backup media (tapes, removable disks, off-site copies) | Carry the same data as primary storage; covered by MP.L1-3.8.3 for disposal. |
+| Facility power and HVAC cutoffs serving the rack | Unauthorised de-energise is a denial-of-service and a risk to in-flight writes. |
+
+Nodes deployed in an unstaffed remote or edge location must be installed in a locked enclosure with tamper-evident seals and, where feasible, a sensor (door contact, accelerometer) feeding the central monitoring system. Record the enclosure location, seal serial, and sensor channel in the asset register.
+
+## 2. Limit Physical Access (PE.L1-3.10.1)
+
+### 2.1 Authorized Individuals ([a])
+
+Maintain a written access list naming every individual authorized to enter each protected space. For each entry record:
+
+- Full name and employing organization.
+- Role justifying access (e.g. "Spinifex operator", "facilities", "vendor: OEM field service").
+- Scope — which protected spaces, and whether escorted or unescorted.
+- Start date, scheduled review date, and end date when access is removed.
+- Approver (named individual, not a role mailbox).
+
+Review the list at least quarterly and immediately on personnel change (role change, departure, contractor end-of-engagement). Revocations must be effective in the access-control system promptly per operator policy.
+
+### 2.2 Enforcement ([b])
+
+Access to protected spaces must require authentication at the barrier — a badge, PIN, key, biometric, or combination. Unaccompanied access by individuals not on the list in [§2.1](#21-authorized-individuals-a) must not be possible. Specifically:
+
+- Doors and cages: electronic access control (badge/PIN) with door-forced and door-held alarms wired to the monitoring system. Mechanical-only locks are acceptable for lab/edge deployments provided key distribution is tracked under [§5](#5-manage-physical-access-devices-pel1-3105).
+- Racks and cabinets holding Spinifex nodes: locked at all times when unattended. Key or combination distribution tracked as an access device under [§5](#5-manage-physical-access-devices-pel1-3105).
+- Remote/edge enclosures: locked with tamper-evident seal; seal integrity checked on every site visit and recorded in the visit log.
+
+Shared credentials (one badge used by several people, a rack key left in the cage) are not acceptable.
+
+## 3. Escort and Monitor Visitors (PE.L1-3.10.3)
+
+A visitor is any individual entering a protected space who is not on the authorized-access list in [§2.1](#21-authorized-individuals-a). This includes vendor field engineers, auditors, janitorial staff, and employees of the operator who do not hold Spinifex access.
+
+### 3.1 Escort ([a])
+
+- Every visitor must be signed in by a named authorized escort before entering the protected space.
+- The escort must remain physically present with the visitor for the duration of the visit. Passing a visitor between escorts is permitted; leaving a visitor unaccompanied is not.
+- Work inside a rack or at a node console requires one escort per visitor.
+
+### 3.2 Monitoring ([b])
+
+Visitor activity must be monitored by at least one of:
+
+- Continuous physical presence of the escort in line-of-sight of the visitor. This is the minimum.
+- CCTV coverage of the protected space with recordings retained for at least 30 days. Cameras must cover rack fronts and rears, door entries, and any console/KVM positions.
+- For vendor maintenance involving system access (BMC, console, disk swap): a second operator witness on-site, or a recorded screen share if the work is performed from the console. Record the witness or session recording ID in the visitor log.
+
+CCTV is strongly recommended for any facility hosting more than a single rack. It also serves as corroborating evidence for [§4](#4-physical-access-logs-pel1-3104) logs and is useful for incident response.
+
+## 4. Physical Access Logs (PE.L1-3.10.4)
+
+### 4.1 What to Log
+
+Every entry into a protected space must produce a log entry capturing:
+
+| Field | Source |
+|-------|--------|
+| Identity (name, badge ID, or visitor record ID) | Access control system or sign-in sheet. |
+| Timestamp in and timestamp out | Access control system; manual for paper logs. |
+| Protected space entered | Door/cage ID. |
+| Purpose | Free-text; required for visitors, recommended for authorized staff. |
+| Escort (for visitors) | Named individual from the authorized list. |
+| Associated change or ticket ID | When the visit is tied to a Spinifex change (upgrade, disk swap, node rebuild). |
+
+### 4.2 Retention and Review
+
+- **Retention:** at least 12 months. CCTV recordings associated with the same visit, where used to discharge [§3.2](#32-monitoring-b), should be retained for the same period.
+- **Review cadence:** monthly spot-check of 10% of entries, quarterly full review against the authorized-access list. Discrepancies (unknown badge ID, visitor with no matching sign-in, escort named who was off-site) must be investigated and the finding recorded.
+- **Alerting:** out-of-hours entries, repeated failed reads at a single door, and door-forced / door-held events must fire an alert to the on-call operator. Treat these as incidents until triaged.
+
+### 4.3 Integration with Spinifex Logs
+
+Forward physical-access events to the same SIEM or log collector used for Spinifex service logs (see [Malware Protection §3](https://docs.mulgadc.com/docs/security/malware-protection#3-scan-schedule-sil1-3145)) so a `master.key` access attempt can be correlated with a physical entry into the rack.
+
+## 5. Manage Physical Access Devices (PE.L1-3.10.5)
+
+Physical access devices are badges, key cards, PINs, mechanical keys, rack keys, safe combinations, and tamper seals. USB tokens or HSMs holding the cluster CA key or master encryption key also belong in the register.
+
+### 5.1 Identification ([a])
+
+Maintain a device register with one entry per issued device. Minimum fields:
+
+| Field | Notes |
+|-------|-------|
+| Device ID | Badge serial, key stamp, seal serial, HSM serial. |
+| Device type | Badge / mechanical key / PIN / combination / seal / hardware token. |
+| Scope | Which barriers it opens, or which asset it protects. |
+| Holder | Named individual. Shared holders are not acceptable for badges or PINs. |
+| Issued date, issued by | Audit trail for [c]. |
+| Returned or revoked date, reason | Populated on personnel change. |
+
+### 5.2 Control ([b])
+
+- Issue only to individuals named in the [§2.1](#21-authorized-individuals-a) access list. Scope of the device must not exceed that individual's authorized scope.
+- Badges and PINs must be unique per holder. Mechanical keys that must be shared (e.g. a single rack key) are permitted only when distribution is tracked by check-out/check-in against the register.
+- Lost or compromised devices trigger immediate revocation: badges deactivated in the access-control system, PINs changed, affected mechanical locks re-keyed within 30 days, tamper seals re-applied on the next site visit. Record the event and remediation in the register.
+- Terminated or reassigned personnel surrender all devices on last day. Mark returned in the register.
+
+### 5.3 Management ([c])
+
+- **Inventory review:** at least quarterly, reconcile the device register against (a) the access-control system's badge database, (b) the physical key count, and (c) the authorized-access list in [§2.1](#21-authorized-individuals-a). Discrepancies must be resolved before the review is marked complete.
+- **Rotation / re-keying:** mechanical locks re-keyed on loss of any key, and at least every five years. Default combinations (safe, cabinet, BMC default passwords at install) must be changed before the device enters service.
+- **Spares and master keys:** held in a secured location (locked cabinet or safe inside a protected space), accessible only to named individuals, with any access itself logged under [§4](#4-physical-access-logs-pel1-3104).
+
+## 6. Evidence and Record Keeping
+
+For CMMC assessment, retain the following for at least 12 months (longer where facility or contract policy requires):
+
+- **Authorized access list** ([§2.1](#21-authorized-individuals-a)) with review dates and approver names.
+- **Access-control system configuration** showing which badges are permitted at which readers, exported or screenshotted at each quarterly review.
+- **Physical access logs** ([§4](#4-physical-access-logs-pel1-3104)) and, where applicable, CCTV retention policy and sample recordings.
+- **Visitor logs** with escort names for every visit.
+- **Device register** ([§5.1](#51-identification-a)) with issue, return, and revocation history.
+- **Incident records** for door-forced/door-held events, lost devices, failed seal checks, and any access-review discrepancies, with remediation outcomes.
+- **Annual review attestation** from the facility or security owner confirming the controls above operated for the prior 12 months.
+
+## 7. Operator Checklist
+
+Use this list to confirm a site meets the four CMMC practices before admitting Spinifex nodes to a production cluster:
+
+- Authorized-access list exists, names every protected space, is reviewed at least quarterly, and has an approver for every entry.
+- Every protected space enforces authentication at the barrier; shared credentials are not in use for badged/PIN'd spaces.
+- Remote/edge enclosures are locked, sealed, and listed in the asset register with seal serials recorded.
+- Visitor procedure is documented: sign-in, escort (one-per-visitor at equipment), sign-out.
+- CCTV or equivalent monitoring covers the rack fronts, rears, and console positions, with 30-day minimum retention.
+- Physical access logs are produced for every entry, retained 12 months, and reviewed on the cadence in [§4.2](#42-retention-and-review).
+- Door-forced, door-held, and out-of-hours events alert the on-call operator.
+- Physical access events forwarded to the SIEM used for Spinifex service logs (see [Malware Protection §3](https://docs.mulgadc.com/docs/security/malware-protection#3-scan-schedule-sil1-3145)).
+- Device register reconciles quarterly against the access-control system and physical key count; discrepancies are closed before sign-off.
+- Lost-device and termination procedures trigger immediate revocation per [§5.2](#52-control-b).
+- System security plan references this guide and records the facility, access-control product, CCTV retention, and log-forwarding destination.

scripts/check-coverage.sh

@@ -13,7 +13,12 @@ if [[ ! -s "$PROFILE" ]]; then
     exit 1
 fi
 
-TOTAL=$(go tool cover -func="$PROFILE" | tail -1 | awk '{print $NF}' | tr -d '%')
+# Filter out packages excluded from coverage requirements (e.g. migrations)
+FILTERED=$(mktemp)
+trap 'rm -f "$FILTERED"' EXIT
+awk 'NR==1 || !/\/migrate\//' "$PROFILE" > "$FILTERED"
+
+TOTAL=$(go tool cover -func="$FILTERED" | tail -1 | awk '{print $NF}' | tr -d '%')
 
 if [[ -z "$TOTAL" ]]; then
     echo "ERROR: No coverage data generated — tests may have failed to compile"

scripts/create-multi-node.sh

@@ -39,7 +39,26 @@ echo "Starting init node (127.0.0.1) and joining nodes (127.0.0.2, 127.0.0.3)...
 INIT_PID=$!
 
 # Wait for formation server to start
-sleep 2
+echo "Waiting for formation server..."
+for i in $(seq 1 30); do
+  if curl -sk "https://127.0.0.1:4432/formation/health" > /dev/null 2>&1; then
+    echo "Formation server ready"
+    break
+  fi
+  if [ "$i" -eq 30 ]; then
+    echo "ERROR: Formation server failed to start"
+    kill $INIT_PID 2>/dev/null || true
+    exit 1
+  fi
+  sleep 1
+done
+
+# Read join token
+JOIN_TOKEN=$(cat ~/node1/config/join-token)
+if [ -z "$JOIN_TOKEN" ]; then
+  echo "ERROR: Join token file is empty"
+  exit 1
+fi
 
 # Join nodes concurrently
 ./bin/spx admin join \
@@ -47,6 +66,7 @@ sleep 2
   --bind 127.0.0.2 \
   --cluster-bind 127.0.0.2 \
   --host 127.0.0.1:4432 \
+  --token "$JOIN_TOKEN" \
   --data-dir ~/node2/ \
   --config-dir ~/node2/config/ \
   --region ap-southeast-2 \
@@ -58,6 +78,7 @@ JOIN2_PID=$!
   --bind 127.0.0.3 \
   --cluster-bind 127.0.0.3 \
   --host 127.0.0.1:4432 \
+  --token "$JOIN_TOKEN" \
   --data-dir ~/node3/ \
   --config-dir ~/node3/config/ \
   --region ap-southeast-2 \

scripts/diff-coverage.sh

@@ -66,7 +66,7 @@ trap 'rm -rf "$TMPDIR"' EXIT
 # Only non-test .go files that were added or modified.
 # -M enables rename detection so file splits/moves only count truly changed lines.
 # -C enables copy detection for the same reason (file split = partial copy).
-git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' | awk '
+git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' ':!spinifex/migrate/*' | awk '
     /^\+\+\+ / {
         file = substr($2, 3)
         next

scripts/iac/configure-cluster.sh

@@ -121,6 +121,14 @@ while ! curl -sk --connect-timeout 3 "https://$NODE1_IP:4432/formation/health" >
 done
 echo "    Formation server ready"
 
+# Read join token from init node
+JOIN_TOKEN=$(remote "$NODE1_IP" "cat ~/spinifex/config/join-token")
+if [ -z "$JOIN_TOKEN" ]; then
+    echo "Error: Failed to read join token from $NODE1_IP"
+    exit 1
+fi
+echo "    Join token loaded from node1"
+
 # Nodes 2..N: join
 if [ "$NODE_COUNT" -gt 1 ]; then
     JOIN_PIDS=()
@@ -134,6 +142,7 @@ if [ "$NODE_COUNT" -gt 1 ]; then
             --cluster-bind $ip \
             --cluster-routes $NODE1_IP:4248 \
             --host $NODE1_IP:4432 \
+            --token $JOIN_TOKEN \
             --data-dir ~/spinifex/ \
             --config-dir ~/spinifex/config/ \
             --region $REGION \

scripts/start-dev.sh

@@ -271,19 +271,33 @@ if has_service "nats"; then
     # Wait for NATS JetStream to be ready before Predastore starts.
     # Predastore lazily opens IAM KV buckets — it starts with config-only auth
     # and activates IAM auth once the spinifex daemon creates the KV buckets.
-    echo "🔍 Waiting for NATS JetStream..."
-    NATS_JS_READY=false
-    for i in $(seq 1 15); do
-        # Send a NATS INFO probe — JetStream-enabled servers include "jetstream" in the INFO response
-        if echo "" | nc -w 1 "$NATS_CHECK_HOST" 4222 2>/dev/null | grep -q "jetstream"; then
-            echo "   ✅ NATS JetStream is ready"
-            NATS_JS_READY=true
-            break
+    # Wait for NATS JetStream to be ready before Predastore starts.
+    # Predastore lazily opens IAM KV buckets — it starts with config-only auth
+    # and activates IAM auth once the spinifex daemon creates the KV buckets.
+    # Use the monitoring HTTP endpoint (plaintext nc probe doesn't work with TLS).
+    # Monitoring is only on the primary node; secondary nodes skip this check.
+    NATS_HAS_MONITORING=false
+    if [ -f "$CONFIG_DIR/nats/nats.conf" ] && grep -q '^http:' "$CONFIG_DIR/nats/nats.conf"; then
+        NATS_HAS_MONITORING=true
+    fi
+
+    if [ "$NATS_HAS_MONITORING" = "true" ]; then
+        echo "🔍 Waiting for NATS JetStream..."
+        NATS_JS_READY=false
+        for i in $(seq 1 15); do
+            if curl -sf "http://127.0.0.1:8222/jsz" > /dev/null 2>&1; then
+                echo "   ✅ NATS JetStream is ready"
+                NATS_JS_READY=true
+                break
+            fi
+            sleep 1
+        done
+        if [ "$NATS_JS_READY" = "false" ]; then
+            echo "   ⚠️  NATS JetStream not confirmed ready — Predastore may fall back to config-only auth"
         fi
-        sleep 1
-    done
-    if [ "$NATS_JS_READY" = "false" ]; then
-        echo "   ⚠️  NATS JetStream not confirmed ready — Predastore may fall back to config-only auth"
+    else
+        echo "🔍 NATS monitoring not configured — skipping JetStream readiness check"
+        sleep 2
     fi
 else
     echo "1️⃣  Skipping NATS (not a local service)"
@@ -329,7 +343,11 @@ if has_service "predastore"; then
     PREDASTORE_CMD="./bin/spx service predastore start"
     start_service "predastore" "$PREDASTORE_CMD"
     set_oom_score "predastore" "-500"
-    check_service "Predastore" "$SPINIFEX_PREDASTORE_HOST" "$SPINIFEX_PREDASTORE_PORT"
+    if is_multinode; then
+        echo "   ⏭️  Skipping Predastore connectivity check (multi-node: needs quorum from peer nodes)"
+    else
+        check_service "Predastore" "$SPINIFEX_PREDASTORE_HOST" "$SPINIFEX_PREDASTORE_PORT"
+    fi
 else
     echo "2️⃣  Skipping Predastore (not a local service)"
 fi

spinifex/admin/admin.go

@@ -678,7 +678,7 @@ func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string,
 		NotBefore:             notBefore,
 		NotAfter:              notAfter,
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
 		BasicConstraintsValid: true,
 		DNSNames:              dnsNames,
 		IPAddresses:           ipAddresses,
@@ -740,7 +740,7 @@ func GenerateSelfSignedCert(certPath, keyPath string) error {
 		NotBefore:             notBefore,
 		NotAfter:              notAfter,
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
 		BasicConstraintsValid: true,
 		DNSNames:              []string{"localhost"},
 		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},

spinifex/config/config.go

@@ -117,9 +117,10 @@ type DaemonConfig struct {
 
 // NATSConfig holds the NATS configuration
 type NATSConfig struct {
-	Host string  `json:"Host" mapstructure:"host"`
-	ACL  NATSACL `json:"ACL" mapstructure:"acl"`
-	Sub  NATSSub `json:"Sub" mapstructure:"sub"`
+	Host   string  `json:"Host" mapstructure:"host"`
+	CACert string  `json:"CACert" mapstructure:"cacert"`
+	ACL    NATSACL `json:"ACL" mapstructure:"acl"`
+	Sub    NATSSub `json:"Sub" mapstructure:"sub"`
 }
 
 // NATSACL holds the NATS ACL configuration

spinifex/daemon/daemon.go

@@ -56,7 +56,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/nats-io/nats.go"
-	"github.com/pelletier/go-toml/v2"
 )
 
 type BlockDeviceMapping struct {
@@ -550,8 +549,8 @@ func (d *Daemon) Start() error {
 		return fmt.Errorf("failed to connect to NATS: %w", err)
 	}
 
-	// ClusterManager must start before JetStream init so other nodes can join
-	// via /join endpoint and help form the NATS cluster.
+	// ClusterManager must start before JetStream init so peers can reach
+	// /health during bootstrap.
 	if err := d.ClusterManager(); err != nil {
 		return fmt.Errorf("failed to start cluster manager: %w", err)
 	}
@@ -851,7 +850,7 @@ func (d *Daemon) connectNATS() error {
 	start := time.Now()
 
 	for {
-		nc, err := utils.ConnectNATS(d.config.NATS.Host, d.config.NATS.ACL.Token)
+		nc, err := utils.ConnectNATS(d.config.NATS.Host, d.config.NATS.ACL.Token, d.config.NATS.CACert)
 		if err == nil {
 			d.natsConn = nc
 			if time.Since(start) > time.Second {
@@ -860,6 +859,11 @@ func (d *Daemon) connectNATS() error {
 			return nil
 		}
 
+		// TLS configuration errors are permanent — retrying will not help.
+		if strings.Contains(err.Error(), "CA cert") {
+			return fmt.Errorf("NATS TLS configuration error: %w", err)
+		}
+
 		elapsed := time.Since(start)
 		if elapsed >= maxWait {
 			return fmt.Errorf("NATS connect failed after %s: %w", elapsed.Round(time.Second), err)
@@ -1383,27 +1387,6 @@ func (d *Daemon) computeConfigHash() (string, error) {
 	return hex.EncodeToString(hash[:]), nil
 }
 
-// saveClusterConfig writes the cluster config to disk in TOML format
-func (d *Daemon) saveClusterConfig() error {
-	if d.configPath == "" {
-		return fmt.Errorf("config path not set")
-	}
-
-	// Marshal to TOML
-	configTOML, err := toml.Marshal(d.clusterConfig)
-	if err != nil {
-		return fmt.Errorf("failed to marshal config to TOML: %w", err)
-	}
-
-	// Write to config file
-	if err := os.WriteFile(d.configPath, configTOML, 0600); err != nil {
-		return fmt.Errorf("failed to write config: %w", err)
-	}
-
-	slog.Info("Cluster config saved", "path", d.configPath, "epoch", d.clusterConfig.Epoch)
-	return nil
-}
-
 // ClusterManager starts the HTTP cluster management server
 func (d *Daemon) ClusterManager() error {
 	// Get daemon host from config
@@ -1471,160 +1454,6 @@ func (d *Daemon) ClusterManager() error {
 		}
 	})
 
-	// Join endpoint - accepts new nodes joining the cluster
-	r.Post("/join", func(w http.ResponseWriter, r *http.Request) {
-		var req types.NodeJoinRequest
-		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusBadRequest)
-			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
-				Success: false,
-				Message: "invalid request body",
-			}); err != nil {
-				slog.Error("Failed to encode join error response", "error", err)
-			}
-			return
-		}
-
-		slog.Info("Node join request received", "node", req.Node, "region", req.Region, "az", req.AZ)
-
-		// Validate request
-		if req.Node == "" || req.Region == "" || req.AZ == "" {
-			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusBadRequest)
-			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
-				Success: false,
-				Message: "node, region, and az are required",
-			}); err != nil {
-				slog.Error("Failed to encode join validation response", "error", err)
-			}
-			return
-		}
-
-		// Check if node already exists
-		if _, exists := d.clusterConfig.Nodes[req.Node]; exists {
-			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusConflict)
-			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
-				Success: false,
-				Message: fmt.Sprintf("node %s already exists in cluster", req.Node),
-			}); err != nil {
-				slog.Error("Failed to encode join conflict response", "error", err)
-			}
-			return
-		}
-
-		// Add new node to cluster config
-		d.mu.Lock()
-		newNodeConfig := config.Config{
-			Node:    req.Node,
-			Region:  req.Region,
-			AZ:      req.AZ,
-			DataDir: req.DataDir,
-			Daemon: config.DaemonConfig{
-				Host: req.DaemonHost,
-			},
-			// Copy shared config from current node
-			NATS:       d.config.NATS,
-			Predastore: d.config.Predastore,
-			AWSGW:      d.config.AWSGW,
-			BaseDir:    req.DataDir,
-		}
-
-		d.clusterConfig.Nodes[req.Node] = newNodeConfig
-		d.clusterConfig.Epoch++ // Increment epoch for version tracking
-		d.mu.Unlock()
-
-		// Save updated config
-		if err := d.saveClusterConfig(); err != nil {
-			slog.Error("Failed to save cluster config", "error", err)
-			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusInternalServerError)
-			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
-				Success: false,
-				Message: "failed to save cluster config",
-			}); err != nil {
-				slog.Error("Failed to encode join save-error response", "error", err)
-			}
-			return
-		}
-
-		configHash, _ := d.computeConfigHash()
-
-		slog.Info("Node joined cluster", "node", req.Node, "epoch", d.clusterConfig.Epoch)
-
-		// Update JetStream KV replicas to match new cluster size
-		// This may fail if the new node's NATS server isn't running yet - that's OK,
-		// replicas can be updated later when the cluster is fully formed
-		if d.jsManager != nil {
-			newReplicaCount := len(d.clusterConfig.Nodes)
-			if err := d.jsManager.UpdateReplicas(newReplicaCount); err != nil {
-				slog.Warn("Failed to update JetStream replicas (new node NATS may not be ready yet)", "targetReplicas", newReplicaCount, "error", err)
-			}
-		}
-
-		// Send only shared cluster data (exclude node-specific top-level fields)
-		sharedData := &types.SharedClusterData{
-			Epoch:   d.clusterConfig.Epoch,
-			Version: d.clusterConfig.Version,
-			Nodes:   d.clusterConfig.Nodes,
-		}
-
-		// Read CA certificate and key to share with joining node for per-node cert generation
-		configDir := filepath.Dir(d.configPath)
-		caCertPath := filepath.Join(configDir, "ca.pem")
-		caKeyPath := filepath.Join(configDir, "ca.key")
-
-		var caCert, caKey string
-		if caCertPEM, err := os.ReadFile(caCertPath); err == nil {
-			caCert = string(caCertPEM)
-		} else {
-			slog.Warn("Could not read CA cert for join response", "error", err)
-		}
-
-		if caKeyPEM, err := os.ReadFile(caKeyPath); err == nil {
-			caKey = string(caKeyPEM)
-		} else {
-			slog.Warn("Could not read CA key for join response", "error", err)
-		}
-
-		// Read predastore.toml to share with joining node (for multi-node predastore)
-		var predastoreConfig string
-		predastorePath := filepath.Join(configDir, "predastore", "predastore.toml")
-		if content, err := os.ReadFile(predastorePath); err == nil {
-			predastoreConfig = string(content)
-		} else {
-			slog.Warn("Could not read predastore.toml for join response", "path", predastorePath, "error", err)
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-		if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
-			Success:          true,
-			Message:          fmt.Sprintf("node %s successfully joined cluster", req.Node),
-			SharedData:       sharedData,
-			ConfigHash:       configHash,
-			JoiningNode:      req.Node,
-			CACert:           caCert,
-			CAKey:            caKey,
-			PredastoreConfig: predastoreConfig,
-		}); err != nil {
-			slog.Error("Failed to encode join success response", "error", err)
-		}
-	})
-
-	// Get cluster config endpoint
-	r.Get("/config", func(w http.ResponseWriter, r *http.Request) {
-		configHash, _ := d.computeConfigHash()
-
-		w.Header().Set("Content-Type", "application/json")
-		if err := json.NewEncoder(w).Encode(map[string]any{
-			"config":      d.clusterConfig,
-			"config_hash": configHash,
-		}); err != nil {
-			slog.Error("Failed to encode config response", "error", err)
-		}
-	})
-
 	// Load TLS certificate (C-5: serve over HTTPS instead of plaintext HTTP)
 	// Resolve relative cert paths against config directory (cert lives alongside spinifex.toml).
 	// For binary installs, systemd sets absolute paths via env vars; for dev, the config
@@ -1650,6 +1479,7 @@ func (d *Daemon) ClusterManager() error {
 
 	tlsConfig := &tls.Config{
 		Certificates: []tls.Certificate{cert},
+		MinVersion:   tls.VersionTLS12,
 	}
 
 	d.clusterServer = &http.Server{

spinifex/daemon/daemon_handlers.go

@@ -207,7 +207,7 @@ func (d *Daemon) queryNATSRole() string {
 	if !d.config.HasService("nats") {
 		return ""
 	}
-	url := "http://" + net.JoinHostPort(d.daemonIP(), strconv.Itoa(natsMonitorPort)) + "/varz"
+	url := "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(natsMonitorPort)) + "/varz"
 	return fetchNATSRole(url, roleHTTPClient)
 }
 

spinifex/daemon/daemon_test.go

@@ -34,7 +34,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
-	"github.com/pelletier/go-toml/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -3087,61 +3086,6 @@ func TestComputeConfigHash_ExcludesNodeField(t *testing.T) {
 	assert.Equal(t, h1, h2, "changing top-level Node should not affect config hash")
 }
 
-// --- saveClusterConfig ---
-
-func TestSaveClusterConfig_WritesToDisk(t *testing.T) {
-	dir := t.TempDir()
-	path := filepath.Join(dir, "spinifex.toml")
-
-	d := &Daemon{
-		configPath: path,
-		clusterConfig: &config.ClusterConfig{
-			Epoch:   5,
-			Version: "2.0",
-			Node:    "test-node",
-			Nodes: map[string]config.Config{
-				"test-node": {Region: "us-west-2"},
-			},
-		},
-	}
-
-	err := d.saveClusterConfig()
-	require.NoError(t, err)
-
-	data, err := os.ReadFile(path)
-	require.NoError(t, err)
-
-	var loaded config.ClusterConfig
-	require.NoError(t, toml.Unmarshal(data, &loaded))
-	assert.Equal(t, uint64(5), loaded.Epoch)
-	assert.Equal(t, "2.0", loaded.Version)
-	assert.Equal(t, "us-west-2", loaded.Nodes["test-node"].Region)
-
-	// Verify permissions
-	info, _ := os.Stat(path)
-	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
-}
-
-func TestSaveClusterConfig_ErrorOnEmptyPath(t *testing.T) {
-	d := &Daemon{
-		configPath:    "",
-		clusterConfig: &config.ClusterConfig{},
-	}
-	err := d.saveClusterConfig()
-	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "config path not set")
-}
-
-func TestSaveClusterConfig_ErrorOnInvalidPath(t *testing.T) {
-	d := &Daemon{
-		configPath:    "/nonexistent/dir/spinifex.toml",
-		clusterConfig: &config.ClusterConfig{},
-	}
-	err := d.saveClusterConfig()
-	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "failed to write config")
-}
-
 // --- instanceTypeVCPUs / instanceTypeMemoryMiB nil safety ---
 
 func TestInstanceTypeVCPUs_NilSafety(t *testing.T) {

spinifex/formation/formation.go

@@ -2,7 +2,10 @@ package formation
 
 import (
 	"context"
+	crypto_rand "crypto/rand"
+	"crypto/subtle"
 	"crypto/tls"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"log/slog"
@@ -98,12 +101,25 @@ type FormationServer struct {
 	caKey         string
 	masterKey     string
 	networkConfig *NetworkConfig
+	joinToken     string
+	tokenExpiry   time.Time
 	done          chan struct{}
 	server        *http.Server
 }
 
+// GenerateJoinToken returns a token of the form "spx_join_<16 base64url chars>".
+// Uses crypto/rand for 12 random bytes (96 bits entropy), base64 URL-safe encoded.
+func GenerateJoinToken() (string, error) {
+	b := make([]byte, 12)
+	if _, err := crypto_rand.Read(b); err != nil {
+		return "", fmt.Errorf("generate join token: %w", err)
+	}
+	return "spx_join_" + base64.RawURLEncoding.EncodeToString(b), nil
+}
+
 // NewFormationServer creates a new formation server expecting the given number of nodes.
-func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string, networkConfig *NetworkConfig) *FormationServer {
+func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string,
+	networkConfig *NetworkConfig, joinToken string, tokenTTL time.Duration) *FormationServer {
 	return &FormationServer{
 		expected:      expected,
 		nodes:         make(map[string]NodeInfo),
@@ -111,6 +127,8 @@ func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey st
 		caCert:        caCert,
 		caKey:         caKey,
 		networkConfig: networkConfig,
+		joinToken:     joinToken,
+		tokenExpiry:   time.Now().Add(tokenTTL),
 		done:          make(chan struct{}),
 	}
 }
@@ -238,7 +256,44 @@ func (fs *FormationServer) Shutdown(ctx context.Context) error {
 	return fs.server.Shutdown(ctx)
 }
 
+// validateToken checks the Authorization header against the join token.
+// Returns an error if the token is missing, invalid, or expired.
+func (fs *FormationServer) validateToken(r *http.Request) error {
+	fs.mu.RLock()
+	defer fs.mu.RUnlock()
+
+	if time.Now().After(fs.tokenExpiry) {
+		return fmt.Errorf("join token has expired")
+	}
+
+	auth := r.Header.Get("Authorization")
+	if auth == "" {
+		return fmt.Errorf("missing Authorization header")
+	}
+
+	const prefix = "Bearer "
+	if len(auth) < len(prefix) || auth[:len(prefix)] != prefix {
+		return fmt.Errorf("invalid Authorization header format")
+	}
+	token := auth[len(prefix):]
+
+	if subtle.ConstantTimeCompare([]byte(token), []byte(fs.joinToken)) != 1 {
+		return fmt.Errorf("invalid join token")
+	}
+
+	return nil
+}
+
 func (fs *FormationServer) handleJoin(w http.ResponseWriter, r *http.Request) {
+	if err := fs.validateToken(r); err != nil {
+		slog.Warn("Formation join rejected", "error", err, "remote_addr", r.RemoteAddr)
+		writeJSON(w, http.StatusUnauthorized, JoinResponse{
+			Success: false,
+			Message: "unauthorized: " + err.Error(),
+		})
+		return
+	}
+
 	r.Body = http.MaxBytesReader(w, r.Body, 1*1024*1024) // 1 MB limit
 	var req JoinRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -270,6 +325,12 @@ func (fs *FormationServer) handleJoin(w http.ResponseWriter, r *http.Request) {
 }
 
 func (fs *FormationServer) handleStatus(w http.ResponseWriter, r *http.Request) {
+	if err := fs.validateToken(r); err != nil {
+		slog.Warn("Formation status rejected", "error", err, "remote_addr", r.RemoteAddr)
+		http.Error(w, "unauthorized: "+err.Error(), http.StatusUnauthorized)
+		return
+	}
+
 	fs.mu.RLock()
 	defer fs.mu.RUnlock()
 

spinifex/formation/formation_test.go

@@ -12,6 +12,9 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const testToken = "spx_join_dGVzdHRva2VuMTIz"
+const testTokenTTL = 1 * time.Hour
+
 func testCreds() *SharedCredentials {
 	return &SharedCredentials{
 		AccessKey:   "AKIATEST1234567890AB",
@@ -33,9 +36,53 @@ func testNode(name, ip string) NodeInfo {
 	}
 }
 
+func testServer(t *testing.T, fs *FormationServer) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /formation/join", fs.handleJoin)
+	mux.HandleFunc("GET /formation/status", fs.handleStatus)
+	mux.HandleFunc("GET /formation/health", fs.handleHealth)
+	return httptest.NewServer(mux)
+}
+
+func authPost(t *testing.T, url, token string, body []byte) *http.Response {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+token)
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	return resp
+}
+
+func authGet(t *testing.T, url, token string) *http.Response {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	require.NoError(t, err)
+	req.Header.Set("Authorization", "Bearer "+token)
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	return resp
+}
+
+func TestGenerateJoinToken(t *testing.T) {
+	t.Parallel()
+	token1, err := GenerateJoinToken()
+	require.NoError(t, err)
+
+	assert.True(t, len(token1) > len("spx_join_"))
+	assert.Equal(t, "spx_join_", token1[:9])
+	assert.Len(t, token1[9:], 16)
+
+	token2, err := GenerateJoinToken()
+	require.NoError(t, err)
+	assert.NotEqual(t, token1, token2)
+}
+
 func TestNewFormationServer(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem", nil)
+	fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem", nil, testToken, testTokenTTL)
 
 	assert.Equal(t, 3, fs.expected)
 	assert.Empty(t, fs.nodes)
@@ -44,7 +91,7 @@ func TestNewFormationServer(t *testing.T) {
 
 func TestRegisterNode(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "", "", nil)
+	fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL)
 
 	err := fs.RegisterNode(testNode("node1", "10.0.0.1"))
 	require.NoError(t, err)
@@ -56,7 +103,7 @@ func TestRegisterNode(t *testing.T) {
 
 func TestRegisterDuplicateName(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "", "", nil)
+	fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL)
 
 	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
 	err := fs.RegisterNode(testNode("node1", "10.0.0.2"))
@@ -66,7 +113,7 @@ func TestRegisterDuplicateName(t *testing.T) {
 
 func TestRegisterDuplicateIP(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "", "", nil)
+	fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL)
 
 	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
 	err := fs.RegisterNode(testNode("node2", "10.0.0.1"))
@@ -76,7 +123,7 @@ func TestRegisterDuplicateIP(t *testing.T) {
 
 func TestIsComplete(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(2, testCreds(), "", "", nil)
+	fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL)
 
 	assert.False(t, fs.IsComplete())
 	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
@@ -87,7 +134,7 @@ func TestIsComplete(t *testing.T) {
 
 func TestWaitForCompletion(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(2, testCreds(), "", "", nil)
+	fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL)
 
 	go func() {
 		time.Sleep(50 * time.Millisecond)
@@ -102,7 +149,7 @@ func TestWaitForCompletion(t *testing.T) {
 
 func TestWaitForCompletionTimeout(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "", "", nil)
+	fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL)
 
 	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
 
@@ -111,18 +158,62 @@ func TestWaitForCompletionTimeout(t *testing.T) {
 	assert.Contains(t, err.Error(), "timed out")
 }
 
-func testServer(t *testing.T, fs *FormationServer) *httptest.Server {
-	t.Helper()
-	mux := http.NewServeMux()
-	mux.HandleFunc("POST /formation/join", fs.handleJoin)
-	mux.HandleFunc("GET /formation/status", fs.handleStatus)
-	mux.HandleFunc("GET /formation/health", fs.handleHealth)
-	return httptest.NewServer(mux)
+func TestValidateToken_Valid(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL)
+
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	assert.NoError(t, fs.validateToken(req))
 }
 
-func TestJoinEndpoint(t *testing.T) {
+func TestValidateToken_Missing(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "", "", nil)
+	fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL)
+
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	err := fs.validateToken(req)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "missing Authorization header")
+}
+
+func TestValidateToken_Invalid(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL)
+
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set("Authorization", "Bearer wrong-token")
+	err := fs.validateToken(req)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid join token")
+}
+
+func TestValidateToken_Expired(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, 1*time.Millisecond)
+	time.Sleep(5 * time.Millisecond)
+
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set("Authorization", "Bearer "+testToken)
+	err := fs.validateToken(req)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "expired")
+}
+
+func TestValidateToken_BadFormat(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(2, testCreds(), "", "", nil, testToken, testTokenTTL)
+
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set("Authorization", "Basic "+testToken)
+	err := fs.validateToken(req)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid Authorization header format")
+}
+
+func TestJoinEndpoint_Unauthorized(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL)
 	ts := testServer(t, fs)
 	defer ts.Close()
 
@@ -131,6 +222,19 @@ func TestJoinEndpoint(t *testing.T) {
 	require.NoError(t, err)
 	defer resp.Body.Close()
 
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
+func TestJoinEndpoint_WithToken(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL)
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	body, _ := json.Marshal(JoinRequest{NodeInfo: testNode("node1", "10.0.0.1")})
+	resp := authPost(t, ts.URL+"/formation/join", testToken, body)
+	defer resp.Body.Close()
+
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 	var jr JoinResponse
@@ -142,33 +246,43 @@ func TestJoinEndpoint(t *testing.T) {
 
 func TestJoinEndpointDuplicate(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "", "", nil)
+	fs := NewFormationServer(3, testCreds(), "", "", nil, testToken, testTokenTTL)
 	ts := testServer(t, fs)
 	defer ts.Close()
 
 	body, _ := json.Marshal(JoinRequest{NodeInfo: testNode("node1", "10.0.0.1")})
 
-	resp1, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body))
-	require.NoError(t, err)
+	resp1 := authPost(t, ts.URL+"/formation/join", testToken, body)
 	resp1.Body.Close()
 	assert.Equal(t, http.StatusOK, resp1.StatusCode)
 
-	resp2, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body))
-	require.NoError(t, err)
+	resp2 := authPost(t, ts.URL+"/formation/join", testToken, body)
 	defer resp2.Body.Close()
 	assert.Equal(t, http.StatusConflict, resp2.StatusCode)
 }
 
+func TestStatusEndpoint_Unauthorized(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil, testToken, testTokenTTL)
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL + "/formation/status")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
 func TestStatusEndpointIncomplete(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil)
+	fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil, testToken, testTokenTTL)
 	ts := testServer(t, fs)
 	defer ts.Close()
 
 	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
 
-	resp, err := http.Get(ts.URL + "/formation/status")
-	require.NoError(t, err)
+	resp := authGet(t, ts.URL+"/formation/status", testToken)
 	defer resp.Body.Close()
 
 	var sr StatusResponse
@@ -184,15 +298,14 @@ func TestStatusEndpointIncomplete(t *testing.T) {
 func TestStatusEndpointComplete(t *testing.T) {
 	t.Parallel()
 	creds := testCreds()
-	fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data", nil)
+	fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data", nil, testToken, testTokenTTL)
 	ts := testServer(t, fs)
 	defer ts.Close()
 
 	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
 	require.NoError(t, fs.RegisterNode(testNode("node2", "10.0.0.2")))
 
-	resp, err := http.Get(ts.URL + "/formation/status")
-	require.NoError(t, err)
+	resp := authGet(t, ts.URL+"/formation/status", testToken)
 	defer resp.Body.Close()
 
 	var sr StatusResponse
@@ -209,14 +322,13 @@ func TestStatusEndpointComplete(t *testing.T) {
 
 func TestStatusEndpointMasterKey(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key", nil)
+	fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key", nil, testToken, testTokenTTL)
 	fs.SetMasterKey("dGVzdC1tYXN0ZXIta2V5LWJhc2U2NC1lbmNvZGVk")
 	ts := testServer(t, fs)
 	defer ts.Close()
 
 	// Before completion: master key should not be exposed
-	resp, err := http.Get(ts.URL + "/formation/status")
-	require.NoError(t, err)
+	resp := authGet(t, ts.URL+"/formation/status", testToken)
 	var sr StatusResponse
 	require.NoError(t, json.NewDecoder(resp.Body).Decode(&sr))
 	resp.Body.Close()
@@ -227,17 +339,16 @@ func TestStatusEndpointMasterKey(t *testing.T) {
 	require.NoError(t, fs.RegisterNode(testNode("node2", "10.0.0.2")))
 
 	// After completion: master key should be present
-	resp, err = http.Get(ts.URL + "/formation/status")
-	require.NoError(t, err)
+	resp = authGet(t, ts.URL+"/formation/status", testToken)
 	defer resp.Body.Close()
 	require.NoError(t, json.NewDecoder(resp.Body).Decode(&sr))
 	assert.True(t, sr.Complete)
 	assert.Equal(t, "dGVzdC1tYXN0ZXIta2V5LWJhc2U2NC1lbmNvZGVk", sr.MasterKey)
 }
 
-func TestHealthEndpoint(t *testing.T) {
+func TestHealthEndpoint_NoAuth(t *testing.T) {
 	t.Parallel()
-	fs := NewFormationServer(1, testCreds(), "", "", nil)
+	fs := NewFormationServer(1, testCreds(), "", "", nil, testToken, testTokenTTL)
 	ts := testServer(t, fs)
 	defer ts.Close()
 
@@ -289,22 +400,21 @@ func TestBuildPredastoreNodes(t *testing.T) {
 	assert.Equal(t, "10.0.0.3", pnodes[2].Host)
 }
 
-func TestFullFormationFlow(t *testing.T) {
+func TestFullFormationFlow_WithToken(t *testing.T) {
 	t.Parallel()
 	creds := testCreds()
-	fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", nil)
+	fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", nil, testToken, testTokenTTL)
 	ts := testServer(t, fs)
 	defer ts.Close()
 
-	// 3 nodes join sequentially
+	// 3 nodes join sequentially with auth
 	for i, n := range []struct{ name, ip string }{
 		{"node1", "10.0.0.1"},
 		{"node2", "10.0.0.2"},
 		{"node3", "10.0.0.3"},
 	} {
 		body, _ := json.Marshal(JoinRequest{NodeInfo: testNode(n.name, n.ip)})
-		resp, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body))
-		require.NoError(t, err)
+		resp := authPost(t, ts.URL+"/formation/join", testToken, body)
 
 		var jr JoinResponse
 		require.NoError(t, json.NewDecoder(resp.Body).Decode(&jr))
@@ -314,9 +424,8 @@ func TestFullFormationFlow(t *testing.T) {
 		assert.Equal(t, i+1, jr.Joined)
 	}
 
-	// All nodes poll status — should be complete
-	resp, err := http.Get(ts.URL + "/formation/status")
-	require.NoError(t, err)
+	// All nodes poll status with auth — should be complete
+	resp := authGet(t, ts.URL+"/formation/status", testToken)
 	defer resp.Body.Close()
 
 	var sr StatusResponse

spinifex/gateway/auth.go

@@ -106,7 +106,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 			ak, err := gw.IAMService.LookupAccessKey(accessKey)
 			if err != nil {
 				if strings.Contains(err.Error(), awserrors.ErrorIAMNoSuchEntity) {
-					slog.Debug("Access key not found", "accessKeyID", accessKey)
+					slog.Warn("Auth failure: access key not found", "accessKeyID", accessKey, "sourceIP", clientIP)
 					gw.RateLimiter.RecordFailure(clientIP)
 					gw.writeSigV4Error(w, r, awserrors.ErrorInvalidClientTokenId)
 					return
@@ -116,7 +116,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 				return
 			}
 			if ak.Status != handlers_iam.AccessKeyStatusActive {
-				slog.Debug("Access key inactive", "accessKeyID", accessKey)
+				slog.Warn("Auth failure: access key inactive", "accessKeyID", accessKey, "sourceIP", clientIP)
 				gw.RateLimiter.RecordFailure(clientIP)
 				gw.writeSigV4Error(w, r, awserrors.ErrorInvalidClientTokenId)
 				return
@@ -172,8 +172,9 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 
 			// Compare signatures using constant-time comparison to prevent timing attacks
 			if subtle.ConstantTimeCompare([]byte(expectedSignature), []byte(providedSignature)) != 1 {
-				slog.Debug("Signature mismatch",
-					"accessKeyId", accessKey,
+				slog.Warn("Auth failure: signature mismatch",
+					"accessKeyID", accessKey,
+					"sourceIP", clientIP,
 				)
 				gw.RateLimiter.RecordFailure(clientIP)
 				gw.writeSigV4Error(w, r, awserrors.ErrorSignatureDoesNotMatch)

spinifex/migrate/003_nats_monitor_localhost.go

@@ -0,0 +1,51 @@
+package migrate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+)
+
+func init() {
+	DefaultRegistry.RegisterConfig("nats.conf", ConfigMigration{
+		FromVersion: 1,
+		ToVersion:   2,
+		Description: "Bind NATS monitoring to localhost only",
+		Run: func(ctx ConfigContext) error {
+			path := filepath.Join(ctx.ConfigDir, "nats", "nats.conf")
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			// Match "http: <anything>:8222" or "http_port: 8222"
+			httpRe := regexp.MustCompile(`(?m)^http(?:_port)?:\s*.*8222\s*$`)
+			if !httpRe.Match(content) {
+				ctx.Logger.Info("nats.conf: no monitoring line found, skipping")
+				return nil
+			}
+
+			updated := httpRe.ReplaceAll(content, []byte("http: 127.0.0.1:8222"))
+
+			info, err := os.Stat(path)
+			if err != nil {
+				return err
+			}
+			if err := os.WriteFile(path, updated, info.Mode()); err != nil {
+				return err
+			}
+
+			// Post-migration validation.
+			result, err := os.ReadFile(path)
+			if err != nil {
+				return fmt.Errorf("post-migration validation read failed: %w", err)
+			}
+			localhostRe := regexp.MustCompile(`(?m)^http:\s*127\.0\.0\.1:8222\s*$`)
+			if !localhostRe.Match(result) {
+				return fmt.Errorf("monitoring line not rewritten to localhost after migration")
+			}
+			return nil
+		},
+	})
+}

spinifex/migrate/004_nats_tls.go

@@ -0,0 +1,148 @@
+package migrate
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+func init() {
+	// nats.conf: version 2 → 3 — add TLS blocks for client and cluster connections.
+	DefaultRegistry.RegisterConfig("nats.conf", ConfigMigration{
+		FromVersion: 2,
+		ToVersion:   3,
+		Description: "Add TLS for client connections and mutual TLS for cluster routes",
+		Run: func(ctx ConfigContext) error {
+			path := filepath.Join(ctx.ConfigDir, "nats", "nats.conf")
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			text := string(content)
+
+			// Skip if TLS is already configured.
+			if strings.Contains(text, "tls {") {
+				ctx.Logger.Info("nats.conf: TLS block already present, skipping")
+				return nil
+			}
+
+			certFile := fmt.Sprintf("%s/server.pem", ctx.ConfigDir)
+			keyFile := fmt.Sprintf("%s/server.key", ctx.ConfigDir)
+			caFile := fmt.Sprintf("%s/ca.pem", ctx.ConfigDir)
+
+			// 1. Inject client TLS block after the top-level listen line (port 4222 only,
+			// not the cluster listen on 4248).
+			clientTLS := fmt.Sprintf("\n\ntls {\n  cert_file: \"%s\"\n  key_file:  \"%s\"\n  ca_file:   \"%s\"\n}", certFile, keyFile, caFile)
+
+			listenRe := regexp.MustCompile(`(?m)^listen:\s*\S+:4222\s*$`)
+			if !listenRe.MatchString(text) {
+				return fmt.Errorf("nats.conf: client listen line (port 4222) not found")
+			}
+			loc := listenRe.FindStringIndex(text)
+			text = text[:loc[1]] + clientTLS + text[loc[1]:]
+
+			// 2. Inject cluster TLS block inside the cluster {} block, after the listen line.
+			clusterTLS := fmt.Sprintf("\n\n  tls {\n    cert_file: \"%s\"\n    key_file:  \"%s\"\n    ca_file:   \"%s\"\n    verify:    true\n  }", certFile, keyFile, caFile)
+
+			// Match the cluster listen line and insert TLS block after it.
+			clusterListenRe := regexp.MustCompile(`(?m)^(\s*listen:\s*\S+:4248)\s*$`)
+			if clusterListenRe.MatchString(text) {
+				text = clusterListenRe.ReplaceAllStringFunc(text, func(match string) string {
+					return strings.TrimRight(match, "\n") + clusterTLS
+				})
+			}
+
+			info, err := os.Stat(path)
+			if err != nil {
+				return err
+			}
+			return os.WriteFile(path, []byte(text), info.Mode())
+		},
+	})
+
+	// spinifex.toml: version 1 → 2 — add cacert under [nodes.*.nats].
+	DefaultRegistry.RegisterConfigTarget("spinifex.toml", "spinifex.toml", &TOMLVersionReader{})
+
+	DefaultRegistry.RegisterConfig("spinifex.toml", ConfigMigration{
+		FromVersion: 1,
+		ToVersion:   2,
+		Description: "Add NATS CA cert path for TLS",
+		Run: func(ctx ConfigContext) error {
+			path := filepath.Join(ctx.ConfigDir, "spinifex.toml")
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			text := string(content)
+
+			// Skip if cacert is already present.
+			if strings.Contains(text, "cacert") {
+				ctx.Logger.Info("spinifex.toml: cacert already present, skipping")
+				return nil
+			}
+
+			// Rewrite 0.0.0.0 → 127.0.0.1 for the NATS host. 0.0.0.0 is a valid
+			// bind address but not a valid TLS connect address (certs have real IPs).
+			text = strings.ReplaceAll(text, `host = "0.0.0.0:4222"`, `host = "127.0.0.1:4222"`)
+
+			// Insert cacert after each host line under [nodes.*.nats].
+			// Pattern: host = "x.x.x.x:4222" followed by a newline.
+			caPath := fmt.Sprintf("%s/ca.pem", ctx.ConfigDir)
+			natsHostRe := regexp.MustCompile(`(?m)^(host\s*=\s*"[^"]+:4222")\s*$`)
+			if !natsHostRe.MatchString(text) {
+				return fmt.Errorf("spinifex.toml: NATS host line (host = \"...:4222\") not found, cannot insert cacert")
+			}
+			text = natsHostRe.ReplaceAllStringFunc(text, func(match string) string {
+				return strings.TrimRight(match, "\n") + fmt.Sprintf("\ncacert = \"%s\"", caPath)
+			})
+
+			info, err := os.Stat(path)
+			if err != nil {
+				return err
+			}
+			return os.WriteFile(path, []byte(text), info.Mode())
+		},
+	})
+
+	// predastore.toml: version 2 → 3 — add nats_ca_cert under [iam].
+	DefaultRegistry.RegisterConfig("predastore.toml", ConfigMigration{
+		FromVersion: 2,
+		ToVersion:   3,
+		Description: "Add NATS CA cert path for TLS",
+		Run: func(ctx ConfigContext) error {
+			path := filepath.Join(ctx.ConfigDir, "predastore", "predastore.toml")
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			text := string(content)
+
+			// Skip if nats_ca_cert is already present.
+			if strings.Contains(text, "nats_ca_cert") {
+				ctx.Logger.Info("predastore.toml: nats_ca_cert already present, skipping")
+				return nil
+			}
+
+			// Insert nats_ca_cert after the nats_token line.
+			caPath := fmt.Sprintf("%s/ca.pem", ctx.ConfigDir)
+			natsTokenRe := regexp.MustCompile(`(?m)^(nats_token\s*=\s*"[^"]*")\s*$`)
+			if !natsTokenRe.MatchString(text) {
+				return fmt.Errorf("predastore.toml: nats_token line not found, cannot insert nats_ca_cert")
+			}
+			text = natsTokenRe.ReplaceAllStringFunc(text, func(match string) string {
+				return strings.TrimRight(match, "\n") + fmt.Sprintf("\nnats_ca_cert = \"%s\"", caPath)
+			})
+
+			info, err := os.Stat(path)
+			if err != nil {
+				return err
+			}
+			return os.WriteFile(path, []byte(text), info.Mode())
+		},
+	})
+}

spinifex/migrate/migrate_test.go

@@ -712,6 +712,67 @@ listen: 0.0.0.0:4222
 	assert.Equal(t, 2, v)
 }
 
+// --- NATS monitoring localhost migration tests ---
+
+func TestNATSMonitorLocalhost(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			"BindIP 0.0.0.0",
+			"# spinifex-config-version: 1\n# NATS\nhttp: 0.0.0.0:8222\n",
+			"http: 127.0.0.1:8222",
+		},
+		{
+			"BindIP specific IP",
+			"# spinifex-config-version: 1\n# NATS\nhttp: 10.0.0.1:8222\n",
+			"http: 127.0.0.1:8222",
+		},
+		{
+			"no BindIP (http_port)",
+			"# spinifex-config-version: 1\n# NATS\nhttp_port: 8222\n",
+			"http: 127.0.0.1:8222",
+		},
+		{
+			"already migrated (idempotent)",
+			"# spinifex-config-version: 1\n# NATS\nhttp: 127.0.0.1:8222\n",
+			"http: 127.0.0.1:8222",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir := t.TempDir()
+			natsDir := filepath.Join(dir, "nats")
+			require.NoError(t, os.MkdirAll(natsDir, 0o755))
+			confPath := filepath.Join(natsDir, "nats.conf")
+			require.NoError(t, os.WriteFile(confPath, []byte(tt.input), 0o640))
+
+			r := NewRegistry()
+			r.RegisterConfigTarget("nats.conf", "nats/nats.conf", &NATSConfVersionReader{})
+			r.RegisterConfig("nats.conf", ConfigMigration{
+				FromVersion: 1,
+				ToVersion:   2,
+				Description: "Bind NATS monitoring to localhost only",
+				Run:         DefaultRegistry.configMigrations["nats.conf"][1].Run,
+			})
+
+			err := r.RunConfig("nats.conf", dir, dir)
+			require.NoError(t, err)
+
+			data, err := os.ReadFile(confPath)
+			require.NoError(t, err)
+			assert.Contains(t, string(data), tt.expected)
+			// Verify no leftover non-localhost monitoring lines
+			assert.NotContains(t, string(data), "0.0.0.0:8222")
+			assert.NotContains(t, string(data), "10.0.0.1:8222")
+			assert.NotContains(t, string(data), "http_port:")
+		})
+	}
+}
+
 func TestRunConfig_RejectsChainGap(t *testing.T) {
 	dir := t.TempDir()
 	natsDir := filepath.Join(dir, "nats")

spinifex/services/awsgw/awsgw.go

@@ -102,7 +102,7 @@ func launchService(config *config.ClusterConfig) error {
 
 	// Connect to NATS for service communication. On concurrent startup the
 	// local NATS server may not be listening yet, so retry with backoff.
-	natsConn, err := connectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token)
+	natsConn, err := connectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token, nodeConfig.NATS.CACert)
 	if err != nil {
 		return err
 	}
@@ -205,13 +205,13 @@ func launchService(config *config.ClusterConfig) error {
 
 // connectNATS establishes a connection to NATS with retry/backoff. On concurrent
 // startup the local NATS server may not be listening yet.
-func connectNATS(host, token string) (*nats.Conn, error) {
+func connectNATS(host, token, caCertPath string) (*nats.Conn, error) {
 	const maxWait = 5 * time.Minute
 	retryDelay := 500 * time.Millisecond
 	start := time.Now()
 
 	for {
-		nc, err := utils.ConnectNATS(host, token)
+		nc, err := utils.ConnectNATS(host, token, caCertPath)
 		if err == nil {
 			if time.Since(start) > time.Second {
 				slog.Info("NATS connection established", "elapsed", time.Since(start).Round(time.Second))
@@ -219,6 +219,11 @@ func connectNATS(host, token string) (*nats.Conn, error) {
 			return nc, nil
 		}
 
+		// TLS configuration errors are permanent — retrying will not help.
+		if strings.Contains(err.Error(), "CA cert") {
+			return nil, fmt.Errorf("NATS TLS configuration error: %w", err)
+		}
+
 		elapsed := time.Since(start)
 		if elapsed >= maxWait {
 			return nil, fmt.Errorf("NATS connect failed after %s: %w", elapsed.Round(time.Second), err)

spinifex/services/nats/nats_test.go

@@ -9,6 +9,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -34,6 +35,14 @@ func TestRenderedConfig_EnforcesAuth(t *testing.T) {
 	tmpDir := t.TempDir()
 	confPath := filepath.Join(tmpDir, "nats.conf")
 
+	// Generate test CA and server certs for TLS.
+	caCertPath := filepath.Join(tmpDir, "ca.pem")
+	caKeyPath := filepath.Join(tmpDir, "ca.key")
+	require.NoError(t, admin.GenerateCACert(caCertPath, caKeyPath))
+	serverCertPath := filepath.Join(tmpDir, "server.pem")
+	serverKeyPath := filepath.Join(tmpDir, "server.key")
+	require.NoError(t, admin.GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, "127.0.0.1"))
+
 	// Read and render the production template.
 	raw, err := os.ReadFile(templatePath(t))
 	require.NoError(t, err)
@@ -50,6 +59,7 @@ func TestRenderedConfig_EnforcesAuth(t *testing.T) {
 		"NatsToken": token,
 		"DataDir":   tmpDir,
 		"LogDir":    tmpDir,
+		"ConfigDir": tmpDir,
 	})
 	f.Close()
 	require.NoError(t, err)
@@ -70,22 +80,30 @@ func TestRenderedConfig_EnforcesAuth(t *testing.T) {
 	require.True(t, ns.ReadyForConnections(5*time.Second))
 	t.Cleanup(func() { ns.Shutdown() })
 
+	// Build TLS option using the test CA cert for client connections.
+	tlsOpt := nats.RootCAs(caCertPath)
+
 	t.Run("no token rejected", func(t *testing.T) {
-		_, err := nats.Connect(ns.ClientURL(), nats.MaxReconnects(0))
+		_, err := nats.Connect(ns.ClientURL(), tlsOpt, nats.MaxReconnects(0))
 		assert.Error(t, err, "unauthenticated connection should be rejected")
 	})
 
 	t.Run("wrong token rejected", func(t *testing.T) {
-		_, err := nats.Connect(ns.ClientURL(), nats.Token("wrong-token"), nats.MaxReconnects(0))
+		_, err := nats.Connect(ns.ClientURL(), tlsOpt, nats.Token("wrong-token"), nats.MaxReconnects(0))
 		assert.Error(t, err, "wrong token should be rejected")
 	})
 
 	t.Run("correct token accepted", func(t *testing.T) {
-		nc, err := nats.Connect(ns.ClientURL(), nats.Token(token), nats.MaxReconnects(0))
+		nc, err := nats.Connect(ns.ClientURL(), tlsOpt, nats.Token(token), nats.MaxReconnects(0))
 		require.NoError(t, err, "correct token should be accepted")
 		defer nc.Close()
 		assert.True(t, nc.IsConnected())
 	})
+
+	t.Run("plaintext rejected", func(t *testing.T) {
+		_, err := nats.Connect(ns.ClientURL(), nats.Token(token), nats.MaxReconnects(0))
+		assert.Error(t, err, "plaintext connection should be rejected when TLS is enabled")
+	})
 }
 
 // TestRenderedConfig_HasMigrationVersionMarker ensures the nats.conf template
@@ -98,6 +116,6 @@ func TestRenderedConfig_HasMigrationVersionMarker(t *testing.T) {
 	require.NoError(t, err)
 
 	firstLine := strings.SplitN(string(raw), "\n", 2)[0]
-	assert.Equal(t, "# spinifex-config-version: 1", firstLine,
+	assert.Equal(t, "# spinifex-config-version: 3", firstLine,
 		"nats.conf template must start with the current migration version marker")
 }

spinifex/services/viperblockd/viperblockd.go

@@ -39,6 +39,7 @@ type Config struct {
 	Debug          bool
 	NatsHost       string
 	NatsToken      string
+	NatsCACert     string
 	MountedVolumes []MountedVolume
 	S3Host         string
 	Bucket         string
@@ -169,7 +170,7 @@ func (svc *Service) Reload() (err error) {
 
 func launchService(cfg *Config) (err error) {
 	// Connect to NATS
-	nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken)
+	nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken, cfg.NatsCACert)
 	if err != nil {
 		slog.Error("Failed to connect to NATS", "err", err)
 		return err

spinifex/services/vpcd/mock_ovn.go

@@ -532,7 +532,7 @@ func (m *MockOVNClient) SetPortGroupPorts(_ context.Context, name string, ports
 
 // ACLs
 
-func (m *MockOVNClient) AddACL(_ context.Context, portGroupName string, direction string, priority int, match string, action string) error {
+func (m *MockOVNClient) AddACL(_ context.Context, portGroupName string, spec ACLSpec) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	pg, exists := m.portGroups[portGroupName]
@@ -541,10 +541,19 @@ func (m *MockOVNClient) AddACL(_ context.Context, portGroupName string, directio
 	}
 	acl := &nbdb.ACL{
 		UUID:      utils.GenerateResourceID("acl"),
-		Direction: direction,
-		Priority:  priority,
-		Match:     match,
-		Action:    action,
+		Direction: spec.Direction,
+		Priority:  spec.Priority,
+		Match:     spec.Match,
+		Action:    spec.Action,
+		Log:       spec.Log,
+	}
+	if spec.Name != "" {
+		name := spec.Name
+		acl.Name = &name
+	}
+	if spec.Severity != "" {
+		severity := spec.Severity
+		acl.Severity = &severity
 	}
 	m.acls[acl.UUID] = acl
 	pg.ACLs = append(pg.ACLs, acl.UUID)

spinifex/services/vpcd/nbdb/model.go

@@ -104,10 +104,14 @@ type PortGroup struct {
 // ACL represents an OVN ACL rule attached to a port group or logical switch.
 type ACL struct {
 	UUID        string            `ovsdb:"_uuid"`
+	Name        *string           `ovsdb:"name"`
 	Direction   string            `ovsdb:"direction"` // "to-lport" or "from-lport"
 	Priority    int               `ovsdb:"priority"`
 	Match       string            `ovsdb:"match"`
 	Action      string            `ovsdb:"action"` // "allow-related", "drop", "allow", "reject"
+	Log         bool              `ovsdb:"log"`
+	Severity    *string           `ovsdb:"severity"` // "alert", "warning", "notice", "info", "debug"
+	Meter       *string           `ovsdb:"meter"`
 	ExternalIDs map[string]string `ovsdb:"external_ids"`
 }
 

spinifex/services/vpcd/ovn.go

@@ -34,6 +34,18 @@ func (c *LiveOVNClient) transactOps(ctx context.Context, ops []ovsdb.Operation)
 	return err
 }
 
+// ACLSpec describes an OVN ACL rule for attachment to a port group.
+// Name, Severity, and Meter are optional — set Severity when Log is true.
+type ACLSpec struct {
+	Direction string // "to-lport" or "from-lport"
+	Priority  int
+	Match     string
+	Action    string // "allow-related", "drop", "allow", "reject"
+	Name      string
+	Log       bool
+	Severity  string // "alert", "warning", "notice", "info", "debug"
+}
+
 // OVNClient defines the interface for interacting with the OVN Northbound Database.
 // This abstraction allows for mock implementations in tests.
 type OVNClient interface {
@@ -88,7 +100,7 @@ type OVNClient interface {
 	SetPortGroupPorts(ctx context.Context, name string, ports []string) error
 
 	// ACLs (attached to port groups)
-	AddACL(ctx context.Context, portGroupName string, direction string, priority int, match string, action string) error
+	AddACL(ctx context.Context, portGroupName string, spec ACLSpec) error
 	ClearACLs(ctx context.Context, portGroupName string) error
 
 	// Gateway Chassis (HA scheduling for gateway router ports)
@@ -821,7 +833,7 @@ func (c *LiveOVNClient) SetPortGroupPorts(_ context.Context, _ string, _ []strin
 	return fmt.Errorf("SetPortGroupPorts: not yet implemented for live OVN client")
 }
 
-func (c *LiveOVNClient) AddACL(_ context.Context, _ string, _ string, _ int, _ string, _ string) error {
+func (c *LiveOVNClient) AddACL(_ context.Context, _ string, _ ACLSpec) error {
 	return fmt.Errorf("AddACL: not yet implemented for live OVN client")
 }
 

spinifex/services/vpcd/security.go

@@ -16,6 +16,12 @@ const (
 	TopicUpdateSG = "vpc.update-sg"
 )
 
+// denyACLSeverity is the syslog severity OVN uses when a packet matches a
+// default-deny ACL. "info" is loud enough to be captured by a syslog forwarder
+// yet quiet enough to avoid paging on every dropped scan packet. Operators can
+// promote it at their log collector if they want higher-priority alerts.
+const denyACLSeverity = "info"
+
 // SGEvent carries security group state from the handler to vpcd.
 type SGEvent struct {
 	GroupId      string         `json:"group_id"`
@@ -48,11 +54,13 @@ func (h *TopologyHandler) handleCreateSG(msg *nats.Msg) {
 		return
 	}
 
-	// Add default deny ACL (priority 900) — drop all traffic not explicitly allowed
-	if err := h.ovn.AddACL(ctx, pgName, "to-lport", 900, fmt.Sprintf("outport == @%s && ip4", pgName), "drop"); err != nil {
+	// Add default deny ACLs (priority 900) — drop all traffic not explicitly
+	// allowed. Logging is enabled so boundary communications are observable via
+	// syslog (CMMC SC.L1-3.13.1).
+	if err := h.ovn.AddACL(ctx, pgName, denyIngressACL(pgName)); err != nil {
 		slog.Warn("vpcd: failed to add default deny ingress ACL", "pg", pgName, "err", err)
 	}
-	if err := h.ovn.AddACL(ctx, pgName, "from-lport", 900, fmt.Sprintf("inport == @%s && ip4", pgName), "drop"); err != nil {
+	if err := h.ovn.AddACL(ctx, pgName, denyEgressACL(pgName)); err != nil {
 		slog.Warn("vpcd: failed to add default deny egress ACL", "pg", pgName, "err", err)
 	}
 
@@ -127,11 +135,11 @@ func (h *TopologyHandler) handleUpdateSG(msg *nats.Msg) {
 		slog.Warn("vpcd: failed to clear ACLs for update", "pg", pgName, "err", err)
 	}
 
-	// Re-add default deny ACLs (priority 900)
-	if err := h.ovn.AddACL(ctx, pgName, "to-lport", 900, fmt.Sprintf("outport == @%s && ip4", pgName), "drop"); err != nil {
+	// Re-add default deny ACLs (priority 900) with logging enabled.
+	if err := h.ovn.AddACL(ctx, pgName, denyIngressACL(pgName)); err != nil {
 		slog.Warn("vpcd: failed to re-add default deny ingress ACL", "pg", pgName, "err", err)
 	}
-	if err := h.ovn.AddACL(ctx, pgName, "from-lport", 900, fmt.Sprintf("inport == @%s && ip4", pgName), "drop"); err != nil {
+	if err := h.ovn.AddACL(ctx, pgName, denyEgressACL(pgName)); err != nil {
 		slog.Warn("vpcd: failed to re-add default deny egress ACL", "pg", pgName, "err", err)
 	}
 
@@ -150,18 +158,49 @@ func (h *TopologyHandler) handleUpdateSG(msg *nats.Msg) {
 
 // addRuleACLs adds OVN ACLs for a set of ingress and egress rules at priority 1000
 // (higher than the default deny at 900, so allow rules take precedence).
+// Allow rules are not logged — accept logging on a private network is
+// high-volume and low-signal. Only denies carry Log=true.
 func (h *TopologyHandler) addRuleACLs(ctx context.Context, pgName string, ingress, egress []SGRuleForACL) {
 	for _, rule := range ingress {
 		match := BuildIngressACLMatch(pgName, rule)
-		if err := h.ovn.AddACL(ctx, pgName, "to-lport", 1000, match, "allow-related"); err != nil {
+		spec := ACLSpec{Direction: "to-lport", Priority: 1000, Match: match, Action: "allow-related"}
+		if err := h.ovn.AddACL(ctx, pgName, spec); err != nil {
 			slog.Warn("vpcd: failed to add ingress ACL", "pg", pgName, "match", match, "err", err)
 		}
 	}
 
 	for _, rule := range egress {
 		match := BuildEgressACLMatch(pgName, rule)
-		if err := h.ovn.AddACL(ctx, pgName, "from-lport", 1000, match, "allow-related"); err != nil {
+		spec := ACLSpec{Direction: "from-lport", Priority: 1000, Match: match, Action: "allow-related"}
+		if err := h.ovn.AddACL(ctx, pgName, spec); err != nil {
 			slog.Warn("vpcd: failed to add egress ACL", "pg", pgName, "match", match, "err", err)
 		}
 	}
 }
+
+// denyIngressACL builds the default-deny ingress ACL for a port group with
+// logging enabled (CMMC SC.L1-3.13.1 boundary monitoring).
+func denyIngressACL(pgName string) ACLSpec {
+	return ACLSpec{
+		Direction: "to-lport",
+		Priority:  900,
+		Match:     fmt.Sprintf("outport == @%s && ip4", pgName),
+		Action:    "drop",
+		Name:      pgName + "-deny-ingress",
+		Log:       true,
+		Severity:  denyACLSeverity,
+	}
+}
+
+// denyEgressACL is the egress counterpart to denyIngressACL.
+func denyEgressACL(pgName string) ACLSpec {
+	return ACLSpec{
+		Direction: "from-lport",
+		Priority:  900,
+		Match:     fmt.Sprintf("inport == @%s && ip4", pgName),
+		Action:    "drop",
+		Name:      pgName + "-deny-egress",
+		Log:       true,
+		Severity:  denyACLSeverity,
+	}
+}

spinifex/services/vpcd/security_test.go

@@ -40,10 +40,44 @@ func TestSecurity_CreatePortGroup(t *testing.T) {
 	assert.NotNil(t, pg)
 
 	// Verify default deny ACLs were created (2 deny ACLs at priority 900)
+	// and that each has logging enabled per CMMC SC.L1-3.13.1.
+	type aclSnapshot struct {
+		action   string
+		log      bool
+		severity string
+		name     string
+	}
+	var snapshots []aclSnapshot
 	mock.mu.Lock()
 	aclCount := len(pg.ACLs)
+	for _, aclUUID := range pg.ACLs {
+		a := mock.acls[aclUUID]
+		if a == nil {
+			continue
+		}
+		snap := aclSnapshot{action: a.Action, log: a.Log}
+		if a.Severity != nil {
+			snap.severity = *a.Severity
+		}
+		if a.Name != nil {
+			snap.name = *a.Name
+		}
+		snapshots = append(snapshots, snap)
+	}
 	mock.mu.Unlock()
+
 	assert.Equal(t, 2, aclCount, "should have 2 default deny ACLs (ingress + egress)")
+	denyCount := 0
+	for _, s := range snapshots {
+		if s.action != "drop" {
+			continue
+		}
+		denyCount++
+		assert.True(t, s.log, "default deny ACL must have log=true for boundary monitoring")
+		assert.Equal(t, "info", s.severity, "default deny ACL must use info severity")
+		assert.Contains(t, s.name, "sg_abc123-deny-", "default deny ACL must have a name for syslog correlation")
+	}
+	assert.Equal(t, 2, denyCount, "should have 2 drop ACLs")
 }
 
 func TestSecurity_DeletePortGroup(t *testing.T) {
@@ -123,30 +157,41 @@ func TestSecurity_UpdateSGAddRules(t *testing.T) {
 	assertSuccess(t, resp, "update SG with ingress rules")
 
 	// Verify ACLs were created: 2 default deny + 2 ingress allow = 4
+	type aclSnapshot struct {
+		action string
+		match  string
+		log    bool
+	}
+	var snapshots []aclSnapshot
 	mock.mu.Lock()
 	pg := mock.portGroups["sg_upd1"]
 	aclCount := len(pg.ACLs)
-
-	// Collect match strings from the ACLs
-	var matches []string
 	for _, aclUUID := range pg.ACLs {
-		acl := mock.acls[aclUUID]
-		if acl != nil {
-			matches = append(matches, acl.Match)
+		a := mock.acls[aclUUID]
+		if a == nil {
+			continue
 		}
+		snapshots = append(snapshots, aclSnapshot{action: a.Action, match: a.Match, log: a.Log})
 	}
 	mock.mu.Unlock()
 
 	assert.Equal(t, 4, aclCount, "should have 4 ACLs (2 deny + 2 ingress allow)")
 
-	// Check that at least one match contains tcp.dst == 22
+	// Check that at least one match contains tcp.dst == 22. Also verify
+	// logging policy: denies logged, allows not logged (CMMC SC.L1-3.13.1).
 	foundSSH := false
 	foundHTTPS := false
-	for _, m := range matches {
-		if containsAll(m, "tcp.dst == 22", "ip4.src == 10.0.0.0/8") {
+	for _, s := range snapshots {
+		switch s.action {
+		case "drop":
+			assert.True(t, s.log, "deny ACL must be logged")
+		case "allow-related":
+			assert.False(t, s.log, "allow ACL must not be logged (high volume, low signal)")
+		}
+		if containsAll(s.match, "tcp.dst == 22", "ip4.src == 10.0.0.0/8") {
 			foundSSH = true
 		}
-		if containsAll(m, "tcp.dst == 443") {
+		if containsAll(s.match, "tcp.dst == 443") {
 			foundHTTPS = true
 		}
 	}

spinifex/services/vpcd/vpcd.go

@@ -44,6 +44,8 @@ type Config struct {
 	NatsHost string
 	// NatsToken is the NATS authentication token.
 	NatsToken string
+	// NatsCACert is the path to the CA certificate for NATS TLS.
+	NatsCACert string
 	// OVNNBAddr is the OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641").
 	OVNNBAddr string
 	// OVNSBAddr is the OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642"), used for monitoring.
@@ -272,7 +274,7 @@ func launchService(cfg *Config) error {
 	slog.Info("OVN preflight passed (br-int exists, ovn-controller running)")
 
 	// Connect to NATS
-	nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken)
+	nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken, cfg.NatsCACert)
 	if err != nil {
 		slog.Error("Failed to connect to NATS", "err", err)
 		return err

spinifex/types/cluster.go

@@ -2,16 +2,6 @@ package types
 
 import "github.com/mulgadc/spinifex/spinifex/config"
 
-// Cluster manager API types
-
-type NodeJoinRequest struct {
-	Node       string `json:"node"`
-	Region     string `json:"region"`
-	AZ         string `json:"az"`
-	DataDir    string `json:"data_dir"`
-	DaemonHost string `json:"daemon_host"` // Host:port where this node's daemon is accessible
-}
-
 // SharedClusterData contains only the shared cluster information (no node-specific top-level fields)
 type SharedClusterData struct {
 	Epoch   uint64                   `json:"epoch" toml:"epoch"`
@@ -19,19 +9,6 @@ type SharedClusterData struct {
 	Nodes   map[string]config.Config `json:"nodes" toml:"nodes"`
 }
 
-type NodeJoinResponse struct {
-	Success     bool               `json:"success"`
-	Message     string             `json:"message"`
-	SharedData  *SharedClusterData `json:"shared_data,omitempty"`
-	ConfigHash  string             `json:"config_hash,omitempty"`
-	JoiningNode string             `json:"joining_node,omitempty"` // The node name that is joining
-	// CA distribution for per-node certificate generation
-	CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate
-	CAKey  string `json:"ca_key,omitempty"`  // PEM-encoded CA private key
-	// Predastore config distribution for multi-node clusters
-	PredastoreConfig string `json:"predastore_config,omitempty"` // Full predastore.toml content
-}
-
 type NodeHealthResponse struct {
 	Node          string            `json:"node"`
 	Status        string            `json:"status"`

spinifex/utils/images.go

@@ -3,11 +3,23 @@ package utils
 import (
 	"bufio"
 	"bytes"
+	"context"
+	"crypto/sha256"
+	"crypto/sha512"
+	"crypto/subtle"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"hash"
 	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
+	"os/signal"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -18,6 +30,230 @@ import (
 
 var ErrQCOWDetected = errors.New("qcow format detected")
 
+const sumsFileMaxSize = 1 * 1024 * 1024 // 1 MB, matches formation.go JoinRequest cap.
+
+var (
+	ErrChecksumMismatch        = errors.New("image checksum mismatch")
+	ErrChecksumNotFound        = errors.New("checksum entry for image filename not found")
+	ErrUnsupportedChecksumType = errors.New("unsupported checksum type")
+	ErrChecksumFetchFailed     = errors.New("checksum fetch failed")
+)
+
+// checksumExtraRootCAs is a test-only hook so unit tests can stand up an
+// httptest.NewTLSServer (which uses a self-signed cert) without relaxing the
+// HTTPS enforcement. Production builds leave this nil and the verification
+// client uses the system trust store.
+var checksumExtraRootCAs *x509.CertPool
+
+// checksumFetchTimeout is a var (not const) so tests can shrink it to exercise
+// the context-deadline path without waiting 30s.
+var checksumFetchTimeout = 30 * time.Second
+
+// VerifyImageChecksum fetches the sums file at checksumURL, locates the entry
+// for imagePath's basename, hashes imagePath with the algorithm named by
+// checksumType ("sha256" or "sha512"), and compares digests.
+//
+// Fails closed: every error path returns without accepting the image. A
+// non-HTTPS scheme, non-2xx status, transport error, response over 1 MB, or
+// cross-scheme redirect all wrap ErrChecksumFetchFailed. Unknown algorithm
+// wraps ErrUnsupportedChecksumType. Missing filename entry wraps
+// ErrChecksumNotFound. Digest mismatch wraps ErrChecksumMismatch and the
+// wrapped error's %v includes expected and actual hex.
+func VerifyImageChecksum(imagePath, checksumURL, checksumType string) error {
+	hasher, err := newHasher(checksumType)
+	if err != nil {
+		return err
+	}
+
+	expected, err := fetchExpectedDigest(checksumURL, filepath.Base(imagePath))
+	if err != nil {
+		slog.Error("image checksum fetch failed", "source", checksumURL, "err", err)
+		return err
+	}
+
+	// Distinct error for a catalog/sums-file algorithm mismatch — without this
+	// it'd surface as "tampering" via ConstantTimeCompare's length-0 return.
+	if len(expected) != hasher.Size()*2 {
+		return fmt.Errorf("%w: digest length %d from sums file does not match %s output length %d",
+			ErrChecksumFetchFailed, len(expected), checksumType, hasher.Size()*2)
+	}
+
+	actual, err := hashImageFile(imagePath, hasher)
+	if err != nil {
+		return fmt.Errorf("hash image file: %w", err)
+	}
+
+	if subtle.ConstantTimeCompare([]byte(expected), []byte(actual)) != 1 {
+		slog.Error("image checksum mismatch",
+			"image", imagePath,
+			"algorithm", checksumType,
+			"expected", expected,
+			"actual", actual,
+			"source", checksumURL,
+		)
+		return fmt.Errorf("%w: expected %s got %s", ErrChecksumMismatch, expected, actual)
+	}
+
+	return nil
+}
+
+func newHasher(checksumType string) (hash.Hash, error) {
+	switch strings.ToLower(checksumType) {
+	case "sha256":
+		return sha256.New(), nil
+	case "sha512":
+		return sha512.New(), nil
+	default:
+		return nil, fmt.Errorf("%w: %q", ErrUnsupportedChecksumType, checksumType)
+	}
+}
+
+// fetchExpectedDigest downloads the sums file and returns the hex digest for
+// the given filename. Enforces HTTPS on initial URL and every redirect hop,
+// caps redirects at 10, and truncates response at sumsFileMaxSize+1 so the
+// size check is unambiguous.
+func fetchExpectedDigest(checksumURL, filename string) (string, error) {
+	parsed, err := url.Parse(checksumURL)
+	if err != nil {
+		return "", fmt.Errorf("%w: parse url: %v", ErrChecksumFetchFailed, err)
+	}
+	if parsed.Scheme != "https" {
+		return "", fmt.Errorf("%w: non-https checksum url scheme %q", ErrChecksumFetchFailed, parsed.Scheme)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), checksumFetchTimeout)
+	defer cancel()
+	intCh := make(chan os.Signal, 1)
+	signal.Notify(intCh, os.Interrupt)
+	defer signal.Stop(intCh)
+	go func() {
+		select {
+		case <-intCh:
+			cancel()
+		case <-ctx.Done():
+		}
+	}()
+
+	transport := &http.Transport{}
+	if checksumExtraRootCAs != nil {
+		transport.TLSClientConfig = &tls.Config{RootCAs: checksumExtraRootCAs}
+	}
+	client := &http.Client{
+		Timeout:   checksumFetchTimeout,
+		Transport: transport,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= 10 {
+				return fmt.Errorf("stopped after 10 redirects")
+			}
+			if req.URL.Scheme != "https" {
+				return fmt.Errorf("refusing non-https redirect to %s", req.URL.Redacted())
+			}
+			return nil
+		},
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, checksumURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("%w: build request: %v", ErrChecksumFetchFailed, err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("%w: %v", ErrChecksumFetchFailed, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return "", fmt.Errorf("%w: unexpected status %s", ErrChecksumFetchFailed, resp.Status)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, sumsFileMaxSize+1))
+	if err != nil {
+		return "", fmt.Errorf("%w: read body: %v", ErrChecksumFetchFailed, err)
+	}
+	if len(body) > sumsFileMaxSize {
+		return "", fmt.Errorf("%w: sums file exceeds %d byte limit", ErrChecksumFetchFailed, sumsFileMaxSize)
+	}
+
+	digest, err := parseSumsFile(body, filename)
+	if err != nil {
+		return "", err
+	}
+	return digest, nil
+}
+
+// parseSumsFile scans a coreutils-style sums file and returns the hex digest
+// matching filename. Filename match is case-sensitive: upstream Debian/Ubuntu/
+// Alpine sums are consistently lowercase and a divergence is a real signal.
+//
+// Accepts three on-the-wire shapes: "<hex>  <name>" (text mode), "<hex> *<name>"
+// (binary mode), and a bare single-token "<hex>" line (single-file .sha512 from
+// Alpine, which does not carry a filename).
+func parseSumsFile(body []byte, filename string) (string, error) {
+	var bareDigest string
+	bareCount := 0
+	scanner := bufio.NewScanner(bytes.NewReader(body))
+	scanner.Buffer(make([]byte, 0, 64*1024), sumsFileMaxSize+1)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		fields := strings.Fields(line)
+		switch len(fields) {
+		case 1:
+			bareDigest = fields[0]
+			bareCount++
+		case 2:
+			name := strings.TrimPrefix(fields[1], "*")
+			if name == filename {
+				return fields[0], nil
+			}
+		default:
+			// More than two fields: not a format we recognise. Skip rather than
+			// reject outright — signed sums files occasionally have trailing
+			// commentary we want to tolerate.
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return "", fmt.Errorf("%w: scan sums file: %v", ErrChecksumFetchFailed, err)
+	}
+
+	// Alpine single-file .sha512 fallback: exactly one bare-digest line in the
+	// whole file. Any more and we refuse, because we can't tell which line is
+	// the right one without a filename.
+	if bareCount == 1 {
+		return bareDigest, nil
+	}
+
+	return "", fmt.Errorf("%w: %s", ErrChecksumNotFound, filename)
+}
+
+// hashImageFile streams the image through hasher and returns the digest as
+// lowercase hex. Zero-byte files are rejected outright so a truncated
+// download surfaces as an explicit error rather than an opaque "checksum
+// mismatch".
+func hashImageFile(imagePath string, hasher hash.Hash) (string, error) {
+	f, err := os.Open(imagePath)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	stat, err := f.Stat()
+	if err != nil {
+		return "", err
+	}
+	if stat.Size() == 0 {
+		return "", fmt.Errorf("image file %s is empty (likely a truncated or failed download)", imagePath)
+	}
+
+	if _, err := io.Copy(hasher, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
 type Images struct {
 	Name         string    `json:"name"`
 	Description  string    `json:"description"`

spinifex/utils/nats.go

@@ -1,10 +1,13 @@
 package utils
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
+	"os"
 	"time"
 
 	"github.com/nats-io/nats.go"
@@ -12,7 +15,8 @@ import (
 
 // ConnectNATS establishes a connection to a NATS server with standard reconnect
 // handling and logging. If token is non-empty, token authentication is used.
-func ConnectNATS(host, token string) (*nats.Conn, error) {
+// If caCertPath is non-empty, TLS is enabled using the given CA certificate.
+func ConnectNATS(host, token, caCertPath string) (*nats.Conn, error) {
 	opts := []nats.Option{
 		nats.ReconnectWait(time.Second),
 		nats.MaxReconnects(-1),
@@ -28,6 +32,20 @@ func ConnectNATS(host, token string) (*nats.Conn, error) {
 		opts = append(opts, nats.Token(token))
 	}
 
+	if caCertPath != "" {
+		caCert, err := os.ReadFile(caCertPath)
+		if err != nil {
+			return nil, fmt.Errorf("read CA cert %s: %w", caCertPath, err)
+		}
+		pool := x509.NewCertPool()
+		if !pool.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("failed to parse CA cert from %s", caCertPath)
+		}
+		opts = append(opts, nats.Secure(&tls.Config{
+			RootCAs: pool,
+		}))
+	}
+
 	nc, err := nats.Connect(host, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("NATS connect failed: %w", err)

spinifex/utils/nats_test.go

@@ -1,7 +1,18 @@
 package utils
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/json"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -21,7 +32,7 @@ func startTestNATSServer(t *testing.T) *server.Server {
 func TestConnectNATS_Success(t *testing.T) {
 	ns := startTestNATSServer(t)
 
-	nc, err := ConnectNATS(ns.ClientURL(), "")
+	nc, err := ConnectNATS(ns.ClientURL(), "", "")
 	require.NoError(t, err)
 	defer nc.Close()
 
@@ -44,7 +55,7 @@ func TestConnectNATS_WithToken(t *testing.T) {
 	t.Cleanup(func() { ns.Shutdown() })
 
 	// With correct token — should succeed
-	nc, err := ConnectNATS(ns.ClientURL(), "test-token-123")
+	nc, err := ConnectNATS(ns.ClientURL(), "test-token-123", "")
 	require.NoError(t, err)
 	defer nc.Close()
 	assert.True(t, nc.IsConnected())
@@ -65,17 +76,156 @@ func TestConnectNATS_WrongToken(t *testing.T) {
 	require.True(t, ns.ReadyForConnections(5*time.Second))
 	t.Cleanup(func() { ns.Shutdown() })
 
-	_, err = ConnectNATS(ns.ClientURL(), "wrong-token")
+	_, err = ConnectNATS(ns.ClientURL(), "wrong-token", "")
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "NATS connect failed")
 }
 
 func TestConnectNATS_BadAddress(t *testing.T) {
-	_, err := ConnectNATS("nats://127.0.0.1:1", "")
+	_, err := ConnectNATS("nats://127.0.0.1:1", "", "")
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "NATS connect failed")
 }
 
+func TestConnectNATS_MissingCACert(t *testing.T) {
+	_, err := ConnectNATS("nats://127.0.0.1:4222", "", "/nonexistent/ca.pem")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "read CA cert")
+	assert.Contains(t, err.Error(), "/nonexistent/ca.pem")
+}
+
+func TestConnectNATS_MalformedCACert(t *testing.T) {
+	tmp := t.TempDir()
+	badCert := filepath.Join(tmp, "bad-ca.pem")
+	require.NoError(t, os.WriteFile(badCert, []byte("not a PEM certificate"), 0o644))
+
+	_, err := ConnectNATS("nats://127.0.0.1:4222", "", badCert)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to parse CA cert")
+}
+
+// generateTestCA creates an ephemeral CA cert+key and writes PEM files to dir.
+func generateTestCA(t *testing.T, dir, name string) (certPath, keyPath string) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: name},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+
+	certPath = filepath.Join(dir, name+".pem")
+	keyPath = filepath.Join(dir, name+".key")
+	require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644))
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600))
+	return certPath, keyPath
+}
+
+// generateTestServerCert creates a server cert signed by the given CA.
+func generateTestServerCert(t *testing.T, dir, caCertPath, caKeyPath string) (certPath, keyPath string) {
+	t.Helper()
+	caCertPEM, err := os.ReadFile(caCertPath)
+	require.NoError(t, err)
+	block, _ := pem.Decode(caCertPEM)
+	caCert, err := x509.ParseCertificate(block.Bytes)
+	require.NoError(t, err)
+
+	caKeyPEM, err := os.ReadFile(caKeyPath)
+	require.NoError(t, err)
+	keyBlock, _ := pem.Decode(caKeyPEM)
+	caKey, err := x509.ParseECPrivateKey(keyBlock.Bytes)
+	require.NoError(t, err)
+
+	serverKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(2),
+		Subject:      pkix.Name{CommonName: "localhost"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &serverKey.PublicKey, caKey)
+	require.NoError(t, err)
+
+	certPath = filepath.Join(dir, "server.pem")
+	keyPath = filepath.Join(dir, "server.key")
+	require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644))
+	keyDER, err := x509.MarshalECPrivateKey(serverKey)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600))
+	return certPath, keyPath
+}
+
+// startTLSNATSServer starts a NATS server with TLS using the given cert files.
+func startTLSNATSServer(t *testing.T, serverCertPath, serverKeyPath, caCertPath string) *server.Server {
+	t.Helper()
+	cert, err := tls.LoadX509KeyPair(serverCertPath, serverKeyPath)
+	require.NoError(t, err)
+	caPEM, err := os.ReadFile(caCertPath)
+	require.NoError(t, err)
+	pool := x509.NewCertPool()
+	require.True(t, pool.AppendCertsFromPEM(caPEM))
+
+	tlsCfg := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    pool,
+	}
+
+	opts := &server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		NoLog:     true,
+		NoSigs:    true,
+		TLSConfig: tlsCfg,
+	}
+
+	ns, err := server.NewServer(opts)
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+	return ns
+}
+
+func TestConnectNATS_TLSSuccess(t *testing.T) {
+	tmp := t.TempDir()
+	caCertPath, caKeyPath := generateTestCA(t, tmp, "ca")
+	serverCertPath, serverKeyPath := generateTestServerCert(t, tmp, caCertPath, caKeyPath)
+
+	ns := startTLSNATSServer(t, serverCertPath, serverKeyPath, caCertPath)
+
+	nc, err := ConnectNATS(ns.ClientURL(), "", caCertPath)
+	require.NoError(t, err)
+	defer nc.Close()
+	assert.True(t, nc.IsConnected())
+}
+
+func TestConnectNATS_WrongCA(t *testing.T) {
+	tmp := t.TempDir()
+	caCertPath, caKeyPath := generateTestCA(t, tmp, "ca")
+	serverCertPath, serverKeyPath := generateTestServerCert(t, tmp, caCertPath, caKeyPath)
+	wrongCACertPath, _ := generateTestCA(t, tmp, "wrong-ca")
+
+	ns := startTLSNATSServer(t, serverCertPath, serverKeyPath, caCertPath)
+
+	_, err := ConnectNATS(ns.ClientURL(), "", wrongCACertPath)
+	assert.Error(t, err, "connection with wrong CA should fail")
+}
+
 func TestNATSRequest_Success(t *testing.T) {
 	ns := startTestNATSServer(t)
 

spinifex/utils/utils.go

@@ -249,9 +249,7 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout
 			WithTotal(int(cl)).
 			Start()
 
-		var written int64
 		reader := io.TeeReader(resp.Body, progressWriter(func(n int) {
-			written += int64(n)
 			// Update progress bar with the number of bytes read in this chunk
 			bar.Add(n)
 		}))
@@ -261,8 +259,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout
 		if err != nil {
 			return fmt.Errorf("copy error: %v", err)
 		}
-
-		pterm.Printf("Saved %s (%s)\n", filename, humanBytes(SafeInt64ToUint64(written)))
 		return err
 	} else {
 		// Unknown size: spinner that shows bytes downloaded
@@ -280,8 +276,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout
 		if err != nil {
 			return fmt.Errorf("copy error: %v", err)
 		}
-
-		pterm.Printf("Saved %s (%s)\n", filename, humanBytes(SafeInt64ToUint64(written)))
 	}
 
 	return nil

spinifex/utils/utils_test.go

@@ -1,12 +1,20 @@
 package utils
 
 import (
+	"crypto/sha256"
+	"crypto/sha512"
+	"crypto/x509"
+	"encoding/hex"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
@@ -1005,3 +1013,264 @@ func TestHashMAC(t *testing.T) {
 	// Different input → different output
 	assert.NotEqual(t, mac, HashMAC("02:00:00", "other-id"))
 }
+
+// writeTempImage writes content to a file named name under t.TempDir() and
+// returns the absolute path.
+func writeTempImage(t *testing.T, name string, content []byte) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), name)
+	require.NoError(t, os.WriteFile(path, content, 0o600))
+	return path
+}
+
+func sha256Hex(b []byte) string {
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])
+}
+
+func sha512Hex(b []byte) string {
+	sum := sha512.Sum512(b)
+	return hex.EncodeToString(sum[:])
+}
+
+// newSumsServer stands up a TLS test server returning body for any request,
+// configures the package-level trust hook, and registers cleanup. Returns the
+// server URL.
+func newSumsServer(t *testing.T, body string) string {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(srv.Close)
+	trustTestServer(t, srv)
+	return srv.URL + "/SUMS"
+}
+
+// trustTestServer adds srv's certificate to the package-level trust pool so
+// VerifyImageChecksum's HTTPS-only client accepts it. Restores the previous
+// pool on test cleanup.
+func trustTestServer(t *testing.T, srv *httptest.Server) {
+	t.Helper()
+	prev := checksumExtraRootCAs
+	pool := x509.NewCertPool()
+	if prev != nil {
+		pool = prev.Clone()
+	}
+	pool.AddCert(srv.Certificate())
+	checksumExtraRootCAs = pool
+	t.Cleanup(func() { checksumExtraRootCAs = prev })
+}
+
+func TestVerifyImageChecksum(t *testing.T) {
+	const debianName = "debian-12-generic-amd64.tar.xz"
+	const ubuntuName = "noble-server-cloudimg-amd64.img"
+	const alpineName = "alb-alpine-3.21.6-x86_64.raw"
+
+	debianBytes := []byte("debian-image-bytes-fixture")
+	ubuntuBytes := []byte("ubuntu-image-bytes-fixture")
+	alpineBytes := []byte("alpine-image-bytes-fixture")
+
+	t.Run("debian sha512 multi-line match", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		body := fmt.Sprintf("%s  %s\n%s  other-file\n",
+			sha512Hex(debianBytes), debianName, sha512Hex([]byte("unrelated")))
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		assert.NoError(t, err)
+	})
+
+	t.Run("ubuntu sha256 binary-mode asterisk match", func(t *testing.T) {
+		img := writeTempImage(t, ubuntuName, ubuntuBytes)
+		body := fmt.Sprintf("%s *%s\n", sha256Hex(ubuntuBytes), ubuntuName)
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha256")
+		assert.NoError(t, err)
+	})
+
+	t.Run("alpine single-line named match", func(t *testing.T) {
+		img := writeTempImage(t, alpineName, alpineBytes)
+		body := fmt.Sprintf("%s  %s\n", sha512Hex(alpineBytes), alpineName)
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		assert.NoError(t, err)
+	})
+
+	t.Run("alpine bare-digest fallback", func(t *testing.T) {
+		img := writeTempImage(t, alpineName, alpineBytes)
+		body := sha512Hex(alpineBytes) + "\n"
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		assert.NoError(t, err)
+	})
+
+	t.Run("case-sensitive filename mismatch", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		// Capitalised "Debian-..." in sums file: legitimate divergence from upstream
+		// convention, treated as not-found.
+		body := fmt.Sprintf("%s  Debian-12-generic-amd64.tar.xz\n", sha512Hex(debianBytes))
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		assert.ErrorIs(t, err, ErrChecksumNotFound)
+	})
+
+	t.Run("comment lines skipped", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		body := fmt.Sprintf("# Hash file generated by upstream\n# PGP-signed below\n%s  %s\n",
+			sha512Hex(debianBytes), debianName)
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		assert.NoError(t, err)
+	})
+
+	t.Run("digest mismatch", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		wrong := sha512Hex([]byte("a different blob entirely"))
+		body := fmt.Sprintf("%s  %s\n", wrong, debianName)
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumMismatch)
+		// Both expected and actual digests should be in the rendered error so
+		// operators can compare without scraping slog.
+		assert.Contains(t, err.Error(), wrong)
+		assert.Contains(t, err.Error(), sha512Hex(debianBytes))
+	})
+
+	t.Run("filename absent from sums file", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		body := fmt.Sprintf("%s  some-other-file.tar.xz\n%s  yet-another.iso\n",
+			sha512Hex([]byte("a")), sha512Hex([]byte("b")))
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		assert.ErrorIs(t, err, ErrChecksumNotFound)
+	})
+
+	t.Run("unsupported checksum type rejected before any fetch", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		// Server URL is gibberish — function must fail before reaching it.
+		err := VerifyImageChecksum(img, "https://example.invalid/SUMS", "md5")
+		assert.ErrorIs(t, err, ErrUnsupportedChecksumType)
+	})
+
+	t.Run("http 404 wraps fetch failure", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.NotFound(w, r)
+		}))
+		t.Cleanup(srv.Close)
+		trustTestServer(t, srv)
+		err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512")
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+	})
+
+	t.Run("non-https initial url rejected before any request", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		// Hit a port nothing listens on — if HTTPS check ever regresses we'd
+		// still get an error, but we want ErrChecksumFetchFailed wrapping the
+		// scheme refusal specifically.
+		err := VerifyImageChecksum(img, "http://127.0.0.1:1/SUMS", "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+		assert.Contains(t, err.Error(), "non-https")
+	})
+
+	t.Run("non-https redirect refused", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		// Plain-HTTP target the TLS server will try to redirect us to.
+		plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprintf(w, "%s  %s\n", sha512Hex(debianBytes), debianName)
+		}))
+		t.Cleanup(plain.Close)
+		srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Redirect(w, r, plain.URL+"/SUMS", http.StatusFound)
+		}))
+		t.Cleanup(srv.Close)
+		trustTestServer(t, srv)
+		err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+	})
+
+	t.Run("redirect cap exceeded", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		var hits atomic.Int32
+		srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			http.Redirect(w, r, r.URL.Path, http.StatusFound)
+		}))
+		t.Cleanup(srv.Close)
+		trustTestServer(t, srv)
+		err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+		assert.GreaterOrEqual(t, int(hits.Load()), 10)
+	})
+
+	t.Run("sums file size cap exceeded", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		// Body of 2 MB of unrelated entries — over the 1 MB cap.
+		var b strings.Builder
+		junk := strings.Repeat("a", 128)
+		for b.Len() <= sumsFileMaxSize+1 {
+			fmt.Fprintf(&b, "%s  %s.bin\n", sha512Hex([]byte(junk)), junk)
+		}
+		url := newSumsServer(t, b.String())
+		err := VerifyImageChecksum(img, url, "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+		assert.Contains(t, err.Error(), "exceeds")
+	})
+
+	t.Run("context deadline exceeded wraps fetch failure", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			<-r.Context().Done()
+		}))
+		t.Cleanup(srv.Close)
+		trustTestServer(t, srv)
+
+		prev := checksumFetchTimeout
+		checksumFetchTimeout = 100 * time.Millisecond
+		t.Cleanup(func() { checksumFetchTimeout = prev })
+
+		err := VerifyImageChecksum(img, srv.URL+"/SUMS", "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+	})
+
+	t.Run("digest length mismatch rejected before hashing", func(t *testing.T) {
+		// sha256 digest served, caller declares sha512: algorithm disagreement,
+		// not tampering. Must be distinct from ErrChecksumMismatch.
+		img := writeTempImage(t, debianName, debianBytes)
+		body := fmt.Sprintf("%s  %s\n", sha256Hex(debianBytes), debianName)
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+		assert.NotErrorIs(t, err, ErrChecksumMismatch)
+		assert.Contains(t, err.Error(), "digest length")
+	})
+
+	t.Run("empty image file rejected with clear error", func(t *testing.T) {
+		img := writeTempImage(t, debianName, nil)
+		body := fmt.Sprintf("%s  %s\n", sha512Hex(nil), debianName)
+		url := newSumsServer(t, body)
+		err := VerifyImageChecksum(img, url, "sha512")
+		require.Error(t, err)
+		assert.NotErrorIs(t, err, ErrChecksumMismatch)
+		assert.Contains(t, err.Error(), "empty")
+	})
+
+	t.Run("transport error wrapped", func(t *testing.T) {
+		img := writeTempImage(t, debianName, debianBytes)
+		// Stand up a TLS server, trust it, then close immediately so any
+		// subsequent request fails at the transport layer.
+		srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+		trustTestServer(t, srv)
+		url := srv.URL + "/SUMS"
+		srv.Close()
+		err := VerifyImageChecksum(img, url, "sha512")
+		require.Error(t, err)
+		assert.ErrorIs(t, err, ErrChecksumFetchFailed)
+	})
+}

tests/e2e/TEST_COVERAGE.md

@@ -306,11 +306,11 @@ Tests that EC2 resources are properly isolated between tenant accounts (Alpha, B
 - Simulated network IPs (no ramdisk — start-dev.sh uses disk-backed WAL/VB in CI)
 
 ### Phase 2: Cluster Initialization
-- `spx admin init` (leader node1)
+- `spx admin init` (leader node1) — generates join token, writes to `~/node1/config/join-token`
 - CA certificate trust
-- Start node1 services
-- `spx admin join` (node2, node3)
-- Start node2 + node3 services
+- Read join token from node1 config
+- `spx admin join` with `--token` (node2, node3 — concurrent)
+- Start all 3 node services concurrently
 
 ### Phase 3: Cluster Health Verification
 - Verify NATS cluster (3 nodes)
@@ -562,7 +562,7 @@ Runs on a real 3-node libvirt cluster provisioned by OpenTofu (`scripts/tofu-clu
 4. `git fetch` + checkout test branch on all nodes (all 3 repos)
 5. `make build` on all nodes (parallel)
 6. `setup-ovn.sh` — primary gets `--management`, secondaries connect to primary's OVN central
-7. `spx admin init` on primary, `spx admin join` on secondaries (formation server handshake)
+7. `spx admin init` on primary (generates join token), read token via SSH, `spx admin join --token` on secondaries
 8. `start-dev.sh` on all nodes
 9. Wait for `/health` (awsgw=ok) on all nodes
 10. Install Spinifex CA certificate on all nodes

tests/e2e/lib/multinode-helpers.sh

@@ -120,7 +120,7 @@ verify_nats_cluster() {
 
     # Check cluster info via monitoring endpoint on node1
     local cluster_info
-    cluster_info=$(curl -s "http://${NODE1_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || {
+    cluster_info=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || {
         echo "  ERROR: Cannot reach NATS monitoring endpoint"
         return 1
     }
@@ -645,8 +645,8 @@ verify_all_services_down() {
             all_down=false
         fi
 
-        # Check NATS
-        if curl -s --connect-timeout 2 "http://${node_ip}:${NATS_MONITOR_PORT}" > /dev/null 2>&1; then
+        # Check NATS (client port — monitoring is localhost-only)
+        if nc -z -w 2 "${node_ip}" ${NATS_CLIENT_PORT} 2>/dev/null; then
             echo "  Node$i: NATS still responding"
             all_down=false
         fi
@@ -779,10 +779,9 @@ init_leader_node() {
     # shellcheck disable=SC2086
     ./bin/spx admin init \
         --node node1 \
+        --nodes 3 \
         --bind "${NODE1_IP}" \
         --cluster-bind "${NODE1_IP}" \
-        --cluster-routes "${NODE1_IP}:${NATS_CLUSTER_PORT}" \
-        --predastore-nodes "${NODE1_IP},${NODE2_IP},${NODE3_IP}" \
         --port ${CLUSTER_PORT} \
         --region ap-southeast-2 \
         --az ap-southeast-2a \
@@ -796,6 +795,16 @@ init_leader_node() {
     for i in $(seq 1 60); do
         if curl -sk "https://${NODE1_IP}:${CLUSTER_PORT}/formation/health" > /dev/null 2>&1; then
             echo "  Formation server is ready (PID: $LEADER_INIT_PID)"
+
+            # Read join token written by admin init
+            JOIN_TOKEN=$(cat "$HOME/node1/config/join-token")
+            if [ -z "$JOIN_TOKEN" ]; then
+                echo "  ERROR: Join token file is empty"
+                return 1
+            fi
+            echo "  Join token loaded"
+            export JOIN_TOKEN
+
             return 0
         fi
         sleep 1
@@ -817,13 +826,12 @@ join_follower_node() {
     # Remove old node directory
     rm -rf "$data_dir/"
 
-    # Route to node1 (seed node) - other nodes discovered via NATS gossip
     ./bin/spx admin join \
         --node "node$node_num" \
         --bind "$node_ip" \
         --cluster-bind "$node_ip" \
-        --cluster-routes "${NODE1_IP}:${NATS_CLUSTER_PORT}" \
         --host "${NODE1_IP}:${CLUSTER_PORT}" \
+        --token "$JOIN_TOKEN" \
         --data-dir "$data_dir/" \
         --config-dir "$data_dir/config/" \
         --region ap-southeast-2 \

tests/e2e/run-multinode-e2e.sh

@@ -333,7 +333,7 @@ echo "========================================"
 
 # NATS cluster: verify 2 unique peers from node1
 echo "Checking NATS cluster..."
-NATS_INFO=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || {
+NATS_INFO=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null) || {
     echo "  ERROR: Cannot reach NATS monitoring endpoint"
     exit 1
 }
@@ -809,7 +809,7 @@ else
 fi
 
 # Check NATS degraded state (should have 1 route instead of 2)
-NATS_DEGRADED=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null)
+NATS_DEGRADED=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null)
 DEGRADED_PEERS=$(echo "$NATS_DEGRADED" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0")
 echo "  NATS peers during failure: $DEGRADED_PEERS (expected: 1)"
 if [ "$DEGRADED_PEERS" -eq 1 ]; then
@@ -849,7 +849,7 @@ echo "  Waiting for NATS cluster to reform..."
 ATTEMPT=0
 REFORMED=false
 while [ $ATTEMPT -lt 60 ]; do
-    NATS_RECOVER=$(curl -s "http://${LOCAL_IP}:${NATS_MONITOR_PORT}/routez" 2>/dev/null)
+    NATS_RECOVER=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null)
     RECOVER_PEERS=$(echo "$NATS_RECOVER" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0")
     if [ "$RECOVER_PEERS" -ge 2 ]; then
         echo "  NATS cluster reformed ($RECOVER_PEERS peers)"

tests/e2e/run-pseudo-multinode-e2e.sh

@@ -134,6 +134,12 @@ wait $JOIN3_PID || { echo "ERROR: Node 3 join failed"; exit 1; }
 wait $LEADER_INIT_PID || { echo "ERROR: Leader init failed"; exit 1; }
 echo "Cluster formation complete — all configs generated"
 
+# Disable monitoring on non-primary nodes to avoid port conflict on 127.0.0.1:8222
+for i in 2 3; do
+    sed -i '/^http:\s*127\.0\.0\.1:8222/d' "$HOME/node${i}/config/nats/nats.conf"
+    echo "  Stripped NATS monitoring from node$i (avoids 127.0.0.1:8222 conflict)"
+done
+
 # Now start services (configs exist for all nodes)
 echo ""
 echo "Starting node services..."