feat(predastore): aes 256 encryption at rest

Sibling changes in Spinifex to support changes made to Predastore in this PR.

build/scripts/predastore-start.sh

@@ -5,17 +5,20 @@
 
 CONF=/etc/spinifex/spinifex.toml
 BIND="0.0.0.0:8443"
-NODE_ID="0"
+# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex
+# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology.
+# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below.
+# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard).
+NODE_ID="-1"
 
 if [ -f "$CONF" ]; then
     H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
     if [ -n "$H" ]; then
         BIND="$H"
-        BACKEND="distributed"
     fi
 
     N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
-    if [ -n "$N" ] && [ "$N" != "0" ]; then
+    if [ -n "$N" ]; then
         NODE_ID="$N"
     fi
 fi

build/systemd/spinifex-predastore.service

@@ -17,6 +17,7 @@ Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.
 Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
+Environment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key
 
 # Filesystem access
 NoNewPrivileges=yes

cmd/spinifex/cmd/admin.go

@@ -904,6 +904,16 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 	fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))
 
+	// Predastore encryption key is per-node and never transmitted; generate
+	// it locally now so the service has it on first start.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Printf("   Key: %s\n", predastoreKeyPath)
+
 	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
 	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
 	fmt.Printf("   Secret Key:  %s\n", bootstrapResult.AdminSecretKey)
@@ -1158,6 +1168,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	fmt.Println("\n🔐 Generated IAM master key")
 	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
+	// Predastore encryption key is per-node and never distributed via the
+	// formation server. Generate only the leader's own key here; each
+	// joiner generates its own during `spx admin join`.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Printf("   Key: %s\n", predastoreKeyPath)
+
 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
 	if err != nil {
@@ -1628,6 +1649,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	fmt.Println("✅ IAM master key received from leader")
 	fmt.Printf("✅ Bootstrap file written: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
+	// Predastore encryption key is per-node: generate locally rather than
+	// receiving from the leader. Each node only opens fragments it sealed
+	// itself, so there is no cluster-wide predastore key to share.
+	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("✅ Predastore encryption key generated: %s\n", predastoreKeyPath)
+
 	// Generate server cert signed by CA with this node's bind IP
 	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
@@ -2099,6 +2130,27 @@ func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, acces
 	}, nil
 }
 
+// writePredastoreEncryptionKey generates a fresh 32-byte AES-256 master key
+// for this node's predastore data dir and writes it to
+// <configDir>/predastore/encryption.key with mode 0600. The key is per-node:
+// every node generates its own at init/join time and never transmits it,
+// because predastore only opens fragments on the node that sealed them.
+func writePredastoreEncryptionKey(configDir string) (string, error) {
+	predastoreDir := filepath.Join(configDir, "predastore")
+	if err := os.MkdirAll(predastoreDir, 0750); err != nil {
+		return "", fmt.Errorf("create predastore config dir: %w", err)
+	}
+	key, err := handlers_iam.GenerateMasterKey()
+	if err != nil {
+		return "", fmt.Errorf("generate predastore encryption key: %w", err)
+	}
+	keyPath := filepath.Join(predastoreDir, "encryption.key")
+	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
+		return "", fmt.Errorf("save predastore encryption key: %w", err)
+	}
+	return keyPath, nil
+}
+
 // writeSystemCredentials writes the system access key to a plaintext JSON file.
 // The daemon reads this at startup to inject credentials into ALB VM cloud-init
 // for SigV4-authenticated communication with the AWS gateway.

cmd/spinifex/cmd/admin_test.go

@@ -262,3 +262,51 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 	require.NoError(t, err)
 	assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`)
 }
+
+// writePredastoreEncryptionKey is called from both `spx admin init` and `spx
+// admin join`; both depend on it producing a 32-byte file at 0600 in the
+// expected layout. Predastore's keyfile loader rejects anything that isn't
+// exactly that, so a regression here would break service startup on every
+// node.
+func TestWritePredastoreEncryptionKey(t *testing.T) {
+	configDir := t.TempDir()
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "predastore", "encryption.key"), keyPath)
+
+	info, err := os.Stat(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(32), info.Size(), "predastore master key must be exactly 32 bytes")
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "predastore master key must be mode 0600")
+
+	// Per-node invariant: two successive calls (on what would be two
+	// different nodes) must produce different key material. If they ever
+	// returned the same bytes we'd have silently lost the per-node
+	// blast-radius property.
+	configDir2 := t.TempDir()
+	keyPath2, err := writePredastoreEncryptionKey(configDir2)
+	require.NoError(t, err)
+
+	key1, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	key2, err := os.ReadFile(keyPath2)
+	require.NoError(t, err)
+	assert.NotEqual(t, key1, key2, "per-node keys must differ")
+}
+
+// Joiners run `spx admin join` against a configDir that doesn't yet contain
+// a predastore/ subdir. The helper must create it (predastore subdir is
+// owned by spinifex-storage in prod, but the helper just needs the dir to
+// exist).
+func TestWritePredastoreEncryptionKey_CreatesMissingDir(t *testing.T) {
+	configDir := t.TempDir()
+	// Don't pre-create the predastore subdir — simulate a fresh joiner.
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+
+	dirInfo, err := os.Stat(filepath.Dir(keyPath))
+	require.NoError(t, err)
+	assert.True(t, dirInfo.IsDir())
+}

cmd/spinifex/cmd/service.go

@@ -115,6 +115,13 @@ var predastoreStartCmd = &cobra.Command{
 			return
 		}
 
+		encryptionKeyFile := viper.GetString("encryption-key-file")
+
+		if encryptionKeyFile == "" {
+			fmt.Println("Encryption key file is not set")
+			return
+		}
+
 		nodeID := viper.GetInt("node-id")
 		pprofEnabled := viper.GetBool("pprof")
 		pprofOutput := viper.GetString("pprof-output")
@@ -128,6 +135,8 @@ var predastoreStartCmd = &cobra.Command{
 			TlsCert:    tlsCert,
 			TlsKey:     tlsKey,
 
+			EncryptionKeyFile: encryptionKeyFile,
+
 			NodeID: nodeID,
 
 			PprofEnabled:    pprofEnabled,
@@ -858,13 +867,20 @@ func init() {
 	viper.BindEnv("tls-key", "SPINIFEX_PREDASTORE_TLS_KEY")
 	viper.BindPFlag("tls-key", predastoreCmd.PersistentFlags().Lookup("tls-key"))
 
+	// Predastore at-rest encryption master key (per node; mode 0600)
+	predastoreCmd.PersistentFlags().String("encryption-key-file", "", "Path to this node's 32-byte AES-256 master key file (required)")
+	viper.BindEnv("encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE")
+	viper.BindPFlag("encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file"))
+
 	// Predastore Backend
 	predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend")
 	viper.BindEnv("backend", "SPINIFEX_PREDASTORE_BACKEND")
 	viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend"))
 
-	// Predastore Node ID
-	predastoreCmd.PersistentFlags().Int("node-id", 0, "Predastore (S3) node ID")
+	// Predastore Node ID. Default -1 is dev mode (launch every configured
+	// QUIC node in-process). Production deployments set this to the node's
+	// real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id.
+	predastoreCmd.PersistentFlags().Int("node-id", -1, "Predastore (S3) node ID (-1 = dev mode, >= 1 = production)")
 	viper.BindEnv("node-id", "SPINIFEX_PREDASTORE_NODE_ID")
 	viper.BindPFlag("node-id", predastoreCmd.PersistentFlags().Lookup("node-id"))
 

scripts/start-dev.sh

@@ -314,6 +314,17 @@ if has_service "predastore"; then
     export SPINIFEX_PREDASTORE_TLS_CERT=$CONFIG_DIR/server.pem
     export SPINIFEX_PREDASTORE_TLS_KEY=$CONFIG_DIR/server.key
 
+    # Per-node predastore encryption key. Self-heal for devs who skipped
+    # `spx admin init`: generate with umask 0177 so the file is 0600 from
+    # creation (predastore rejects group/other-readable keys outright).
+    ENCRYPTION_KEY="$CONFIG_DIR/predastore/encryption.key"
+    if [ ! -f "$ENCRYPTION_KEY" ]; then
+        mkdir -p "$CONFIG_DIR/predastore"
+        ( umask 0177 && openssl rand -out "$ENCRYPTION_KEY" 32 )
+        echo "   Generated predastore encryption key: $ENCRYPTION_KEY"
+    fi
+    export SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE="$ENCRYPTION_KEY"
+
     # Auto-detect Predastore host:port from spinifex.toml [nodes.<name>.predastore] section
     PREDASTORE_BIND="0.0.0.0:8443"
     if [ -f "$CONFIG_DIR/spinifex.toml" ]; then

spinifex/services/predastore/predastore.go

@@ -25,6 +25,11 @@ type Config struct {
 	TlsCert    string
 	TlsKey     string
 
+	// EncryptionKeyFile is the path to this node's 32-byte AES-256 master
+	// key for predastore at-rest encryption. Each node has its own key;
+	// fragments are only ever opened on the node that sealed them.
+	EncryptionKeyFile string
+
 	NodeID int
 
 	// Profiling
@@ -52,6 +57,10 @@ func New(config any) (svc *Service, err error) {
 
 // Start starts the predastore service
 func (svc *Service) Start() (int, error) {
+	if svc.Config.EncryptionKeyFile == "" {
+		return 0, fmt.Errorf("predastore encryption key file is required (set EncryptionKeyFile)")
+	}
+
 	if err := utils.WritePidFileTo(svc.Config.BasePath, serviceName, os.Getpid()); err != nil {
 		return 0, fmt.Errorf("write pid file: %w", err)
 	}
@@ -64,6 +73,7 @@ func (svc *Service) Start() (int, error) {
 		s3.WithDebug(svc.Config.Debug),
 		s3.WithNodeID(svc.Config.NodeID),
 		s3.WithPprof(svc.Config.PprofEnabled, svc.Config.PprofOutputPath),
+		s3.WithEncryptionKeyFile(svc.Config.EncryptionKeyFile),
 	)
 	if err != nil {
 		slog.Error("Failed to create predastore server", "error", err)

spinifex/services/predastore/predastore_integration_test.go

@@ -147,6 +147,17 @@ func startPredastoreServer(t *testing.T) *Config {
 		t.Fatalf("Failed to generate certificate: %v", err)
 	}
 
+	// Predastore mandates a 32-byte master key at mode 0600 (rejected otherwise
+	// by internal/keyfile.Load).
+	encryptionKeyPath := filepath.Join(testDir, "encryption.key")
+	testEncryptionKey := make([]byte, 32)
+	if _, err := rand.Read(testEncryptionKey); err != nil {
+		t.Fatalf("Failed to generate test encryption key: %v", err)
+	}
+	if err := os.WriteFile(encryptionKeyPath, testEncryptionKey, 0600); err != nil {
+		t.Fatalf("Failed to write test encryption key: %v", err)
+	}
+
 	// Create config file. Five storage nodes are declared so predastore's dev-mode
 	// path launches all QUIC servers locally as goroutines (server.go:629). Buckets
 	// are created via the S3 API after startup — config buckets are no longer
@@ -206,13 +217,19 @@ account_id = "123456789012"
 	}
 
 	cfg := &Config{
-		ConfigPath: configPath,
-		Port:       18443,
-		Host:       "127.0.0.1",
-		Debug:      false,
-		BasePath:   testDir,
-		TlsCert:    certPath,
-		TlsKey:     keyPath,
+		ConfigPath:        configPath,
+		Port:              18443,
+		Host:              "127.0.0.1",
+		Debug:             false,
+		BasePath:          testDir,
+		TlsCert:           certPath,
+		TlsKey:            keyPath,
+		EncryptionKeyFile: encryptionKeyPath,
+		// Predastore rejects NodeID 0; -1 triggers dev mode, which launches
+		// every configured QUIC node in-process. The test config defines
+		// five nodes, so this is the only sane choice for a single-process
+		// integration test.
+		NodeID: -1,
 	}
 	sharedConfig = cfg
 

spinifex/services/predastore/predastore_test.go

@@ -9,13 +9,14 @@ import (
 // TestNew tests the service constructor
 func TestNew(t *testing.T) {
 	cfg := &Config{
-		ConfigPath: "/tmp/test-config.toml",
-		Port:       8443,
-		Host:       "0.0.0.0",
-		Debug:      false,
-		BasePath:   "/tmp/predastore",
-		TlsCert:    "/tmp/cert.pem",
-		TlsKey:     "/tmp/key.pem",
+		ConfigPath:        "/tmp/test-config.toml",
+		Port:              8443,
+		Host:              "0.0.0.0",
+		Debug:             false,
+		BasePath:          "/tmp/predastore",
+		TlsCert:           "/tmp/cert.pem",
+		TlsKey:            "/tmp/key.pem",
+		EncryptionKeyFile: "/tmp/encryption.key",
 	}
 
 	svc, err := New(cfg)
@@ -30,6 +31,7 @@ func TestNew(t *testing.T) {
 	assert.Equal(t, "/tmp/predastore", svc.Config.BasePath)
 	assert.Equal(t, "/tmp/cert.pem", svc.Config.TlsCert)
 	assert.Equal(t, "/tmp/key.pem", svc.Config.TlsKey)
+	assert.Equal(t, "/tmp/encryption.key", svc.Config.EncryptionKeyFile)
 }
 
 // TestNewWithNilConfig tests that New handles nil config correctly
@@ -53,6 +55,7 @@ func TestConfigDefaults(t *testing.T) {
 	assert.Empty(t, cfg.BasePath)
 	assert.Empty(t, cfg.TlsCert)
 	assert.Empty(t, cfg.TlsKey)
+	assert.Empty(t, cfg.EncryptionKeyFile)
 }
 
 // TestConfigWithDebug tests Config with debug enabled
@@ -71,13 +74,14 @@ func TestConfigWithDebug(t *testing.T) {
 // TestConfigFullyPopulated tests a fully populated config
 func TestConfigFullyPopulated(t *testing.T) {
 	cfg := &Config{
-		ConfigPath: "/etc/predastore/config.toml",
-		Port:       9443,
-		Host:       "127.0.0.1",
-		Debug:      true,
-		BasePath:   "/var/lib/predastore",
-		TlsCert:    "/etc/ssl/certs/predastore.crt",
-		TlsKey:     "/etc/ssl/private/predastore.key",
+		ConfigPath:        "/etc/predastore/config.toml",
+		Port:              9443,
+		Host:              "127.0.0.1",
+		Debug:             true,
+		BasePath:          "/var/lib/predastore",
+		TlsCert:           "/etc/ssl/certs/predastore.crt",
+		TlsKey:            "/etc/ssl/private/predastore.key",
+		EncryptionKeyFile: "/etc/spinifex/predastore/encryption.key",
 	}
 
 	assert.Equal(t, "/etc/predastore/config.toml", cfg.ConfigPath)
@@ -87,6 +91,7 @@ func TestConfigFullyPopulated(t *testing.T) {
 	assert.Equal(t, "/var/lib/predastore", cfg.BasePath)
 	assert.Equal(t, "/etc/ssl/certs/predastore.crt", cfg.TlsCert)
 	assert.Equal(t, "/etc/ssl/private/predastore.key", cfg.TlsKey)
+	assert.Equal(t, "/etc/spinifex/predastore/encryption.key", cfg.EncryptionKeyFile)
 }
 
 // TestServiceMethods tests the service interface methods
@@ -232,6 +237,28 @@ func TestServiceStartWithoutConfig(t *testing.T) {
 	t.Skip("Skipping test that requires actual predastore config file - covered by integration tests")
 }
 
+// Start() must fail fast (before touching the pid file or trying to bind a
+// port) when no encryption key file is configured. Predastore itself
+// enforces this at the s3.Server layer, but a clear pre-check error at the
+// spinifex service boundary saves operators an opaque downstream failure.
+func TestServiceStartRejectsMissingEncryptionKey(t *testing.T) {
+	cfg := &Config{
+		ConfigPath: "/tmp/test-config.toml",
+		Port:       18443,
+		Host:       "127.0.0.1",
+		BasePath:   t.TempDir(),
+		TlsCert:    "/tmp/cert.pem",
+		TlsKey:     "/tmp/key.pem",
+		// EncryptionKeyFile deliberately left empty.
+	}
+	svc, err := New(cfg)
+	assert.NoError(t, err)
+
+	_, err = svc.Start()
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "encryption key file is required")
+}
+
 // TestConfigDebugFlag tests debug flag behavior
 func TestConfigDebugFlag(t *testing.T) {
 	// Debug enabled