diff --git a/build/scripts/predastore-start.sh b/build/scripts/predastore-start.sh index e83f5b2f6..044754098 100755 --- a/build/scripts/predastore-start.sh +++ b/build/scripts/predastore-start.sh @@ -5,17 +5,20 @@ CONF=/etc/spinifex/spinifex.toml BIND="0.0.0.0:8443" -NODE_ID="0" +# -1 = co-located mode (all [[db]] peers in one process). Single-node spinifex +# templates list 3 DB peers on 127.0.0.1; only -1 can serve that topology. +# Multi-node spinifex.toml emits node_id = N (N >= 1) which overrides below. +# predastore rejects NODE_ID=0 since d3a02f0 (strconv.Atoi-of-garbage guard). +NODE_ID="-1" if [ -f "$CONF" ]; then H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF") if [ -n "$H" ]; then BIND="$H" - BACKEND="distributed" fi N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF") - if [ -n "$N" ] && [ "$N" != "0" ]; then + if [ -n "$N" ]; then NODE_ID="$N" fi fi diff --git a/build/systemd/spinifex-predastore.service b/build/systemd/spinifex-predastore.service index 73614d31f..71ef6aad8 100644 --- a/build/systemd/spinifex-predastore.service +++ b/build/systemd/spinifex-predastore.service @@ -17,6 +17,7 @@ Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore. Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/ Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key +Environment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key # Filesystem access NoNewPrivileges=yes diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 5280b9de0..8c161b406 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -904,6 +904,16 @@ func runAdminInit(cmd *cobra.Command, args []string) { fmt.Printf(" Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json")) fmt.Printf(" System creds: %s\n", filepath.Join(configDir, "system-credentials.json")) + // Predastore encryption key is per-node and never transmitted; generate + // it locally now so the service has it on first start. + predastoreKeyPath, err := writePredastoreEncryptionKey(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating predastore encryption key: %v\n", err) + os.Exit(1) + } + fmt.Println("\nšŸ” Generated predastore encryption key (per-node, never transmitted)") + fmt.Printf(" Key: %s\n", predastoreKeyPath) + fmt.Printf("\nšŸ”‘ Generated admin credentials (save these ā€” they won't be shown again):\n") fmt.Printf(" Access Key: %s\n", bootstrapResult.AdminAccessKey) fmt.Printf(" Secret Key: %s\n", bootstrapResult.AdminSecretKey) @@ -1158,6 +1168,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, fmt.Println("\nšŸ” Generated IAM master key") fmt.Printf(" Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json")) + // Predastore encryption key is per-node and never distributed via the + // formation server. Generate only the leader's own key here; each + // joiner generates its own during `spx admin join`. + predastoreKeyPath, err := writePredastoreEncryptionKey(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "āŒ Error generating predastore encryption key: %v\n", err) + os.Exit(1) + } + fmt.Println("\nšŸ” Generated predastore encryption key (per-node, never transmitted)") + fmt.Printf(" Key: %s\n", predastoreKeyPath) + // Read CA cert/key for distribution to joining nodes caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem")) if err != nil { @@ -1628,6 +1649,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) { fmt.Println("āœ… IAM master key received from leader") fmt.Printf("āœ… Bootstrap file written: %s\n", filepath.Join(bootstrapDir, "bootstrap.json")) + // Predastore encryption key is per-node: generate locally rather than + // receiving from the leader. Each node only opens fragments it sealed + // itself, so there is no cluster-wide predastore key to share. + predastoreKeyPath, err := writePredastoreEncryptionKey(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "āŒ Error generating predastore encryption key: %v\n", err) + os.Exit(1) + } + fmt.Printf("āœ… Predastore encryption key generated: %s\n", predastoreKeyPath) + // Generate server cert signed by CA with this node's bind IP if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil { fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err) @@ -2099,6 +2130,27 @@ func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, acces }, nil } +// writePredastoreEncryptionKey generates a fresh 32-byte AES-256 master key +// for this node's predastore data dir and writes it to +// /predastore/encryption.key with mode 0600. The key is per-node: +// every node generates its own at init/join time and never transmits it, +// because predastore only opens fragments on the node that sealed them. +func writePredastoreEncryptionKey(configDir string) (string, error) { + predastoreDir := filepath.Join(configDir, "predastore") + if err := os.MkdirAll(predastoreDir, 0750); err != nil { + return "", fmt.Errorf("create predastore config dir: %w", err) + } + key, err := handlers_iam.GenerateMasterKey() + if err != nil { + return "", fmt.Errorf("generate predastore encryption key: %w", err) + } + keyPath := filepath.Join(predastoreDir, "encryption.key") + if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil { + return "", fmt.Errorf("save predastore encryption key: %w", err) + } + return keyPath, nil +} + // writeSystemCredentials writes the system access key to a plaintext JSON file. // The daemon reads this at startup to inject credentials into ALB VM cloud-init // for SigV4-authenticated communication with the AWS gateway. diff --git a/cmd/spinifex/cmd/admin_test.go b/cmd/spinifex/cmd/admin_test.go index 39c8ee971..4089c2e25 100644 --- a/cmd/spinifex/cmd/admin_test.go +++ b/cmd/spinifex/cmd/admin_test.go @@ -262,3 +262,51 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) { require.NoError(t, err) assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`) } + +// writePredastoreEncryptionKey is called from both `spx admin init` and `spx +// admin join`; both depend on it producing a 32-byte file at 0600 in the +// expected layout. Predastore's keyfile loader rejects anything that isn't +// exactly that, so a regression here would break service startup on every +// node. +func TestWritePredastoreEncryptionKey(t *testing.T) { + configDir := t.TempDir() + + keyPath, err := writePredastoreEncryptionKey(configDir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(configDir, "predastore", "encryption.key"), keyPath) + + info, err := os.Stat(keyPath) + require.NoError(t, err) + assert.Equal(t, int64(32), info.Size(), "predastore master key must be exactly 32 bytes") + assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "predastore master key must be mode 0600") + + // Per-node invariant: two successive calls (on what would be two + // different nodes) must produce different key material. If they ever + // returned the same bytes we'd have silently lost the per-node + // blast-radius property. + configDir2 := t.TempDir() + keyPath2, err := writePredastoreEncryptionKey(configDir2) + require.NoError(t, err) + + key1, err := os.ReadFile(keyPath) + require.NoError(t, err) + key2, err := os.ReadFile(keyPath2) + require.NoError(t, err) + assert.NotEqual(t, key1, key2, "per-node keys must differ") +} + +// Joiners run `spx admin join` against a configDir that doesn't yet contain +// a predastore/ subdir. The helper must create it (predastore subdir is +// owned by spinifex-storage in prod, but the helper just needs the dir to +// exist). +func TestWritePredastoreEncryptionKey_CreatesMissingDir(t *testing.T) { + configDir := t.TempDir() + // Don't pre-create the predastore subdir ā€” simulate a fresh joiner. + + keyPath, err := writePredastoreEncryptionKey(configDir) + require.NoError(t, err) + + dirInfo, err := os.Stat(filepath.Dir(keyPath)) + require.NoError(t, err) + assert.True(t, dirInfo.IsDir()) +} diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index e63bb5240..0ac6b9fbd 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -115,6 +115,13 @@ var predastoreStartCmd = &cobra.Command{ return } + encryptionKeyFile := viper.GetString("encryption-key-file") + + if encryptionKeyFile == "" { + fmt.Println("Encryption key file is not set") + return + } + nodeID := viper.GetInt("node-id") pprofEnabled := viper.GetBool("pprof") pprofOutput := viper.GetString("pprof-output") @@ -128,6 +135,8 @@ var predastoreStartCmd = &cobra.Command{ TlsCert: tlsCert, TlsKey: tlsKey, + EncryptionKeyFile: encryptionKeyFile, + NodeID: nodeID, PprofEnabled: pprofEnabled, @@ -858,13 +867,20 @@ func init() { viper.BindEnv("tls-key", "SPINIFEX_PREDASTORE_TLS_KEY") viper.BindPFlag("tls-key", predastoreCmd.PersistentFlags().Lookup("tls-key")) + // Predastore at-rest encryption master key (per node; mode 0600) + predastoreCmd.PersistentFlags().String("encryption-key-file", "", "Path to this node's 32-byte AES-256 master key file (required)") + viper.BindEnv("encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE") + viper.BindPFlag("encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file")) + // Predastore Backend predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend") viper.BindEnv("backend", "SPINIFEX_PREDASTORE_BACKEND") viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend")) - // Predastore Node ID - predastoreCmd.PersistentFlags().Int("node-id", 0, "Predastore (S3) node ID") + // Predastore Node ID. Default -1 is dev mode (launch every configured + // QUIC node in-process). Production deployments set this to the node's + // real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id. + predastoreCmd.PersistentFlags().Int("node-id", -1, "Predastore (S3) node ID (-1 = dev mode, >= 1 = production)") viper.BindEnv("node-id", "SPINIFEX_PREDASTORE_NODE_ID") viper.BindPFlag("node-id", predastoreCmd.PersistentFlags().Lookup("node-id")) diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index 114916b9a..c12a0889a 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -314,6 +314,17 @@ if has_service "predastore"; then export SPINIFEX_PREDASTORE_TLS_CERT=$CONFIG_DIR/server.pem export SPINIFEX_PREDASTORE_TLS_KEY=$CONFIG_DIR/server.key + # Per-node predastore encryption key. Self-heal for devs who skipped + # `spx admin init`: generate with umask 0177 so the file is 0600 from + # creation (predastore rejects group/other-readable keys outright). + ENCRYPTION_KEY="$CONFIG_DIR/predastore/encryption.key" + if [ ! -f "$ENCRYPTION_KEY" ]; then + mkdir -p "$CONFIG_DIR/predastore" + ( umask 0177 && openssl rand -out "$ENCRYPTION_KEY" 32 ) + echo " Generated predastore encryption key: $ENCRYPTION_KEY" + fi + export SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE="$ENCRYPTION_KEY" + # Auto-detect Predastore host:port from spinifex.toml [nodes..predastore] section PREDASTORE_BIND="0.0.0.0:8443" if [ -f "$CONFIG_DIR/spinifex.toml" ]; then diff --git a/spinifex/services/predastore/predastore.go b/spinifex/services/predastore/predastore.go index e25d5f30b..7ba811212 100644 --- a/spinifex/services/predastore/predastore.go +++ b/spinifex/services/predastore/predastore.go @@ -25,6 +25,11 @@ type Config struct { TlsCert string TlsKey string + // EncryptionKeyFile is the path to this node's 32-byte AES-256 master + // key for predastore at-rest encryption. Each node has its own key; + // fragments are only ever opened on the node that sealed them. + EncryptionKeyFile string + NodeID int // Profiling @@ -52,6 +57,10 @@ func New(config any) (svc *Service, err error) { // Start starts the predastore service func (svc *Service) Start() (int, error) { + if svc.Config.EncryptionKeyFile == "" { + return 0, fmt.Errorf("predastore encryption key file is required (set EncryptionKeyFile)") + } + if err := utils.WritePidFileTo(svc.Config.BasePath, serviceName, os.Getpid()); err != nil { return 0, fmt.Errorf("write pid file: %w", err) } @@ -64,6 +73,7 @@ func (svc *Service) Start() (int, error) { s3.WithDebug(svc.Config.Debug), s3.WithNodeID(svc.Config.NodeID), s3.WithPprof(svc.Config.PprofEnabled, svc.Config.PprofOutputPath), + s3.WithEncryptionKeyFile(svc.Config.EncryptionKeyFile), ) if err != nil { slog.Error("Failed to create predastore server", "error", err) diff --git a/spinifex/services/predastore/predastore_integration_test.go b/spinifex/services/predastore/predastore_integration_test.go index 455983676..a58b54b0b 100644 --- a/spinifex/services/predastore/predastore_integration_test.go +++ b/spinifex/services/predastore/predastore_integration_test.go @@ -147,6 +147,17 @@ func startPredastoreServer(t *testing.T) *Config { t.Fatalf("Failed to generate certificate: %v", err) } + // Predastore mandates a 32-byte master key at mode 0600 (rejected otherwise + // by internal/keyfile.Load). + encryptionKeyPath := filepath.Join(testDir, "encryption.key") + testEncryptionKey := make([]byte, 32) + if _, err := rand.Read(testEncryptionKey); err != nil { + t.Fatalf("Failed to generate test encryption key: %v", err) + } + if err := os.WriteFile(encryptionKeyPath, testEncryptionKey, 0600); err != nil { + t.Fatalf("Failed to write test encryption key: %v", err) + } + // Create config file. Five storage nodes are declared so predastore's dev-mode // path launches all QUIC servers locally as goroutines (server.go:629). Buckets // are created via the S3 API after startup ā€” config buckets are no longer @@ -206,13 +217,19 @@ account_id = "123456789012" } cfg := &Config{ - ConfigPath: configPath, - Port: 18443, - Host: "127.0.0.1", - Debug: false, - BasePath: testDir, - TlsCert: certPath, - TlsKey: keyPath, + ConfigPath: configPath, + Port: 18443, + Host: "127.0.0.1", + Debug: false, + BasePath: testDir, + TlsCert: certPath, + TlsKey: keyPath, + EncryptionKeyFile: encryptionKeyPath, + // Predastore rejects NodeID 0; -1 triggers dev mode, which launches + // every configured QUIC node in-process. The test config defines + // five nodes, so this is the only sane choice for a single-process + // integration test. + NodeID: -1, } sharedConfig = cfg diff --git a/spinifex/services/predastore/predastore_test.go b/spinifex/services/predastore/predastore_test.go index df7458265..4f2ed5e1d 100644 --- a/spinifex/services/predastore/predastore_test.go +++ b/spinifex/services/predastore/predastore_test.go @@ -9,13 +9,14 @@ import ( // TestNew tests the service constructor func TestNew(t *testing.T) { cfg := &Config{ - ConfigPath: "/tmp/test-config.toml", - Port: 8443, - Host: "0.0.0.0", - Debug: false, - BasePath: "/tmp/predastore", - TlsCert: "/tmp/cert.pem", - TlsKey: "/tmp/key.pem", + ConfigPath: "/tmp/test-config.toml", + Port: 8443, + Host: "0.0.0.0", + Debug: false, + BasePath: "/tmp/predastore", + TlsCert: "/tmp/cert.pem", + TlsKey: "/tmp/key.pem", + EncryptionKeyFile: "/tmp/encryption.key", } svc, err := New(cfg) @@ -30,6 +31,7 @@ func TestNew(t *testing.T) { assert.Equal(t, "/tmp/predastore", svc.Config.BasePath) assert.Equal(t, "/tmp/cert.pem", svc.Config.TlsCert) assert.Equal(t, "/tmp/key.pem", svc.Config.TlsKey) + assert.Equal(t, "/tmp/encryption.key", svc.Config.EncryptionKeyFile) } // TestNewWithNilConfig tests that New handles nil config correctly @@ -53,6 +55,7 @@ func TestConfigDefaults(t *testing.T) { assert.Empty(t, cfg.BasePath) assert.Empty(t, cfg.TlsCert) assert.Empty(t, cfg.TlsKey) + assert.Empty(t, cfg.EncryptionKeyFile) } // TestConfigWithDebug tests Config with debug enabled @@ -71,13 +74,14 @@ func TestConfigWithDebug(t *testing.T) { // TestConfigFullyPopulated tests a fully populated config func TestConfigFullyPopulated(t *testing.T) { cfg := &Config{ - ConfigPath: "/etc/predastore/config.toml", - Port: 9443, - Host: "127.0.0.1", - Debug: true, - BasePath: "/var/lib/predastore", - TlsCert: "/etc/ssl/certs/predastore.crt", - TlsKey: "/etc/ssl/private/predastore.key", + ConfigPath: "/etc/predastore/config.toml", + Port: 9443, + Host: "127.0.0.1", + Debug: true, + BasePath: "/var/lib/predastore", + TlsCert: "/etc/ssl/certs/predastore.crt", + TlsKey: "/etc/ssl/private/predastore.key", + EncryptionKeyFile: "/etc/spinifex/predastore/encryption.key", } assert.Equal(t, "/etc/predastore/config.toml", cfg.ConfigPath) @@ -87,6 +91,7 @@ func TestConfigFullyPopulated(t *testing.T) { assert.Equal(t, "/var/lib/predastore", cfg.BasePath) assert.Equal(t, "/etc/ssl/certs/predastore.crt", cfg.TlsCert) assert.Equal(t, "/etc/ssl/private/predastore.key", cfg.TlsKey) + assert.Equal(t, "/etc/spinifex/predastore/encryption.key", cfg.EncryptionKeyFile) } // TestServiceMethods tests the service interface methods @@ -232,6 +237,28 @@ func TestServiceStartWithoutConfig(t *testing.T) { t.Skip("Skipping test that requires actual predastore config file - covered by integration tests") } +// Start() must fail fast (before touching the pid file or trying to bind a +// port) when no encryption key file is configured. Predastore itself +// enforces this at the s3.Server layer, but a clear pre-check error at the +// spinifex service boundary saves operators an opaque downstream failure. +func TestServiceStartRejectsMissingEncryptionKey(t *testing.T) { + cfg := &Config{ + ConfigPath: "/tmp/test-config.toml", + Port: 18443, + Host: "127.0.0.1", + BasePath: t.TempDir(), + TlsCert: "/tmp/cert.pem", + TlsKey: "/tmp/key.pem", + // EncryptionKeyFile deliberately left empty. + } + svc, err := New(cfg) + assert.NoError(t, err) + + _, err = svc.Start() + assert.Error(t, err) + assert.Contains(t, err.Error(), "encryption key file is required") +} + // TestConfigDebugFlag tests debug flag behavior func TestConfigDebugFlag(t *testing.T) { // Debug enabled