From 744cb1929dc5cbbeae132df4b443d57e7fc0bd35 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:46:12 +1000 Subject: [PATCH] fix: admin init --force preserves keys, credentials and CA (#536) --- cmd/spinifex/cmd/admin.go | 259 +++++++++++++++++++-------------- cmd/spinifex/cmd/admin_test.go | 193 +++++++++++++++++++++--- docs/COMMANDS.md | 2 +- spinifex/admin/admin.go | 54 ++++--- spinifex/admin/admin_test.go | 54 ++++++- 5 files changed, 410 insertions(+), 152 deletions(-) diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 828e67672..07ad80382 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -1267,66 +1267,91 @@ func runAdminInit(cmd *cobra.Command, args []string) { } fmt.Printf("āœ… Created config directory: %s\n", configDir) - // Generate system credentials (for service-to-service auth in config files) - accessKey, err := admin.GenerateAWSAccessKey() + // Identity and crypto material is load-or-generate: a fresh install mints a + // new identity bundle, but a --force re-init preserves the existing one so + // data sealed under it (NATS KV secrets, sealed fragments, encrypted volumes) + // stays decryptable. + masterKey, masterKeyExisted, err := ensureMasterKey(configDir) if err != nil { - fmt.Fprintf(os.Stderr, "Error generating access key: %v\n", err) - os.Exit(1) - } - secretKey, err := admin.GenerateAWSSecretKey() - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating secret key: %v\n", err) + fmt.Fprintf(os.Stderr, "Error preparing IAM master key: %v\n", err) os.Exit(1) } accountID := admin.SystemAccountID() - - // Generate IAM master key (AES-256, used to encrypt secrets in NATS KV) - masterKey, err := handlers_iam.GenerateMasterKey() - if err != nil { - fmt.Fprintf(os.Stderr, "Error generating IAM master key: %v\n", err) - os.Exit(1) - } bootstrapDir := filepath.Join(spxRoot, "awsgw") - bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID) - if err != nil { - fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err) - os.Exit(1) - } - if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil { - fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err) - os.Exit(1) - } - fmt.Println("\nšŸ” Generated IAM master key") - fmt.Printf(" Master key: %s\n", filepath.Join(configDir, "master.key")) - fmt.Printf(" Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json")) - fmt.Printf(" System creds: %s\n", filepath.Join(configDir, "system-credentials.json")) - // Predastore encryption key is per-node and never transmitted; generate - // it locally now so the service has it on first start. + var accessKey, secretKey, adminAccessKey, adminSecretKey string + if masterKeyExisted { + // Preserve path: reuse the existing identity. The system credentials must + // match what seeded the NATS KV `system` secret, so load them rather than + // mint new ones. The admin credentials are not recovered: bootstrap.json is + // consumed and deleted by awsgw after first boot, and the operator's copy + // already lives in ~/.aws/credentials. Leaving them empty makes + // finalizeNodeSetup refresh only ~/.aws/config (endpoint/CA for a changed + // bind IP), not the credentials. + accessKey, secretKey, err = loadSystemCredentials(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error loading preserved system credentials: %v\n", err) + os.Exit(1) + } + fmt.Println("\nšŸ” Preserved existing identity (master key, credentials and CA unchanged)") + fmt.Printf(" Master key: %s\n", filepath.Join(configDir, "master.key")) + } else { + // Fresh install: mint system + admin credentials and seed the bootstrap files. + accessKey, err = admin.GenerateAWSAccessKey() + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating access key: %v\n", err) + os.Exit(1) + } + secretKey, err = admin.GenerateAWSSecretKey() + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating secret key: %v\n", err) + os.Exit(1) + } + bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err) + os.Exit(1) + } + if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil { + fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err) + os.Exit(1) + } + adminAccessKey = bootstrapResult.AdminAccessKey + adminSecretKey = bootstrapResult.AdminSecretKey + fmt.Println("\nšŸ” Generated IAM master key") + fmt.Printf(" Master key: %s\n", filepath.Join(configDir, "master.key")) + fmt.Printf(" Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json")) + fmt.Printf(" System creds: %s\n", filepath.Join(configDir, "system-credentials.json")) + } + + // Predastore encryption key is per-node and never transmitted; load-or-generate + // so the service has it on first start and a re-init keeps sealed fragments. predastoreKeyPath, err := writePredastoreEncryptionKey(configDir) if err != nil { - fmt.Fprintf(os.Stderr, "Error generating predastore encryption key: %v\n", err) + fmt.Fprintf(os.Stderr, "Error preparing predastore encryption key: %v\n", err) os.Exit(1) } - fmt.Println("\nšŸ” Generated predastore encryption key (per-node, never transmitted)") + fmt.Println("\nšŸ” Predastore encryption key ready (per-node, never transmitted)") fmt.Printf(" Key: %s\n", predastoreKeyPath) - // Viperblock at-rest encryption key is cluster-wide; on a single-node init - // there are no joiners, so generate it locally and enable encryption by - // default for all volumes created on this install. - viperblockKeyPath, err := writeViperblockEncryptionKey(configDir) + // Viperblock at-rest encryption key is cluster-wide; load-or-generate so a + // re-init keeps the existing key (and its sealed volumes). The bytes feed the + // multi-node leader's key distribution below. + viperblockKey, viperblockKeyPath, err := ensureViperblockEncryptionKey(configDir) if err != nil { - fmt.Fprintf(os.Stderr, "Error generating viperblock encryption key: %v\n", err) + fmt.Fprintf(os.Stderr, "Error preparing viperblock encryption key: %v\n", err) os.Exit(1) } - fmt.Println("\nšŸ” Generated viperblock at-rest encryption key") + fmt.Println("\nšŸ” Viperblock at-rest encryption key ready") fmt.Printf(" Key: %s\n", viperblockKeyPath) - fmt.Printf("\nšŸ”‘ Generated admin credentials (save these ā€” they won't be shown again):\n") - fmt.Printf(" Access Key: %s\n", bootstrapResult.AdminAccessKey) - fmt.Printf(" Secret Key: %s\n", bootstrapResult.AdminSecretKey) - fmt.Printf(" Account: %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID()) - fmt.Printf(" AWS Profile: spinifex\n") + if !masterKeyExisted { + fmt.Printf("\nšŸ”‘ Generated admin credentials (save these ā€” they won't be shown again):\n") + fmt.Printf(" Access Key: %s\n", adminAccessKey) + fmt.Printf(" Secret Key: %s\n", adminSecretKey) + fmt.Printf(" Account: %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID()) + fmt.Printf(" AWS Profile: spinifex\n") + } // Generate SSL certificates (with bind IP in SANs for multi-node support) certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix) @@ -1392,7 +1417,8 @@ func runAdminInit(cmd *cobra.Command, args []string) { networkConfig.BootstrapSubnetCidr = handlers_ec2_vpc.DefaultSubnetCidr } - runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName, + runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey, + masterKey, viperblockKey, natsToken, clusterName, configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email, port, nodes, formationTimeoutStr, tokenTTLStr, services, networkConfig) return @@ -1544,7 +1570,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { os.Exit(1) } - finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP) + finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP) // Write node.conf so spx admin banner works on source installs (not just ISO). nodeHostname, _ := os.Hostname() @@ -1575,7 +1601,8 @@ func runAdminInit(cmd *cobra.Command, args []string) { // runAdminInitMultiNode handles the multi-node formation path for admin init. // It starts a formation server, registers this node, waits for all nodes to join, // then generates configs with complete cluster topology. -func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName, +func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string, + masterKey, viperblockKey []byte, natsToken, clusterName, configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email string, port, expectedNodes int, formationTimeoutStr, tokenTTLStr string, services []string, networkConfig *formation.NetworkConfig) { formationTimeout, err := time.ParseDuration(formationTimeoutStr) @@ -1602,47 +1629,11 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, os.Exit(1) } - // Generate IAM master key for the cluster - masterKey, err := handlers_iam.GenerateMasterKey() - if err != nil { - fmt.Fprintf(os.Stderr, "āŒ Error generating IAM master key: %v\n", err) - os.Exit(1) - } - bootstrapDir := filepath.Join(spxRoot, "awsgw") - bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID) - if err != nil { - fmt.Fprintf(os.Stderr, "āŒ Error writing bootstrap files: %v\n", err) - os.Exit(1) - } - fmt.Println("\nšŸ” Generated IAM master key") - fmt.Printf(" Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json")) - - // Predastore encryption key is per-node and never distributed via the - // formation server. Generate only the leader's own key here; each - // joiner generates its own during `spx admin join`. - predastoreKeyPath, err := writePredastoreEncryptionKey(configDir) - if err != nil { - fmt.Fprintf(os.Stderr, "āŒ Error generating predastore encryption key: %v\n", err) - os.Exit(1) - } - fmt.Println("\nšŸ” Generated predastore encryption key (per-node, never transmitted)") - fmt.Printf(" Key: %s\n", predastoreKeyPath) - - // Viperblock at-rest encryption key is cluster-wide and shared: the leader - // generates it once and distributes it to joiners via the formation server - // so a volume sealed on any node can be opened on any other. - viperblockKey, err := handlers_iam.GenerateMasterKey() - if err != nil { - fmt.Fprintf(os.Stderr, "āŒ Error generating viperblock encryption key: %v\n", err) - os.Exit(1) - } - viperblockKeyPath, err := saveViperblockEncryptionKey(configDir, viperblockKey) - if err != nil { - fmt.Fprintf(os.Stderr, "āŒ Error saving viperblock encryption key: %v\n", err) - os.Exit(1) - } - fmt.Println("\nšŸ” Generated viperblock at-rest encryption key (cluster-wide, shared with joiners)") - fmt.Printf(" Key: %s\n", viperblockKeyPath) + // Identity, predastore and viperblock keys were prepared load-or-generate by + // the caller: reuse them here so a re-init never rotates cluster crypto and + // the leader distributes the same viperblock key to joiners. The path is + // deterministic and already written on disk. + viperblockKeyPath := filepath.Join(configDir, "viperblock", "encryption.key") // Read CA cert/key for distribution to joining nodes caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem")) @@ -1663,8 +1654,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, NatsToken: natsToken, ClusterName: clusterName, Region: region, - AdminAccessKey: bootstrapResult.AdminAccessKey, - AdminSecretKey: bootstrapResult.AdminSecretKey, + AdminAccessKey: adminAccessKey, + AdminSecretKey: adminSecretKey, } fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig, joinToken, tokenTTL) @@ -1801,7 +1792,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, os.Exit(1) } - finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP) + finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP) // Keep formation server running briefly so joining nodes can fetch complete status fmt.Println("\nā³ Waiting for joining nodes to fetch cluster data...") @@ -2625,6 +2616,48 @@ type writeBootstrapResult struct { AdminSecretKey string } +// ensureMasterKey load-or-generates the IAM master key at /master.key. +// It returns the key bytes and whether the key already existed on disk. On a +// --force re-init the existing key is preserved: rotating it would orphan every +// IAM secret in NATS KV (e.g. the ECR signing key) that was encrypted under it. +func ensureMasterKey(configDir string) (key []byte, existed bool, err error) { + keyPath := filepath.Join(configDir, "master.key") + if admin.FileExists(keyPath) { + key, err = handlers_iam.LoadMasterKey(keyPath) + if err != nil { + return nil, false, fmt.Errorf("load master key: %w", err) + } + return key, true, nil + } + key, err = handlers_iam.GenerateMasterKey() + if err != nil { + return nil, false, fmt.Errorf("generate master key: %w", err) + } + return key, false, nil +} + +// loadSystemCredentials reads the preserved system access/secret key from +// /system-credentials.json. On a re-init the configs must render the +// same system credentials that seeded the NATS KV `system` secret, else SigV4 +// auth between services fails. +func loadSystemCredentials(configDir string) (accessKey, secretKey string, err error) { + data, err := os.ReadFile(filepath.Join(configDir, "system-credentials.json")) + if err != nil { + return "", "", fmt.Errorf("read system credentials: %w", err) + } + var creds struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + } + if err := json.Unmarshal(data, &creds); err != nil { + return "", "", fmt.Errorf("parse system credentials: %w", err) + } + if creds.AccessKeyID == "" || creds.SecretAccessKey == "" { + return "", "", fmt.Errorf("system credentials missing access/secret key") + } + return creds.AccessKeyID, creds.SecretAccessKey, nil +} + // writeBootstrapFiles generates new admin credentials and writes the bootstrap // files (master.key to configDir, bootstrap.json to bootstrapDir). // Used by init flows (single and multi-node). @@ -2646,47 +2679,57 @@ func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, acces }, nil } -// writePredastoreEncryptionKey generates a fresh 32-byte AES-256 master key -// for this node's predastore data dir and writes it to -// /predastore/encryption.key with mode 0600. The key is per-node: -// every node generates its own at init/join time and never transmits it, -// because predastore only opens fragments on the node that sealed them. +// writePredastoreEncryptionKey load-or-generates this node's per-node predastore +// key at /predastore/encryption.key (mode 0600). The key is per-node: +// every node generates its own at init/join time and never transmits it, because +// predastore only opens fragments on the node that sealed them. An existing key +// is preserved so a --force re-init does not orphan already-sealed fragments. func writePredastoreEncryptionKey(configDir string) (string, error) { predastoreDir := filepath.Join(configDir, "predastore") if err := os.MkdirAll(predastoreDir, 0750); err != nil { return "", fmt.Errorf("create predastore config dir: %w", err) } + keyPath := filepath.Join(predastoreDir, "encryption.key") + if admin.FileExists(keyPath) { + return keyPath, nil + } key, err := handlers_iam.GenerateMasterKey() if err != nil { return "", fmt.Errorf("generate predastore encryption key: %w", err) } - keyPath := filepath.Join(predastoreDir, "encryption.key") if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil { return "", fmt.Errorf("save predastore encryption key: %w", err) } return keyPath, nil } -// writeViperblockEncryptionKey generates a fresh 32-byte AES-256 master key for -// viperblock at-rest encryption and writes it to -// /viperblock/encryption.key with mode 0600. Unlike the predastore -// key, this key is cluster-wide and shared: the leader generates it once at init -// and distributes it to joiners via the formation server, so a volume sealed on -// any node can be opened on any other. Loaded at startup via masterkey.LoadShared. -func writeViperblockEncryptionKey(configDir string) (string, error) { +// ensureViperblockEncryptionKey load-or-generates the cluster-wide viperblock +// at-rest key at /viperblock/encryption.key (mode 0600) and returns +// its bytes and path. Unlike the predastore key it is shared: the leader loads +// these bytes to distribute the same key to joiners via the formation server, so +// a volume sealed on any node can be opened on any other. An existing key is +// preserved so a --force re-init does not orphan already-encrypted volumes. +func ensureViperblockEncryptionKey(configDir string) ([]byte, string, error) { viperblockDir := filepath.Join(configDir, "viperblock") if err := os.MkdirAll(viperblockDir, 0750); err != nil { - return "", fmt.Errorf("create viperblock config dir: %w", err) + return nil, "", fmt.Errorf("create viperblock config dir: %w", err) + } + keyPath := filepath.Join(viperblockDir, "encryption.key") + if admin.FileExists(keyPath) { + key, err := handlers_iam.LoadMasterKey(keyPath) + if err != nil { + return nil, "", fmt.Errorf("load viperblock encryption key: %w", err) + } + return key, keyPath, nil } key, err := handlers_iam.GenerateMasterKey() if err != nil { - return "", fmt.Errorf("generate viperblock encryption key: %w", err) + return nil, "", fmt.Errorf("generate viperblock encryption key: %w", err) } - keyPath := filepath.Join(viperblockDir, "encryption.key") if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil { - return "", fmt.Errorf("save viperblock encryption key: %w", err) + return nil, "", fmt.Errorf("save viperblock encryption key: %w", err) } - return keyPath, nil + return key, keyPath, nil } // saveViperblockEncryptionKey writes an already-generated 32-byte viperblock diff --git a/cmd/spinifex/cmd/admin_test.go b/cmd/spinifex/cmd/admin_test.go index 60f513873..c0fcc6efe 100644 --- a/cmd/spinifex/cmd/admin_test.go +++ b/cmd/spinifex/cmd/admin_test.go @@ -11,6 +11,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/formation" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/utils" toml "github.com/pelletier/go-toml/v2" "github.com/spf13/cobra" @@ -255,23 +256,6 @@ func TestWritePredastoreEncryptionKey(t *testing.T) { assert.NotEqual(t, key1, key2, "per-node keys must differ") } -// The viperblock key gates at-rest encryption for every volume on a fresh -// install. masterkey.LoadShared rejects anything that isn't exactly 32 bytes at -// 0640-or-stricter, so a regression in size/mode would silently fall back to -// cleartext (empty path) or fail service startup. -func TestWriteViperblockEncryptionKey(t *testing.T) { - configDir := t.TempDir() - - keyPath, err := writeViperblockEncryptionKey(configDir) - require.NoError(t, err) - assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath) - - info, err := os.Stat(keyPath) - require.NoError(t, err) - assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes") - assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600") -} - // Joiners persist the leader's shared key via saveViperblockEncryptionKey // against a configDir with no viperblock/ subdir yet. The bytes round-trip // verbatim ā€” the whole cluster shares one key, so any mutation would orphan @@ -292,6 +276,181 @@ func TestSaveViperblockEncryptionKey_RoundTrip(t *testing.T) { assert.Equal(t, key, got, "shared key must be written verbatim") } +// A re-init must not rotate the per-node predastore key: rotating it would orphan +// every fragment already sealed under the old key. The helper preserves an +// existing key byte-for-byte. +func TestWritePredastoreEncryptionKey_PreservesExisting(t *testing.T) { + configDir := t.TempDir() + + keyPath, err := writePredastoreEncryptionKey(configDir) + require.NoError(t, err) + orig, err := os.ReadFile(keyPath) + require.NoError(t, err) + + keyPath2, err := writePredastoreEncryptionKey(configDir) + require.NoError(t, err) + assert.Equal(t, keyPath, keyPath2) + after, err := os.ReadFile(keyPath2) + require.NoError(t, err) + assert.Equal(t, orig, after, "existing predastore key must be preserved on re-init") +} + +// ensureViperblockEncryptionKey generates a shared 32-byte 0600 key on a fresh +// dir and returns bytes that match the on-disk file; a second call preserves it +// so a re-init keeps every encrypted volume readable. +func TestEnsureViperblockEncryptionKey(t *testing.T) { + configDir := t.TempDir() + + key, keyPath, err := ensureViperblockEncryptionKey(configDir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath) + assert.Len(t, key, 32, "viperblock key must be 32 bytes") + + info, err := os.Stat(keyPath) + require.NoError(t, err) + assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes") + assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600") + + onDisk, err := os.ReadFile(keyPath) + require.NoError(t, err) + assert.Equal(t, key, onDisk, "returned bytes must match the file") + + key2, _, err := ensureViperblockEncryptionKey(configDir) + require.NoError(t, err) + assert.Equal(t, key, key2, "existing viperblock key must be preserved on re-init") +} + +// A present-but-corrupt viperblock key must fail loud rather than regenerate, +// which would orphan every already-encrypted volume in the cluster. +func TestEnsureViperblockEncryptionKey_CorruptFailsLoud(t *testing.T) { + configDir := t.TempDir() + keyDir := filepath.Join(configDir, "viperblock") + require.NoError(t, os.MkdirAll(keyDir, 0750)) + keyPath := filepath.Join(keyDir, "encryption.key") + require.NoError(t, os.WriteFile(keyPath, []byte("bad"), 0600)) + + _, _, err := ensureViperblockEncryptionKey(configDir) + require.Error(t, err, "a truncated viperblock key must error, not regenerate") + + after, err := os.ReadFile(keyPath) + require.NoError(t, err) + assert.Equal(t, []byte("bad"), after, "corrupt viperblock key must not be overwritten") +} + +// ensureMasterKey generates a fresh in-memory key without touching disk (the +// bootstrap writer persists it), and preserves an existing on-disk key so a +// re-init never rotates the key that encrypts IAM secrets in NATS KV. +func TestEnsureMasterKey(t *testing.T) { + t.Run("FreshGeneratesWithoutWriting", func(t *testing.T) { + configDir := t.TempDir() + key, existed, err := ensureMasterKey(configDir) + require.NoError(t, err) + assert.False(t, existed, "no master.key present yet") + assert.Len(t, key, 32) + assert.NoFileExists(t, filepath.Join(configDir, "master.key"), + "ensureMasterKey must not persist on the fresh path") + }) + + t.Run("PreservesExisting", func(t *testing.T) { + configDir := t.TempDir() + seed := make([]byte, 32) + for i := range seed { + seed[i] = byte(200 - i) + } + require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(configDir, "master.key"), seed)) + + key, existed, err := ensureMasterKey(configDir) + require.NoError(t, err) + assert.True(t, existed, "existing master.key must be detected") + assert.Equal(t, seed, key, "existing master key must be loaded verbatim") + }) + + // A present-but-corrupt master.key must fail loud, never be silently + // regenerated: rotating it would orphan every IAM secret in NATS KV. + t.Run("CorruptFailsLoud", func(t *testing.T) { + configDir := t.TempDir() + keyPath := filepath.Join(configDir, "master.key") + require.NoError(t, os.WriteFile(keyPath, []byte("too-short"), 0600)) + + _, _, err := ensureMasterKey(configDir) + require.Error(t, err, "a truncated master.key must error, not regenerate") + + after, err := os.ReadFile(keyPath) + require.NoError(t, err) + assert.Equal(t, []byte("too-short"), after, "corrupt master key must not be overwritten") + }) +} + +// The identity bundle is preserved holistically on re-init: the load helpers +// recover the exact system credentials and never rewrite bootstrap.json. Admin +// credentials are not recovered from disk ā€” awsgw consumes and deletes +// bootstrap.json after first boot, so the preserve path must not read it back. +func TestPreservedIdentityBundle(t *testing.T) { + configDir := t.TempDir() + bootstrapDir := filepath.Join(configDir, "awsgw") + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + const ( + sysAccess = "AKIASYSTEM0000000000" + sysSecret = "system-secret-value" + adminAccess = "AKIAADMIN00000000000" + adminSecret = "admin-secret-value" + accountID = "123456789012" + ) + require.NoError(t, writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKey, + sysAccess, sysSecret, accountID, adminAccess, adminSecret)) + require.NoError(t, writeSystemCredentials(configDir, sysAccess, sysSecret)) + + bootstrapPath := filepath.Join(bootstrapDir, "bootstrap.json") + bootstrapBefore, err := os.ReadFile(bootstrapPath) + require.NoError(t, err) + + // master.key round-trips and is flagged as pre-existing. + loadedKey, existed, err := ensureMasterKey(configDir) + require.NoError(t, err) + assert.True(t, existed) + assert.Equal(t, masterKey, loadedKey) + + // System credentials come back verbatim (they must match the NATS KV seed). + gotSysAccess, gotSysSecret, err := loadSystemCredentials(configDir) + require.NoError(t, err) + assert.Equal(t, sysAccess, gotSysAccess) + assert.Equal(t, sysSecret, gotSysSecret) + + // The preserve helpers must not rewrite the seed file. + bootstrapAfter, err := os.ReadFile(bootstrapPath) + require.NoError(t, err) + assert.Equal(t, bootstrapBefore, bootstrapAfter, "bootstrap.json must not be rewritten on re-init") +} + +// loadSystemCredentials surfaces a clear error when the file is absent rather +// than returning empty credentials that would silently break SigV4 auth. +func TestLoadSystemCredentials_Missing(t *testing.T) { + _, _, err := loadSystemCredentials(t.TempDir()) + require.Error(t, err) +} + +// A present-but-empty or malformed system-credentials.json must error rather +// than yield blank credentials that would silently break SigV4 between services. +func TestLoadSystemCredentials_Invalid(t *testing.T) { + t.Run("EmptyValues", func(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"), + []byte(`{"access_key_id":"","secret_access_key":""}`), 0600)) + _, _, err := loadSystemCredentials(configDir) + require.Error(t, err) + }) + + t.Run("MalformedJSON", func(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"), + []byte("not json"), 0600)) + _, _, err := loadSystemCredentials(configDir) + require.Error(t, err) + }) +} + // A fresh install must render encryption_key_file so viperblockd enables // at-rest encryption; an empty field (upgrade / legacy) must omit the line // entirely so the volume stays in cleartext legacy mode. diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 908a47258..1e1f06047 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -54,7 +54,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | Command | Flags | Description | |---------|-------|-------------| -| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway ` or `--external-source=dhcp [--external-bind-bridge ]` for a public pool with full EIP support; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 ā€” they require `pool` mode), `--no-external` | Generates root IAM credentials (AKIA-prefixed access key + secret) ā†’ creates master.key (AES-256, 32 bytes, 0600) ā†’ writes bootstrap.json (consumed on first start) ā†’ generates CA + server TLS certificates ā†’ generates join token (written to `join-token` file, displayed in join command) ā†’ creates NATS config with auth token ā†’ writes spinifex.toml, awsgw.toml, predastore.toml ā†’ configures AWS CLI `spx` profile ā†’ creates directory structure under `~/spinifex/` | +| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway ` or `--external-source=dhcp [--external-bind-bridge ]` for a public pool with full EIP support; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 ā€” they require `pool` mode), `--no-external` | Generates root IAM credentials (AKIA-prefixed access key + secret) ā†’ creates master.key (AES-256, 32 bytes, 0600) ā†’ writes bootstrap.json (consumed on first start) ā†’ generates CA + server TLS certificates ā†’ generates join token (written to `join-token` file, displayed in join command) ā†’ creates NATS config with auth token ā†’ writes spinifex.toml, awsgw.toml, predastore.toml ā†’ configures AWS CLI `spx` profile ā†’ creates directory structure under `~/spinifex/`. `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. | | `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Connects to leader node with join token (Authorization: Bearer header) ā†’ retrieves cluster configuration ā†’ configures local node to join cluster and participate in distributed operations | ### Version diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index b95696e1b..2915ce4d2 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -185,20 +185,19 @@ func AWSGWServiceDNSNames(region, suffix string) []string { } } +// GenerateCertificatesIfNeeded prepares the TLS material for a node. The CA is +// preserved across re-inits ā€” it anchors trust for already-joined nodes, IPsec +// peer certs, and CA-baked AMIs ā€” so it is regenerated only when absent, never on +// force. The CA-signed server cert is cheap to reissue, so force refreshes it to +// pick up a changed bind IP / SANs while keeping the CA (and all trust) intact. func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, awsRegion, internalSuffix string) (caCertPath string) { caCertPath = filepath.Join(configDir, "ca.pem") caKeyPath := filepath.Join(configDir, "ca.key") serverCertPath := filepath.Join(configDir, "server.pem") serverKeyPath := filepath.Join(configDir, "server.key") - // Check if we need to generate certificates - needsGeneration := force || - !FileExists(caCertPath) || !FileExists(caKeyPath) || - !FileExists(serverCertPath) || !FileExists(serverKeyPath) - - if needsGeneration { - fmt.Println("\nšŸ” Generating Certificate Authority and SSL certificates...") - + if !FileExists(caCertPath) || !FileExists(caKeyPath) { + fmt.Println("\nšŸ” Generating Certificate Authority...") if err := GenerateCACert(caCertPath, caKeyPath); err != nil { fmt.Fprintf(os.Stderr, "Error generating CA certificate: %v\n", err) os.Exit(1) @@ -207,15 +206,6 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, a fmt.Printf(" CA Certificate: %s\n", caCertPath) fmt.Printf(" CA Key: %s\n", caKeyPath) - extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix) - if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS); err != nil { - fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err) - os.Exit(1) - } - fmt.Printf("āœ… Server certificate generated (signed by CA):\n") - fmt.Printf(" Certificate: %s\n", serverCertPath) - fmt.Printf(" Key: %s\n", serverKeyPath) - // Print manual instructions only when not root (root gets auto-install) if os.Getuid() != 0 { fmt.Println("\nšŸ“‹ To trust the Spinifex CA system-wide (recommended):") @@ -224,7 +214,20 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, a fmt.Println("\n This allows AWS CLI and other tools to trust Spinifex services automatically.") } } else { - fmt.Println("\nāœ… CA and SSL certificates already exist") + fmt.Println("\nāœ… Certificate Authority already exists (preserved)") + } + + if force || !FileExists(serverCertPath) || !FileExists(serverKeyPath) { + extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix) + if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS); err != nil { + fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err) + os.Exit(1) + } + fmt.Printf("āœ… Server certificate generated (signed by CA):\n") + fmt.Printf(" Certificate: %s\n", serverCertPath) + fmt.Printf(" Key: %s\n", serverKeyPath) + } else { + fmt.Println("āœ… Server certificate already exists") } return caCertPath @@ -786,11 +789,16 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string) profileName := "spinifex" - if err := UpdateAWSINIFile(credPath, profileName, map[string]string{ - "aws_access_key_id": accessKey, - "aws_secret_access_key": secretKey, - }); err != nil { - return err + // Empty credentials mean a --force re-init preserving the existing identity: + // the admin secret is not recoverable from disk and is unchanged, so leave the + // credentials file as-is and refresh only the config (endpoint/CA/region). + if accessKey != "" && secretKey != "" { + if err := UpdateAWSINIFile(credPath, profileName, map[string]string{ + "aws_access_key_id": accessKey, + "aws_secret_access_key": secretKey, + }); err != nil { + return err + } } configSection := profileName diff --git a/spinifex/admin/admin_test.go b/spinifex/admin/admin_test.go index 6a793f8bf..91084c5ad 100644 --- a/spinifex/admin/admin_test.go +++ b/spinifex/admin/admin_test.go @@ -277,6 +277,32 @@ func TestSetupAWSCredentials_FallsBackToLocalhostForWildcard(t *testing.T) { assert.Contains(t, string(configData), "https://localhost:9999") } +// On a --force re-init the preserve path passes empty admin credentials: the +// existing ~/.aws/credentials must be left intact while ~/.aws/config is still +// refreshed (endpoint/CA for a changed bind IP). +func TestSetupAWSCredentials_EmptyCredsRefreshesConfigOnly(t *testing.T) { + dir := t.TempDir() + t.Setenv("HOME", dir) + + awsDir := filepath.Join(dir, ".aws") + require.NoError(t, os.MkdirAll(awsDir, 0700)) + require.NoError(t, UpdateAWSINIFile(filepath.Join(awsDir, "credentials"), "spinifex", map[string]string{ + "aws_access_key_id": "PRESERVED_KEY", + "aws_secret_access_key": "PRESERVED_SECRET", + })) + + err := SetupAWSCredentials("", "", "ap-southeast-2", "/new/ca.pem", "10.11.12.5") + require.NoError(t, err) + + credData, _ := os.ReadFile(filepath.Join(awsDir, "credentials")) + assert.Contains(t, string(credData), "PRESERVED_KEY", "existing admin credentials must be preserved on re-init") + assert.Contains(t, string(credData), "PRESERVED_SECRET") + + configData, _ := os.ReadFile(filepath.Join(awsDir, "config")) + assert.Contains(t, string(configData), "https://10.11.12.5:9999", "config endpoint must be refreshed") + assert.Contains(t, string(configData), "/new/ca.pem") +} + // --- Certificate generation --- func TestGenerateCACert_CreatesValidCA(t *testing.T) { @@ -617,12 +643,34 @@ func TestGenerateCertificatesIfNeeded(t *testing.T) { assert.Equal(t, origModTime, caInfo2.ModTime()) }) - t.Run("ForceRegenerates", func(t *testing.T) { + // --force must preserve the CA (trust anchor for joined nodes / baked AMIs) + // and only re-sign the server cert, which stays verifiable against that CA. + t.Run("ForcePreservesCARegeneratesServerCert", func(t *testing.T) { origCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem")) + origCAKey, _ := os.ReadFile(filepath.Join(dir, "ca.key")) + origServer, _ := os.ReadFile(filepath.Join(dir, "server.pem")) + + GenerateCertificatesIfNeeded(dir, true, "10.9.8.7", "us-east-1", "spinifex.internal") - GenerateCertificatesIfNeeded(dir, true, "", "us-east-1", "spinifex.internal") newCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem")) - assert.NotEqual(t, origCA, newCA) + newCAKey, _ := os.ReadFile(filepath.Join(dir, "ca.key")) + newServer, _ := os.ReadFile(filepath.Join(dir, "server.pem")) + assert.Equal(t, origCA, newCA, "CA cert must be preserved on --force") + assert.Equal(t, origCAKey, newCAKey, "CA key must be preserved on --force") + assert.NotEqual(t, origServer, newServer, "server cert must be re-signed on --force") + + // The re-signed server cert must verify against the preserved CA. + caBlock, _ := pem.Decode(newCA) + caCert, err := x509.ParseCertificate(caBlock.Bytes) + require.NoError(t, err) + pool := x509.NewCertPool() + pool.AddCert(caCert) + + srvBlock, _ := pem.Decode(newServer) + srvCert, err := x509.ParseCertificate(srvBlock.Bytes) + require.NoError(t, err) + _, err = srvCert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}) + assert.NoError(t, err, "re-signed server cert must verify against preserved CA") }) }