From ef50392f9ed385bc1ccdeb2f739e34747f43482b Mon Sep 17 00:00:00 2001 From: brynmailer-mdc Date: Thu, 28 May 2026 14:10:44 +1000 Subject: [PATCH] feat: wire viperblock at-rest encryption key through daemon and AWS handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds EncryptionKeyFile to ViperblockConfig (cluster config) and viperblockd.Config. viperblockd loads the master key once at launchService start via masterkey.LoadShared and passes MasterKey + EncryptionEnabled into the VB constructed for every mounted volume. AWS handlers (volume.CreateVolume, instance.newViperblock, image.snapshotStoppedVolume) and the spx admin AMI-import path now resolve the key via utils.LoadViperblockMasterKey — a process-wide cache keyed by path so the per-VB lookup is one map hit after the first load. volume.mergeVolumeConfig: switch from json.Unmarshal to json.Decoder so a trailing AES-GCM tag cannot route the body to the wrapper-only path, and refuse outright when state.EncryptionEnabled — re-marshaling an encrypted VBState here would strip the tag and brick the volume. cmd/spinifex viperblock CLI grows a --encryption-key-file flag (env SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE) under a distinct viper key so it does not collide with predastore's identically-named flag. --- cmd/spinifex/cmd/admin.go | 10 ++++- cmd/spinifex/cmd/service.go | 38 +++++++++++------ spinifex/config/config.go | 8 ++++ spinifex/handlers/ec2/image/service_impl.go | 16 ++++++-- .../handlers/ec2/instance/service_impl.go | 17 +++++--- spinifex/handlers/ec2/volume/service_impl.go | 35 +++++++++++++--- spinifex/services/viperblockd/viperblockd.go | 24 ++++++++++- spinifex/utils/encryption.go | 41 +++++++++++++++++++ 8 files changed, 160 insertions(+), 29 deletions(-) create mode 100644 spinifex/utils/encryption.go diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 4149aa48b..44f201de7 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -603,6 +603,12 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { Host: appConfig.Nodes[appConfig.Node].Predastore.Host, } + mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err) + os.Exit(1) + } + vbConfig := viperblock.VB{ VolumeName: volumeId, VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()), @@ -612,7 +618,9 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { Size: 0, }, }, - VolumeConfig: manifest, + VolumeConfig: manifest, + MasterKey: mkey, + EncryptionEnabled: mkey != nil, } err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath) diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index e6a360c26..8cdb0d879 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -283,19 +283,25 @@ var viperblockStartCmd = &cobra.Command{ shardWAL = *nodeConfig.Viperblock.ShardWAL } + encryptionKeyFile := nodeConfig.Viperblock.EncryptionKeyFile + if envKey := viper.GetString("viperblock-encryption-key-file"); envKey != "" { + encryptionKeyFile = envKey + } + service, err := service.New("viperblock", &viperblockd.Config{ - NatsHost: nodeConfig.NATS.Host, - NatsToken: nodeConfig.NATS.ACL.Token, - NatsCACert: nodeConfig.NATS.CACert, - PluginPath: pluginPath, - S3Host: nodeConfig.Predastore.Host, - Bucket: nodeConfig.Predastore.Bucket, - Region: nodeConfig.Predastore.Region, - AccessKey: nodeConfig.Predastore.AccessKey, - SecretKey: nodeConfig.Predastore.SecretKey, - BaseDir: nodeConfig.Predastore.BaseDir, - NodeName: clusterConfig.Node, - ShardWAL: shardWAL, + NatsHost: nodeConfig.NATS.Host, + NatsToken: nodeConfig.NATS.ACL.Token, + NatsCACert: nodeConfig.NATS.CACert, + PluginPath: pluginPath, + S3Host: nodeConfig.Predastore.Host, + Bucket: nodeConfig.Predastore.Bucket, + Region: nodeConfig.Predastore.Region, + AccessKey: nodeConfig.Predastore.AccessKey, + SecretKey: nodeConfig.Predastore.SecretKey, + BaseDir: nodeConfig.Predastore.BaseDir, + NodeName: clusterConfig.Node, + ShardWAL: shardWAL, + EncryptionKeyFile: encryptionKeyFile, }) if err != nil { @@ -911,6 +917,14 @@ func init() { viper.BindEnv("plugin-path", "SPINIFEX_VIPERBLOCK_PLUGIN_PATH") viper.BindPFlag("plugin-path", predastoreCmd.PersistentFlags().Lookup("plugin-path")) + // Viperblock at-rest encryption master key (shared with other on-node + // services via group ownership; mode 0640 or stricter). Distinct viper + // key from predastore's encryption-key-file so the two BindPFlag calls + // don't collide globally. + viperblockCmd.PersistentFlags().String("encryption-key-file", "", "Path to the 32-byte AES-256 master key file for at-rest encryption (empty disables encryption)") + viper.BindEnv("viperblock-encryption-key-file", "SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE") + viper.BindPFlag("viperblock-encryption-key-file", viperblockCmd.PersistentFlags().Lookup("encryption-key-file")) + viperblockCmd.AddCommand(viperblockStartCmd) viperblockCmd.AddCommand(viperblockStopCmd) viperblockCmd.AddCommand(viperblockStatusCmd) diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 29063b2a6..2d00d1022 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -101,6 +101,14 @@ type AWSGWConfig struct { type ViperblockConfig struct { ShardWAL *bool `json:"ShardWAL" mapstructure:"shardwal"` // Enable sharded WAL (default false when nil) + + // EncryptionKeyFile is the path to the shared 32-byte AES-256 master key + // for viperblock at-rest encryption (chunks, WAL records, VBState/snapshot + // metadata). When empty, volumes are written in cleartext (legacy mode). + // When set, every VB constructed on this node — daemons and AWS handlers + // alike — must load it via masterkey.LoadShared so existing volumes can + // still be opened. + EncryptionKeyFile string `json:"EncryptionKeyFile" mapstructure:"encryption_key_file"` } // VPCDConfig holds the VPC daemon (vpcd) configuration. diff --git a/spinifex/handlers/ec2/image/service_impl.go b/spinifex/handlers/ec2/image/service_impl.go index f49a80423..5f100e6c0 100644 --- a/spinifex/handlers/ec2/image/service_impl.go +++ b/spinifex/handlers/ec2/image/service_impl.go @@ -520,11 +520,19 @@ func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID string) er Host: s.config.Predastore.Host, } + mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) + if err != nil { + slog.Error("snapshotStoppedVolume: failed to load encryption key", "volumeId", volumeID, "err", err) + return errors.New(awserrors.ErrorServerInternal) + } + vbconfig := viperblock.VB{ - VolumeName: volumeID, - VolumeSize: volumeSize, - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, + VolumeName: volumeID, + VolumeSize: volumeSize, + BaseDir: s.config.WalDir, + Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, + MasterKey: mkey, + EncryptionEnabled: mkey != nil, } vb, err := viperblock.New(&vbconfig, "s3", cfg) diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index e8dceee38..2888f5ac1 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -1357,12 +1357,19 @@ func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeC Host: s.config.Predastore.Host, } + mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) + if err != nil { + return nil, err + } + vbconfig := viperblock.VB{ - VolumeName: volumeName, - VolumeSize: utils.SafeIntToUint64(size), - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - VolumeConfig: volumeConfig, + VolumeName: volumeName, + VolumeSize: utils.SafeIntToUint64(size), + BaseDir: s.config.WalDir, + Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, + VolumeConfig: volumeConfig, + MasterKey: mkey, + EncryptionEnabled: mkey != nil, } vb, err := viperblock.New(&vbconfig, "s3", cfg) diff --git a/spinifex/handlers/ec2/volume/service_impl.go b/spinifex/handlers/ec2/volume/service_impl.go index c47a1cf25..1305fc91e 100644 --- a/spinifex/handlers/ec2/volume/service_impl.go +++ b/spinifex/handlers/ec2/volume/service_impl.go @@ -177,12 +177,20 @@ func (s *VolumeServiceImpl) CreateVolume(input *ec2.CreateVolumeInput, accountID Host: s.config.Predastore.Host, } + mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) + if err != nil { + slog.Error("CreateVolume failed to load encryption key", "err", err) + return nil, errors.New(awserrors.ErrorServerInternal) + } + vbconfig := viperblock.VB{ - VolumeName: volumeID, - VolumeSize: volumeSizeBytes, - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - VolumeConfig: volumeConfig, + VolumeName: volumeID, + VolumeSize: volumeSizeBytes, + BaseDir: s.config.WalDir, + Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, + VolumeConfig: volumeConfig, + MasterKey: mkey, + EncryptionEnabled: mkey != nil, } // If created from a snapshot, set the snapshot fields so viperblock's @@ -1022,6 +1030,13 @@ func (s *VolumeServiceImpl) putVolumeConfig(volumeID string, cfg *viperblock.Vol // mergeVolumeConfig reads existing config.json from S3 and merges the new // VolumeConfig into it, preserving full VBState when present. If no existing // VBState is found, it returns a plain volumeConfigWrapper. +// +// Refuses to merge an encrypted-at-rest VBState: the persisted blob carries a +// trailing 16-byte AES-GCM tag bound to the JSON bytes, and re-marshaling +// here without the master key would strip the tag and brick the volume. +// Spinifex does not currently hold the viperblock master key, so the safe +// behavior is to fail loud rather than silently corrupt the on-disk state. +// Track full encryption-aware merge under a separate bead. func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.VolumeConfig) ([]byte, error) { getResult, err := s.store.GetObject(&s3.GetObjectInput{ Bucket: aws.String(s.bucketName), @@ -1041,12 +1056,20 @@ func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock. return nil, fmt.Errorf("failed to read existing config: %w", err) } + // Decoder (not Unmarshal) so a trailing 16-byte AES-GCM tag does not + // produce a parse error and route the body to the wrapper-only path — + // that would silently overwrite an encrypted config.json with a tag-less + // blob and brick the volume. var state viperblock.VBState - if json.Unmarshal(body, &state) != nil || state.BlockSize == 0 { + if decodeErr := json.NewDecoder(bytes.NewReader(body)).Decode(&state); decodeErr != nil || state.BlockSize == 0 { // Not a full VBState (new volume or wrapper-only) -- write wrapper return json.Marshal(volumeConfigWrapper{VolumeConfig: *cfg}) } + if state.EncryptionEnabled { + return nil, fmt.Errorf("mergeVolumeConfig: refusing to merge encrypted VBState for %s without master key (would strip AES-GCM tag and brick volume)", configKey) + } + // Full VBState exists -- update VolumeConfig and reconcile VolumeSize state.VolumeConfig = *cfg configSizeBytes := cfg.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024 diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index 15f76ea44..310abeac9 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -13,6 +13,7 @@ import ( "syscall" "time" + "github.com/mulgadc/predastore/pkg/masterkey" "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/nbd" "github.com/mulgadc/spinifex/spinifex/types" @@ -108,6 +109,14 @@ type Config struct { // ShardWAL enables sharded WAL for mounted volumes (default false) ShardWAL bool + // EncryptionKeyFile is the path to the shared 32-byte AES-256 master key + // used to seal viperblock WAL records, chunk blocks, and metadata. When + // empty the daemon mounts volumes in cleartext mode (legacy). Loaded once + // at launchService start; the parsed key is cached in masterKey. + EncryptionKeyFile string + + masterKey *masterkey.Key + mu sync.Mutex } @@ -224,6 +233,17 @@ func launchService(cfg *Config) (err error) { return err } + if cfg.EncryptionKeyFile != "" { + mkey, err := masterkey.LoadShared(cfg.EncryptionKeyFile) + if err != nil { + return fmt.Errorf("load viperblock encryption key %s: %w", cfg.EncryptionKeyFile, err) + } + cfg.masterKey = mkey + slog.Info("Viperblock at-rest encryption enabled", "key_fingerprint", mkey.Fingerprint) + } else { + slog.Warn("Viperblock at-rest encryption disabled (no EncryptionKeyFile configured)") + } + slog.Info("Viperblock config", "shardwal", cfg.ShardWAL) if cfg.NodeName != "" { @@ -468,7 +488,9 @@ func launchService(cfg *Config) (err error) { Size: defaultCache, }, }, - VolumeConfig: viperblock.VolumeConfig{}, + VolumeConfig: viperblock.VolumeConfig{}, + MasterKey: cfg.masterKey, + EncryptionEnabled: cfg.masterKey != nil, } vb, err := viperblock.New(&vbconfig, "s3", s3cfg) diff --git a/spinifex/utils/encryption.go b/spinifex/utils/encryption.go new file mode 100644 index 000000000..c824c4f1a --- /dev/null +++ b/spinifex/utils/encryption.go @@ -0,0 +1,41 @@ +// Copyright 2026 Mulga Defense Corporation (MDC). All rights reserved. +// Use of this source code is governed by an Apache 2.0 license +// that can be found in the LICENSE file. + +package utils + +import ( + "fmt" + "sync" + + "github.com/mulgadc/predastore/pkg/masterkey" +) + +// viperblockKeyCache memoises masterkey.LoadShared by path so the AWS gateway +// handlers (volume / instance / image) do not re-stat and re-parse the same +// 32-byte file on every VB.New. Process-wide; the underlying *Key holds an +// AEAD which is safe for concurrent use per the crypto/cipher contract. +var ( + viperblockKeyCacheMu sync.Mutex + viperblockKeyCache = map[string]*masterkey.Key{} +) + +// LoadViperblockMasterKey returns the cached *masterkey.Key for the given +// path, loading it via masterkey.LoadShared on first use. An empty path +// returns (nil, nil) — encryption is treated as disabled. +func LoadViperblockMasterKey(path string) (*masterkey.Key, error) { + if path == "" { + return nil, nil + } + viperblockKeyCacheMu.Lock() + defer viperblockKeyCacheMu.Unlock() + if k, ok := viperblockKeyCache[path]; ok { + return k, nil + } + k, err := masterkey.LoadShared(path) + if err != nil { + return nil, fmt.Errorf("load viperblock encryption key %s: %w", path, err) + } + viperblockKeyCache[path] = k + return k, nil +}