mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
feat: wire viperblock at-rest encryption key through daemon and AWS handlers
Adds EncryptionKeyFile to ViperblockConfig (cluster config) and viperblockd.Config. viperblockd loads the master key once at launchService start via masterkey.LoadShared and passes MasterKey + EncryptionEnabled into the VB constructed for every mounted volume. AWS handlers (volume.CreateVolume, instance.newViperblock, image.snapshotStoppedVolume) and the spx admin AMI-import path now resolve the key via utils.LoadViperblockMasterKey — a process-wide cache keyed by path so the per-VB lookup is one map hit after the first load. volume.mergeVolumeConfig: switch from json.Unmarshal to json.Decoder so a trailing AES-GCM tag cannot route the body to the wrapper-only path, and refuse outright when state.EncryptionEnabled — re-marshaling an encrypted VBState here would strip the tag and brick the volume. cmd/spinifex viperblock CLI grows a --encryption-key-file flag (env SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE) under a distinct viper key so it does not collide with predastore's identically-named flag.
This commit is contained in:
committed by
Bryn Mailer
parent
8883984f3f
commit
ef50392f9e
@@ -603,6 +603,12 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
|
||||
Host: appConfig.Nodes[appConfig.Node].Predastore.Host,
|
||||
}
|
||||
|
||||
mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
vbConfig := viperblock.VB{
|
||||
VolumeName: volumeId,
|
||||
VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
|
||||
@@ -612,7 +618,9 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
|
||||
Size: 0,
|
||||
},
|
||||
},
|
||||
VolumeConfig: manifest,
|
||||
VolumeConfig: manifest,
|
||||
MasterKey: mkey,
|
||||
EncryptionEnabled: mkey != nil,
|
||||
}
|
||||
|
||||
err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath)
|
||||
|
||||
+26
-12
@@ -283,19 +283,25 @@ var viperblockStartCmd = &cobra.Command{
|
||||
shardWAL = *nodeConfig.Viperblock.ShardWAL
|
||||
}
|
||||
|
||||
encryptionKeyFile := nodeConfig.Viperblock.EncryptionKeyFile
|
||||
if envKey := viper.GetString("viperblock-encryption-key-file"); envKey != "" {
|
||||
encryptionKeyFile = envKey
|
||||
}
|
||||
|
||||
service, err := service.New("viperblock", &viperblockd.Config{
|
||||
NatsHost: nodeConfig.NATS.Host,
|
||||
NatsToken: nodeConfig.NATS.ACL.Token,
|
||||
NatsCACert: nodeConfig.NATS.CACert,
|
||||
PluginPath: pluginPath,
|
||||
S3Host: nodeConfig.Predastore.Host,
|
||||
Bucket: nodeConfig.Predastore.Bucket,
|
||||
Region: nodeConfig.Predastore.Region,
|
||||
AccessKey: nodeConfig.Predastore.AccessKey,
|
||||
SecretKey: nodeConfig.Predastore.SecretKey,
|
||||
BaseDir: nodeConfig.Predastore.BaseDir,
|
||||
NodeName: clusterConfig.Node,
|
||||
ShardWAL: shardWAL,
|
||||
NatsHost: nodeConfig.NATS.Host,
|
||||
NatsToken: nodeConfig.NATS.ACL.Token,
|
||||
NatsCACert: nodeConfig.NATS.CACert,
|
||||
PluginPath: pluginPath,
|
||||
S3Host: nodeConfig.Predastore.Host,
|
||||
Bucket: nodeConfig.Predastore.Bucket,
|
||||
Region: nodeConfig.Predastore.Region,
|
||||
AccessKey: nodeConfig.Predastore.AccessKey,
|
||||
SecretKey: nodeConfig.Predastore.SecretKey,
|
||||
BaseDir: nodeConfig.Predastore.BaseDir,
|
||||
NodeName: clusterConfig.Node,
|
||||
ShardWAL: shardWAL,
|
||||
EncryptionKeyFile: encryptionKeyFile,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -911,6 +917,14 @@ func init() {
|
||||
viper.BindEnv("plugin-path", "SPINIFEX_VIPERBLOCK_PLUGIN_PATH")
|
||||
viper.BindPFlag("plugin-path", predastoreCmd.PersistentFlags().Lookup("plugin-path"))
|
||||
|
||||
// Viperblock at-rest encryption master key (shared with other on-node
|
||||
// services via group ownership; mode 0640 or stricter). Distinct viper
|
||||
// key from predastore's encryption-key-file so the two BindPFlag calls
|
||||
// don't collide globally.
|
||||
viperblockCmd.PersistentFlags().String("encryption-key-file", "", "Path to the 32-byte AES-256 master key file for at-rest encryption (empty disables encryption)")
|
||||
viper.BindEnv("viperblock-encryption-key-file", "SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE")
|
||||
viper.BindPFlag("viperblock-encryption-key-file", viperblockCmd.PersistentFlags().Lookup("encryption-key-file"))
|
||||
|
||||
viperblockCmd.AddCommand(viperblockStartCmd)
|
||||
viperblockCmd.AddCommand(viperblockStopCmd)
|
||||
viperblockCmd.AddCommand(viperblockStatusCmd)
|
||||
|
||||
@@ -101,6 +101,14 @@ type AWSGWConfig struct {
|
||||
|
||||
type ViperblockConfig struct {
|
||||
ShardWAL *bool `json:"ShardWAL" mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
|
||||
|
||||
// EncryptionKeyFile is the path to the shared 32-byte AES-256 master key
|
||||
// for viperblock at-rest encryption (chunks, WAL records, VBState/snapshot
|
||||
// metadata). When empty, volumes are written in cleartext (legacy mode).
|
||||
// When set, every VB constructed on this node — daemons and AWS handlers
|
||||
// alike — must load it via masterkey.LoadShared so existing volumes can
|
||||
// still be opened.
|
||||
EncryptionKeyFile string `json:"EncryptionKeyFile" mapstructure:"encryption_key_file"`
|
||||
}
|
||||
|
||||
// VPCDConfig holds the VPC daemon (vpcd) configuration.
|
||||
|
||||
@@ -520,11 +520,19 @@ func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID string) er
|
||||
Host: s.config.Predastore.Host,
|
||||
}
|
||||
|
||||
mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile)
|
||||
if err != nil {
|
||||
slog.Error("snapshotStoppedVolume: failed to load encryption key", "volumeId", volumeID, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
vbconfig := viperblock.VB{
|
||||
VolumeName: volumeID,
|
||||
VolumeSize: volumeSize,
|
||||
BaseDir: s.config.WalDir,
|
||||
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
|
||||
VolumeName: volumeID,
|
||||
VolumeSize: volumeSize,
|
||||
BaseDir: s.config.WalDir,
|
||||
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
|
||||
MasterKey: mkey,
|
||||
EncryptionEnabled: mkey != nil,
|
||||
}
|
||||
|
||||
vb, err := viperblock.New(&vbconfig, "s3", cfg)
|
||||
|
||||
@@ -1357,12 +1357,19 @@ func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeC
|
||||
Host: s.config.Predastore.Host,
|
||||
}
|
||||
|
||||
mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vbconfig := viperblock.VB{
|
||||
VolumeName: volumeName,
|
||||
VolumeSize: utils.SafeIntToUint64(size),
|
||||
BaseDir: s.config.WalDir,
|
||||
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
|
||||
VolumeConfig: volumeConfig,
|
||||
VolumeName: volumeName,
|
||||
VolumeSize: utils.SafeIntToUint64(size),
|
||||
BaseDir: s.config.WalDir,
|
||||
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
|
||||
VolumeConfig: volumeConfig,
|
||||
MasterKey: mkey,
|
||||
EncryptionEnabled: mkey != nil,
|
||||
}
|
||||
|
||||
vb, err := viperblock.New(&vbconfig, "s3", cfg)
|
||||
|
||||
@@ -177,12 +177,20 @@ func (s *VolumeServiceImpl) CreateVolume(input *ec2.CreateVolumeInput, accountID
|
||||
Host: s.config.Predastore.Host,
|
||||
}
|
||||
|
||||
mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile)
|
||||
if err != nil {
|
||||
slog.Error("CreateVolume failed to load encryption key", "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
vbconfig := viperblock.VB{
|
||||
VolumeName: volumeID,
|
||||
VolumeSize: volumeSizeBytes,
|
||||
BaseDir: s.config.WalDir,
|
||||
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
|
||||
VolumeConfig: volumeConfig,
|
||||
VolumeName: volumeID,
|
||||
VolumeSize: volumeSizeBytes,
|
||||
BaseDir: s.config.WalDir,
|
||||
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
|
||||
VolumeConfig: volumeConfig,
|
||||
MasterKey: mkey,
|
||||
EncryptionEnabled: mkey != nil,
|
||||
}
|
||||
|
||||
// If created from a snapshot, set the snapshot fields so viperblock's
|
||||
@@ -1022,6 +1030,13 @@ func (s *VolumeServiceImpl) putVolumeConfig(volumeID string, cfg *viperblock.Vol
|
||||
// mergeVolumeConfig reads existing config.json from S3 and merges the new
|
||||
// VolumeConfig into it, preserving full VBState when present. If no existing
|
||||
// VBState is found, it returns a plain volumeConfigWrapper.
|
||||
//
|
||||
// Refuses to merge an encrypted-at-rest VBState: the persisted blob carries a
|
||||
// trailing 16-byte AES-GCM tag bound to the JSON bytes, and re-marshaling
|
||||
// here without the master key would strip the tag and brick the volume.
|
||||
// Spinifex does not currently hold the viperblock master key, so the safe
|
||||
// behavior is to fail loud rather than silently corrupt the on-disk state.
|
||||
// Track full encryption-aware merge under a separate bead.
|
||||
func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.VolumeConfig) ([]byte, error) {
|
||||
getResult, err := s.store.GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
@@ -1041,12 +1056,20 @@ func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.
|
||||
return nil, fmt.Errorf("failed to read existing config: %w", err)
|
||||
}
|
||||
|
||||
// Decoder (not Unmarshal) so a trailing 16-byte AES-GCM tag does not
|
||||
// produce a parse error and route the body to the wrapper-only path —
|
||||
// that would silently overwrite an encrypted config.json with a tag-less
|
||||
// blob and brick the volume.
|
||||
var state viperblock.VBState
|
||||
if json.Unmarshal(body, &state) != nil || state.BlockSize == 0 {
|
||||
if decodeErr := json.NewDecoder(bytes.NewReader(body)).Decode(&state); decodeErr != nil || state.BlockSize == 0 {
|
||||
// Not a full VBState (new volume or wrapper-only) -- write wrapper
|
||||
return json.Marshal(volumeConfigWrapper{VolumeConfig: *cfg})
|
||||
}
|
||||
|
||||
if state.EncryptionEnabled {
|
||||
return nil, fmt.Errorf("mergeVolumeConfig: refusing to merge encrypted VBState for %s without master key (would strip AES-GCM tag and brick volume)", configKey)
|
||||
}
|
||||
|
||||
// Full VBState exists -- update VolumeConfig and reconcile VolumeSize
|
||||
state.VolumeConfig = *cfg
|
||||
configSizeBytes := cfg.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/predastore/pkg/masterkey"
|
||||
"github.com/mulgadc/spinifex/spinifex/admin"
|
||||
"github.com/mulgadc/spinifex/spinifex/nbd"
|
||||
"github.com/mulgadc/spinifex/spinifex/types"
|
||||
@@ -108,6 +109,14 @@ type Config struct {
|
||||
// ShardWAL enables sharded WAL for mounted volumes (default false)
|
||||
ShardWAL bool
|
||||
|
||||
// EncryptionKeyFile is the path to the shared 32-byte AES-256 master key
|
||||
// used to seal viperblock WAL records, chunk blocks, and metadata. When
|
||||
// empty the daemon mounts volumes in cleartext mode (legacy). Loaded once
|
||||
// at launchService start; the parsed key is cached in masterKey.
|
||||
EncryptionKeyFile string
|
||||
|
||||
masterKey *masterkey.Key
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
@@ -224,6 +233,17 @@ func launchService(cfg *Config) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.EncryptionKeyFile != "" {
|
||||
mkey, err := masterkey.LoadShared(cfg.EncryptionKeyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load viperblock encryption key %s: %w", cfg.EncryptionKeyFile, err)
|
||||
}
|
||||
cfg.masterKey = mkey
|
||||
slog.Info("Viperblock at-rest encryption enabled", "key_fingerprint", mkey.Fingerprint)
|
||||
} else {
|
||||
slog.Warn("Viperblock at-rest encryption disabled (no EncryptionKeyFile configured)")
|
||||
}
|
||||
|
||||
slog.Info("Viperblock config", "shardwal", cfg.ShardWAL)
|
||||
|
||||
if cfg.NodeName != "" {
|
||||
@@ -468,7 +488,9 @@ func launchService(cfg *Config) (err error) {
|
||||
Size: defaultCache,
|
||||
},
|
||||
},
|
||||
VolumeConfig: viperblock.VolumeConfig{},
|
||||
VolumeConfig: viperblock.VolumeConfig{},
|
||||
MasterKey: cfg.masterKey,
|
||||
EncryptionEnabled: cfg.masterKey != nil,
|
||||
}
|
||||
|
||||
vb, err := viperblock.New(&vbconfig, "s3", s3cfg)
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2026 Mulga Defense Corporation (MDC). All rights reserved.
|
||||
// Use of this source code is governed by an Apache 2.0 license
|
||||
// that can be found in the LICENSE file.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/mulgadc/predastore/pkg/masterkey"
|
||||
)
|
||||
|
||||
// viperblockKeyCache memoises masterkey.LoadShared by path so the AWS gateway
|
||||
// handlers (volume / instance / image) do not re-stat and re-parse the same
|
||||
// 32-byte file on every VB.New. Process-wide; the underlying *Key holds an
|
||||
// AEAD which is safe for concurrent use per the crypto/cipher contract.
|
||||
var (
|
||||
viperblockKeyCacheMu sync.Mutex
|
||||
viperblockKeyCache = map[string]*masterkey.Key{}
|
||||
)
|
||||
|
||||
// LoadViperblockMasterKey returns the cached *masterkey.Key for the given
|
||||
// path, loading it via masterkey.LoadShared on first use. An empty path
|
||||
// returns (nil, nil) — encryption is treated as disabled.
|
||||
func LoadViperblockMasterKey(path string) (*masterkey.Key, error) {
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
viperblockKeyCacheMu.Lock()
|
||||
defer viperblockKeyCacheMu.Unlock()
|
||||
if k, ok := viperblockKeyCache[path]; ok {
|
||||
return k, nil
|
||||
}
|
||||
k, err := masterkey.LoadShared(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load viperblock encryption key %s: %w", path, err)
|
||||
}
|
||||
viperblockKeyCache[path] = k
|
||||
return k, nil
|
||||
}
|
||||
Reference in New Issue
Block a user