feat: wire viperblock at-rest encryption key through daemon and AWS handlers

Adds EncryptionKeyFile to ViperblockConfig (cluster config) and
viperblockd.Config. viperblockd loads the master key once at
launchService start via masterkey.LoadShared and passes
MasterKey + EncryptionEnabled into the VB constructed for every
mounted volume.

AWS handlers (volume.CreateVolume, instance.newViperblock,
image.snapshotStoppedVolume) and the spx admin AMI-import path now
resolve the key via utils.LoadViperblockMasterKey — a process-wide
cache keyed by path so the per-VB lookup is one map hit after the
first load.

volume.mergeVolumeConfig: switch from json.Unmarshal to json.Decoder
so a trailing AES-GCM tag cannot route the body to the wrapper-only
path, and refuse outright when state.EncryptionEnabled — re-marshaling
an encrypted VBState here would strip the tag and brick the volume.

cmd/spinifex viperblock CLI grows a --encryption-key-file flag (env
SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE) under a distinct viper key
so it does not collide with predastore's identically-named flag.
This commit is contained in:
brynmailer-mdc
2026-05-28 14:10:44 +10:00
committed by Bryn Mailer
parent 8883984f3f
commit ef50392f9e
8 changed files with 160 additions and 29 deletions
+9 -1
View File
@@ -603,6 +603,12 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
Host: appConfig.Nodes[appConfig.Node].Predastore.Host,
}
mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
os.Exit(1)
}
vbConfig := viperblock.VB{
VolumeName: volumeId,
VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
@@ -612,7 +618,9 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
Size: 0,
},
},
VolumeConfig: manifest,
VolumeConfig: manifest,
MasterKey: mkey,
EncryptionEnabled: mkey != nil,
}
err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath)
+26 -12
View File
@@ -283,19 +283,25 @@ var viperblockStartCmd = &cobra.Command{
shardWAL = *nodeConfig.Viperblock.ShardWAL
}
encryptionKeyFile := nodeConfig.Viperblock.EncryptionKeyFile
if envKey := viper.GetString("viperblock-encryption-key-file"); envKey != "" {
encryptionKeyFile = envKey
}
service, err := service.New("viperblock", &viperblockd.Config{
NatsHost: nodeConfig.NATS.Host,
NatsToken: nodeConfig.NATS.ACL.Token,
NatsCACert: nodeConfig.NATS.CACert,
PluginPath: pluginPath,
S3Host: nodeConfig.Predastore.Host,
Bucket: nodeConfig.Predastore.Bucket,
Region: nodeConfig.Predastore.Region,
AccessKey: nodeConfig.Predastore.AccessKey,
SecretKey: nodeConfig.Predastore.SecretKey,
BaseDir: nodeConfig.Predastore.BaseDir,
NodeName: clusterConfig.Node,
ShardWAL: shardWAL,
NatsHost: nodeConfig.NATS.Host,
NatsToken: nodeConfig.NATS.ACL.Token,
NatsCACert: nodeConfig.NATS.CACert,
PluginPath: pluginPath,
S3Host: nodeConfig.Predastore.Host,
Bucket: nodeConfig.Predastore.Bucket,
Region: nodeConfig.Predastore.Region,
AccessKey: nodeConfig.Predastore.AccessKey,
SecretKey: nodeConfig.Predastore.SecretKey,
BaseDir: nodeConfig.Predastore.BaseDir,
NodeName: clusterConfig.Node,
ShardWAL: shardWAL,
EncryptionKeyFile: encryptionKeyFile,
})
if err != nil {
@@ -911,6 +917,14 @@ func init() {
viper.BindEnv("plugin-path", "SPINIFEX_VIPERBLOCK_PLUGIN_PATH")
viper.BindPFlag("plugin-path", predastoreCmd.PersistentFlags().Lookup("plugin-path"))
// Viperblock at-rest encryption master key (shared with other on-node
// services via group ownership; mode 0640 or stricter). Distinct viper
// key from predastore's encryption-key-file so the two BindPFlag calls
// don't collide globally.
viperblockCmd.PersistentFlags().String("encryption-key-file", "", "Path to the 32-byte AES-256 master key file for at-rest encryption (empty disables encryption)")
viper.BindEnv("viperblock-encryption-key-file", "SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE")
viper.BindPFlag("viperblock-encryption-key-file", viperblockCmd.PersistentFlags().Lookup("encryption-key-file"))
viperblockCmd.AddCommand(viperblockStartCmd)
viperblockCmd.AddCommand(viperblockStopCmd)
viperblockCmd.AddCommand(viperblockStatusCmd)
+8
View File
@@ -101,6 +101,14 @@ type AWSGWConfig struct {
type ViperblockConfig struct {
ShardWAL *bool `json:"ShardWAL" mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
// EncryptionKeyFile is the path to the shared 32-byte AES-256 master key
// for viperblock at-rest encryption (chunks, WAL records, VBState/snapshot
// metadata). When empty, volumes are written in cleartext (legacy mode).
// When set, every VB constructed on this node — daemons and AWS handlers
// alike — must load it via masterkey.LoadShared so existing volumes can
// still be opened.
EncryptionKeyFile string `json:"EncryptionKeyFile" mapstructure:"encryption_key_file"`
}
// VPCDConfig holds the VPC daemon (vpcd) configuration.
+12 -4
View File
@@ -520,11 +520,19 @@ func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID string) er
Host: s.config.Predastore.Host,
}
mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile)
if err != nil {
slog.Error("snapshotStoppedVolume: failed to load encryption key", "volumeId", volumeID, "err", err)
return errors.New(awserrors.ErrorServerInternal)
}
vbconfig := viperblock.VB{
VolumeName: volumeID,
VolumeSize: volumeSize,
BaseDir: s.config.WalDir,
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
VolumeName: volumeID,
VolumeSize: volumeSize,
BaseDir: s.config.WalDir,
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
MasterKey: mkey,
EncryptionEnabled: mkey != nil,
}
vb, err := viperblock.New(&vbconfig, "s3", cfg)
+12 -5
View File
@@ -1357,12 +1357,19 @@ func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeC
Host: s.config.Predastore.Host,
}
mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile)
if err != nil {
return nil, err
}
vbconfig := viperblock.VB{
VolumeName: volumeName,
VolumeSize: utils.SafeIntToUint64(size),
BaseDir: s.config.WalDir,
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
VolumeConfig: volumeConfig,
VolumeName: volumeName,
VolumeSize: utils.SafeIntToUint64(size),
BaseDir: s.config.WalDir,
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
VolumeConfig: volumeConfig,
MasterKey: mkey,
EncryptionEnabled: mkey != nil,
}
vb, err := viperblock.New(&vbconfig, "s3", cfg)
+29 -6
View File
@@ -177,12 +177,20 @@ func (s *VolumeServiceImpl) CreateVolume(input *ec2.CreateVolumeInput, accountID
Host: s.config.Predastore.Host,
}
mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile)
if err != nil {
slog.Error("CreateVolume failed to load encryption key", "err", err)
return nil, errors.New(awserrors.ErrorServerInternal)
}
vbconfig := viperblock.VB{
VolumeName: volumeID,
VolumeSize: volumeSizeBytes,
BaseDir: s.config.WalDir,
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
VolumeConfig: volumeConfig,
VolumeName: volumeID,
VolumeSize: volumeSizeBytes,
BaseDir: s.config.WalDir,
Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
VolumeConfig: volumeConfig,
MasterKey: mkey,
EncryptionEnabled: mkey != nil,
}
// If created from a snapshot, set the snapshot fields so viperblock's
@@ -1022,6 +1030,13 @@ func (s *VolumeServiceImpl) putVolumeConfig(volumeID string, cfg *viperblock.Vol
// mergeVolumeConfig reads existing config.json from S3 and merges the new
// VolumeConfig into it, preserving full VBState when present. If no existing
// VBState is found, it returns a plain volumeConfigWrapper.
//
// Refuses to merge an encrypted-at-rest VBState: the persisted blob carries a
// trailing 16-byte AES-GCM tag bound to the JSON bytes, and re-marshaling
// here without the master key would strip the tag and brick the volume.
// Spinifex does not currently hold the viperblock master key, so the safe
// behavior is to fail loud rather than silently corrupt the on-disk state.
// Track full encryption-aware merge under a separate bead.
func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.VolumeConfig) ([]byte, error) {
getResult, err := s.store.GetObject(&s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
@@ -1041,12 +1056,20 @@ func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.
return nil, fmt.Errorf("failed to read existing config: %w", err)
}
// Decoder (not Unmarshal) so a trailing 16-byte AES-GCM tag does not
// produce a parse error and route the body to the wrapper-only path —
// that would silently overwrite an encrypted config.json with a tag-less
// blob and brick the volume.
var state viperblock.VBState
if json.Unmarshal(body, &state) != nil || state.BlockSize == 0 {
if decodeErr := json.NewDecoder(bytes.NewReader(body)).Decode(&state); decodeErr != nil || state.BlockSize == 0 {
// Not a full VBState (new volume or wrapper-only) -- write wrapper
return json.Marshal(volumeConfigWrapper{VolumeConfig: *cfg})
}
if state.EncryptionEnabled {
return nil, fmt.Errorf("mergeVolumeConfig: refusing to merge encrypted VBState for %s without master key (would strip AES-GCM tag and brick volume)", configKey)
}
// Full VBState exists -- update VolumeConfig and reconcile VolumeSize
state.VolumeConfig = *cfg
configSizeBytes := cfg.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024
+23 -1
View File
@@ -13,6 +13,7 @@ import (
"syscall"
"time"
"github.com/mulgadc/predastore/pkg/masterkey"
"github.com/mulgadc/spinifex/spinifex/admin"
"github.com/mulgadc/spinifex/spinifex/nbd"
"github.com/mulgadc/spinifex/spinifex/types"
@@ -108,6 +109,14 @@ type Config struct {
// ShardWAL enables sharded WAL for mounted volumes (default false)
ShardWAL bool
// EncryptionKeyFile is the path to the shared 32-byte AES-256 master key
// used to seal viperblock WAL records, chunk blocks, and metadata. When
// empty the daemon mounts volumes in cleartext mode (legacy). Loaded once
// at launchService start; the parsed key is cached in masterKey.
EncryptionKeyFile string
masterKey *masterkey.Key
mu sync.Mutex
}
@@ -224,6 +233,17 @@ func launchService(cfg *Config) (err error) {
return err
}
if cfg.EncryptionKeyFile != "" {
mkey, err := masterkey.LoadShared(cfg.EncryptionKeyFile)
if err != nil {
return fmt.Errorf("load viperblock encryption key %s: %w", cfg.EncryptionKeyFile, err)
}
cfg.masterKey = mkey
slog.Info("Viperblock at-rest encryption enabled", "key_fingerprint", mkey.Fingerprint)
} else {
slog.Warn("Viperblock at-rest encryption disabled (no EncryptionKeyFile configured)")
}
slog.Info("Viperblock config", "shardwal", cfg.ShardWAL)
if cfg.NodeName != "" {
@@ -468,7 +488,9 @@ func launchService(cfg *Config) (err error) {
Size: defaultCache,
},
},
VolumeConfig: viperblock.VolumeConfig{},
VolumeConfig: viperblock.VolumeConfig{},
MasterKey: cfg.masterKey,
EncryptionEnabled: cfg.masterKey != nil,
}
vb, err := viperblock.New(&vbconfig, "s3", s3cfg)
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2026 Mulga Defense Corporation (MDC). All rights reserved.
// Use of this source code is governed by an Apache 2.0 license
// that can be found in the LICENSE file.
package utils
import (
"fmt"
"sync"
"github.com/mulgadc/predastore/pkg/masterkey"
)
// viperblockKeyCache memoises masterkey.LoadShared by path so the AWS gateway
// handlers (volume / instance / image) do not re-stat and re-parse the same
// 32-byte file on every VB.New. Process-wide; the underlying *Key holds an
// AEAD which is safe for concurrent use per the crypto/cipher contract.
var (
viperblockKeyCacheMu sync.Mutex
viperblockKeyCache = map[string]*masterkey.Key{}
)
// LoadViperblockMasterKey returns the cached *masterkey.Key for the given
// path, loading it via masterkey.LoadShared on first use. An empty path
// returns (nil, nil) — encryption is treated as disabled.
func LoadViperblockMasterKey(path string) (*masterkey.Key, error) {
if path == "" {
return nil, nil
}
viperblockKeyCacheMu.Lock()
defer viperblockKeyCacheMu.Unlock()
if k, ok := viperblockKeyCache[path]; ok {
return k, nil
}
k, err := masterkey.LoadShared(path)
if err != nil {
return nil, fmt.Errorf("load viperblock encryption key %s: %w", path, err)
}
viperblockKeyCache[path] = k
return k, nil
}