feat: add configurable shard WAL toggle, bump viperblock v1.0.11

Add [viperblock] section to hive.toml with shardwal config (default
true). The setting flows from config through viperblockd to nbdkit
plugin args, allowing operators to disable sharded WAL if needed.

Bumps viperblock dependency from v1.0.10 to v1.0.11 which includes
the sharded WAL implementation (UseShardedWAL/OpenShardedWAL), fixing
CI build failures.

cmd/hive/cmd/service.go

@@ -266,6 +266,12 @@ var viperblockStartCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
+		// Resolve sharded WAL setting: default true unless explicitly set to false
+		shardWAL := true
+		if nodeConfig.Viperblock.ShardWAL != nil {
+			shardWAL = *nodeConfig.Viperblock.ShardWAL
+		}
+
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:   nodeConfig.NATS.Host,
 			PluginPath: pluginPath,
@@ -276,6 +282,7 @@ var viperblockStartCmd = &cobra.Command{
 			SecretKey:  nodeConfig.Predastore.SecretKey,
 			BaseDir:    nodeConfig.Predastore.BaseDir,
 			NodeName:   clusterConfig.Node,
+			ShardWAL:   shardWAL,
 		})
 
 		if err != nil {

cmd/hive/cmd/templates/hive.toml

@@ -36,6 +36,9 @@ token = "{{.NatsToken}}"
 
 [nodes.{{.Node}}.nats.sub]
 
+[nodes.{{.Node}}.viperblock]
+shardwal = true
+
 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"
 region = "{{.Region}}"

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
 	github.com/mulgadc/predastore v1.0.0
-	github.com/mulgadc/viperblock v1.0.10
+	github.com/mulgadc/viperblock v1.0.11
 	github.com/nats-io/nats-server/v2 v2.11.6
 	github.com/nats-io/nats.go v1.43.0
 	github.com/pelletier/go-toml/v2 v2.2.4

go.sum

@@ -276,6 +276,8 @@ github.com/mulgadc/predastore v1.0.0 h1:IZ5V/EL+k+zz0zGoTYp+1BgODb43JxHsb/kvORuy
 github.com/mulgadc/predastore v1.0.0/go.mod h1:x46A1z091hlZkMt3kkcuLSy2PAzSgROr6ft04qcYOU4=
 github.com/mulgadc/viperblock v1.0.10 h1:nQJGD44cQ6gW2oWt7IZGjovLNdhLpxjX+WO1EDT+lgY=
 github.com/mulgadc/viperblock v1.0.10/go.mod h1:1ZKEGGbjmiMi3rfmQ5jMZJB3gIws0pMnAreDbjoCodE=
+github.com/mulgadc/viperblock v1.0.11 h1:819xzoQ1DNbu/fz3flUSyu+A60WkfrfoYGPUo796ELU=
+github.com/mulgadc/viperblock v1.0.11/go.mod h1:dV1eRKaIjH8YfuCk2Bd4hxKd3Rx5mWfO6gelJ9SbbeY=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI=

hive/config/config.go

@@ -31,6 +31,7 @@ type Config struct {
 	Daemon     DaemonConfig     `mapstructure:"daemon"`
 	NATS       NATSConfig       `mapstructure:"nats"`
 	Predastore PredastoreConfig `mapstructure:"predastore"`
+	Viperblock ViperblockConfig `mapstructure:"viperblock"`
 	AWSGW      AWSGWConfig      `mapstructure:"awsgw"`
 
 	// Authentication
@@ -77,6 +78,10 @@ type AWSGWConfig struct {
 	ExpectedNodes int  `mapstructure:"expected_nodes"` // TODO: Replace with root cluster config
 }
 
+type ViperblockConfig struct {
+	ShardWAL *bool `mapstructure:"shardwal"` // Enable sharded WAL (default true when nil)
+}
+
 type PredastoreConfig struct {
 	Host      string `mapstructure:"host"`
 	Bucket    string `mapstructure:"bucket"`

hive/config/config_test.go

@@ -263,3 +263,73 @@ func TestNBDTransportConstants(t *testing.T) {
 	assert.Equal(t, NBDTransport("socket"), NBDTransportSocket)
 	assert.Equal(t, NBDTransport("tcp"), NBDTransportTCP)
 }
+
+// Tests for ViperblockConfig
+
+func TestLoadConfig_ViperblockShardWAL_Explicit(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1]
+region = "us-east-1"
+
+[nodes.n1.viperblock]
+shardwal = false
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	n := cfg.Nodes["n1"]
+	require.NotNil(t, n.Viperblock.ShardWAL, "ShardWAL should be set when explicitly configured")
+	assert.False(t, *n.Viperblock.ShardWAL)
+}
+
+func TestLoadConfig_ViperblockShardWAL_DefaultNil(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1]
+region = "us-east-1"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	n := cfg.Nodes["n1"]
+	assert.Nil(t, n.Viperblock.ShardWAL, "ShardWAL should be nil when not configured (defaults to true in service)")
+}
+
+func TestLoadConfig_ViperblockShardWAL_True(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1]
+region = "us-east-1"
+
+[nodes.n1.viperblock]
+shardwal = true
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	n := cfg.Nodes["n1"]
+	require.NotNil(t, n.Viperblock.ShardWAL)
+	assert.True(t, *n.Viperblock.ShardWAL)
+}

hive/nbd/nbd.go

@@ -23,7 +23,8 @@ type NBDKitConfig struct {
 	BaseDir    string `json:"base_dir"`
 	Host       string `json:"host"`
 	CacheSize  int    `json:"cache_size"`
-	UseTCP     bool   `json:"use_tcp"` // If true, use TCP transport; otherwise use Unix socket
+	ShardWAL   bool   `json:"shardwal"` // Enable sharded WAL (default true)
+	UseTCP     bool   `json:"use_tcp"`  // If true, use TCP transport; otherwise use Unix socket
 }
 
 func (cfg *NBDKitConfig) Execute() (*exec.Cmd, error) {
@@ -68,6 +69,7 @@ func (cfg *NBDKitConfig) Execute() (*exec.Cmd, error) {
 		fmt.Sprintf("base_dir=%s", cfg.BaseDir),
 		fmt.Sprintf("host=%s", cfg.Host),
 		fmt.Sprintf("cache_size=%d", cfg.CacheSize),
+		fmt.Sprintf("shardwal=%t", cfg.ShardWAL),
 	}
 
 	args = append(args, pluginArgs...)

hive/services/viperblockd/viperblockd.go

@@ -56,6 +56,9 @@ type Config struct {
 	// Socket is faster for local connections, TCP required for remote/DPU scenarios
 	NBDTransport config.NBDTransport
 
+	// ShardWAL enables sharded WAL for mounted volumes (default true)
+	ShardWAL bool
+
 	mu sync.Mutex
 }
 
@@ -155,6 +158,8 @@ func launchService(cfg *Config) (err error) {
 		return err
 	}
 
+	slog.Info("Viperblock config", "shardwal", cfg.ShardWAL)
+
 	if cfg.NodeName != "" {
 		slog.Info("Waiting for EBS events", "node", cfg.NodeName)
 	} else {
@@ -586,6 +591,7 @@ func launchService(cfg *Config) (err error) {
 			AccessKey:  cfg.AccessKey,
 			SecretKey:  cfg.SecretKey,
 			CacheSize:  nbdCacheSize,
+			ShardWAL:   cfg.ShardWAL,
 		}
 
 		// Create a unique error channel for this specific mount request