diff --git a/cmd/hive/cmd/service.go b/cmd/hive/cmd/service.go index 55cc6c6c9..f8df69473 100644 --- a/cmd/hive/cmd/service.go +++ b/cmd/hive/cmd/service.go @@ -266,6 +266,12 @@ var viperblockStartCmd = &cobra.Command{ os.Exit(1) } + // Resolve sharded WAL setting: default true unless explicitly set to false + shardWAL := true + if nodeConfig.Viperblock.ShardWAL != nil { + shardWAL = *nodeConfig.Viperblock.ShardWAL + } + service, err := service.New("viperblock", &viperblockd.Config{ NatsHost: nodeConfig.NATS.Host, PluginPath: pluginPath, @@ -276,6 +282,7 @@ var viperblockStartCmd = &cobra.Command{ SecretKey: nodeConfig.Predastore.SecretKey, BaseDir: nodeConfig.Predastore.BaseDir, NodeName: clusterConfig.Node, + ShardWAL: shardWAL, }) if err != nil { diff --git a/cmd/hive/cmd/templates/hive.toml b/cmd/hive/cmd/templates/hive.toml index 3214acf39..25af32add 100644 --- a/cmd/hive/cmd/templates/hive.toml +++ b/cmd/hive/cmd/templates/hive.toml @@ -36,6 +36,9 @@ token = "{{.NatsToken}}" [nodes.{{.Node}}.nats.sub] +[nodes.{{.Node}}.viperblock] +shardwal = true + [nodes.{{.Node}}.predastore] host = "{{.BindIP}}:8443" region = "{{.Region}}" diff --git a/go.mod b/go.mod index 4f0ad3487..cc95453af 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/kdomanski/iso9660 v0.4.0 github.com/klauspost/cpuid/v2 v2.3.0 github.com/mulgadc/predastore v1.0.0 - github.com/mulgadc/viperblock v1.0.10 + github.com/mulgadc/viperblock v1.0.11 github.com/nats-io/nats-server/v2 v2.11.6 github.com/nats-io/nats.go v1.43.0 github.com/pelletier/go-toml/v2 v2.2.4 diff --git a/go.sum b/go.sum index bd0d8f152..f84ed4749 100644 --- a/go.sum +++ b/go.sum @@ -276,6 +276,8 @@ github.com/mulgadc/predastore v1.0.0 h1:IZ5V/EL+k+zz0zGoTYp+1BgODb43JxHsb/kvORuy github.com/mulgadc/predastore v1.0.0/go.mod h1:x46A1z091hlZkMt3kkcuLSy2PAzSgROr6ft04qcYOU4= github.com/mulgadc/viperblock v1.0.10 h1:nQJGD44cQ6gW2oWt7IZGjovLNdhLpxjX+WO1EDT+lgY= github.com/mulgadc/viperblock v1.0.10/go.mod h1:1ZKEGGbjmiMi3rfmQ5jMZJB3gIws0pMnAreDbjoCodE= +github.com/mulgadc/viperblock v1.0.11 h1:819xzoQ1DNbu/fz3flUSyu+A60WkfrfoYGPUo796ELU= +github.com/mulgadc/viperblock v1.0.11/go.mod h1:dV1eRKaIjH8YfuCk2Bd4hxKd3Rx5mWfO6gelJ9SbbeY= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= diff --git a/hive/config/config.go b/hive/config/config.go index f79354058..500e83b6c 100644 --- a/hive/config/config.go +++ b/hive/config/config.go @@ -31,6 +31,7 @@ type Config struct { Daemon DaemonConfig `mapstructure:"daemon"` NATS NATSConfig `mapstructure:"nats"` Predastore PredastoreConfig `mapstructure:"predastore"` + Viperblock ViperblockConfig `mapstructure:"viperblock"` AWSGW AWSGWConfig `mapstructure:"awsgw"` // Authentication @@ -77,6 +78,10 @@ type AWSGWConfig struct { ExpectedNodes int `mapstructure:"expected_nodes"` // TODO: Replace with root cluster config } +type ViperblockConfig struct { + ShardWAL *bool `mapstructure:"shardwal"` // Enable sharded WAL (default true when nil) +} + type PredastoreConfig struct { Host string `mapstructure:"host"` Bucket string `mapstructure:"bucket"` diff --git a/hive/config/config_test.go b/hive/config/config_test.go index 715e3b1f2..57c79f43d 100644 --- a/hive/config/config_test.go +++ b/hive/config/config_test.go @@ -263,3 +263,73 @@ func TestNBDTransportConstants(t *testing.T) { assert.Equal(t, NBDTransport("socket"), NBDTransportSocket) assert.Equal(t, NBDTransport("tcp"), NBDTransportTCP) } + +// Tests for ViperblockConfig + +func TestLoadConfig_ViperblockShardWAL_Explicit(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "hive.toml") + + toml := ` +node = "n1" + +[nodes.n1] +region = "us-east-1" + +[nodes.n1.viperblock] +shardwal = false +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.NoError(t, err) + + n := cfg.Nodes["n1"] + require.NotNil(t, n.Viperblock.ShardWAL, "ShardWAL should be set when explicitly configured") + assert.False(t, *n.Viperblock.ShardWAL) +} + +func TestLoadConfig_ViperblockShardWAL_DefaultNil(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "hive.toml") + + toml := ` +node = "n1" + +[nodes.n1] +region = "us-east-1" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.NoError(t, err) + + n := cfg.Nodes["n1"] + assert.Nil(t, n.Viperblock.ShardWAL, "ShardWAL should be nil when not configured (defaults to true in service)") +} + +func TestLoadConfig_ViperblockShardWAL_True(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "hive.toml") + + toml := ` +node = "n1" + +[nodes.n1] +region = "us-east-1" + +[nodes.n1.viperblock] +shardwal = true +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.NoError(t, err) + + n := cfg.Nodes["n1"] + require.NotNil(t, n.Viperblock.ShardWAL) + assert.True(t, *n.Viperblock.ShardWAL) +} diff --git a/hive/nbd/nbd.go b/hive/nbd/nbd.go index 65938b7e0..709b08a24 100644 --- a/hive/nbd/nbd.go +++ b/hive/nbd/nbd.go @@ -23,7 +23,8 @@ type NBDKitConfig struct { BaseDir string `json:"base_dir"` Host string `json:"host"` CacheSize int `json:"cache_size"` - UseTCP bool `json:"use_tcp"` // If true, use TCP transport; otherwise use Unix socket + ShardWAL bool `json:"shardwal"` // Enable sharded WAL (default true) + UseTCP bool `json:"use_tcp"` // If true, use TCP transport; otherwise use Unix socket } func (cfg *NBDKitConfig) Execute() (*exec.Cmd, error) { @@ -68,6 +69,7 @@ func (cfg *NBDKitConfig) Execute() (*exec.Cmd, error) { fmt.Sprintf("base_dir=%s", cfg.BaseDir), fmt.Sprintf("host=%s", cfg.Host), fmt.Sprintf("cache_size=%d", cfg.CacheSize), + fmt.Sprintf("shardwal=%t", cfg.ShardWAL), } args = append(args, pluginArgs...) diff --git a/hive/services/viperblockd/viperblockd.go b/hive/services/viperblockd/viperblockd.go index 10e4e8c8a..3e10c1e6c 100644 --- a/hive/services/viperblockd/viperblockd.go +++ b/hive/services/viperblockd/viperblockd.go @@ -56,6 +56,9 @@ type Config struct { // Socket is faster for local connections, TCP required for remote/DPU scenarios NBDTransport config.NBDTransport + // ShardWAL enables sharded WAL for mounted volumes (default true) + ShardWAL bool + mu sync.Mutex } @@ -155,6 +158,8 @@ func launchService(cfg *Config) (err error) { return err } + slog.Info("Viperblock config", "shardwal", cfg.ShardWAL) + if cfg.NodeName != "" { slog.Info("Waiting for EBS events", "node", cfg.NodeName) } else { @@ -586,6 +591,7 @@ func launchService(cfg *Config) (err error) { AccessKey: cfg.AccessKey, SecretKey: cfg.SecretKey, CacheSize: nbdCacheSize, + ShardWAL: cfg.ShardWAL, } // Create a unique error channel for this specific mount request