gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-23 12:47:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: predastore multinode (#19)

Browse Source
This commit is contained in:
Julian Sommer
2026-02-03 12:07:06 +11:00
committed by GitHub
parent fdb5cca77e
commit 917b66beb9
11 changed files with 467 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
INSTALL.md
+17 -11
View File
@@ -557,26 +557,31 @@ ip addr add 10.11.12.2/24 dev eth0
ip addr add 10.11.12.3/24 dev eth0
```
### Initialize the Leader Node
Next, `hive` will be installed and initiated in `$HOME/node1` for configuration files.
```bash
# Initialize node1
./bin/hive admin init \
--node node1 \
--bind 10.11.12.1 \
--cluster-bind 10.11.12.1 \
--cluster-routes 10.11.12.1:4248 \
--predastore-nodes 10.11.12.1,10.11.12.2,10.11.12.3 \
--port 4432 \
--hive-dir ~/node1/ \
--config-dir ~/node1/config/ \
--region ap-southeast-2 \
--az ap-southeast-2a \
./scripts/start-dev.sh ~/node1/
```
### Join Follower Nodes
Two other nodes will be created, joining node1 to exchange configuration files and identify as a new node.
```bash
# Join cluster
./bin/hive admin join \
--node node2 \
--bind 10.11.12.2 \
@@ -588,11 +593,10 @@ Two other nodes will be created, joining node1 to exchange configuration files a
--region ap-southeast-2 \
--az ap-southeast-2a \
# Start node2
./scripts/start-dev.sh ~/node2/
```
echo "Node3 Setup:"
```bash
./bin/hive admin join \
--node node3 \
--bind 10.11.12.3 \
@@ -600,22 +604,24 @@ echo "Node3 Setup:"
--cluster-routes 10.11.12.1:4248 \
--host 10.11.12.1:4432 \
--data-dir ~/node3/ \
--config-dir ~/node3/config/
--config-dir ~/node3/config/ \
--region ap-southeast-2 \
--az ap-southeast-2a \
./scripts/start-dev.sh ~/node3/
```
Once configured, each node will have its own storage for config, data and state in `~/node[1,2,3]` to simulate a unique node and network.
### Verify the Cluster
Once the multi-node is configured, follow the original installation instructions above to:
Each node will have its own storage for config, data and state in `~/node[1,2,3]` to simulate a unique node and network.
Once the multi-node cluster is configured, follow the original installation instructions above to:
- Import an SSH key
- Clone an AMI
- Launch an instance
- Import an AMI
- Launch instances
Note - when using the AWS CLI tool you must specify the IP address of the node (10.11.12.1, 10.11.12.2, or 10.11.12.3).
Note - when using the AWS CLI tool you must specify the IP address of a gateway node (10.11.12.1, 10.11.12.2, or 10.11.12.3).
```bash
aws --endpoint-url https://10.11.12.3:9999 ec2 describe-instances --instance-ids i-9f5f648adc57ea46d
cmd/hive/cmd/admin.go
+107 -53
View File
@@ -55,6 +55,9 @@ var predastoreTomlTemplate string
//go:embed templates/nats.conf
var natsConfTemplate string
//go:embed templates/predastore-multinode.toml
var predastoreMultiNodeTemplate string
var supportedArchs = map[string]bool{
"x86_64": true,
"aarch64": true, // alias for arm64
@@ -157,6 +160,7 @@ func init() {
adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node)")
adminInitCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
adminInitCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
// Flags for admin join
adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node")
@@ -474,6 +478,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
clusterBind, _ := cmd.Flags().GetString("cluster-bind")
clusterRoutesStr, _ := cmd.Flags().GetString("cluster-routes")
clusterRoutes := strings.Split(clusterRoutesStr, ",")
predastoreNodesStr, _ := cmd.Flags().GetString("predastore-nodes")
// Validate IP address format
if net.ParseIP(bindIP) == nil {
@@ -561,6 +566,49 @@ func runAdminInit(cmd *cobra.Command, args []string) {
portStr := fmt.Sprintf("%d", port)
// Parse multi-node predastore configuration
var predastoreNodeID int
if predastoreNodesStr != "" {
ips := strings.Split(predastoreNodesStr, ",")
if len(ips) < 3 {
fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 3 IPs, got %d\n", len(ips))
os.Exit(1)
}
var predastoreNodes []admin.PredastoreNodeConfig
for i, ip := range ips {
ip = strings.TrimSpace(ip)
if net.ParseIP(ip) == nil {
fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP in --predastore-nodes: %s\n", ip)
os.Exit(1)
}
predastoreNodes = append(predastoreNodes, admin.PredastoreNodeConfig{
ID: i + 1,
Host: ip,
})
}
predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
if predastoreNodeID == 0 {
fmt.Fprintf(os.Stderr, "❌ Error: --bind IP %s not found in --predastore-nodes list\n", bindIP)
os.Exit(1)
}
// Generate multi-node predastore.toml
predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region)
if err != nil {
fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
os.Exit(1)
}
predastorePath := filepath.Join(predastoreDir, "predastore.toml")
if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil {
fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
os.Exit(1)
}
fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
}
configSettings := admin.ConfigSettings{
AccessKey: accessKey,
SecretKey: secretKey,
@@ -575,30 +623,21 @@ func runAdminInit(cmd *cobra.Command, args []string) {
BindIP: bindIP,
ClusterBindIP: clusterBind,
ClusterRoutes: clusterRoutes,
PredastoreNodeID: predastoreNodeID,
}
// Generate all config files
// Generate config files
configs := []admin.ConfigFile{
{
Name: "hive.toml",
Path: hiveTomlPath,
Template: hiveTomlTemplate,
},
{
Name: filepath.Join(awsgwDir, "awsgw.toml"),
Path: filepath.Join(awsgwDir, "awsgw.toml"),
Template: awsgwTomlTemplate,
},
{
Name: filepath.Join(predastoreDir, "predastore.toml"),
Path: filepath.Join(predastoreDir, "predastore.toml"),
Template: predastoreTomlTemplate,
},
{
Name: filepath.Join(natsDir, "nats.conf"),
Path: filepath.Join(natsDir, "nats.conf"),
Template: natsConfTemplate,
},
{Name: "hive.toml", Path: hiveTomlPath, Template: hiveTomlTemplate},
{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
}
// Skip template-based predastore.toml if multi-node was already generated
if predastoreNodesStr == "" {
configs = append(configs, admin.ConfigFile{
Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
})
}
err := admin.GenerateConfigFiles(configs, configSettings)
@@ -836,6 +875,41 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
// Write individual node config files
portStr := fmt.Sprintf("%d", port)
awsgwDir := filepath.Join(configDir, "awsgw")
predastoreDir := filepath.Join(configDir, "predastore")
natsDir := filepath.Join(configDir, "nats")
for _, dir := range []string{awsgwDir, predastoreDir, natsDir} {
if err := os.MkdirAll(dir, 0700); err != nil {
fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
os.Exit(1)
}
}
// Handle multi-node predastore config from leader
var predastoreNodeID int
hasLeaderPredastoreConfig := joinResp.PredastoreConfig != ""
if hasLeaderPredastoreConfig {
// Write the leader's predastore.toml
predastorePath := filepath.Join(predastoreDir, "predastore.toml")
if err := os.WriteFile(predastorePath, []byte(joinResp.PredastoreConfig), 0600); err != nil {
fmt.Fprintf(os.Stderr, "Error writing predastore config from leader: %v\n", err)
os.Exit(1)
}
fmt.Printf("✅ Predastore config received from leader: %s\n", predastorePath)
predastoreNodeID = admin.ParsePredastoreNodeIDFromConfig(joinResp.PredastoreConfig, bindIP)
if predastoreNodeID > 0 {
fmt.Printf("✅ Detected Predastore node ID: %d (for bind IP %s)\n", predastoreNodeID, bindIP)
} else {
fmt.Fprintf(os.Stderr, "ERROR: Could not detect Predastore node ID for bind IP %s in leader config\n", bindIP)
fmt.Fprintf(os.Stderr, "This node's IP must be listed in the --predastore-nodes used during init\n")
os.Exit(1)
}
}
configSettings := admin.ConfigSettings{
AccessKey: localConfig.Nodes[node].AccessKey,
SecretKey: localConfig.Nodes[node].SecretKey,
@@ -849,41 +923,21 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
BindIP: bindIP,
ClusterBindIP: clusterBind,
ClusterRoutes: clusterRoutes,
PredastoreNodeID: predastoreNodeID,
}
awsgwDir := filepath.Join(configDir, "awsgw")
predastoreDir := filepath.Join(configDir, "predastore")
natsDir := filepath.Join(configDir, "nats")
for _, dir := range []string{awsgwDir, predastoreDir, natsDir} {
if err := os.MkdirAll(dir, 0700); err != nil {
fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
os.Exit(1)
}
}
// Generate all config files
// Generate config files
configs := []admin.ConfigFile{
{
Name: "hive.toml",
Path: hiveTomlPath,
Template: hiveTomlTemplate,
},
{
Name: filepath.Join(awsgwDir, "awsgw.toml"),
Path: filepath.Join(awsgwDir, "awsgw.toml"),
Template: awsgwTomlTemplate,
},
{
Name: filepath.Join(predastoreDir, "predastore.toml"),
Path: filepath.Join(predastoreDir, "predastore.toml"),
Template: predastoreTomlTemplate,
},
{
Name: filepath.Join(natsDir, "nats.conf"),
Path: filepath.Join(natsDir, "nats.conf"),
Template: natsConfTemplate,
},
{Name: "hive.toml", Path: hiveTomlPath, Template: hiveTomlTemplate},
{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
}
// Skip template-based predastore.toml if received from leader
if !hasLeaderPredastoreConfig {
configs = append(configs, admin.ConfigFile{
Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
})
}
err = admin.GenerateConfigFiles(configs, configSettings)
cmd/hive/cmd/templates/hive.toml
+4 -1
View File
@@ -37,4 +37,7 @@ host = "{{.BindIP}}:8443"
region = "{{.Region}}"
bucket = "predastore"
accesskey = "{{.AccessKey}}"
secretkey = "{{.SecretKey}}"
secretkey = "{{.SecretKey}}"
{{- if gt .PredastoreNodeID 0}}
node_id = {{.PredastoreNodeID}}
{{- end}}
cmd/hive/cmd/templates/predastore-multinode.toml
+45
View File
@@ -0,0 +1,45 @@
# Predastore S3-compatible storage configuration file in TOML format.
# Generated for multi-node cluster
version = "1.0"
region = "{{.Region}}"
host = "0.0.0.0"
port = 8443
debug = false
[rs]
data = 2
parity = 1
{{range .Nodes}}
[[db]]
id = {{.ID}}
host = "{{.Host}}"
port = 6660
path = "distributed/db/node-{{.ID}}/"
access_key_id = "{{$.AccessKey}}"
secret_access_key = "{{$.SecretKey}}"
{{- if eq .ID 1}}
leader = true
{{- end}}
{{end}}
{{- range .Nodes}}
[[nodes]]
id = {{.ID}}
host = "{{.Host}}"
port = 9991
path = "distributed/nodes/node-{{.ID}}/"
{{end}}
[[buckets]]
name = "predastore"
region = "{{.Region}}"
type = "distributed"
public = false
encryption = ""
[[auth]]
access_key_id = "{{.AccessKey}}"
secret_access_key = "{{.SecretKey}}"
policy = [
{ bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
]
docs/development/features/multi-node-predastore-config.md
+154
View File
@@ -0,0 +1,154 @@
# Multi-Node Predastore Configuration
## Overview
Predastore uses Reed-Solomon erasure coding RS(2,1) -- 2 data shards and 1 parity shard -- requiring a minimum of 3 distinct nodes. Each shard is placed on a separate node via a consistent hash ring (`GetClosestN(hash, 3)`), which means every node must know the full cluster topology at startup to make identical placement decisions.
This is fundamentally different from NATS, which uses gossip-based discovery and can add members dynamically. Predastore's hash ring is static -- if nodes disagree about membership, they'll compute different shard placements and data becomes unreachable.
## How It Works
### Init (leader node)
```
hive admin init --predastore-nodes "10.11.12.1,10.11.12.2,10.11.12.3" --bind 10.11.12.1 ...
```
1. The comma-separated IPs are parsed and validated. Each gets a sequential node ID (1, 2, 3).
2. The leader's `--bind` IP is matched against the list to determine its own node ID. If the bind IP isn't in the list, init fails.
3. `GenerateMultiNodePredastoreConfig` renders a single `predastore.toml` containing the full cluster topology -- all DB entries (port 6660), all shard entries (port 9991), RS config, auth, and buckets. Node ID 1 is marked as the Raft bootstrap leader.
4. The predastore.toml is written to `config/predastore/predastore.toml`.
5. The leader's node ID is written into `hive.toml` as `node_id = 1` under the `[predastore]` section.
### Join (follower nodes)
```
hive admin join --leader https://10.11.12.1:7777 --bind 10.11.12.2 ...
```
1. The follower contacts the leader's `/join` HTTP endpoint.
2. The leader reads its `predastore.toml` from disk and includes the full content in the join response (`PredastoreConfig` field).
3. The follower writes this verbatim to its own `config/predastore/predastore.toml`. Now all nodes have an identical copy.
4. `ParsePredastoreNodeIDFromConfig` unmarshals the TOML, extracts the `[[db]]` array, and matches the follower's `--bind` IP to find its node ID. If the IP isn't found, join fails with an error.
5. The detected node ID is written into the follower's `hive.toml` (e.g., `node_id = 2`).
### Startup
When `start-dev.sh` runs, it reads `node_id` from `hive.toml` using awk and exports it as `HIVE_PREDASTORE_NODE_ID`. Predastore reads this environment variable to know which DB and shard entries in the shared config belong to it.
## Key Design Decisions
### Shared config, per-node identity
Every node gets the **same** `predastore.toml` (the full cluster topology). The only difference per node is `node_id` in `hive.toml`. This separation exists because:
- Predastore needs all members known at startup for consistent hash ring agreement.
- The `predastore.toml` describes the cluster. The `node_id` in `hive.toml` says "I am this member."
- This makes config distribution simple: the leader just sends one file, and the follower figures out its own identity by matching its bind IP.
### Why IPs must be specified at init time
Unlike NATS (gossip discovery), Predastore requires static membership. The hash ring maps `GetClosestN(hash, 3)` to determine which 3 nodes store each object's shards. If a node joins without being in the ring, existing nodes won't route data to it. If a node is in the ring but missing, data placed on it becomes unavailable.
This means all Predastore node IPs must be known when the leader is initialized. The `--predastore-nodes` flag captures this.
### Node ID 1 is the Raft bootstrap leader
The generated config marks `leader = true` on the `[[db]]` entry with `id = 1`. This is the node that bootstraps the Raft cluster. It's always the first IP in the `--predastore-nodes` list, which is also the `--bind` IP of the init command (the leader node).
### Config distribution via join protocol
Rather than requiring operators to manually copy `predastore.toml` to each node, the existing HTTP join protocol distributes it. The leader's daemon reads its predastore.toml and includes it in the join response. This is the same pattern used for CA certificates.
The leader currently sends its predastore.toml unconditionally (even in single-node setups). The follower treats any non-empty `PredastoreConfig` in the response as "leader provided a multi-node config" and skips generating its own template-based config.
### Node ID detection uses proper TOML parsing
The follower determines its node ID by parsing the received predastore.toml with `go-toml/v2` and calling `FindNodeIDByIP` against the `[[db]]` entries. If parsing fails or the bind IP isn't found, join exits with an error. This mirrors the init path, which also exits on failure.
## Backward Compatibility
When `--predastore-nodes` is omitted from `hive admin init`:
- No multi-node predastore.toml is generated.
- `PredastoreNodeID` stays at 0, so `node_id` is not emitted in hive.toml.
- The default single-node predastore.toml template is used instead.
- `start-dev.sh` finds no `node_id` in hive.toml and leaves `HIVE_PREDASTORE_NODE_ID` empty.
- Predastore runs in its default single-node mode.
Existing single-node deployments are unaffected.
## Files Involved
| File | Role |
|------|------|
| `cmd/hive/cmd/admin.go` | CLI flag parsing, init/join orchestration |
| `hive/admin/admin.go` | `GenerateMultiNodePredastoreConfig`, `FindNodeIDByIP`, `ParsePredastoreNodeIDFromConfig`, `BaseConfigFiles` |
| `hive/config/config.go` | `PredastoreConfig.NodeID` field, `NodeJoinResponse.PredastoreConfig` field |
| `hive/daemon/daemon.go` | `/join` endpoint reads and sends predastore.toml |
| `cmd/hive/cmd/templates/hive.toml` | Conditional `node_id` rendering |
| `scripts/start-dev.sh` | Auto-detects `NODE_ID` from hive.toml at startup |
## Generated Config Example
For `--predastore-nodes "10.11.12.1,10.11.12.2,10.11.12.3"`:
```toml
version = "1.0"
region = "ap-southeast-2"
host = "0.0.0.0"
port = 8443
[rs]
data = 2
parity = 1
[[db]]
id = 1
host = "10.11.12.1"
port = 6660
path = "distributed/db/node-1/"
leader = true
[[db]]
id = 2
host = "10.11.12.2"
port = 6660
path = "distributed/db/node-2/"
[[db]]
id = 3
host = "10.11.12.3"
port = 6660
path = "distributed/db/node-3/"
[[nodes]]
id = 1
host = "10.11.12.1"
port = 9991
path = "distributed/nodes/node-1/"
[[nodes]]
id = 2
host = "10.11.12.2"
port = 9991
path = "distributed/nodes/node-2/"
[[nodes]]
id = 3
host = "10.11.12.3"
port = 9991
path = "distributed/nodes/node-3/"
[[buckets]]
name = "predastore"
region = "ap-southeast-2"
type = "distributed"
[[auth]]
access_key_id = "..."
secret_access_key = "..."
```
Each node receives this identical file. Node 1's `hive.toml` has `node_id = 1`, node 2 has `node_id = 2`, etc.
hive/admin/admin.go
+65
View File
@@ -8,13 +8,16 @@ import (
"encoding/base64"
"encoding/pem"
"fmt"
"log/slog"
"math/big"
"net"
"os"
"path/filepath"
"strings"
"text/template"
"time"
toml "github.com/pelletier/go-toml/v2"
"gopkg.in/ini.v1"
)
@@ -35,6 +38,15 @@ type ConfigSettings struct {
// Cluster settings
ClusterBindIP string
ClusterRoutes []string
// Predastore multi-node
PredastoreNodeID int
}
// PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
type PredastoreNodeConfig struct {
ID int
Host string
}
type ConfigFile struct {
@@ -563,3 +575,56 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath string) error {
return nil
}
// GenerateMultiNodePredastoreConfig produces a complete predastore.toml for a
// multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
// and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region string) (string, error) {
if len(nodes) < 3 {
return "", fmt.Errorf("multi-node predastore requires at least 3 nodes, got %d", len(nodes))
}
data := struct {
Nodes []PredastoreNodeConfig
AccessKey string
SecretKey string
Region string
}{nodes, accessKey, secretKey, region}
tmpl, err := template.New("predastore-multinode").Parse(templateStr)
if err != nil {
return "", fmt.Errorf("failed to parse predastore template: %w", err)
}
var b strings.Builder
if err := tmpl.Execute(&b, data); err != nil {
return "", fmt.Errorf("failed to execute predastore template: %w", err)
}
return b.String(), nil
}
// FindNodeIDByIP returns the node ID for the given IP in the node list,
// or 0 if the IP is not found.
func FindNodeIDByIP(nodes []PredastoreNodeConfig, ip string) int {
for _, n := range nodes {
if n.Host == ip {
return n.ID
}
}
return 0
}
// ParsePredastoreNodeIDFromConfig parses a predastore.toml string and returns
// the node ID whose host matches the given IP, or 0 if not found.
func ParsePredastoreNodeIDFromConfig(tomlContent string, ip string) int {
var cfg struct {
DB []PredastoreNodeConfig `toml:"db"`
}
if err := toml.Unmarshal([]byte(tomlContent), &cfg); err != nil {
slog.Warn("Failed to parse predastore.toml content", "error", err)
return 0
}
return FindNodeIDByIP(cfg.DB, ip)
}
hive/config/config.go
+3
View File
@@ -57,6 +57,7 @@ type PredastoreConfig struct {
AccessKey string `mapstructure:"accesskey"`
SecretKey string `mapstructure:"secretkey"`
BaseDir string `mapstructure:"base_dir"`
NodeID int `mapstructure:"node_id"`
}
// DaemonConfig holds the daemon configuration
@@ -188,6 +189,8 @@ type NodeJoinResponse struct {
// CA distribution for per-node certificate generation
CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate
CAKey string `json:"ca_key,omitempty"` // PEM-encoded CA private key
// Predastore config distribution for multi-node clusters
PredastoreConfig string `json:"predastore_config,omitempty"` // Full predastore.toml content
}
type NodeHealthResponse struct {
hive/daemon/daemon.go
+20 -9
View File
@@ -766,8 +766,9 @@ func (d *Daemon) ClusterManager() error {
}
// Read CA certificate and key to share with joining node for per-node cert generation
caCertPath := filepath.Join(d.config.BaseDir, "config", "ca.pem")
caKeyPath := filepath.Join(d.config.BaseDir, "config", "ca.key")
configDir := filepath.Dir(d.configPath)
caCertPath := filepath.Join(configDir, "ca.pem")
caKeyPath := filepath.Join(configDir, "ca.key")
var caCert, caKey string
if caCertPEM, err := os.ReadFile(caCertPath); err == nil {
@@ -782,14 +783,24 @@ func (d *Daemon) ClusterManager() error {
slog.Warn("Could not read CA key for join response", "error", err)
}
// Read predastore.toml to share with joining node (for multi-node predastore)
var predastoreConfig string
predastorePath := filepath.Join(configDir, "predastore", "predastore.toml")
if content, err := os.ReadFile(predastorePath); err == nil {
predastoreConfig = string(content)
} else {
slog.Warn("Could not read predastore.toml for join response", "path", predastorePath, "error", err)
}
return c.JSON(config.NodeJoinResponse{
Success: true,
Message: fmt.Sprintf("node %s successfully joined cluster", req.Node),
SharedData: sharedData,
ConfigHash: configHash,
JoiningNode: req.Node,
CACert: caCert,
CAKey: caKey,
Success: true,
Message: fmt.Sprintf("node %s successfully joined cluster", req.Node),
SharedData: sharedData,
ConfigHash: configHash,
JoiningNode: req.Node,
CACert: caCert,
CAKey: caKey,
PredastoreConfig: predastoreConfig,
})
})
scripts/start-dev.sh
+12 -1
View File
@@ -180,7 +180,18 @@ export HIVE_PREDASTORE_PORT=8443
# Default, distributed backend. For testing, all nodes running locally.
# Specify NODE_ID to run a specific node (e.g multi-server)
export HIVE_PREDASTORE_BACKEND=distributed
export HIVE_PREDASTORE_NODE_ID=
# Auto-detect Predastore NODE_ID from hive.toml if not already set
if [ -z "$HIVE_PREDASTORE_NODE_ID" ]; then
if [ -f "$CONFIG_DIR/hive.toml" ]; then
DETECTED_NODE_ID=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2); print $2; exit}' "$CONFIG_DIR/hive.toml")
if [ -n "$DETECTED_NODE_ID" ] && [ "$DETECTED_NODE_ID" != "0" ]; then
export HIVE_PREDASTORE_NODE_ID="$DETECTED_NODE_ID"
echo " Auto-detected Predastore NODE_ID=$DETECTED_NODE_ID from hive.toml"
fi
fi
fi
export HIVE_PREDASTORE_NODE_ID="${HIVE_PREDASTORE_NODE_ID:-}"
# Use air for hot reloading (dev!)
#PREDASTORE_CMD="air -c .air-predastore.toml"
tests/e2e/lib/multinode-helpers.sh
+32 -3
View File
@@ -454,6 +454,7 @@ init_leader_node() {
--bind "${NODE1_IP}" \
--cluster-bind "${NODE1_IP}" \
--cluster-routes "${NODE1_IP}:${NATS_CLUSTER_PORT}" \
--predastore-nodes "${NODE1_IP},${NODE2_IP},${NODE3_IP}" \
--port ${CLUSTER_PORT} \
--region ap-southeast-2 \
--az ap-southeast-2a \
@@ -488,8 +489,36 @@ join_follower_node() {
--region ap-southeast-2 \
--az "ap-southeast-2a"
# Point predastore to node1 (only node1 runs predastore until multi-node predastore is implemented)
sed -i "s|host = \"${node_ip}:8443\"|host = \"${NODE1_IP}:8443\"|" "$data_dir/config/hive.toml"
echo "Node$node_num joined cluster"
}
# Verify Predastore cluster health
# Checks that Predastore is reachable on all node IPs
# Usage: verify_predastore_cluster [expected_nodes]
verify_predastore_cluster() {
local expected_nodes="${1:-3}"
local healthy=0
echo "Verifying Predastore cluster health (expecting $expected_nodes nodes)..."
for i in $(seq 1 "$expected_nodes"); do
local node_ip="${SIMULATED_NETWORK}.$i"
if curl -k -s "https://${node_ip}:${PREDASTORE_PORT}" > /dev/null 2>&1; then
echo " Node$i ($node_ip:${PREDASTORE_PORT}): reachable"
healthy=$((healthy + 1))
else
echo " Node$i ($node_ip:${PREDASTORE_PORT}): NOT reachable"
fi
done
echo " Healthy Predastore nodes: $healthy/$expected_nodes"
if [ "$healthy" -ge "$expected_nodes" ]; then
echo " Predastore cluster is healthy"
return 0
else
echo " WARNING: Predastore cluster may not be fully formed"
return 1
fi
}
tests/e2e/run-multinode-e2e.sh
+8
View File
@@ -129,6 +129,14 @@ verify_nats_cluster 3 || {
echo "WARNING: NATS cluster verification failed, continuing anyway..."
}
# Verify Predastore cluster
echo ""
verify_predastore_cluster 3 || {
echo "ERROR: Predastore cluster verification failed"
dump_all_node_logs
exit 1
}
# Wait for gateway on node1 (primary gateway)
echo ""
wait_for_gateway "${NODE1_IP}" 30