mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-24 05:04:03 +00:00
Compare commits
177 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 09a1e142eb | |||
| ca14ecf2bc | |||
| 0c65d75371 | |||
| b5d2c22e68 | |||
| 135f02c386 | |||
| 4986e3fb87 | |||
| 20e67051b0 | |||
| 5f370ab75e | |||
| a1a801b26a | |||
| c2265920eb | |||
| bb1fe5e485 | |||
| 1b431b29f7 | |||
| c645fc46b1 | |||
| 83b1435c99 | |||
| 7128a22fdb | |||
| b7396b97c0 | |||
| 1735a46f11 | |||
| 8d9da69abb | |||
| c6b89e873a | |||
| 88ee6e3171 | |||
| dad0df0e33 | |||
| aeb802448b | |||
| b648fe7390 | |||
| 0e47a376c7 | |||
| 81bf36bfc1 | |||
| 93c5b09ac5 | |||
| dd7bce20e6 | |||
| 2e9ede516a | |||
| 244cb69715 | |||
| bb533dfbb6 | |||
| dae9d1bb42 | |||
| e4db948910 | |||
| 962d85edf2 | |||
| 24924be941 | |||
| c5e561abbf | |||
| 774655eff0 | |||
| 5863aeed13 | |||
| febeae8c47 | |||
| 3095a90090 | |||
| d80dd0996a | |||
| 25d98dda7e | |||
| 6df221309d | |||
| db0d46ca5b | |||
| cf12a3272e | |||
| 8774fb1e95 | |||
| fb361dfe49 | |||
| 9d3747bf26 | |||
| 5b4bad3cb3 | |||
| 6c4caec652 | |||
| 22af6c4a46 | |||
| 815e9a8c36 | |||
| c7d832a823 | |||
| 32e2f7bf78 | |||
| 202fffc8db | |||
| 72709dda12 | |||
| 0ff03be0dd | |||
| 998649cbab | |||
| efe07a9ed4 | |||
| 27530ba270 | |||
| 467eb0252d | |||
| 2d30743ba5 | |||
| 10e65ab66a | |||
| c514d6b937 | |||
| df70750f09 | |||
| 210ca92b2c | |||
| 5ce3773c42 | |||
| 39b9e0dcd1 | |||
| b1762f445c | |||
| 221a1ef258 | |||
| e95c2536da | |||
| 2937c8f425 | |||
| 1b424928f4 | |||
| 7a0d148e71 | |||
| 6b4728e702 | |||
| 63e476323d | |||
| 6abae11122 | |||
| 8d3808c364 | |||
| 51ffc4c3ab | |||
| b443bbd3e2 | |||
| e545e81e59 | |||
| 2491c6f2c2 | |||
| 59c57cc074 | |||
| c5bfedcd98 | |||
| 05a608855c | |||
| 513fd07ced | |||
| 2977b03cef | |||
| 560e5cb58c | |||
| 09f78fe29b | |||
| f0323bc081 | |||
| dcb08c0bd1 | |||
| 9ba912b745 | |||
| a554b9b884 | |||
| e15c077d3a | |||
| 88f07a9f3a | |||
| 45a466f71d | |||
| e5038d4a35 | |||
| 25dd435bc6 | |||
| daf5f0c780 | |||
| e0fea35b19 | |||
| e31e523380 | |||
| 0efa570cc5 | |||
| ef87b41858 | |||
| 1c2174a10f | |||
| 801f0f8135 | |||
| d0dfea56e8 | |||
| 2566d9d528 | |||
| a3d7e20e31 | |||
| ba990ffd97 | |||
| d81e417410 | |||
| 45ac7c4357 | |||
| fca82939f5 | |||
| 65ad19e604 | |||
| d14158a619 | |||
| 3ab1a839db | |||
| 10ded4608b | |||
| 2de2899724 | |||
| 130e4b9662 | |||
| 941aaa6504 | |||
| 917b66beb9 | |||
| fdb5cca77e | |||
| b772b2e0bf | |||
| 875328cfef | |||
| 6baf49885b | |||
| 6067f0ccd4 | |||
| e4a2e2972a | |||
| 731ba4fbe8 | |||
| 7e7d358135 | |||
| ad763eb41d | |||
| 1a8e273ac6 | |||
| 91384856ec | |||
| 0e2b715d60 | |||
| f45a106dda | |||
| 7aa688e564 | |||
| 50a0a2e1f1 | |||
| 7f39da8718 | |||
| d1d6e6d004 | |||
| 6d24ec2111 | |||
| 230335adca | |||
| 0b261743d2 | |||
| f04d553a02 | |||
| dfd36370b0 | |||
| 2fae29d486 | |||
| 98701e5208 | |||
| 3232d45ff8 | |||
| d6bfef02ac | |||
| 5b55e1d1e0 | |||
| 2aaed28404 | |||
| e92d67caf0 | |||
| 0b1c44d908 | |||
| 5e53ab1f7e | |||
| 3bca0e69df | |||
| 20de0a7989 | |||
| 118fb35a1e | |||
| dc77e2547b | |||
| 936828ed53 | |||
| 454e8dcace | |||
| 785d2d6c7b | |||
| 5cd5189b56 | |||
| a4e7b71d6c | |||
| adfc54a7f0 | |||
| 82fa85f992 | |||
| de0e9a915e | |||
| 1d816721e8 | |||
| 84c0b3e715 | |||
| a423de3380 | |||
| 21a8e2b004 | |||
| 04197d1adc | |||
| bd0c12d45e | |||
| a3a32d37c3 | |||
| 07056f42a9 | |||
| 528cbc0acd | |||
| 8f0e8d7ead | |||
| 772da5a181 | |||
| 3912a2a84d | |||
| bf99270095 | |||
| 11c1d74a79 | |||
| 7dca292e67 |
@@ -2,7 +2,17 @@ name: Hive E2E Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, e2e]
|
||||
branches:
|
||||
- main
|
||||
- e2e
|
||||
paths:
|
||||
- "**/*.go"
|
||||
- go.mod
|
||||
- go.sum
|
||||
- Makefile
|
||||
- tests/e2e/**
|
||||
- scripts/**
|
||||
- .github/workflows/e2e.yml
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
|
||||
+81
-18
@@ -2,39 +2,102 @@ name: Hive Build and Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches:
|
||||
- main
|
||||
- dev
|
||||
paths:
|
||||
- "**/*.go"
|
||||
- go.mod
|
||||
- go.sum
|
||||
- Makefile
|
||||
- .github/workflows/hive.yml
|
||||
pull_request:
|
||||
paths:
|
||||
- "**/*.go"
|
||||
- go.mod
|
||||
- go.sum
|
||||
- Makefile
|
||||
- .github/workflows/hive.yml
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
hive_build_and_test:
|
||||
build_and_test:
|
||||
name: Hive Build and Test
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
- name: Checkout Hive
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
path: hive
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: dev
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: dev
|
||||
path: predastore
|
||||
|
||||
- name: Setup Go Workspace
|
||||
run: go work init ./hive ./viperblock ./predastore
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
cache-dependency-path: "go.sum"
|
||||
go-version-file: "go.mod"
|
||||
cache-dependency-path: "hive/go.sum"
|
||||
go-version-file: "hive/go.mod"
|
||||
|
||||
- name: Build Hive
|
||||
run: make -C hive build
|
||||
|
||||
- name: Run Tests
|
||||
run: LOG_IGNORE=1 go test -v -timeout 120s ./...
|
||||
run: make -C hive test
|
||||
|
||||
lint_and_security:
|
||||
name: Hive Lint and Security
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Checkout Hive
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
path: hive
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: dev
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: dev
|
||||
path: predastore
|
||||
|
||||
- name: Setup Go Workspace
|
||||
run: go work init ./hive ./viperblock ./predastore
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
cache-dependency-path: "hive/go.sum"
|
||||
go-version-file: "hive/go.mod"
|
||||
|
||||
- name: Check Format
|
||||
run: make -C hive check-format
|
||||
|
||||
- name: Go Vet
|
||||
run: go vet ./...
|
||||
run: make -C hive vet
|
||||
|
||||
- name: Static Check
|
||||
run: go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./...
|
||||
|
||||
- name: Go Vulnerability Check
|
||||
run: go tool govulncheck ./...
|
||||
|
||||
# - name: Gosec Security Scanner
|
||||
# run: go tool gosec -exclude-generated ./...
|
||||
|
||||
- name: Build
|
||||
run: make go_build
|
||||
- name: Security Checks
|
||||
run: make -C hive security-check
|
||||
|
||||
+20
-43
@@ -2,72 +2,49 @@ name: UI Build and Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, ui]
|
||||
paths: ["hive-ui/**", ".github/workflows/ui.yml"]
|
||||
branches:
|
||||
- main
|
||||
- ui
|
||||
paths:
|
||||
- hive/services/hiveui/frontend/**
|
||||
- .github/workflows/ui.yml
|
||||
pull_request:
|
||||
paths:
|
||||
- hive/services/hiveui/frontend/**
|
||||
- .github/workflows/ui.yml
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
name: Frontend Build and Test
|
||||
ui_build_and_test:
|
||||
name: UI Build and Test
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: hive-ui/frontend
|
||||
working-directory: hive/services/hiveui/frontend
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
sparse-checkout: hive/services/hiveui/frontend
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
package_json_file: hive-ui/frontend/package.json
|
||||
package_json_file: hive/services/hiveui/frontend/package.json
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: hive-ui/frontend/pnpm-lock.yaml
|
||||
node-version-file: hive-ui/frontend/.nvmrc
|
||||
cache-dependency-path: hive/services/hiveui/frontend/pnpm-lock.yaml
|
||||
node-version-file: hive/services/hiveui/frontend/.nvmrc
|
||||
|
||||
- name: Install frontend dependencies
|
||||
- name: Install Dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Run Biome formatter check
|
||||
- name: Run Biome Check
|
||||
run: pnpm check
|
||||
|
||||
- name: Build frontend
|
||||
- name: Build Frontend
|
||||
run: pnpm build
|
||||
|
||||
webserver:
|
||||
name: Webserver Build and Test
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: hive-ui/webserver
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
cache-dependency-path: hive-ui/webserver/go.sum
|
||||
go-version-file: hive-ui/webserver/go.mod
|
||||
|
||||
- name: Go Vet
|
||||
run: go vet ./...
|
||||
|
||||
- name: Static Check
|
||||
run: go tool staticcheck ./...
|
||||
|
||||
- name: Go Vulnerability Check
|
||||
run: go tool govulncheck ./...
|
||||
|
||||
- name: Gosec Security Scanner
|
||||
run: go tool gosec -exclude-generated ./...
|
||||
|
||||
- name: Build
|
||||
run: go build ./...
|
||||
|
||||
+18
-2
@@ -1,3 +1,12 @@
|
||||
# Terraform/OpenTofu
|
||||
.terraform/
|
||||
.terraform.lock.hcl
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
*.tfvars
|
||||
!example.tfvars
|
||||
scripts/iac/proxmox/clusters/
|
||||
|
||||
# Go workspace
|
||||
go.work
|
||||
go.work.sum
|
||||
@@ -21,5 +30,12 @@ Thumbs.db
|
||||
# Test reports
|
||||
*-report.txt
|
||||
|
||||
# hive-ui
|
||||
hive-ui/certs/
|
||||
# Internal docs (moved to private mulga repo)
|
||||
docs/INTERNAL.md
|
||||
docs/PRIVATE.md
|
||||
docs/DEVNOTES.md
|
||||
docs/HIVE_DEVELOPMENT_PLAN.md
|
||||
docs/PLAN.md
|
||||
docs/TODO.md
|
||||
docs/development/
|
||||
docs/internal/
|
||||
|
||||
Vendored
-7
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"biomejs.biome",
|
||||
"bradlc.vscode-tailwindcss",
|
||||
"golang.go"
|
||||
]
|
||||
}
|
||||
Vendored
-36
@@ -1,36 +0,0 @@
|
||||
{
|
||||
"files.readonlyInclude": {
|
||||
"**/routeTree.gen.ts": true
|
||||
},
|
||||
"files.watcherExclude": {
|
||||
"**/routeTree.gen.ts": true
|
||||
},
|
||||
"search.exclude": {
|
||||
"**/routeTree.gen.ts": true
|
||||
},
|
||||
"[css][json][jsonc]": {
|
||||
"editor.defaultFormatter": "biomejs.biome",
|
||||
"editor.tabSize": 2
|
||||
},
|
||||
"[javascript][javascriptreact][typescript][typescriptreact]": {
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.fixAll.biome": "explicit",
|
||||
"source.organizeImports.biome": "explicit"
|
||||
},
|
||||
"editor.defaultFormatter": "biomejs.biome",
|
||||
"editor.tabSize": 2
|
||||
},
|
||||
"[go]": {
|
||||
"editor.defaultFormatter": "golang.go",
|
||||
"editor.tabSize": 4
|
||||
},
|
||||
"biome.configurationPath": "./hive-ui/frontend/biome.jsonc",
|
||||
"biome.requireConfiguration": true,
|
||||
"biome.lsp.bin": "./hive-ui/frontend/node_modules/.bin/biome",
|
||||
"editor.formatOnSave": true,
|
||||
"editor.formatOnPaste": true,
|
||||
"files.insertFinalNewline": true,
|
||||
"files.trimFinalNewlines": true,
|
||||
"files.trimTrailingWhitespace": true,
|
||||
"typescript.tsdk": "./hive-ui/frontend/node_modules/typescript/lib"
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
This is the Hive platform - an AWS-compatible infrastructure stack for bare-metal, edge, and on-premise deployments. Hive orchestrates multiple independent components that work together to provide AWS-compatible services.
|
||||
|
||||
## Component Repositories
|
||||
|
||||
The Hive platform consists of these independent repositories:
|
||||
|
||||
- **[viperblock](../viperblock/)** - High-performance block storage service (EBS alternative) with WAL-backed storage
|
||||
- **[predastore](../predastore/)** - S3-compatible object storage service with Reed-Solomon erasure coding
|
||||
- **nbdkit/** - Network Block Device (NBD) server for exporting block devices (upstream project)
|
||||
- **hive/** (this repo) - VM orchestration service and platform coordinator
|
||||
|
||||
Each component can be used independently or as part of the integrated Hive stack. See component-specific `CLAUDE.md` files for detailed development guidance:
|
||||
- `../viperblock/CLAUDE.md` - Viperblock development guide
|
||||
- `../predastore/CLAUDE.md` - Predastore development guide
|
||||
|
||||
## Cross-Repository Development Setup
|
||||
|
||||
For platform development, clone all repositories to the same parent directory:
|
||||
```bash
|
||||
mkdir mulga
|
||||
cd mulga
|
||||
git clone <hive-repo-url> hive
|
||||
git clone <viperblock-repo-url> viperblock
|
||||
git clone <predastore-repo-url> predastore
|
||||
```
|
||||
|
||||
The `hive/go.mod` uses local replace directives for cross-component development:
|
||||
```go
|
||||
replace github.com/mulgadc/viperblock => ../viperblock
|
||||
replace github.com/mulgadc/predastore => ../predastore
|
||||
```
|
||||
|
||||
## Build Commands
|
||||
|
||||
### Hive (Platform Orchestrator)
|
||||
```bash
|
||||
make build # Build the hive binary
|
||||
make test # Run Go tests
|
||||
make bench # Run benchmarks
|
||||
make clean # Clean build artifacts
|
||||
```
|
||||
|
||||
### **MANDATORY: Unit Testing Requirements**
|
||||
|
||||
**⚠️ CRITICAL: Always run unit tests before any commit or push operation.**
|
||||
|
||||
```bash
|
||||
# REQUIRED before any git commit or push
|
||||
make test # Must pass with zero failures
|
||||
|
||||
# Example workflow:
|
||||
make test # Verify all tests pass
|
||||
git add . # Stage changes
|
||||
git commit -m "..." # Only after tests pass
|
||||
git push origin main # Only after tests pass
|
||||
```
|
||||
|
||||
**Testing Policy:**
|
||||
- **All unit tests MUST pass** before committing changes
|
||||
- **No exceptions** - failing tests block commits
|
||||
- Tests must complete without errors or panics
|
||||
- Use `make test` command to run the full test suite
|
||||
- If tests fail, fix issues before proceeding with git operations
|
||||
|
||||
### Component Build Commands
|
||||
For detailed build instructions, see component-specific CLAUDE.md files:
|
||||
|
||||
**Viperblock** (`../viperblock/CLAUDE.md`):
|
||||
```bash
|
||||
cd ../viperblock && make build # Build sfs, vblock binaries + NBD plugin
|
||||
```
|
||||
|
||||
**Predastore** (`../predastore/CLAUDE.md`):
|
||||
```bash
|
||||
cd ../predastore && make build # Build s3d binary
|
||||
```
|
||||
|
||||
**NBDkit** (system dependency):
|
||||
```bash
|
||||
cd nbdkit && autoreconf -i && ./configure && make && make install
|
||||
```
|
||||
|
||||
## Architecture Notes
|
||||
|
||||
### Message-Driven Microservices Architecture
|
||||
|
||||
Hive uses a **scalable message-driven architecture** that separates API handling from request processing:
|
||||
|
||||
```
|
||||
AWS SDK (custom endpoint) → Hive API Gateway → NATS Topics → Specialized Daemons
|
||||
↑ ↓
|
||||
AWS-compatible response ← NATS Topics ← Processing Results
|
||||
```
|
||||
|
||||
#### Core Components:
|
||||
|
||||
1. **Hive API Gateway Service** - TLS endpoint that handles AWS SDK requests
|
||||
2. **Specialized Daemon Services** - Horizontally scalable workers for each AWS service type
|
||||
3. **NATS Message Broker** - Topic-based routing with queue groups for load balancing
|
||||
|
||||
#### Current Implementation (daemon.go):
|
||||
- **Gateway Logic**: `StartAWSGateway()` handles AWS API calls on port 9999
|
||||
- **NATS Integration**: Uses request-response pattern: `d.natsConn.Request("ec2.runinstances", data, 30s)`
|
||||
- **Queue Groups**: Load balancing via `"hive-workers"` queue group
|
||||
- **Service Coordination**: All inter-service communication via NATS topics
|
||||
|
||||
### Inter-component Dependencies
|
||||
- **Service Startup Order**: NATS → Predastore → Viperblock → NBDkit → Gateway → Daemons
|
||||
- **Message Flow**: Gateway translates AWS calls to NATS messages, daemons process and respond
|
||||
- **Scaling**: Multiple daemon instances can subscribe to same topics for horizontal scaling
|
||||
- All Go projects use local module replacements for development
|
||||
|
||||
### Storage Backend Integration
|
||||
- **Viperblock Integration**: EBS volumes mounted via NATS messaging (`ebs.mount` topic)
|
||||
- **Predastore Integration**: S3 operations proxied through dedicated S3 daemons
|
||||
- **NBDkit Integration**: Block device access for VM storage
|
||||
- **Volume Lifecycle**: Coordinated through NATS between EC2 and EBS daemons
|
||||
|
||||
### Key Design Patterns
|
||||
- **Message-Driven**: All service communication via NATS topics (ec2.*, ebs.*, s3.*, vpc.*)
|
||||
- **Queue Groups**: Load balancing and fault tolerance via NATS queue groups
|
||||
- **Request-Response**: 30-second timeout pattern for AWS API compatibility
|
||||
- **Service Specialization**: Dedicated daemon types for each AWS service
|
||||
- **Horizontal Scaling**: Multiple daemon instances handle same topic types
|
||||
|
||||
### NATS Topic Structure
|
||||
|
||||
The system uses structured NATS topics for service communication:
|
||||
|
||||
```bash
|
||||
# EC2 Service Topics:
|
||||
ec2.runinstances # Launch new instances
|
||||
ec2.describeinstances # Query instance status
|
||||
ec2.startinstances # Start stopped instances
|
||||
ec2.stopinstances # Stop running instances
|
||||
ec2.terminateinstances # Terminate instances
|
||||
|
||||
# EBS Service Topics:
|
||||
ebs.createvolume # Create new EBS volume
|
||||
ebs.attachvolume # Attach volume to instance
|
||||
ebs.detachvolume # Detach volume
|
||||
ebs.describevolumes # List volumes
|
||||
ebs.mount # Mount volume (internal)
|
||||
ebs.unmount # Unmount volume (internal)
|
||||
|
||||
# S3 Service Topics:
|
||||
s3.createbucket # Create S3 bucket
|
||||
s3.putobject # Store object
|
||||
s3.getobject # Retrieve object
|
||||
s3.listbuckets # List buckets
|
||||
|
||||
# VPC Service Topics:
|
||||
vpc.createvpc # Create VPC
|
||||
vpc.createsubnet # Create subnet
|
||||
vpc.describevpcs # List VPCs
|
||||
|
||||
# Health and Discovery:
|
||||
health.ec2.daemon.{id} # Daemon health heartbeats
|
||||
discovery.services # Service registration
|
||||
```
|
||||
|
||||
### Go Module Structure
|
||||
All Go projects use:
|
||||
- Go 1.23+ with module mode
|
||||
- Local replace directives for cross-component development
|
||||
- Standard build flags: `-ldflags "-s -w"` for optimized binaries
|
||||
- Test environment variable: `LOG_IGNORE=1` to suppress logs during testing
|
||||
|
||||
### Development Workflow
|
||||
|
||||
#### Quick Setup (Recommended)
|
||||
```bash
|
||||
# 1. Clone dependencies (if not already done)
|
||||
./scripts/clone-deps.sh
|
||||
|
||||
# 2. Setup development environment
|
||||
./scripts/dev-setup.sh
|
||||
|
||||
# 3. Start all services
|
||||
./scripts/start-dev.sh
|
||||
|
||||
# 4. Stop all services when done
|
||||
./scripts/stop-dev.sh
|
||||
```
|
||||
|
||||
#### Manual Development Process
|
||||
1. **Cross-repo Setup**: Clone all repositories to same parent directory
|
||||
2. **Service Dependencies**: Start services in order: NATS → Predastore → Viperblock → NBDkit → Gateway → Daemons
|
||||
3. **Build Order**: Build components: predastore → viperblock → hive
|
||||
4. **Testing**: Use `make test` in each component to verify changes
|
||||
5. **Message-Driven Development**: Test NATS message flows between gateway and daemons
|
||||
6. **Scaling**: Test with multiple daemon instances subscribing to same topics
|
||||
|
||||
#### Development Tools
|
||||
- **Hot Reloading**: Use `air` for automatic restarts during development
|
||||
- **TLS Certificates**: Auto-generated self-signed certificates for HTTPS endpoints
|
||||
- **Log Monitoring**: Centralized logs in `data/logs/` directory
|
||||
- **AWS CLI Testing**: Test endpoints with `aws --endpoint-url https://localhost:9999`
|
||||
|
||||
### Development Roadmap
|
||||
For major feature development and architectural changes, refer to `HIVE_DEVELOPMENT_PLAN.md` which contains:
|
||||
- Comprehensive roadmap for transforming Hive from proof-of-concept to beta release
|
||||
- Detailed implementation phases with specific tasks and file structures
|
||||
- AWS API compatibility implementation guidelines
|
||||
- Development environment automation plans
|
||||
- Testing and validation frameworks
|
||||
|
||||
When implementing new AWS services or enhancing existing functionality, follow the structured approach outlined in the development plan.
|
||||
File diff suppressed because it is too large
Load Diff
+429
-171
@@ -4,13 +4,14 @@ Notes for development environment installation.
|
||||
|
||||
## Dependencies
|
||||
|
||||
The current supported OS that is validated and tested includes:
|
||||
The currently supported operating systems that are validated and tested include:
|
||||
|
||||
* Ubuntu 22.04
|
||||
* Ubuntu 24.04
|
||||
* Ubuntu 25.10
|
||||
- Ubuntu 22.04
|
||||
- Ubuntu 24.04
|
||||
- Ubuntu 25.10
|
||||
- Debian 12.13
|
||||
|
||||
For the development preview please use one of the supported versions above.
|
||||
For the development preview, please use one of the supported versions above.
|
||||
|
||||
## Download
|
||||
|
||||
@@ -24,24 +25,30 @@ git clone https://github.com/mulgadc/hive.git
|
||||
|
||||
### Quick Install
|
||||
|
||||
To bootstrap the dependencies of Hive in one simple step (QEMU, Go, AWS CLI)
|
||||
To bootstrap the dependencies of Hive in one simple step (QEMU, Go, AWS CLI):
|
||||
|
||||
```bash
|
||||
sudo make -C hive quickinstall
|
||||
```
|
||||
|
||||
Ensure Go is available in your `PATH` if not previously installed:
|
||||
|
||||
```bash
|
||||
export PATH=$PATH:/usr/local/go/bin/
|
||||
```
|
||||
|
||||
### Manual Install
|
||||
|
||||
Alternatively, run the following steps to manually setup the required dependencies.
|
||||
Alternatively, run the following steps to manually set up the required dependencies.
|
||||
|
||||
```bash
|
||||
sudo add-apt-repository universe
|
||||
sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file
|
||||
```
|
||||
|
||||
Ensure the Go toolkit is installed for version 1.25.5 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
|
||||
Ensure the Go toolkit is installed for version 1.25.7 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
|
||||
|
||||
Confirm go is correctly installed, and set in your $PATH.
|
||||
Confirm Go is correctly installed, and set in your $PATH.
|
||||
|
||||
```bash
|
||||
go version
|
||||
@@ -55,7 +62,7 @@ unzip awscliv2.zip
|
||||
sudo ./aws/install
|
||||
```
|
||||
|
||||
Confirm awscli version > 2.0 is installed which is required by Hive.
|
||||
Confirm awscli version > 2.0 is installed.
|
||||
|
||||
[https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
|
||||
|
||||
@@ -73,7 +80,7 @@ cd hive
|
||||
./scripts/dev-setup.sh # Setup complete development environment
|
||||
```
|
||||
|
||||
Once complete, confirm `./bin/hive` exists and executable.
|
||||
Once complete, confirm `./bin/hive` exists and is executable.
|
||||
|
||||
# Single Node Installation
|
||||
|
||||
@@ -104,8 +111,6 @@ Verify the certificate was added:
|
||||
ls -la /usr/local/share/ca-certificates/hive-ca.crt
|
||||
```
|
||||
|
||||
This step is required for AWS CLI 2.33+ which requires proper CA trust chains. Without this, you would need to use `--no-verify-ssl` for every AWS CLI command.
|
||||
|
||||
Next, set the AWS profile to use `hive` which points to the local environment.
|
||||
|
||||
```bash
|
||||
@@ -148,7 +153,7 @@ aws ec2 create-key-pair \
|
||||
| jq -r '.KeyMaterial | rtrimstr("\n")' > ~/.ssh/hive-key
|
||||
```
|
||||
|
||||
Update permissions for the key for ssh to accept reading the file.
|
||||
Update permissions for the key for SSH to accept reading the file.
|
||||
|
||||
```bash
|
||||
chmod 600 ~/.ssh/hive-key
|
||||
@@ -160,7 +165,7 @@ Next, generate a public key from the specified private key pair.
|
||||
ssh-keygen -y -f ~/.ssh/hive-key > ~/.ssh/hive-key.pub
|
||||
```
|
||||
|
||||
### Validate New Key
|
||||
### Verify New Key
|
||||
|
||||
```bash
|
||||
aws ec2 describe-key-pairs
|
||||
@@ -183,13 +188,13 @@ aws ec2 describe-key-pairs
|
||||
|
||||
## Create AMI Template
|
||||
|
||||
Use the hive CLI tool to import a selected OS image. Note, the source file can be compressed (e.g image.tar.gz, image.gz, image.tar.xz) and the tool will automatically extract and upload the raw OS image as an AMI after validation the disk image contains a UEFI/BIOS boot capability.
|
||||
The Hive CLI tool offers 2 ways of importing an image. You can use automatic image importing, and select from our range of images. Alternatively, provide your own image source file (e.g image.tar.gz, image.gz, image.tar.xz) and the tool will automatically extract and upload the raw OS image as an AMI (after validating the image contains UEFI/BIOS boot capability).
|
||||
|
||||
Note, when downloading OS images, use supported platforms that support the `cloud-init` feature to automatically bootstrap when using the Hive EC2 functionality to access SSH and networking services.
|
||||
**Note:** When downloading OS images, use platforms that support the `cloud-init` feature to automatically bootstrap when using the Hive EC2 functionality to access SSH and networking services.
|
||||
|
||||
### Automatic Image Import
|
||||
|
||||
Discover available images to automatically download and install. This will pull the images from the distro official mirror and simplify the process to bootstrap a Hive installation with AMIs that include common operating systems.
|
||||
Discover available images to automatically download and install. This will pull the images from the distro official mirror and simplify the process to bootstrap a Hive installation with AMIs for common operating systems.
|
||||
|
||||
```bash
|
||||
./bin/hive admin images list
|
||||
@@ -207,19 +212,7 @@ ubuntu-24.04-x86_64 | ubuntu | 24.04 | x86_64 | bios
|
||||
Next, choose the image you would like to import as an AMI. Choose the appropriate `x86_64` or `arm64` architecture depending on your platform.
|
||||
|
||||
```bash
|
||||
./bin/hive admin images import --name debian-12-arm64 --force
|
||||
```
|
||||
|
||||
Make note of the Image-ID `ami-XXX`
|
||||
|
||||
```bash
|
||||
✅ Image import complete. Image-ID (AMI): ami-e29fcc65734aec9ea
|
||||
```
|
||||
|
||||
Next, verify available disk images to confirm the import was successful, replace `ami-XXX` with the value from the command output.
|
||||
|
||||
```bash
|
||||
aws ec2 describe-images --image-ids ami-XXX
|
||||
./bin/hive admin images import --name debian-12-arm64
|
||||
```
|
||||
|
||||
### Manual Image Import
|
||||
@@ -238,25 +231,29 @@ Import as an AMI to the backend store:
|
||||
./bin/hive admin images import --file ~/debian-12-genericcloud-arm64.tar.xz --arch arm64 --distro debian --version 12
|
||||
```
|
||||
|
||||
Next, verify available disk images to confirm the import was successful.
|
||||
### Verify Image Import
|
||||
|
||||
```bash
|
||||
aws ec2 describe-images
|
||||
```
|
||||
|
||||
## Run Instance
|
||||
|
||||
Once Hive is successfully installed and bootstrapped with a system AMI and SSH keys, proceed to run an instance. Replace `ami-XXX` with your imported ImageId above.
|
||||
Export the AMI.
|
||||
|
||||
```bash
|
||||
export HIVE_AMI="ami-XXX"
|
||||
```
|
||||
|
||||
### Query instance types available
|
||||
Next, verify available disk images to confirm the import was successful.
|
||||
|
||||
Depending on the host platform Hive is installed for the CPU capabilities and vendor (e.g AMD, Intel, ARM), different compute instances will be available.
|
||||
```bash
|
||||
aws ec2 describe-images --image-ids $HIVE_AMI
|
||||
```
|
||||
|
||||
Query available instance types and choose an instance type available on your host with 2 vCPUs:
|
||||
## Run Instance
|
||||
|
||||
Once Hive is successfully installed and bootstrapped with a system AMI and SSH keys, proceed to run an instance.
|
||||
|
||||
### Query Instance Types
|
||||
|
||||
Depending on the host platform Hive is installed, different compute instances will be available.
|
||||
|
||||
Query available instance types and choose an instance type available on your host:
|
||||
|
||||
```bash
|
||||
aws ec2 describe-instance-types
|
||||
@@ -266,28 +263,28 @@ aws ec2 describe-instance-types
|
||||
{
|
||||
"InstanceTypes": [
|
||||
{
|
||||
"InstanceType": "m8a.small",
|
||||
"InstanceType": "t3.small",
|
||||
"CurrentGeneration": true,
|
||||
"SupportedRootDeviceTypes": [
|
||||
"ebs"
|
||||
"ebs"
|
||||
],
|
||||
"SupportedVirtualizationTypes": [
|
||||
"hvm"
|
||||
"hvm"
|
||||
],
|
||||
"Hypervisor": "kvm",
|
||||
"ProcessorInfo": {
|
||||
"SupportedArchitectures": [
|
||||
"x86_64"
|
||||
]
|
||||
"SupportedArchitectures": [
|
||||
"x86_64"
|
||||
]
|
||||
},
|
||||
"VCpuInfo": {
|
||||
"DefaultVCpus": 2
|
||||
"DefaultVCpus": 2
|
||||
},
|
||||
"MemoryInfo": {
|
||||
"SizeInMiB": 1024
|
||||
"SizeInMiB": 2048
|
||||
},
|
||||
"BurstablePerformanceSupported": false
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -295,10 +292,12 @@ aws ec2 describe-instance-types
|
||||
Export the instance-type:
|
||||
|
||||
```sh
|
||||
export HIVE_INSTANCE="m8a.small"
|
||||
export HIVE_INSTANCE="t3.small"
|
||||
```
|
||||
|
||||
Next, launch a new instance, note `hive-key` is the SSH key specified in the previous stage. Be sure to specify the instance-type that is available from your host as above.
|
||||
### Run Instance
|
||||
|
||||
Next, launch a new instance, note `hive-key` is the SSH key specified in the previous stage.
|
||||
|
||||
```bash
|
||||
aws ec2 run-instances \
|
||||
@@ -332,12 +331,14 @@ A sample response is below from the `RunInstance` request, note the `InstanceId`
|
||||
}
|
||||
```
|
||||
|
||||
Export the instance ID for following the rest of the tutorial.
|
||||
Export the instance ID.
|
||||
|
||||
```bash
|
||||
export INSTANCE_ID="i-XXX"
|
||||
```
|
||||
|
||||
### Verify Instance
|
||||
|
||||
Next, validate the running instance is ready.
|
||||
|
||||
```bash
|
||||
@@ -372,7 +373,7 @@ Confirm the `State.Name` attribute is set as `running`.
|
||||
|
||||
## SSH Connection (development)
|
||||
|
||||
For a Hive development environment (toggled off for production), a local SSH port forwaring will be active to connect directly to the instance, regardless of the VPC and network settings.
|
||||
For a Hive development environment (toggled off for production), a local SSH port forwarding will be active to connect directly to the instance, regardless of the VPC and network settings.
|
||||
|
||||
Determine the SSH port allocated.
|
||||
|
||||
@@ -395,7 +396,7 @@ Linux hive-vm-36765eb0 6.1.0-40-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.153-1 (2
|
||||
ec2-user@hive-vm-36765eb0:~$
|
||||
```
|
||||
|
||||
Congratulations, your first AMI image is imported, a new EC2 instance launched, and successfully connected via SSH for the configured SSH key, using the OS `cloud-init` procedure.
|
||||
Congratulations! Your first AMI image is imported, a new EC2 instance launched, and successfully connected via SSH for the configured SSH key, using the OS `cloud-init` procedure.
|
||||
|
||||
## Managing Instances
|
||||
|
||||
@@ -409,19 +410,20 @@ aws ec2 stop-instances --instance-ids $INSTANCE_ID
|
||||
|
||||
```json
|
||||
{
|
||||
"StoppingInstances": [
|
||||
{
|
||||
"InstanceId": "i-36765eb0c6609e4d2",
|
||||
"CurrentState": {
|
||||
"Code": 64,
|
||||
"Name": "stopping"
|
||||
},
|
||||
"PreviousState": {
|
||||
"Code": 16,
|
||||
"Name": "running"
|
||||
}
|
||||
}
|
||||
]
|
||||
"StoppingInstances": [
|
||||
{
|
||||
"InstanceId": "i-36765eb0c6609e4d2",
|
||||
"CurrentState": {
|
||||
"Code": 64,
|
||||
"Name": "stopping"
|
||||
},
|
||||
"PreviousState": {
|
||||
"Code": 16,
|
||||
"Name": "running"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Next, confirm the instance has stopped as requested.
|
||||
@@ -464,19 +466,20 @@ aws ec2 start-instances --instance-ids $INSTANCE_ID
|
||||
|
||||
```json
|
||||
{
|
||||
"StartingInstances": [
|
||||
{
|
||||
"InstanceId": "i-36765eb0c6609e4d2",
|
||||
"CurrentState": {
|
||||
"Code": 0,
|
||||
"Name": "pending"
|
||||
},
|
||||
"PreviousState": {
|
||||
"Code": 80,
|
||||
"Name": "stopped"
|
||||
}
|
||||
}
|
||||
]
|
||||
"StartingInstances": [
|
||||
{
|
||||
"InstanceId": "i-36765eb0c6609e4d2",
|
||||
"CurrentState": {
|
||||
"Code": 0,
|
||||
"Name": "pending"
|
||||
},
|
||||
"PreviousState": {
|
||||
"Code": 80,
|
||||
"Name": "stopped"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Next, validate the instance is running as expected.
|
||||
@@ -511,7 +514,7 @@ aws ec2 describe-instances --instance-ids "i-36765eb0c6609e4d2"
|
||||
|
||||
### Terminate Instance
|
||||
|
||||
To terminate an instance, which will first stop the instance, and on success, remove the EBS volumes and permanately remove the instance data.
|
||||
To terminate an instance, which will first stop the instance, and on success, remove the EBS volumes and permanently remove the instance data.
|
||||
|
||||
```bash
|
||||
aws ec2 terminate-instances --instance-ids $INSTANCE_ID
|
||||
@@ -535,7 +538,7 @@ aws ec2 terminate-instances --instance-ids $INSTANCE_ID
|
||||
}
|
||||
```
|
||||
|
||||
Next, validate the instance is removed, this may take a few minutes depending the instance volume size.
|
||||
Next, validate the instance is removed, this may take a few minutes depending on the instance volume size.
|
||||
|
||||
```bash
|
||||
aws ec2 describe-instances --instance-ids $INSTANCE_ID
|
||||
@@ -543,127 +546,382 @@ aws ec2 describe-instances --instance-ids $INSTANCE_ID
|
||||
|
||||
On success no data will be returned, since the instance is no longer available.
|
||||
|
||||
# Multi-node Configuration
|
||||
## UI Management Panel
|
||||
|
||||
To create a simulate multi-node installation on a single server (e.g node1, node2, node3) review the script `./scripts/create-multi-node.sh`.
|
||||
The Hive Platform also offers a web interface for managing hive services. Simply have the hive server running and go to [https://localhost:3000](https://localhost:3000) in your browser to continue.
|
||||
|
||||
Specifically this will create 3 IPs on a specified ethernet adapter.
|
||||
**Note:** If you are not viewing the website on the same machine that is running the hive server, you will need to go to [https://localhost:9999](https://localhost:9999) and [https://localhost:8443](https://localhost:8443) and accept the certificates. If you have added the CA to your local machine this is not needed.
|
||||
|
||||
# Multi-Node Configuration
|
||||
|
||||
Hive is designed for distributed, multi-server deployments. A Hive cluster operates as a fully distributed infrastructure region — similar to an AWS Region — where multiple nodes work together to provide high availability, data durability, and fault tolerance across the platform.
|
||||
|
||||
When deploying a multi-node cluster, you define a **region** (e.g., `ap-southeast-2`, `us-east-1`) and **availability zones** to organize your infrastructure. Services are distributed across nodes: NATS provides a clustered message bus for request routing and JetStream replication, Predastore uses Raft consensus with Reed-Solomon erasure coding (RS 2+1) for durable S3-compatible object storage, and the Hive daemon on each node can independently serve AWS API requests. The result is a platform where compute, storage, and API services are replicated — an AMI stored on one node is available from any node, an EC2 instance launched via any gateway is visible cluster-wide, and the loss of a single node does not compromise data or availability.
|
||||
|
||||
Cluster formation is automatic. When initializing with `--nodes 3`, the init node starts a formation server and waits for other nodes to join. Once all nodes have registered, each node receives the full cluster topology — credentials, CA certificates, NATS routes, Predastore peer lists — and generates its own configuration files. No manual configuration synchronization is required.
|
||||
|
||||
## Choose a Deployment Mode
|
||||
|
||||
| | Simulated (Option A) | Real Multi-Server (Option B) |
|
||||
|---|---|---|
|
||||
| **Servers** | 1 machine, loopback IPs | 3+ physical/virtual servers |
|
||||
| **Use case** | Development & testing | Production & staging |
|
||||
| **Network** | `127.0.0.1/2/3` (no setup) | Real LAN IPs |
|
||||
| **Data dirs** | `~/node1/`, `~/node2/`, `~/node3/` | `~/hive/` on each server |
|
||||
|
||||
Both options use `HIVE_NODE1`, `HIVE_NODE2`, and `HIVE_NODE3` environment variables. Set them in the option you choose, then all subsequent sections (verification, usage, shutdown) reference them.
|
||||
|
||||
### Option A: Simulated Multi-Node (Single Server)
|
||||
|
||||
For development and testing, a 3-node cluster can be simulated on a single machine using loopback addresses. Linux handles the full `127.0.0.0/8` range natively, so `127.0.0.1`, `127.0.0.2`, and `127.0.0.3` all work without any network configuration. Each node uses a separate data directory to isolate configuration, logs, and state.
|
||||
|
||||
#### 1. Build and Set Variables
|
||||
|
||||
```bash
|
||||
ip addr add 10.11.12.1/24 dev eth0
|
||||
ip addr add 10.11.12.2/24 dev eth0
|
||||
ip addr add 10.11.12.3/24 dev eth0
|
||||
make build
|
||||
|
||||
export HIVE_NODE1=127.0.0.1
|
||||
export HIVE_NODE2=127.0.0.2
|
||||
export HIVE_NODE3=127.0.0.3
|
||||
```
|
||||
|
||||
Next, `hive` will be installed and initiated in `$HOME/node1` for configuration files.
|
||||
#### 2. Form the Cluster
|
||||
|
||||
The formation process requires running init and join commands concurrently. The init node blocks until all expected nodes have joined.
|
||||
|
||||
**Terminal 1 — Initialize the leader node:**
|
||||
|
||||
```bash
|
||||
# Initialize node1
|
||||
./bin/hive admin init \
|
||||
--region ap-southeast-2 \
|
||||
--az ap-southeast-2a \
|
||||
--node node1 \
|
||||
--bind 10.11.12.1 \
|
||||
--cluster-bind 10.11.12.1 \
|
||||
--port 4432 \
|
||||
--hive-dir ~/node1/ \
|
||||
--config-dir ~/node1/config/
|
||||
--node node1 \
|
||||
--nodes 3 \
|
||||
--bind $HIVE_NODE1 \
|
||||
--cluster-bind $HIVE_NODE1 \
|
||||
--port 4432 \
|
||||
--hive-dir ~/node1/ \
|
||||
--config-dir ~/node1/config/ \
|
||||
--region ap-southeast-2 \
|
||||
--az ap-southeast-2a
|
||||
```
|
||||
|
||||
Two other nodes will be created, joining node1 to exchange configuration files and identify as a new node.
|
||||
This starts a formation server on `$HIVE_NODE1:4432` and waits for 2 more nodes to join.
|
||||
|
||||
**Terminal 2 — Join node 2 (while init is running):**
|
||||
|
||||
```bash
|
||||
# Join cluster
|
||||
./bin/hive admin join \
|
||||
--region ap-southeast-2 \
|
||||
--az ap-southeast-2a \
|
||||
--node node2 \
|
||||
--bind 10.11.12.2 \
|
||||
--cluster-bind 10.11.12.2 \
|
||||
--cluster-routes 10.11.12.1:4248 \
|
||||
--host 10.11.12.1:4432 \
|
||||
--data-dir ~/node2/ \
|
||||
--config-dir ~/node2/config/ \
|
||||
--node node2 \
|
||||
--bind $HIVE_NODE2 \
|
||||
--cluster-bind $HIVE_NODE2 \
|
||||
--host $HIVE_NODE1:4432 \
|
||||
--data-dir ~/node2/ \
|
||||
--config-dir ~/node2/config/ \
|
||||
--region ap-southeast-2 \
|
||||
--az ap-southeast-2a
|
||||
```
|
||||
|
||||
# Start node2
|
||||
./scripts/start-dev.sh ~/node2/
|
||||
**Terminal 3 — Join node 3 (while init is running):**
|
||||
|
||||
echo "Node3 Setup:"
|
||||
```bash
|
||||
./bin/hive admin join \
|
||||
--node node3 \
|
||||
--bind $HIVE_NODE3 \
|
||||
--cluster-bind $HIVE_NODE3 \
|
||||
--host $HIVE_NODE1:4432 \
|
||||
--data-dir ~/node3/ \
|
||||
--config-dir ~/node3/config/ \
|
||||
--region ap-southeast-2 \
|
||||
--az ap-southeast-2a
|
||||
```
|
||||
|
||||
All three processes will exit with a cluster summary once formation is complete.
|
||||
|
||||
#### 3. Start Services
|
||||
|
||||
Start services on each node (in separate terminals or background):
|
||||
|
||||
```bash
|
||||
HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node1/
|
||||
HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node2/
|
||||
HIVE_SKIP_BUILD=true UI=false ./scripts/start-dev.sh ~/node3/
|
||||
```
|
||||
|
||||
### Option B: Real Multi-Node (Physical Servers)
|
||||
|
||||
For production or production-like deployments across multiple physical servers or VMs.
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
On **each server**:
|
||||
|
||||
1. Install Hive dependencies (see [Dependencies](#dependencies) above)
|
||||
2. Clone the repository and build:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/Development/mulga/
|
||||
cd ~/Development/mulga/
|
||||
git clone https://github.com/mulgadc/hive.git
|
||||
cd hive
|
||||
./scripts/clone-deps.sh
|
||||
make build
|
||||
```
|
||||
|
||||
3. Identify the bind IP for each server:
|
||||
|
||||
```bash
|
||||
ip addr show | grep "inet "
|
||||
```
|
||||
|
||||
#### 1. Set Variables
|
||||
|
||||
Export the IPs for your three servers (replace with your actual IPs) on each server:
|
||||
|
||||
```bash
|
||||
export HIVE_NODE1=192.168.1.10
|
||||
export HIVE_NODE2=192.168.1.11
|
||||
export HIVE_NODE3=192.168.1.12
|
||||
```
|
||||
|
||||
#### 2. Form the Cluster
|
||||
|
||||
**Server 1 — Initialize the cluster:**
|
||||
|
||||
```bash
|
||||
cd ~/Development/mulga/hive
|
||||
|
||||
./bin/hive admin init \
|
||||
--node node1 \
|
||||
--nodes 3 \
|
||||
--bind $HIVE_NODE1 \
|
||||
--cluster-bind $HIVE_NODE1 \
|
||||
--port 4432 \
|
||||
--hive-dir ~/hive/ \
|
||||
--config-dir ~/hive/config/ \
|
||||
--region us-east-1 \
|
||||
--az us-east-1a
|
||||
```
|
||||
|
||||
**Server 2 — Join the cluster (while init is running):**
|
||||
|
||||
```bash
|
||||
cd ~/Development/mulga/hive
|
||||
|
||||
./bin/hive admin join \
|
||||
--region ap-southeast-2 \
|
||||
--az ap-southeast-2a \
|
||||
--node node3 \
|
||||
--bind 10.11.12.3 \
|
||||
--cluster-bind 10.11.12.3 \
|
||||
--cluster-routes 10.11.12.1:4248 \
|
||||
--host 10.11.12.1:4432 \
|
||||
--data-dir ~/node3/ \
|
||||
--config-dir ~/node3/config/
|
||||
|
||||
./scripts/start-dev.sh ~/node3/
|
||||
--node node2 \
|
||||
--bind $HIVE_NODE2 \
|
||||
--cluster-bind $HIVE_NODE2 \
|
||||
--host $HIVE_NODE1:4432 \
|
||||
--data-dir ~/hive/ \
|
||||
--config-dir ~/hive/config/ \
|
||||
--region us-east-1 \
|
||||
--az us-east-1a
|
||||
```
|
||||
|
||||
Once configured, each node will have it's own storage for config, data and state in `~/node[1,2,3]` to simulate a unique node and network.
|
||||
|
||||
Once the multi-node is configured, follow the original installation instructions above to:
|
||||
|
||||
- Import an SSH key
|
||||
- Clone an AMI
|
||||
- Launch an instance
|
||||
|
||||
Note - when using the AWS CLI tool you must specify the IP address of the node (10.11.12.1, 10.11.12.2, or 10.11.12.3).
|
||||
|
||||
**Important**: For multi-node setups, the server certificates need to include the additional IP addresses. You may need to regenerate certificates with the `--force` flag after configuring multi-node, or ensure the CA certificate is trusted system-wide (see "Trust CA Certificate" section above).
|
||||
**Server 3 — Join the cluster (while init is running):**
|
||||
|
||||
```bash
|
||||
aws --endpoint-url https://10.11.12.3:9999/ ec2 describe-instances --instance-ids i-9f5f648adc57ea46d
|
||||
cd ~/Development/mulga/hive
|
||||
|
||||
./bin/hive admin join \
|
||||
--node node3 \
|
||||
--bind $HIVE_NODE3 \
|
||||
--cluster-bind $HIVE_NODE3 \
|
||||
--host $HIVE_NODE1:4432 \
|
||||
--data-dir ~/hive/ \
|
||||
--config-dir ~/hive/config/ \
|
||||
--region us-east-1 \
|
||||
--az us-east-1a
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"Reservations": [
|
||||
{
|
||||
"ReservationId": "r-9d26866c3b0d53bf4",
|
||||
"OwnerId": "123456789012",
|
||||
"Instances": [
|
||||
{
|
||||
"InstanceId": "i-9f5f648adc57ea46d",
|
||||
"ImageId": "ami-d0de73dd18fa33ac9",
|
||||
"State": {
|
||||
"Code": 16,
|
||||
"Name": "running"
|
||||
},
|
||||
"KeyName": "hive-key",
|
||||
"InstanceType": "t3.micro",
|
||||
"LaunchTime": "2025-11-21T10:36:42.834000+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
All three processes will exit with a cluster summary once formation is complete.
|
||||
|
||||
For debugging, reference the node unique configuration files, e.g
|
||||
#### 3. Start Services
|
||||
|
||||
After formation completes, start services on **all servers**:
|
||||
|
||||
```bash
|
||||
tail -n 100 ~/node3/logs/hive.log
|
||||
./scripts/start-dev.sh
|
||||
```
|
||||
|
||||
## Verify the Cluster
|
||||
|
||||
From any node, view cluster status and resource capacity:
|
||||
|
||||
```bash
|
||||
...
|
||||
2025/11/21 22:59:32 Received message on subject: ec2.DescribeInstances
|
||||
2025/11/21 22:59:32 Message data: {"DryRun":null,"Filters":null,"InstanceIds":["i-9f5f648adc57ea46d"],"MaxResults":null,"NextToken":null}
|
||||
2025/11/21 22:59:32 INFO Processing DescribeInstances request from this node
|
||||
2025/11/21 22:59:32 INFO handleEC2DescribeInstances completed count=1
|
||||
...
|
||||
./bin/hive get nodes
|
||||
# NAME STATUS IP REGION AZ UPTIME VMs SERVICES
|
||||
# node1 Ready $HIVE_NODE1 ... ... 2m 0 nats,predastore,viperblock,daemon,...
|
||||
# node2 Ready $HIVE_NODE2 ... ... 2m 0 nats,predastore,viperblock,daemon
|
||||
# node3 Ready $HIVE_NODE3 ... ... 2m 0 nats,predastore,viperblock,daemon
|
||||
|
||||
./bin/hive top nodes
|
||||
# NAME CPU (used/total) MEM (used/total) VMs
|
||||
# node1 0/16 0.0Gi/30.6Gi 0
|
||||
# node2 0/16 0.0Gi/30.6Gi 0
|
||||
# node3 0/16 0.0Gi/30.6Gi 0
|
||||
```
|
||||
|
||||
### Stop multi-node Instance
|
||||
All nodes should show `Ready` status. Use `hive get vms` to see running instances across the cluster.
|
||||
|
||||
To stop a simulated multi-node instance:
|
||||
Check individual daemon health endpoints:
|
||||
|
||||
```bash
|
||||
./scripts/stop-multi-node.sh
|
||||
curl -s http://$HIVE_NODE1:4432/health
|
||||
curl -s http://$HIVE_NODE2:4432/health
|
||||
curl -s http://$HIVE_NODE3:4432/health
|
||||
```
|
||||
|
||||
Note if multiple EC2 instances are running, it make take a few minutes to gracefully terminate the instance, unmount the attached EBS volume (via NBD) and push the write-ahead-log (WAL) to the S3 server (predastore).
|
||||
Check NATS cluster routing (from any node):
|
||||
|
||||
```bash
|
||||
grep -i "route\|cluster" ~/node1/logs/nats.log # simulated
|
||||
grep -i "route\|cluster" ~/hive/logs/nats.log # real multi-server
|
||||
```
|
||||
|
||||
Check Predastore Raft consensus:
|
||||
|
||||
```bash
|
||||
grep -i "leader\|election" ~/node1/logs/predastore.log # simulated
|
||||
grep -i "leader\|election" ~/hive/logs/predastore.log # real multi-server
|
||||
```
|
||||
|
||||
## Trust CA Certificate
|
||||
|
||||
The CA certificate is generated by the init node and distributed to all joining nodes during formation. On **each node** (or the single machine for simulated mode), trust the CA:
|
||||
|
||||
```bash
|
||||
sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
|
||||
sudo update-ca-certificates
|
||||
```
|
||||
|
||||
For simulated mode, the config directory is under each node's data directory — copy from any node since the CA is the same:
|
||||
|
||||
```bash
|
||||
sudo cp ~/node1/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
|
||||
sudo update-ca-certificates
|
||||
```
|
||||
|
||||
Set the AWS profile:
|
||||
|
||||
```bash
|
||||
export AWS_PROFILE=hive
|
||||
```
|
||||
|
||||
## Using the Cluster
|
||||
|
||||
Connect to any node's AWS Gateway:
|
||||
|
||||
```bash
|
||||
AWS_ENDPOINT_URL=https://$HIVE_NODE1:9999 aws ec2 describe-instances
|
||||
AWS_ENDPOINT_URL=https://$HIVE_NODE2:9999 aws ec2 describe-instances
|
||||
```
|
||||
|
||||
Check logs on each node for debugging. Log locations depend on your deployment mode — `~/node{1,2,3}/logs/` for simulated, `~/hive/logs/` for real multi-server.
|
||||
|
||||
## Advanced Configuration
|
||||
|
||||
### Service Co-location
|
||||
|
||||
Every node that runs EC2 instances (compute) **must** also run the viperblock service. EBS volumes are mounted locally via Unix domain sockets (NBD) — the daemon publishes to a node-specific NATS topic (`ebs.{nodeName}.mount`) which is handled by the viperblock instance on the same server. Nodes without viperblock cannot run EC2 instances.
|
||||
|
||||
In the default deployment, every node runs all services (NATS, Predastore, Viperblock, Daemon, Gateway), which satisfies this requirement.
|
||||
|
||||
### Dual NIC Configuration
|
||||
|
||||
If servers have separate management and cluster network interfaces, use `--cluster-bind` to specify the cluster network IP:
|
||||
|
||||
```bash
|
||||
# Server 1: management=192.168.1.10, cluster=10.0.0.10
|
||||
./bin/hive admin init \
|
||||
--node node1 \
|
||||
--nodes 3 \
|
||||
--bind 192.168.1.10 \
|
||||
--cluster-bind 10.0.0.10 \
|
||||
--port 4432 \
|
||||
--hive-dir ~/hive/ \
|
||||
--config-dir ~/hive/config/ \
|
||||
--region us-east-1 \
|
||||
--az us-east-1a
|
||||
```
|
||||
|
||||
The `--bind` IP is used for the formation server, daemon, and AWS gateway. The `--cluster-bind` IP is used for NATS cluster routing and Predastore Raft consensus. If `--cluster-bind` is not specified, it defaults to the `--bind` IP.
|
||||
|
||||
### IP Aliases (Simulated Mode)
|
||||
|
||||
For a more production-like simulation using dedicated IP addresses on a network interface:
|
||||
|
||||
```bash
|
||||
sudo ip addr add 10.11.12.1/24 dev eth0
|
||||
sudo ip addr add 10.11.12.2/24 dev eth0
|
||||
sudo ip addr add 10.11.12.3/24 dev eth0
|
||||
```
|
||||
|
||||
Replace `eth0` with your network interface name (e.g., `enp0s3`, `ens33`). Then export these as your `HIVE_NODE` variables instead of the loopback addresses.
|
||||
|
||||
To remove the aliases:
|
||||
|
||||
```bash
|
||||
sudo ip addr del 10.11.12.1/24 dev eth0
|
||||
sudo ip addr del 10.11.12.2/24 dev eth0
|
||||
sudo ip addr del 10.11.12.3/24 dev eth0
|
||||
```
|
||||
|
||||
## Shutdown
|
||||
|
||||
When running in the multi-server cluster mode a graceful shutdown is required to synchronize and coordinate nodes.
|
||||
|
||||
On any node, issue the following command to shutdown nodes cluster wide:
|
||||
|
||||
```sh
|
||||
./bin/hive admin cluster shutdown
|
||||
```
|
||||
|
||||
Example output:
|
||||
```bash
|
||||
Starting coordinated cluster shutdown (3 nodes)
|
||||
Phases: gate -> drain -> storage -> persist -> infra
|
||||
Timeout per phase: 2m0s
|
||||
|
||||
[GATE] Sending to 3 node(s)...
|
||||
[GATE] node1: stopped awsgw, hive-ui
|
||||
[GATE] node3: stopped awsgw, hive-ui
|
||||
[GATE] node2: stopped awsgw, hive-ui
|
||||
[GATE] Complete (3/3 nodes, 2.005s)
|
||||
|
||||
[DRAIN] Sending to 3 node(s)...
|
||||
[DRAIN] node1: 0/0 VMs remaining
|
||||
[DRAIN] node1: 0/0 VMs remaining
|
||||
[DRAIN] node2: 0/0 VMs remaining
|
||||
[DRAIN] node2: 0/0 VMs remaining
|
||||
[DRAIN] node3: 0/0 VMs remaining
|
||||
[DRAIN] node3: 0/0 VMs remaining
|
||||
[DRAIN] node1: OK
|
||||
[DRAIN] node3: OK
|
||||
[DRAIN] node2: OK
|
||||
[DRAIN] Complete (3/3 nodes, 5ms)
|
||||
|
||||
[STORAGE] Sending to 3 node(s)...
|
||||
[STORAGE] node1: stopped viperblock
|
||||
[STORAGE] node3: stopped viperblock
|
||||
[STORAGE] node2: stopped viperblock
|
||||
[STORAGE] Complete (3/3 nodes, 1.09s)
|
||||
|
||||
[PERSIST] Sending to 3 node(s)...
|
||||
[PERSIST] node1: stopped predastore
|
||||
[PERSIST] node3: stopped predastore
|
||||
[PERSIST] node2: stopped predastore
|
||||
[PERSIST] Complete (3/3 nodes, 1.003s)
|
||||
|
||||
[INFRA] Sending final shutdown to all nodes...
|
||||
[INFRA] Complete
|
||||
Cluster shutdown complete (6.104s)
|
||||
|
||||
```
|
||||
|
||||
If EC2 instances are running, the stop process will gracefully terminate them, unmount attached EBS volumes (via NBD), and flush the write-ahead-log (WAL) to the S3 server (Predastore). This may take a few minutes depending on instance volume sizes.
|
||||
|
||||
To fully remove a simulated cluster's data:
|
||||
|
||||
```bash
|
||||
rm -rf ~/node1/ ~/node2/ ~/node3/
|
||||
```
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
GO_PROJECT_NAME := hive
|
||||
SHELL := /bin/bash
|
||||
|
||||
# Detect architecture for cross-platform support
|
||||
ARCH := $(shell uname -m)
|
||||
@@ -35,21 +36,32 @@ endif
|
||||
build:
|
||||
$(MAKE) go_build
|
||||
|
||||
# Build hive-ui frontend (requires pnpm)
|
||||
build-ui:
|
||||
@echo -e "\n....Building hive-ui frontend...."
|
||||
cd hive/services/hiveui/frontend && pnpm build
|
||||
|
||||
# GO commands
|
||||
go_build:
|
||||
@echo "\n....Building $(GO_PROJECT_NAME)"
|
||||
GOEXPERIMENT=greenteagc go build $(GO_BUILD_MOD) -ldflags "-s -w" -o ./bin/$(GO_PROJECT_NAME) cmd/hive/main.go
|
||||
@echo -e "\n....Building $(GO_PROJECT_NAME)"
|
||||
go build $(GO_BUILD_MOD) -ldflags "-s -w" -o ./bin/$(GO_PROJECT_NAME) cmd/hive/main.go
|
||||
|
||||
go_run:
|
||||
@echo "\n....Running $(GO_PROJECT_NAME)...."
|
||||
@echo -e "\n....Running $(GO_PROJECT_NAME)...."
|
||||
$(GOPATH)/bin/$(GO_PROJECT_NAME)
|
||||
|
||||
# Preflight — runs the same checks as GitHub Actions (format + lint + security + tests).
|
||||
# Use this before committing to catch CI failures locally.
|
||||
preflight: check-format vet security-check test
|
||||
@echo -e "\n ✅ Preflight passed — safe to commit."
|
||||
|
||||
# Run unit tests
|
||||
test:
|
||||
@echo "\n....Running tests for $(GO_PROJECT_NAME)...."
|
||||
@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
|
||||
LOG_IGNORE=1 go test -v -timeout 120s ./...
|
||||
|
||||
bench:
|
||||
@echo "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
|
||||
@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
|
||||
$(MAKE) easyjson
|
||||
LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
|
||||
|
||||
@@ -58,10 +70,11 @@ run:
|
||||
$(MAKE) go_run
|
||||
|
||||
clean:
|
||||
rm ./bin/$(GO_PROJECT_NAME)
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
rm -rf hive/services/hiveui/frontend/dist
|
||||
|
||||
install-system:
|
||||
@echo "\n....Installing system dependencies for $(ARCH)...."
|
||||
@echo -e "\n....Installing system dependencies for $(ARCH)...."
|
||||
@echo "QEMU packages: $(QEMU_PACKAGES)"
|
||||
apt-get update && sudo apt-get install -y \
|
||||
nbdkit nbdkit-plugin-dev pkg-config $(QEMU_PACKAGES) qemu-utils qemu-kvm \
|
||||
@@ -69,16 +82,16 @@ install-system:
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file
|
||||
|
||||
install-go:
|
||||
@echo "\n....Installing Go 1.25.5 for $(ARCH) ($(GO_ARCH))...."
|
||||
@echo -e "\n....Installing Go 1.26.0 for $(ARCH) ($(GO_ARCH))...."
|
||||
@if [ ! -d "/usr/local/go" ]; then \
|
||||
curl -L https://go.dev/dl/go1.25.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
|
||||
curl -L https://go.dev/dl/go1.26.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
|
||||
else \
|
||||
echo "Go already installed in /usr/local/go"; \
|
||||
fi
|
||||
@echo "Go version: $$(go version)"
|
||||
|
||||
install-aws:
|
||||
@echo "\n....Installing AWS CLI v2 for $(ARCH) ($(AWS_ARCH))...."
|
||||
@echo -e "\n....Installing AWS CLI v2 for $(ARCH) ($(AWS_ARCH))...."
|
||||
@if ! command -v aws >/dev/null 2>&1; then \
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-$(AWS_ARCH).zip" -o "awscliv2.zip"; \
|
||||
unzip -o awscliv2.zip; \
|
||||
@@ -89,23 +102,76 @@ install-aws:
|
||||
fi
|
||||
|
||||
quickinstall: install-system install-go install-aws
|
||||
@echo "\n✅ Quickinstall complete for $(ARCH)."
|
||||
@echo -e "\n✅ Quickinstall complete for $(ARCH)."
|
||||
@echo " Please ensure /usr/local/go/bin is in your PATH."
|
||||
@echo " Installed: Go ($(GO_ARCH)), AWS CLI ($(AWS_ARCH)), QEMU ($(QEMU_PACKAGES))"
|
||||
|
||||
security:
|
||||
@echo "\n....Running security checks for $(GO_PROJECT_NAME)...."
|
||||
# Format all Go files in place
|
||||
format:
|
||||
gofmt -w .
|
||||
|
||||
go tool govulncheck ./... > tests/govulncheck-report.txt || true
|
||||
@echo "Govulncheck report saved to tests/govulncheck-report.txt"
|
||||
# Check that all Go files are formatted (CI-compatible, fails on diff)
|
||||
check-format:
|
||||
@echo "Checking gofmt..."
|
||||
@UNFORMATTED=$$(gofmt -l .); \
|
||||
if [ -n "$$UNFORMATTED" ]; then \
|
||||
echo "Files not formatted:"; \
|
||||
echo "$$UNFORMATTED"; \
|
||||
echo "Run 'make format' to fix."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo " gofmt ok"
|
||||
|
||||
go tool gosec ./... > tests/gosec-report.txt || true
|
||||
@echo "Gosec report saved to tests/gosec-report.txt"
|
||||
# Go vet (fails on issues, matches CI)
|
||||
vet:
|
||||
@echo "Running go vet..."
|
||||
go vet ./...
|
||||
@echo " go vet ok"
|
||||
|
||||
go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./... > tests/staticcheck-report.txt || true
|
||||
@echo "Staticcheck report saved to tests/staticcheck-report.txt"
|
||||
# Security checks — each tool fails the build on findings (matches CI).
|
||||
# Reports are also saved to tests/ for review.
|
||||
security-check:
|
||||
@echo -e "\n....Running security checks for $(GO_PROJECT_NAME)...."
|
||||
set -o pipefail && go tool govulncheck ./... 2>&1 | tee tests/govulncheck-report.txt
|
||||
@echo " govulncheck ok"
|
||||
set -o pipefail && go tool gosec -exclude=G204,G304,G402 -exclude-generated -exclude-dir=cmd ./... 2>&1 | tee tests/gosec-report.txt
|
||||
@echo " gosec ok"
|
||||
set -o pipefail && go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./... 2>&1 | tee tests/staticcheck-report.txt
|
||||
@echo " staticcheck ok"
|
||||
|
||||
go vet ./... 2>&1 | tee tests/govet-report.txt || true
|
||||
@echo "Go vet report saved to tests/govet-report.txt"
|
||||
# Docker E2E tests (mirrors GitHub Actions e2e.yml)
|
||||
# Usage: make test-docker # both suites
|
||||
# make test-docker-single # single-node only
|
||||
# make test-docker-multi # multi-node only
|
||||
PARENT_DIR := $(shell cd .. && pwd)
|
||||
E2E_IMAGE := hive-e2e:latest
|
||||
|
||||
.PHONY: build go_build go_run test bench run clean install-system install-go install-aws quickinstall security
|
||||
test-docker-build:
|
||||
@echo -e "\n....Building E2E Docker image...."
|
||||
@for dep in viperblock predastore; do \
|
||||
if [ ! -d "$(PARENT_DIR)/$$dep" ]; then \
|
||||
echo "Missing sibling repo $$dep — running clone-deps.sh"; \
|
||||
./scripts/clone-deps.sh; \
|
||||
break; \
|
||||
fi; \
|
||||
done
|
||||
docker build -t $(E2E_IMAGE) -f tests/e2e/Dockerfile.e2e $(PARENT_DIR)
|
||||
|
||||
test-docker-single: test-docker-build
|
||||
@echo -e "\n....Running Single-Node E2E Tests...."
|
||||
docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
|
||||
|
||||
test-docker-multi: test-docker-build
|
||||
@echo -e "\n....Running Multi-Node E2E Tests...."
|
||||
docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
|
||||
|
||||
test-docker: test-docker-build
|
||||
@echo -e "\n....Running Single-Node E2E Tests...."
|
||||
docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
|
||||
@echo -e "\n....Running Multi-Node E2E Tests...."
|
||||
docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
|
||||
|
||||
.PHONY: build build-ui go_build go_run preflight test bench run clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
format check-format vet security-check \
|
||||
test-docker-build test-docker-single test-docker-multi test-docker
|
||||
|
||||
@@ -79,7 +79,7 @@ Hive coordinates these independent components:
|
||||
|
||||
Each component can be developed independently. See component-specific documentation for focused development guides.
|
||||
|
||||
## Detailed Installation method
|
||||
## Detailed Installation Method
|
||||
|
||||
See [Installation Guide](INSTALL.md) on how to setup a Hive node to get started.
|
||||
|
||||
@@ -118,7 +118,7 @@ While Hive is architected and implemented by experienced engineers, we leverage
|
||||
|
||||
**Development Artifacts:**
|
||||
|
||||
- `CLAUDE.md` - Instructions for Claude Code when working on this codebase
|
||||
- `.claude/CLAUDE.md` - Instructions for Claude Code when working on this codebase
|
||||
- `HIVE_DEVELOPMENT_PLAN.md` - Comprehensive roadmap and implementation strategy
|
||||
- Component-specific guidance in `viperblock/CLAUDE.md` and `predastore/CLAUDE.md`
|
||||
|
||||
|
||||
@@ -1,444 +0,0 @@
|
||||
# Hive Development Roadmap
|
||||
|
||||
## Update Jan 2026
|
||||
|
||||
Big rocks:
|
||||
|
||||
- [DONE] Add multi-node support for predastore (S3) using reed solomon encoding
|
||||
- [DONE] Implement KV store for object lookup, WAL files, offset. Use hash-ring to determine which nodes an object belongs to.
|
||||
- Implement VPC support using Open vSwitch across multiple nodes, core VPC functionality included
|
||||
- Add support for NVIDIA Bluefield DPU with Open vSwitch
|
||||
- Implement basic IAM using NATS Jetstream as KV store, vs IAM/access-keys in local config/TOML files for beta.
|
||||
- Move `daemon.go` instances.json state to Jetstream KV
|
||||
- Add support using the `hive` CLI tool to provision a new user with AWS access-keys/IAM.
|
||||
- Support multi-tenant operations and isolation
|
||||
- Add support to include capabilities when adding a new hardware node to MulgaOS (e.g EC2 target, S3, EBS, NATs, etc) - Features can be turned on/off depending on hardware scope.
|
||||
- [DONE] Add simple Web UI console, using the AWS JS SDK, communicating to local AWS gateway.
|
||||
- Implement ShadCNblocks for UI framework
|
||||
- Simple Go webserver, static files, easy build process.
|
||||
|
||||
Implementation gaps:
|
||||
|
||||
- EC2 - Support extended features for `run-instance`
|
||||
- Volume resize of AMI. Note `nbd` does not support live resize events, instance needs to be stopped, resized, and started.
|
||||
- Confirm cloud-init will resize volume on boot and configured correctly.
|
||||
- Attach additional volumes
|
||||
- Attach to VPC / Security group (required Open vSwitch implementation)
|
||||
|
||||
### Issues to investigate
|
||||
|
||||
- When the host disk volume is full and a new instance is launched, disk corruption will occur within the instance since the host volume is out of space. Consider a pre-flight check, check available storage on the S3 server, local viperblock for WAL cache availability and provide improved guard rails to prevent a new instance running, if available disk space is nearing to be exceeded.
|
||||
|
||||
Host:
|
||||
|
||||
```json
|
||||
{"time":"2026-01-27T21:30:29.368164556+10:00","level":"ERROR","msg":"WAL sync failed","error":"write /home/hive/hive/predastore/distributed/nodes/node-2/0000000000000737.wal: no space left on device"}
|
||||
```
|
||||
|
||||
Instance `dmesg`:
|
||||
|
||||
```
|
||||
[ 1445.573495] EXT4-fs error (device vda1): ext4_journal_check_start:83: comm systemd-journal: Detected aborted journal
|
||||
[ 1445.583565] EXT4-fs (vda1): Remounting filesystem read-only
|
||||
```
|
||||
|
||||
|
||||
### EC2
|
||||
|
||||
- [DONE] - describe-instances
|
||||
- [DONE] - run-instances (count not implemented)
|
||||
- [DONE] - start-instances
|
||||
- [DONE] - stop-instances
|
||||
- [DONE] - terminate-instances
|
||||
- [DONE] - describe-instance-types
|
||||
- [DONE] - create-key-pair
|
||||
- [DONE] - delete-key-pair
|
||||
- [DONE] - describe-key-pairs
|
||||
- [DONE] - import-key-pair
|
||||
- [DONE] - describe-images
|
||||
- [DONE] - describe-regions
|
||||
- [DONE] - describe-volumes
|
||||
|
||||
### To implement
|
||||
|
||||
Easier methods to implement
|
||||
|
||||
- attach-volume
|
||||
- copy-image
|
||||
- copy-snapshot
|
||||
- copy-volumes
|
||||
- create-image
|
||||
- create-snapshot
|
||||
- create-snapshots
|
||||
- create-tags
|
||||
- create-volume
|
||||
- delete-snapshot
|
||||
- describe-snapshots
|
||||
- describe-subnets
|
||||
- describe-tags
|
||||
- describe-volume-attribute
|
||||
- describe-volume-status
|
||||
- detach-network-interface
|
||||
- detach-volume
|
||||
- get-console-output
|
||||
- monitor-instances
|
||||
|
||||
TODO:
|
||||
|
||||
- allocate-address
|
||||
- allocate-hosts
|
||||
- assign-private-ip-addresses
|
||||
- assign-private-nat-gateway-address
|
||||
- associate-address
|
||||
- associate-nat-gateway-address
|
||||
- associate-route-server
|
||||
- associate-route-table
|
||||
- associate-security-group-vpc
|
||||
- associate-subnet-cidr-block
|
||||
- attach-internet-gateway
|
||||
- attach-network-interface
|
||||
- authorize-security-group-egress
|
||||
- authorize-security-group-ingress
|
||||
- create-customer-gateway
|
||||
- create-default-subnet
|
||||
- create-default-vpc
|
||||
- create-dhcp-options
|
||||
- create-egress-only-internet-gateway
|
||||
- create-internet-gateway
|
||||
- create-launch-template
|
||||
- create-launch-template-version
|
||||
- create-local-gateway-route
|
||||
- create-local-gateway-route-table
|
||||
- create-local-gateway-route-table-virtual-interface-group-association
|
||||
- create-local-gateway-virtual-interface
|
||||
- create-local-gateway-virtual-interface-group
|
||||
- create-nat-gateway
|
||||
- create-network-acl
|
||||
- create-network-acl-entry
|
||||
- create-public-ipv4-pool
|
||||
- create-route
|
||||
- create-route-server
|
||||
- create-route-server-endpoint
|
||||
- create-route-server-peer
|
||||
- create-route-table
|
||||
- create-security-group
|
||||
- create-subnet
|
||||
- create-subnet-cidr-reservation
|
||||
- delete-coip-cidr
|
||||
- delete-customer-gateway
|
||||
- delete-dhcp-options
|
||||
- delete-egress-only-internet-gateway
|
||||
- delete-internet-gateway
|
||||
- delete-local-gateway-route
|
||||
- delete-local-gateway-route-table
|
||||
- delete-local-gateway-route-table-virtual-interface-group-association
|
||||
- delete-local-gateway-virtual-interface
|
||||
- delete-local-gateway-virtual-interface-group
|
||||
- delete-nat-gateway
|
||||
- delete-network-acl
|
||||
- delete-network-interface
|
||||
- delete-network-interface-permission
|
||||
- delete-public-ipv4-pool
|
||||
- delete-route
|
||||
- delete-route-server
|
||||
- delete-route-server-endpoint
|
||||
- delete-route-server-peer
|
||||
- delete-route-table
|
||||
- delete-security-group
|
||||
- delete-subnet
|
||||
- delete-subnet-cidr-reservation
|
||||
- delete-tags
|
||||
- delete-volume
|
||||
- describe-account-attributes
|
||||
- describe-addresses
|
||||
- describe-customer-gateways
|
||||
- describe-dhcp-options
|
||||
- describe-egress-only-internet-gateways
|
||||
- describe-hosts
|
||||
- describe-internet-gateways
|
||||
- describe-managed-prefix-lists
|
||||
- describe-nat-gateways
|
||||
- describe-network-acls
|
||||
- describe-route-server-endpoints
|
||||
- describe-route-server-peers
|
||||
- describe-route-servers
|
||||
- describe-route-tables
|
||||
- describe-security-group-references
|
||||
- describe-security-group-rules
|
||||
- describe-security-groups
|
||||
- describe-volumes-modifications
|
||||
- detach-internet-gateway
|
||||
- detach-vpn-gateway
|
||||
- disable-image
|
||||
- disable-route-server-propagation
|
||||
- disable-serial-console-access
|
||||
- enable-address-transfer
|
||||
- enable-capacity-manager
|
||||
- enable-ebs-encryption-by-default
|
||||
- enable-image
|
||||
- enable-image-block-public-access
|
||||
- export-image
|
||||
- get-console-screenshot
|
||||
- get-instance-metadata-defaults
|
||||
- get-instance-tpm-ek-pub
|
||||
- get-instance-types-from-instance-requirements
|
||||
- get-instance-uefi-data
|
||||
- get-launch-template-data
|
||||
- get-route-server-associations
|
||||
- get-route-server-propagations
|
||||
- get-route-server-routing-database
|
||||
- get-security-groups-for-vpc
|
||||
- get-serial-console-access-status
|
||||
- get-snapshot-block-public-access-state
|
||||
- get-spot-placement-scores
|
||||
- get-subnet-cidr-reservations
|
||||
- import-image
|
||||
- import-snapshot
|
||||
- modify-instance-attribute
|
||||
- modify-instance-capacity-reservation-attributes
|
||||
- modify-instance-connect-endpoint
|
||||
- modify-instance-cpu-options
|
||||
- modify-instance-credit-specification
|
||||
- modify-instance-placement
|
||||
- modify-network-interface-attribute
|
||||
- modify-volume
|
||||
- modify-volume-attribute
|
||||
- modify-vpn-connection
|
||||
- modify-vpn-connection-options
|
||||
- reboot-instances
|
||||
- register-image
|
||||
- release-address
|
||||
- release-hosts
|
||||
- replace-network-acl-association
|
||||
- replace-network-acl-entry
|
||||
- replace-route
|
||||
- replace-route-table-association
|
||||
- report-instance-status
|
||||
- revoke-security-group-egress
|
||||
- revoke-security-group-ingress
|
||||
- run-instances
|
||||
|
||||
### EBS
|
||||
|
||||
- complete-snapshot
|
||||
- get-snapshot-block
|
||||
- list-changed-blocks
|
||||
- list-snapshot-blocks
|
||||
- put-snapshot-block
|
||||
- start-snapshot
|
||||
|
||||
### VPC
|
||||
|
||||
- create-vpc
|
||||
- create-vpc-block-public-access-exclusion
|
||||
- create-vpc-encryption-control
|
||||
- create-vpc-endpoint
|
||||
- create-vpc-endpoint-connection-notification
|
||||
- create-vpc-endpoint-service-configuration
|
||||
- create-vpc-peering-connection
|
||||
- delete-vpc
|
||||
- delete-vpc-block-public-access-exclusion
|
||||
- delete-vpc-encryption-control
|
||||
- delete-vpc-endpoint-connection-notifications
|
||||
- delete-vpc-endpoint-service-configurations
|
||||
- delete-vpc-endpoints
|
||||
- delete-vpc-peering-connection
|
||||
- associate-vpc-cidr-block
|
||||
- delete-local-gateway-route-table-vpc-association
|
||||
- create-local-gateway-route-table-vpc-association
|
||||
- describe-security-group-vpc-associations
|
||||
- describe-vpc-attribute
|
||||
- describe-vpc-block-public-access-exclusions
|
||||
- describe-vpc-block-public-access-options
|
||||
- describe-vpc-encryption-controls
|
||||
- describe-vpc-endpoint-associations
|
||||
- describe-vpc-endpoint-connection-notifications
|
||||
- describe-vpc-endpoint-connections
|
||||
- describe-vpc-endpoint-service-configurations
|
||||
- describe-vpc-endpoint-service-permissions
|
||||
- describe-vpc-endpoint-services
|
||||
- describe-vpc-endpoints
|
||||
- describe-vpc-peering-connections
|
||||
- describe-vpcs
|
||||
- modify-vpc-attribute
|
||||
- modify-vpc-encryption-control
|
||||
- modify-vpc-endpoint
|
||||
- modify-vpc-endpoint-connection-notification
|
||||
- modify-vpc-endpoint-service-configuration
|
||||
- modify-vpc-tenancy
|
||||
|
||||
### S3
|
||||
|
||||
- Consider moving S3 control/data plane, from predastore, to Hive format.
|
||||
|
||||
## Update Nov 2025
|
||||
|
||||
[PARTIAL] _ Implement multi-tenant support
|
||||
[PARTIAL] _ Move config settings from ~/hive/\*.toml, to using Nats Jetstream for core config which can be synced between nodes.
|
||||
|
||||
- Implement a lightweight IAM using Nats Jetstream, vs current config files for auth settings.
|
||||
[ONGOING] _ Implement Reed Solomon Encoding for Predastore (S3) objects and a WAL implementation, for storing multiple objects in a single shard (e.g 4MB), with the WAL referencing the location of each object (e.g 4kb min)
|
||||
_ Implement basic KY lookup, object key sha512 (bucket/key), location to shard on S3 (obj.0000124.bin), read WAL (e.g first 4096 bytes) to determine location of the object. e.g key-1234 => obj.0000124.bin, wal header (4096 bytes) > key-1234 == offset location (seek) 16384, len 32768.
|
||||
_ Read multiple nodes (e.g 5 predastore instances, k = 3 (data shards), n = 5 (total shards), n - k = 2 (parity shards) )
|
||||
[ONGOING] _ Complete core scaffolding AWS SDK/API requirements (ec2 describe-instances, run-instances, etc)
|
||||
- Implement UEFI support for image downloads and `qemu` exec in `vm.go`
|
||||
- Confirm Alpine Linux, fails import image AMI > (run-instance) ""Failed to read block from AMI source" err="request out of range" - Block size correct?
|
||||
- Improve shutdown gracefully, `./scripts/stop-dev.sh` waits 60 seconds, while qemu/nbd could still be shutting down.
|
||||
- Add delete-volume support via EBS (s3 vol-\*) for terminated instance
|
||||
- Add default LRU cache support for viperblock, depending on the instance type / volume size and system memory available.
|
||||
|
||||
## Multi-node setup
|
||||
|
||||
[DONE] - Development release - v0.1.0 (single node)
|
||||
[PARTIAL] - Production release - v0.2.0 (multi node)
|
||||
|
||||
Design layout for multi-node configuration.
|
||||
|
||||
```
|
||||
# node1
|
||||
hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 3 --hive-dir ~/hive/
|
||||
|
||||
# node2
|
||||
hive admin join --region ap-southeast-2 --az ap-southeast-2a --node node2 --host node1.local:8443 --data-dir ~/hive/
|
||||
|
||||
# node3, optionally toggle EBS/EC2/NATs support only
|
||||
hive admin join --region ap-southeast-2 --az ap-southeast-2a --node node3 --host node1.local:8443 --data-dir ~/hive/ --cap ebs,ec2,nats
|
||||
```
|
||||
|
||||
If --host is missing, the join command tries multicast broadcast to find parent (leader) node.
|
||||
|
||||
Set region with `--region ap-southeast-2` which will create a new Hive cluster for the specified region.
|
||||
|
||||
For production, it is recommended to run Hive on at least three physical nodes. In this minimum setup, Hive S3 and EBS volumes use 2 data shards and 1 parity shard RS(2,1) to distribute each chunk across the cluster. This configuration tolerates a full node failure while adding only 1.5x storage overhead.
|
||||
|
||||
### Overview:
|
||||
|
||||
- init creates a cluster-id, node-id, and a short join token, starts a tiny control server on :8443, and writes DNS hint files if available.
|
||||
|
||||
- init sets target size to 3 and waits until all 3 nodes join and ack.
|
||||
|
||||
- join nodes contact node1, present the token or cluster-id, and advertise capabilities.
|
||||
|
||||
- node1 appends them to the member set and immediately pushes the current settings bundle to them.
|
||||
|
||||
- When member count reaches 3, node1 broadcasts the final cluster settings to all nodes.
|
||||
|
||||
- Every node writes the same cluster.json and starts the services that match its capabilities.
|
||||
|
||||
Node exchange payload
|
||||
|
||||
```json
|
||||
{
|
||||
"cluster_id": "c-82d5",
|
||||
"node_id": "n-rpi2",
|
||||
"addr": "rpi2.local:8443",
|
||||
"version": "0.1.0",
|
||||
"caps": ["ec2", "s3"],
|
||||
"ts": 1731388800
|
||||
}
|
||||
```
|
||||
|
||||
Settings bundle (identical on all 3 once committed)
|
||||
|
||||
```json
|
||||
{
|
||||
"cluster_id": "c-82d5",
|
||||
"target_size": 3,
|
||||
"members": [
|
||||
{
|
||||
"node_id": "n-rpi1",
|
||||
"addr": "rpi1.local:8443",
|
||||
"caps": ["ec2", "s3", "nats", "ebs"]
|
||||
},
|
||||
{ "node_id": "n-rpi2", "addr": "rpi2.local:8443", "caps": ["ec2", "s3"] },
|
||||
{ "node_id": "n-rpi3", "addr": "rpi3.local:8443", "caps": ["nats", "ebs"] }
|
||||
],
|
||||
"services": {
|
||||
"ec2": { "api_bind": ":9001" },
|
||||
"s3": { "api_bind": ":9002", "replicas": 2 },
|
||||
"nats": { "cluster": "c-82d5", "quorum": 2 },
|
||||
"ebs": { "replicas": 3, "block_size": 4096 }
|
||||
},
|
||||
"epoch": 1,
|
||||
"sig": "ed25519:..." // signed by rpi1 during init
|
||||
}
|
||||
```
|
||||
|
||||
## Original TODO Items → Development Plan Integration
|
||||
|
||||
### ✅ **Completed Integration**
|
||||
|
||||
All original TODO items have been incorporated into the structured development phases:
|
||||
|
||||
| Original TODO | Status | Integrated Into | Phase |
|
||||
| ------------------------------------- | ---------- | ---------------------------------------------- | --------- |
|
||||
| #1: Binary compile and install.sh | ✅ Planned | Phase 7: Task 7.1 (Binary Compilation) | Phase 7 |
|
||||
| #2: Move daemon.go to services/hived/ | ✅ Planned | Phase 3: Task 3.2 (Specialized Services) | Phase 3 |
|
||||
| #3: VPC with openvs-switch as `vpcd` | ✅ Planned | Phase 5: Task 5.1 (VPC with Open vSwitch) | Phase 5 |
|
||||
| #4: AWS HTTP gateway (`awsd`) | ✅ Planned | Phase 3: Task 3.1 (AWS Gateway Service) | Phase 3 |
|
||||
| #5: AWS SDK v2 input/output | ✅ Planned | Phase 2: Task 2.1 & Phase 3: Task 3.1 | Phase 2-3 |
|
||||
| #6: `hive.service` for systemd | ✅ Planned | Phase 7: Task 7.2 (System Service Integration) | Phase 7 |
|
||||
| #7: Gossip and RAFT protocols | ✅ Planned | Phase 0: Task 0.1 (Service Registry) | Phase 0 |
|
||||
| #8: etcd/KV for configuration sync | ✅ Planned | Phase 0: Task 0.1 (Distributed Config) | Phase 0 |
|
||||
| #9: Smithy model code generation | ✅ Planned | Phase 2: Task 2.1 (Smithy Code Generation) | Phase 2 |
|
||||
|
||||
## Development Phase Overview
|
||||
|
||||
**Phase 0**: Distributed Systems Foundation (1-2 weeks)
|
||||
|
||||
- Gossip, RAFT, and distributed configuration (#7, #8)
|
||||
|
||||
**Phase 1**: Development Environment Automation (2-3 weeks)
|
||||
|
||||
- Multi-service orchestration and hot reloading
|
||||
|
||||
**Phase 2**: AWS API Model Implementation (3-4 weeks)
|
||||
|
||||
- Smithy-based code generation (#9)
|
||||
- AWS SDK v2 integration (#5)
|
||||
|
||||
**Phase 3**: Scalable Gateway and Daemon Architecture (2-3 weeks)
|
||||
|
||||
- AWS gateway service `awsd` (#4)
|
||||
- Service refactoring to `hived` (#2)
|
||||
- VPC daemon `vpcd` foundation (#3)
|
||||
|
||||
**Phase 4**: Service Integration and Orchestration (3-4 weeks)
|
||||
|
||||
- NATS clustering and cross-service coordination
|
||||
|
||||
**Phase 5**: Infrastructure Services (4-5 weeks)
|
||||
|
||||
- VPC networking with Open vSwitch (#3)
|
||||
- Advanced VM features
|
||||
|
||||
**Phase 6**: Testing and Validation Framework (2-3 weeks)
|
||||
|
||||
- AWS CLI compatibility testing
|
||||
|
||||
**Phase 7**: Production Deployment and Packaging (2-3 weeks)
|
||||
|
||||
- Binary compilation and distribution (#1)
|
||||
- Systemd service integration (#6)
|
||||
- Production configuration management
|
||||
|
||||
## Service Architecture
|
||||
|
||||
The development plan implements these services:
|
||||
|
||||
- **`awsd`** - AWS API Gateway (TODO #4)
|
||||
- **`hived`** - Main Hive compute daemon (TODO #2)
|
||||
- **`vpcd`** - VPC networking daemon (TODO #3)
|
||||
- **Predastore** - S3 service (existing)
|
||||
- **Viperblock** - EBS service (existing)
|
||||
|
||||
## Getting Started
|
||||
|
||||
For development setup:
|
||||
|
||||
```bash
|
||||
./scripts/dev-setup.sh # Setup complete development environment
|
||||
./scripts/start-dev.sh # Start all services
|
||||
```
|
||||
|
||||
For detailed implementation guidance, see [HIVE_DEVELOPMENT_PLAN.md](HIVE_DEVELOPMENT_PLAN.md).
|
||||
+488
-141
@@ -17,7 +17,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -33,14 +33,13 @@ import (
|
||||
|
||||
"github.com/mulgadc/hive/hive/admin"
|
||||
"github.com/mulgadc/hive/hive/config"
|
||||
"github.com/mulgadc/hive/hive/formation"
|
||||
"github.com/mulgadc/hive/hive/utils"
|
||||
"github.com/mulgadc/viperblock/viperblock"
|
||||
"github.com/mulgadc/viperblock/viperblock/backends/s3"
|
||||
"github.com/mulgadc/viperblock/viperblock/v_utils"
|
||||
"github.com/pelletier/go-toml/v2"
|
||||
"github.com/pterm/pterm"
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
//go:embed templates/hive.toml
|
||||
@@ -55,6 +54,9 @@ var predastoreTomlTemplate string
|
||||
//go:embed templates/nats.conf
|
||||
var natsConfTemplate string
|
||||
|
||||
//go:embed templates/predastore-multinode.toml
|
||||
var predastoreMultiNodeTemplate string
|
||||
|
||||
var supportedArchs = map[string]bool{
|
||||
"x86_64": true,
|
||||
"aarch64": true, // alias for arm64
|
||||
@@ -73,6 +75,21 @@ var adminCmd = &cobra.Command{
|
||||
Long: `Administrative commands for initializing and managing the Hive platform.`,
|
||||
}
|
||||
|
||||
var clusterCmd = &cobra.Command{
|
||||
Use: "cluster",
|
||||
Short: "Cluster-wide operations",
|
||||
Long: `Cluster-wide administrative operations such as coordinated shutdown.`,
|
||||
}
|
||||
|
||||
var clusterShutdownCmd = &cobra.Command{
|
||||
Use: "shutdown",
|
||||
Short: "Gracefully shut down the entire cluster",
|
||||
Long: `Perform a coordinated, phased shutdown of the entire cluster.
|
||||
Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon).
|
||||
Each phase waits for all nodes to ACK before proceeding to the next.`,
|
||||
Run: runClusterShutdown,
|
||||
}
|
||||
|
||||
var adminInitCmd = &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "Initialize Hive platform configuration",
|
||||
@@ -135,6 +152,12 @@ func init() {
|
||||
adminCmd.AddCommand(adminInitCmd)
|
||||
adminCmd.AddCommand(adminJoinCmd)
|
||||
|
||||
adminCmd.AddCommand(clusterCmd)
|
||||
clusterCmd.AddCommand(clusterShutdownCmd)
|
||||
clusterShutdownCmd.Flags().Bool("force", false, "Force shutdown even if nodes don't respond")
|
||||
clusterShutdownCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
|
||||
clusterShutdownCmd.Flags().Bool("dry-run", false, "Print phase plan without executing")
|
||||
|
||||
adminCmd.AddCommand(imagesCmd)
|
||||
imagesCmd.AddCommand(imagesImportCmd)
|
||||
imagesCmd.AddCommand(imagesListCmd)
|
||||
@@ -157,6 +180,10 @@ func init() {
|
||||
adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node)")
|
||||
adminInitCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
|
||||
adminInitCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
|
||||
adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
|
||||
adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
|
||||
adminInitCmd.Flags().String("cluster-name", "hive", "NATS cluster name")
|
||||
adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")
|
||||
|
||||
// Flags for admin join
|
||||
adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node")
|
||||
@@ -168,6 +195,7 @@ func init() {
|
||||
adminJoinCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.2 for multi-node on single host)")
|
||||
adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
|
||||
adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
|
||||
adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
|
||||
adminJoinCmd.MarkFlagRequired("node")
|
||||
adminJoinCmd.MarkFlagRequired("host")
|
||||
|
||||
@@ -340,7 +368,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
|
||||
// Calculate the size
|
||||
|
||||
manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
|
||||
volumeId := viperblock.GenerateVolumeID("ami", manifest.AMIMetadata.Name, "", time.Now().Unix()) // TODO: Replace with bucket
|
||||
volumeId := utils.GenerateResourceID("ami")
|
||||
manifest.AMIMetadata.ImageID = volumeId
|
||||
|
||||
manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Hive", manifest.AMIMetadata.Name)
|
||||
@@ -373,7 +401,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
|
||||
|
||||
manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, manifest.AMIMetadata.Name)
|
||||
// Write to file
|
||||
err = os.WriteFile(manifestFilename, jsonData, 0644)
|
||||
err = os.WriteFile(manifestFilename, jsonData, 0600)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Could not write manifest: %v\n", err)
|
||||
os.Exit(1)
|
||||
@@ -469,11 +497,19 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
region, _ := cmd.Flags().GetString("region")
|
||||
az, _ := cmd.Flags().GetString("az")
|
||||
node, _ := cmd.Flags().GetString("node")
|
||||
nodes, _ := cmd.Flags().GetInt("nodes")
|
||||
port, _ := cmd.Flags().GetInt("port")
|
||||
bindIP, _ := cmd.Flags().GetString("bind")
|
||||
clusterBind, _ := cmd.Flags().GetString("cluster-bind")
|
||||
clusterRoutesStr, _ := cmd.Flags().GetString("cluster-routes")
|
||||
clusterRoutes := strings.Split(clusterRoutesStr, ",")
|
||||
var clusterRoutes []string
|
||||
if clusterRoutesStr != "" {
|
||||
clusterRoutes = strings.Split(clusterRoutesStr, ",")
|
||||
}
|
||||
predastoreNodesStr, _ := cmd.Flags().GetString("predastore-nodes")
|
||||
formationTimeoutStr, _ := cmd.Flags().GetString("formation-timeout")
|
||||
clusterName, _ := cmd.Flags().GetString("cluster-name")
|
||||
services, _ := cmd.Flags().GetStringSlice("services")
|
||||
|
||||
// Validate IP address format
|
||||
if net.ParseIP(bindIP) == nil {
|
||||
@@ -487,6 +523,11 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Default cluster-bind to bind IP if not specified
|
||||
if clusterBind == "" {
|
||||
clusterBind = bindIP
|
||||
}
|
||||
|
||||
fmt.Printf("Initializing Hive with bind IP: %s, port: %d\n", bindIP, port)
|
||||
|
||||
// Default config directory
|
||||
@@ -529,8 +570,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf(" Secret Key: %s\n", secretKey)
|
||||
fmt.Printf(" Account ID: %s\n", accountID)
|
||||
|
||||
// Generate SSL certificates
|
||||
certPath := admin.GenerateCertificatesIfNeeded(configDir, force)
|
||||
// Generate SSL certificates (with bind IP in SANs for multi-node support)
|
||||
certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)
|
||||
|
||||
// Generate NATS token
|
||||
natsToken := admin.GenerateNATSToken()
|
||||
@@ -540,9 +581,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
// Get home directory for data path
|
||||
homeDir, _ := os.UserHomeDir()
|
||||
hiveRoot = filepath.Join(homeDir, "hive")
|
||||
|
||||
}
|
||||
|
||||
// Determine if this is a multi-node formation
|
||||
isMultiNode := nodes >= 2 && bindIP != "0.0.0.0"
|
||||
|
||||
if isMultiNode {
|
||||
runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName,
|
||||
configDir, hiveRoot, certPath, region, az, node, bindIP, clusterBind,
|
||||
port, nodes, formationTimeoutStr, services)
|
||||
return
|
||||
}
|
||||
|
||||
// --- Single-node path (existing behavior) ---
|
||||
|
||||
// Create config files from embedded templates
|
||||
fmt.Println("\n📝 Creating configuration files...")
|
||||
|
||||
@@ -561,6 +613,49 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
|
||||
portStr := fmt.Sprintf("%d", port)
|
||||
|
||||
// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
|
||||
var predastoreNodeID int
|
||||
if predastoreNodesStr != "" {
|
||||
ips := strings.Split(predastoreNodesStr, ",")
|
||||
if len(ips) < 3 {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 3 IPs, got %d\n", len(ips))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var predastoreNodes []admin.PredastoreNodeConfig
|
||||
for i, ip := range ips {
|
||||
ip = strings.TrimSpace(ip)
|
||||
if net.ParseIP(ip) == nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP in --predastore-nodes: %s\n", ip)
|
||||
os.Exit(1)
|
||||
}
|
||||
predastoreNodes = append(predastoreNodes, admin.PredastoreNodeConfig{
|
||||
ID: i + 1,
|
||||
Host: ip,
|
||||
})
|
||||
}
|
||||
|
||||
predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
|
||||
if predastoreNodeID == 0 {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: --bind IP %s not found in --predastore-nodes list\n", bindIP)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Generate multi-node predastore.toml
|
||||
predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
predastorePath := filepath.Join(predastoreDir, "predastore.toml")
|
||||
if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
|
||||
}
|
||||
|
||||
configSettings := admin.ConfigSettings{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
@@ -575,30 +670,23 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
BindIP: bindIP,
|
||||
ClusterBindIP: clusterBind,
|
||||
ClusterRoutes: clusterRoutes,
|
||||
ClusterName: clusterName,
|
||||
|
||||
PredastoreNodeID: predastoreNodeID,
|
||||
Services: services,
|
||||
}
|
||||
|
||||
// Generate all config files
|
||||
// Generate config files
|
||||
configs := []admin.ConfigFile{
|
||||
{
|
||||
Name: "hive.toml",
|
||||
Path: hiveTomlPath,
|
||||
Template: hiveTomlTemplate,
|
||||
},
|
||||
{
|
||||
Name: filepath.Join(awsgwDir, "awsgw.toml"),
|
||||
Path: filepath.Join(awsgwDir, "awsgw.toml"),
|
||||
Template: awsgwTomlTemplate,
|
||||
},
|
||||
{
|
||||
Name: filepath.Join(predastoreDir, "predastore.toml"),
|
||||
Path: filepath.Join(predastoreDir, "predastore.toml"),
|
||||
Template: predastoreTomlTemplate,
|
||||
},
|
||||
{
|
||||
Name: filepath.Join(natsDir, "nats.conf"),
|
||||
Path: filepath.Join(natsDir, "nats.conf"),
|
||||
Template: natsConfTemplate,
|
||||
},
|
||||
{Name: "hive.toml", Path: hiveTomlPath, Template: hiveTomlTemplate},
|
||||
{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
|
||||
{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
|
||||
}
|
||||
// Skip template-based predastore.toml if multi-node was already generated
|
||||
if predastoreNodesStr == "" {
|
||||
configs = append(configs, admin.ConfigFile{
|
||||
Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
|
||||
})
|
||||
}
|
||||
|
||||
err := admin.GenerateConfigFiles(configs, configSettings)
|
||||
@@ -609,7 +697,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
|
||||
// Update ~/.aws/credentials and ~/.aws/config
|
||||
fmt.Println("\n🔧 Configuring AWS credentials...")
|
||||
if err := admin.SetupAWSCredentials(accessKey, secretKey, region, certPath); err != nil {
|
||||
if err := admin.SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
|
||||
} else {
|
||||
fmt.Println("✅ AWS credentials configured")
|
||||
@@ -633,6 +721,189 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// runAdminInitMultiNode handles the multi-node formation path for admin init.
|
||||
// It starts a formation server, registers this node, waits for all nodes to join,
|
||||
// then generates configs with complete cluster topology.
|
||||
func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName,
|
||||
configDir, hiveRoot, certPath, region, az, node, bindIP, clusterBind string,
|
||||
port, expectedNodes int, formationTimeoutStr string, services []string) {
|
||||
|
||||
formationTimeout, err := time.ParseDuration(formationTimeoutStr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Read CA cert/key for distribution to joining nodes
|
||||
caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error reading CA cert: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
caKeyData, err := os.ReadFile(filepath.Join(configDir, "ca.key"))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error reading CA key: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
creds := &formation.SharedCredentials{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
AccountID: accountID,
|
||||
NatsToken: natsToken,
|
||||
ClusterName: clusterName,
|
||||
Region: region,
|
||||
}
|
||||
|
||||
fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData))
|
||||
|
||||
// Register self (init node) as the first node
|
||||
selfNode := formation.NodeInfo{
|
||||
Name: node,
|
||||
BindIP: bindIP,
|
||||
ClusterIP: clusterBind,
|
||||
Region: region,
|
||||
AZ: az,
|
||||
Port: port,
|
||||
}
|
||||
if err := fs.RegisterNode(selfNode); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error registering self: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Start formation server
|
||||
formationAddr := fmt.Sprintf("%s:%d", bindIP, port)
|
||||
if err := fs.Start(formationAddr); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error starting formation server: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("\n📡 Formation server started on %s\n", formationAddr)
|
||||
fmt.Printf(" Waiting for %d more node(s) to join...\n", expectedNodes-1)
|
||||
fmt.Printf(" Other nodes should run: hive admin join --host %s --node <name> --bind <ip>\n\n", formationAddr)
|
||||
|
||||
// Wait for all nodes to register
|
||||
if err := fs.WaitForCompletion(formationTimeout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ %v\n", err)
|
||||
fs.Shutdown(context.Background())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("✅ All %d nodes joined!\n", expectedNodes)
|
||||
|
||||
// Build cluster topology from formation data
|
||||
allNodes := fs.Nodes()
|
||||
clusterRoutes := formation.BuildClusterRoutes(allNodes)
|
||||
predastoreNodes := formation.BuildPredastoreNodes(allNodes)
|
||||
|
||||
fmt.Println("\n📝 Creating configuration files...")
|
||||
|
||||
// Create subdirectories
|
||||
awsgwDir := filepath.Join(configDir, "awsgw")
|
||||
predastoreDir := filepath.Join(configDir, "predastore")
|
||||
natsDir := filepath.Join(configDir, "nats")
|
||||
hiveDir := filepath.Join(configDir, "hive")
|
||||
|
||||
for _, dir := range []string{awsgwDir, predastoreDir, natsDir, hiveDir} {
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
portStr := fmt.Sprintf("%d", port)
|
||||
|
||||
// Generate multi-node predastore config
|
||||
var predastoreNodeID int
|
||||
if len(predastoreNodes) >= 3 {
|
||||
predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
predastorePath := filepath.Join(predastoreDir, "predastore.toml")
|
||||
if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
|
||||
fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
|
||||
}
|
||||
|
||||
hiveTomlPath := filepath.Join(configDir, "hive.toml")
|
||||
|
||||
configSettings := admin.ConfigSettings{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
AccountID: accountID,
|
||||
Region: region,
|
||||
NatsToken: natsToken,
|
||||
DataDir: hiveRoot,
|
||||
|
||||
Node: node,
|
||||
Az: az,
|
||||
Port: portStr,
|
||||
BindIP: bindIP,
|
||||
ClusterBindIP: clusterBind,
|
||||
ClusterRoutes: clusterRoutes,
|
||||
ClusterName: clusterName,
|
||||
|
||||
PredastoreNodeID: predastoreNodeID,
|
||||
Services: services,
|
||||
RemoteNodes: buildRemoteNodes(allNodes, node),
|
||||
}
|
||||
|
||||
// Generate config files
|
||||
configs := []admin.ConfigFile{
|
||||
{Name: "hive.toml", Path: hiveTomlPath, Template: hiveTomlTemplate},
|
||||
{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
|
||||
{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
|
||||
}
|
||||
// Skip template-based predastore.toml if multi-node was generated
|
||||
if len(predastoreNodes) < 3 {
|
||||
configs = append(configs, admin.ConfigFile{
|
||||
Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
|
||||
})
|
||||
}
|
||||
|
||||
if err := admin.GenerateConfigFiles(configs, configSettings); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error generating configuration files: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Update ~/.aws/credentials and ~/.aws/config
|
||||
fmt.Println("\n🔧 Configuring AWS credentials...")
|
||||
if err := admin.SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
|
||||
} else {
|
||||
fmt.Println("✅ AWS credentials configured")
|
||||
}
|
||||
|
||||
admin.CreateServiceDirectories(hiveRoot)
|
||||
|
||||
// Keep formation server running briefly so joining nodes can fetch complete status
|
||||
fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...")
|
||||
time.Sleep(15 * time.Second)
|
||||
|
||||
// Shutdown formation server
|
||||
fs.Shutdown(context.Background())
|
||||
|
||||
// Print cluster summary
|
||||
fmt.Println("\n🎉 Cluster formation complete!")
|
||||
fmt.Printf(" Cluster: %s (%d nodes)\n", clusterName, expectedNodes)
|
||||
fmt.Printf(" Region: %s\n", region)
|
||||
fmt.Println(" Nodes:")
|
||||
for name, n := range allNodes {
|
||||
fmt.Printf(" - %s (%s)\n", name, n.BindIP)
|
||||
}
|
||||
fmt.Println("\n📋 Next steps:")
|
||||
fmt.Println(" 1. Start services on ALL nodes:")
|
||||
fmt.Println(" ./scripts/start-dev.sh")
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
node, _ := cmd.Flags().GetString("node")
|
||||
leaderHost, _ := cmd.Flags().GetString("host")
|
||||
@@ -643,12 +914,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
bindIP, _ := cmd.Flags().GetString("bind")
|
||||
configDir, _ := cmd.Flags().GetString("config-dir")
|
||||
clusterBind, _ := cmd.Flags().GetString("cluster-bind")
|
||||
// Receive as string, split later
|
||||
// TODO: Use GetStringArray
|
||||
clusterRoutesStr, _ := cmd.Flags().GetString("cluster-routes")
|
||||
|
||||
// Supply as array to template
|
||||
clusterRoutes := strings.Split(clusterRoutesStr, ",")
|
||||
services, _ := cmd.Flags().GetStringSlice("services")
|
||||
|
||||
// Validate required parameters
|
||||
if node == "" {
|
||||
@@ -672,6 +938,11 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Default cluster-bind to bind IP if not specified
|
||||
if clusterBind == "" {
|
||||
clusterBind = bindIP
|
||||
}
|
||||
|
||||
// Set default data directory
|
||||
if dataDir == "" {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
@@ -682,9 +953,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
dataDir = filepath.Join(homeDir, "hive")
|
||||
}
|
||||
|
||||
// Set daemon host for this node
|
||||
daemonHost := fmt.Sprintf("%s:%d", bindIP, port)
|
||||
|
||||
fmt.Println("🚀 Joining Hive cluster...")
|
||||
fmt.Printf("Node: %s\n", node)
|
||||
fmt.Printf("Leader: %s\n", leaderHost)
|
||||
@@ -693,13 +961,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("Bind IP: %s\n", bindIP)
|
||||
fmt.Printf("Port: %d\n\n", port)
|
||||
|
||||
// Create join request
|
||||
joinReq := config.NodeJoinRequest{
|
||||
Node: node,
|
||||
Region: region,
|
||||
AZ: az,
|
||||
DataDir: dataDir,
|
||||
DaemonHost: daemonHost,
|
||||
// POST join request to formation server
|
||||
joinReq := formation.JoinRequest{
|
||||
NodeInfo: formation.NodeInfo{
|
||||
Name: node,
|
||||
BindIP: bindIP,
|
||||
ClusterIP: clusterBind,
|
||||
Region: region,
|
||||
AZ: az,
|
||||
Port: port,
|
||||
},
|
||||
}
|
||||
|
||||
reqBody, err := json.Marshal(joinReq)
|
||||
@@ -708,46 +979,24 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Send join request to leader with HTTP/2 support
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true, // Skip TLS verification for self-signed certs
|
||||
NextProtos: []string{"h2", "http/1.1"},
|
||||
},
|
||||
ForceAttemptHTTP2: true,
|
||||
}
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
// Configure HTTP/2 support with custom TLS config
|
||||
http2.ConfigureTransport(tr)
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: tr,
|
||||
}
|
||||
|
||||
joinURL := fmt.Sprintf("http://%s/join", leaderHost)
|
||||
joinURL := fmt.Sprintf("http://%s/formation/join", leaderHost)
|
||||
resp, err := client.Post(joinURL, "application/json", bytes.NewBuffer(reqBody))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error connecting to leader node: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "Make sure the leader node is running and accessible at %s\n", leaderHost)
|
||||
fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "Make sure the leader node has run 'hive admin init' and is accessible at %s\n", leaderHost)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error reading response body: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
var errResp config.NodeJoinResponse
|
||||
json.Unmarshal(body, &errResp)
|
||||
fmt.Fprintf(os.Stderr, "❌ Failed to join cluster: %s\n", errResp.Message)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var joinResp config.NodeJoinResponse
|
||||
var joinResp formation.JoinResponse
|
||||
if err := json.Unmarshal(body, &joinResp); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error parsing join response: %v\n", err)
|
||||
os.Exit(1)
|
||||
@@ -758,12 +1007,49 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Println("✅ Successfully joined cluster!")
|
||||
fmt.Printf("Epoch: %d\n", joinResp.SharedData.Epoch)
|
||||
fmt.Printf("Config hash: %s\n\n", joinResp.ConfigHash)
|
||||
fmt.Printf("✅ Registered with formation server (%d/%d nodes joined)\n", joinResp.Joined, joinResp.Expected)
|
||||
|
||||
// Save cluster config locally
|
||||
// Confirm if default config directory required
|
||||
// Poll status until formation is complete
|
||||
statusURL := fmt.Sprintf("http://%s/formation/status", leaderHost)
|
||||
var statusResp formation.StatusResponse
|
||||
|
||||
for {
|
||||
sResp, err := client.Get(statusURL)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error polling formation status: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sBody, err := io.ReadAll(sResp.Body)
|
||||
sResp.Body.Close()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error reading status response: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(sBody, &statusResp); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error parsing status response: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if statusResp.Complete {
|
||||
break
|
||||
}
|
||||
|
||||
fmt.Printf(" Waiting for cluster formation... (%d/%d nodes joined)\n", statusResp.Joined, statusResp.Expected)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
fmt.Printf("✅ Cluster formation complete! (%d nodes)\n\n", statusResp.Expected)
|
||||
|
||||
// Extract credentials and CA from formation status
|
||||
creds := statusResp.Credentials
|
||||
if creds == nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: formation server did not return credentials\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Set up config directory
|
||||
if configDir == "" {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
@@ -773,50 +1059,91 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
configDir = filepath.Join(homeDir, "hive", "config")
|
||||
}
|
||||
|
||||
hiveTomlPath := filepath.Join(configDir, "hive.toml")
|
||||
|
||||
// Create config directory if needed
|
||||
if err := os.MkdirAll(configDir, 0700); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error creating config directory: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Construct local node config with THIS node's name at top level
|
||||
localConfig := config.ClusterConfig{
|
||||
Epoch: joinResp.SharedData.Epoch,
|
||||
Node: node, // THIS node's name, not the leader's
|
||||
Version: joinResp.SharedData.Version,
|
||||
Nodes: joinResp.SharedData.Nodes,
|
||||
}
|
||||
// Write CA cert and key
|
||||
caCertPath := filepath.Join(configDir, "ca.pem")
|
||||
caKeyPath := filepath.Join(configDir, "ca.key")
|
||||
|
||||
// Marshal config to TOML format
|
||||
configTOML, err := toml.Marshal(&localConfig)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error marshaling config to TOML: %v\n", err)
|
||||
if statusResp.CACert == "" || statusResp.CAKey == "" {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: formation server did not return CA certificate\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Write config to file
|
||||
if err := os.WriteFile(hiveTomlPath, configTOML, 0600); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing config file: %v\n", err)
|
||||
if err := os.WriteFile(caCertPath, []byte(statusResp.CACert), 0600); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing CA cert: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(caKeyPath, []byte(statusResp.CAKey), 0600); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing CA key: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("✅ CA certificate received from leader: %s\n", caCertPath)
|
||||
|
||||
fmt.Printf("✅ Config saved to: %s\n\n", hiveTomlPath)
|
||||
// Generate server cert signed by CA with this node's bind IP
|
||||
if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("✅ Server certificate generated with bind IP: %s\n\n", bindIP)
|
||||
|
||||
// Generate certificates if needed
|
||||
certPath := admin.GenerateCertificatesIfNeeded(configDir, false)
|
||||
// Build cluster topology from formation data
|
||||
clusterRoutes := formation.BuildClusterRoutes(statusResp.Nodes)
|
||||
predastoreNodes := formation.BuildPredastoreNodes(statusResp.Nodes)
|
||||
|
||||
fmt.Printf("✅ SSL certificates available at: %s\n\n", certPath)
|
||||
fmt.Println("📝 Creating configuration files...")
|
||||
|
||||
// Create subdirectories
|
||||
awsgwDir := filepath.Join(configDir, "awsgw")
|
||||
predastoreDir := filepath.Join(configDir, "predastore")
|
||||
natsDir := filepath.Join(configDir, "nats")
|
||||
hiveDir := filepath.Join(configDir, "hive")
|
||||
|
||||
for _, dir := range []string{awsgwDir, predastoreDir, natsDir, hiveDir} {
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Write individual node config files
|
||||
portStr := fmt.Sprintf("%d", port)
|
||||
|
||||
// Generate multi-node predastore config
|
||||
var predastoreNodeID int
|
||||
hasPredastoreConfig := len(predastoreNodes) >= 3
|
||||
|
||||
if hasPredastoreConfig {
|
||||
predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
predastorePath := filepath.Join(predastoreDir, "predastore.toml")
|
||||
if err := os.WriteFile(predastorePath, []byte(predastoreContent), 0600); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
|
||||
if predastoreNodeID == 0 {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: bind IP %s not found in predastore node list\n", bindIP)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
|
||||
}
|
||||
|
||||
hiveTomlPath := filepath.Join(configDir, "hive.toml")
|
||||
|
||||
configSettings := admin.ConfigSettings{
|
||||
AccessKey: localConfig.Nodes[node].AccessKey,
|
||||
SecretKey: localConfig.Nodes[node].SecretKey,
|
||||
Region: region,
|
||||
NatsToken: localConfig.Nodes[node].NATS.ACL.Token,
|
||||
AccessKey: creds.AccessKey,
|
||||
SecretKey: creds.SecretKey,
|
||||
AccountID: creds.AccountID,
|
||||
Region: creds.Region,
|
||||
NatsToken: creds.NatsToken,
|
||||
DataDir: dataDir,
|
||||
|
||||
Node: node,
|
||||
@@ -825,41 +1152,24 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
BindIP: bindIP,
|
||||
ClusterBindIP: clusterBind,
|
||||
ClusterRoutes: clusterRoutes,
|
||||
ClusterName: creds.ClusterName,
|
||||
|
||||
PredastoreNodeID: predastoreNodeID,
|
||||
Services: services,
|
||||
RemoteNodes: buildRemoteNodes(statusResp.Nodes, node),
|
||||
}
|
||||
|
||||
awsgwDir := filepath.Join(configDir, "awsgw")
|
||||
predastoreDir := filepath.Join(configDir, "predastore")
|
||||
natsDir := filepath.Join(configDir, "nats")
|
||||
|
||||
for _, dir := range []string{awsgwDir, predastoreDir, natsDir} {
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error creating directory %s: %v\n", dir, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Generate all config files
|
||||
// Generate config files
|
||||
configs := []admin.ConfigFile{
|
||||
{
|
||||
Name: "hive.toml",
|
||||
Path: hiveTomlPath,
|
||||
Template: hiveTomlTemplate,
|
||||
},
|
||||
{
|
||||
Name: filepath.Join(awsgwDir, "awsgw.toml"),
|
||||
Path: filepath.Join(awsgwDir, "awsgw.toml"),
|
||||
Template: awsgwTomlTemplate,
|
||||
},
|
||||
{
|
||||
Name: filepath.Join(predastoreDir, "predastore.toml"),
|
||||
Path: filepath.Join(predastoreDir, "predastore.toml"),
|
||||
Template: predastoreTomlTemplate,
|
||||
},
|
||||
{
|
||||
Name: filepath.Join(natsDir, "nats.conf"),
|
||||
Path: filepath.Join(natsDir, "nats.conf"),
|
||||
Template: natsConfTemplate,
|
||||
},
|
||||
{Name: "hive.toml", Path: hiveTomlPath, Template: hiveTomlTemplate},
|
||||
{Name: filepath.Join(awsgwDir, "awsgw.toml"), Path: filepath.Join(awsgwDir, "awsgw.toml"), Template: awsgwTomlTemplate},
|
||||
{Name: filepath.Join(natsDir, "nats.conf"), Path: filepath.Join(natsDir, "nats.conf"), Template: natsConfTemplate},
|
||||
}
|
||||
// Skip template-based predastore.toml if multi-node was generated
|
||||
if !hasPredastoreConfig {
|
||||
configs = append(configs, admin.ConfigFile{
|
||||
Name: filepath.Join(predastoreDir, "predastore.toml"), Path: filepath.Join(predastoreDir, "predastore.toml"), Template: predastoreTomlTemplate,
|
||||
})
|
||||
}
|
||||
|
||||
err = admin.GenerateConfigFiles(configs, configSettings)
|
||||
@@ -868,11 +1178,48 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Update ~/.aws/credentials and ~/.aws/config
|
||||
fmt.Println("\n🔧 Configuring AWS credentials...")
|
||||
if err := admin.SetupAWSCredentials(creds.AccessKey, creds.SecretKey, creds.Region, caCertPath, bindIP); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
|
||||
} else {
|
||||
fmt.Println("✅ AWS credentials configured")
|
||||
}
|
||||
|
||||
admin.CreateServiceDirectories(dataDir)
|
||||
|
||||
fmt.Println("🎉 Node successfully joined cluster!")
|
||||
// Print cluster summary
|
||||
fmt.Println("\n🎉 Node successfully joined cluster!")
|
||||
fmt.Printf(" Cluster: %s (%d nodes)\n", creds.ClusterName, len(statusResp.Nodes))
|
||||
fmt.Println(" Nodes:")
|
||||
for name, n := range statusResp.Nodes {
|
||||
fmt.Printf(" - %s (%s)\n", name, n.BindIP)
|
||||
}
|
||||
fmt.Println("\n📋 Next steps:")
|
||||
fmt.Println(" 1. Start the hive service:")
|
||||
fmt.Printf(" hive service hive start --config %s\n", hiveTomlPath)
|
||||
fmt.Println(" 1. Start services:")
|
||||
fmt.Println(" ./scripts/start-dev.sh")
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// buildRemoteNodes converts formation NodeInfo into RemoteNode entries,
|
||||
// excluding the local node. This puts all cluster members into hive.toml
|
||||
// so config is the source of truth for expected cluster membership.
|
||||
func buildRemoteNodes(allNodes map[string]formation.NodeInfo, localNode string) []admin.RemoteNode {
|
||||
var remote []admin.RemoteNode
|
||||
for name, n := range allNodes {
|
||||
if name == localNode {
|
||||
continue
|
||||
}
|
||||
remote = append(remote, admin.RemoteNode{
|
||||
Name: name,
|
||||
Host: n.BindIP,
|
||||
Region: n.Region,
|
||||
AZ: n.AZ,
|
||||
Services: n.Services,
|
||||
})
|
||||
}
|
||||
sort.Slice(remote, func(i, j int) bool {
|
||||
return remote[i].Name < remote[j].Name
|
||||
})
|
||||
return remote
|
||||
}
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/hive/hive/daemon"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// runClusterShutdown orchestrates a phased, coordinated shutdown of the cluster.
|
||||
func runClusterShutdown(cmd *cobra.Command, args []string) {
|
||||
force, _ := cmd.Flags().GetBool("force")
|
||||
timeout, _ := cmd.Flags().GetDuration("timeout")
|
||||
dryRun, _ := cmd.Flags().GetBool("dry-run")
|
||||
|
||||
cfg, nc, err := loadConfigAndConnect()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer nc.Close()
|
||||
|
||||
nodeCount := len(cfg.Nodes)
|
||||
phases := []string{"gate", "drain", "storage", "persist", "infra"}
|
||||
|
||||
if dryRun {
|
||||
fmt.Println("Cluster Shutdown Plan (dry-run)")
|
||||
fmt.Printf(" Nodes: %d\n", nodeCount)
|
||||
for name, nodeCfg := range cfg.Nodes {
|
||||
fmt.Printf(" - %s (services: %s)\n", name, strings.Join(nodeCfg.GetServices(), ", "))
|
||||
}
|
||||
fmt.Printf(" Phases: %s\n", strings.Join(phases, " -> "))
|
||||
fmt.Printf(" Timeout per phase: %s\n", timeout)
|
||||
fmt.Printf(" Force: %v\n", force)
|
||||
fmt.Println("\nPhase details:")
|
||||
fmt.Println(" 1. GATE - Stop API gateway and UI, reject new work")
|
||||
fmt.Println(" 2. DRAIN - Gracefully stop all VMs, persist state")
|
||||
fmt.Println(" 3. STORAGE - Stop viperblock, cleanup nbdkit")
|
||||
fmt.Println(" 4. PERSIST - Stop predastore")
|
||||
fmt.Println(" 5. INFRA - Stop NATS, exit daemons")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("Starting coordinated cluster shutdown (%d nodes)\n", nodeCount)
|
||||
fmt.Printf("Phases: %s\n", strings.Join(phases, " -> "))
|
||||
fmt.Printf("Timeout per phase: %s\n\n", timeout)
|
||||
|
||||
// Write cluster shutdown marker to KV
|
||||
jsm, err := daemon.NewJetStreamManager(nc, 1)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to create JetStream manager: %v\n", err)
|
||||
} else {
|
||||
if err := jsm.InitClusterStateBucket(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to init cluster state bucket: %v\n", err)
|
||||
} else {
|
||||
state := &daemon.ClusterShutdownState{
|
||||
Initiator: cfg.Node,
|
||||
Phase: "starting",
|
||||
Started: time.Now().UTC().Format(time.RFC3339),
|
||||
Timeout: timeout.String(),
|
||||
Force: force,
|
||||
NodesTotal: nodeCount,
|
||||
NodesAcked: make(map[string]string),
|
||||
}
|
||||
if err := jsm.WriteClusterShutdown(state); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to write shutdown state: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
// Execute phases sequentially (except INFRA which is fire-and-forget)
|
||||
for _, phase := range phases {
|
||||
topic := "hive.cluster.shutdown." + phase
|
||||
req := daemon.ShutdownRequest{
|
||||
Phase: phase,
|
||||
Force: force,
|
||||
Timeout: int(timeout.Seconds()),
|
||||
}
|
||||
|
||||
reqData, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error marshaling request: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if phase == "infra" {
|
||||
// INFRA is fire-and-forget — NATS is going down, no ACKs possible
|
||||
fmt.Printf("[INFRA] Sending final shutdown to all nodes...\n")
|
||||
if err := nc.Publish(topic, reqData); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error publishing infra shutdown: %v\n", err)
|
||||
}
|
||||
nc.Flush()
|
||||
// Wait briefly for messages to propagate
|
||||
time.Sleep(2 * time.Second)
|
||||
fmt.Printf("[INFRA] Complete\n")
|
||||
break
|
||||
}
|
||||
|
||||
// For DRAIN phase, subscribe to progress updates
|
||||
var progressSub *nats.Subscription
|
||||
if phase == "drain" {
|
||||
progressSub, err = nc.Subscribe("hive.cluster.shutdown.progress", func(msg *nats.Msg) {
|
||||
var progress daemon.ShutdownProgress
|
||||
if err := json.Unmarshal(msg.Data, &progress); err != nil {
|
||||
return
|
||||
}
|
||||
fmt.Printf(" [%s] %s: %d/%d VMs remaining\n", strings.ToUpper(phase), progress.Node, progress.Remaining, progress.Total)
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to subscribe to progress: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Collect ACKs from all nodes
|
||||
phaseStart := time.Now()
|
||||
fmt.Printf("[%s] Sending to %d node(s)...\n", strings.ToUpper(phase), nodeCount)
|
||||
|
||||
acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, timeout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[%s] Error: %v\n", strings.ToUpper(phase), err)
|
||||
if !force {
|
||||
fmt.Fprintf(os.Stderr, "Aborting shutdown. Use --force to continue despite errors.\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Unsubscribe from progress
|
||||
if progressSub != nil {
|
||||
progressSub.Unsubscribe()
|
||||
}
|
||||
|
||||
// Print results
|
||||
for _, ack := range acks {
|
||||
if ack.Error != "" {
|
||||
fmt.Printf(" [%s] %s: ERROR - %s\n", strings.ToUpper(phase), ack.Node, ack.Error)
|
||||
} else if len(ack.Stopped) > 0 {
|
||||
fmt.Printf(" [%s] %s: stopped %s\n", strings.ToUpper(phase), ack.Node, strings.Join(ack.Stopped, ", "))
|
||||
} else {
|
||||
fmt.Printf(" [%s] %s: OK\n", strings.ToUpper(phase), ack.Node)
|
||||
}
|
||||
}
|
||||
|
||||
ackedCount := len(acks)
|
||||
if ackedCount < nodeCount && !force {
|
||||
fmt.Fprintf(os.Stderr, "[%s] Only %d/%d nodes responded. Use --force to continue.\n",
|
||||
strings.ToUpper(phase), ackedCount, nodeCount)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("[%s] Complete (%d/%d nodes, %s)\n\n", strings.ToUpper(phase), ackedCount, nodeCount, time.Since(phaseStart).Round(time.Millisecond))
|
||||
}
|
||||
|
||||
fmt.Printf("Cluster shutdown complete (%s)\n", time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
|
||||
// collectShutdownACKs publishes a shutdown request and collects ACKs from nodes.
|
||||
func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, timeout time.Duration) ([]daemon.ShutdownACK, error) {
|
||||
inbox := nats.NewInbox()
|
||||
sub, err := nc.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to subscribe to inbox: %w", err)
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
if err := nc.PublishRequest(topic, inbox, reqData); err != nil {
|
||||
return nil, fmt.Errorf("failed to publish request: %w", err)
|
||||
}
|
||||
nc.Flush()
|
||||
|
||||
var acks []daemon.ShutdownACK
|
||||
deadline := time.Now().Add(timeout)
|
||||
for len(acks) < nodeCount {
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
break
|
||||
}
|
||||
msg, err := sub.NextMsg(remaining)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
var ack daemon.ShutdownACK
|
||||
if err := json.Unmarshal(msg.Data, &ack); err != nil {
|
||||
continue
|
||||
}
|
||||
acks = append(acks, ack)
|
||||
}
|
||||
return acks, nil
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
/*
|
||||
Copyright © 2025 Mulga Defense Corporation
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/hive/hive/config"
|
||||
"github.com/mulgadc/hive/hive/utils"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/pterm/pterm"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
var getCmd = &cobra.Command{
|
||||
Use: "get",
|
||||
Short: "Display cluster resources",
|
||||
Long: `Display cluster resources such as nodes and VMs.`,
|
||||
}
|
||||
|
||||
var getNodesCmd = &cobra.Command{
|
||||
Use: "nodes",
|
||||
Short: "Display cluster nodes",
|
||||
Long: `Display all physical nodes in the cluster with status, IP, region, and services.`,
|
||||
Run: runGetNodes,
|
||||
}
|
||||
|
||||
var getVMsCmd = &cobra.Command{
|
||||
Use: "vms",
|
||||
Aliases: []string{"instances"},
|
||||
Short: "Display VMs across the cluster",
|
||||
Long: `Display all VMs running across the cluster with instance type, resources, and placement.`,
|
||||
Run: runGetVMs,
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(getCmd)
|
||||
getCmd.AddCommand(getNodesCmd)
|
||||
getCmd.AddCommand(getVMsCmd)
|
||||
|
||||
getCmd.PersistentFlags().Duration("timeout", 3*time.Second, "Timeout for collecting responses from nodes")
|
||||
}
|
||||
|
||||
// loadConfigAndConnect loads the cluster config and connects to NATS.
|
||||
func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) {
|
||||
cfgPath := viper.GetString("config")
|
||||
if cfgPath == "" {
|
||||
homeDir, _ := os.UserHomeDir()
|
||||
cfgPath = fmt.Sprintf("%s/hive/config/hive.toml", homeDir)
|
||||
}
|
||||
|
||||
cfg, err := config.LoadConfig(cfgPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to load config: %w", err)
|
||||
}
|
||||
|
||||
nodeConfig := cfg.Nodes[cfg.Node]
|
||||
nc, err := utils.ConnectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to connect to NATS: %w", err)
|
||||
}
|
||||
|
||||
return cfg, nc, nil
|
||||
}
|
||||
|
||||
// collectResponses publishes to a fan-out topic and collects all responses within the timeout.
|
||||
func collectResponses(nc *nats.Conn, topic string, timeout time.Duration) ([][]byte, error) {
|
||||
inbox := nats.NewInbox()
|
||||
sub, err := nc.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to subscribe to inbox: %w", err)
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
if err := nc.PublishRequest(topic, inbox, nil); err != nil {
|
||||
return nil, fmt.Errorf("failed to publish request: %w", err)
|
||||
}
|
||||
nc.Flush()
|
||||
|
||||
var responses [][]byte
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
break
|
||||
}
|
||||
msg, err := sub.NextMsg(remaining)
|
||||
if err != nil {
|
||||
break // timeout or other error — done collecting
|
||||
}
|
||||
responses = append(responses, msg.Data)
|
||||
}
|
||||
return responses, nil
|
||||
}
|
||||
|
||||
func formatUptime(seconds int64) string {
|
||||
if seconds <= 0 {
|
||||
return "-"
|
||||
}
|
||||
d := time.Duration(seconds) * time.Second
|
||||
days := int(d.Hours()) / 24
|
||||
hours := int(d.Hours()) % 24
|
||||
mins := int(d.Minutes()) % 60
|
||||
|
||||
if days > 0 {
|
||||
return fmt.Sprintf("%dd%dh", days, hours)
|
||||
}
|
||||
if hours > 0 {
|
||||
return fmt.Sprintf("%dh%dm", hours, mins)
|
||||
}
|
||||
return fmt.Sprintf("%dm", mins)
|
||||
}
|
||||
|
||||
func formatMemGB(gb float64) string {
|
||||
if gb >= 1.0 {
|
||||
return fmt.Sprintf("%.1fGi", gb)
|
||||
}
|
||||
return fmt.Sprintf("%dMi", int(gb*1024))
|
||||
}
|
||||
|
||||
func runGetNodes(cmd *cobra.Command, args []string) {
|
||||
cfg, nc, err := loadConfigAndConnect()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer nc.Close()
|
||||
|
||||
timeout, _ := cmd.Flags().GetDuration("timeout")
|
||||
responses, err := collectResponses(nc, "hive.node.status", timeout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Parse responses into a map by node name
|
||||
respondedNodes := make(map[string]config.NodeStatusResponse)
|
||||
for _, data := range responses {
|
||||
var resp config.NodeStatusResponse
|
||||
if err := json.Unmarshal(data, &resp); err != nil {
|
||||
continue
|
||||
}
|
||||
respondedNodes[resp.Node] = resp
|
||||
}
|
||||
|
||||
// Build table: union of config-known nodes + NATS responders
|
||||
tableData := pterm.TableData{
|
||||
{"NAME", "STATUS", "IP", "REGION", "AZ", "UPTIME", "VMs", "SERVICES"},
|
||||
}
|
||||
|
||||
// Collect all node names: config + responded (union)
|
||||
nodeSet := make(map[string]struct{})
|
||||
for name := range cfg.Nodes {
|
||||
nodeSet[name] = struct{}{}
|
||||
}
|
||||
for name := range respondedNodes {
|
||||
nodeSet[name] = struct{}{}
|
||||
}
|
||||
nodeNames := make([]string, 0, len(nodeSet))
|
||||
for name := range nodeSet {
|
||||
nodeNames = append(nodeNames, name)
|
||||
}
|
||||
sort.Strings(nodeNames)
|
||||
|
||||
for _, name := range nodeNames {
|
||||
nodeCfg := cfg.Nodes[name]
|
||||
if resp, ok := respondedNodes[name]; ok {
|
||||
tableData = append(tableData, []string{
|
||||
resp.Node,
|
||||
resp.Status,
|
||||
resp.Host,
|
||||
resp.Region,
|
||||
resp.AZ,
|
||||
formatUptime(resp.Uptime),
|
||||
fmt.Sprintf("%d", resp.VMCount),
|
||||
strings.Join(resp.Services, ","),
|
||||
})
|
||||
} else {
|
||||
tableData = append(tableData, []string{
|
||||
name,
|
||||
"NotReady",
|
||||
nodeCfg.Host,
|
||||
nodeCfg.Region,
|
||||
nodeCfg.AZ,
|
||||
"-",
|
||||
"-",
|
||||
strings.Join(nodeCfg.GetServices(), ","),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Render()
|
||||
}
|
||||
|
||||
func runGetVMs(cmd *cobra.Command, args []string) {
|
||||
_, nc, err := loadConfigAndConnect()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer nc.Close()
|
||||
|
||||
timeout, _ := cmd.Flags().GetDuration("timeout")
|
||||
responses, err := collectResponses(nc, "hive.node.vms", timeout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
type vmRow struct {
|
||||
config.VMInfo
|
||||
Node string
|
||||
Host string
|
||||
}
|
||||
|
||||
var allVMs []vmRow
|
||||
for _, data := range responses {
|
||||
var resp config.NodeVMsResponse
|
||||
if err := json.Unmarshal(data, &resp); err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range resp.VMs {
|
||||
allVMs = append(allVMs, vmRow{VMInfo: v, Node: resp.Node, Host: resp.Host})
|
||||
}
|
||||
}
|
||||
|
||||
if len(allVMs) == 0 {
|
||||
fmt.Println("No VMs found.")
|
||||
return
|
||||
}
|
||||
|
||||
// Sort by node then instance ID
|
||||
sort.Slice(allVMs, func(i, j int) bool {
|
||||
if allVMs[i].Node != allVMs[j].Node {
|
||||
return allVMs[i].Node < allVMs[j].Node
|
||||
}
|
||||
return allVMs[i].InstanceID < allVMs[j].InstanceID
|
||||
})
|
||||
|
||||
tableData := pterm.TableData{
|
||||
{"INSTANCE", "STATUS", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
|
||||
}
|
||||
|
||||
for _, v := range allVMs {
|
||||
age := "-"
|
||||
if v.LaunchTime > 0 {
|
||||
age = formatUptime(time.Now().Unix() - v.LaunchTime)
|
||||
}
|
||||
tableData = append(tableData, []string{
|
||||
v.InstanceID,
|
||||
v.Status,
|
||||
v.InstanceType,
|
||||
fmt.Sprintf("%d", v.VCPU),
|
||||
formatMemGB(v.MemoryGB),
|
||||
v.Node,
|
||||
v.Host,
|
||||
age,
|
||||
})
|
||||
}
|
||||
|
||||
pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Render()
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
"github.com/mulgadc/hive/hive/config"
|
||||
"github.com/mulgadc/hive/hive/service"
|
||||
"github.com/mulgadc/hive/hive/services/hiveui"
|
||||
"github.com/mulgadc/hive/hive/services/nats"
|
||||
"github.com/mulgadc/hive/hive/services/predastore"
|
||||
"github.com/mulgadc/hive/hive/services/viperblockd"
|
||||
@@ -61,6 +62,12 @@ var awsgwCmd = &cobra.Command{
|
||||
Short: "Manage the awsgw (AWS gateway) service",
|
||||
}
|
||||
|
||||
var hiveUICmd = &cobra.Command{
|
||||
Use: "hive-ui",
|
||||
Aliases: []string{"ui", "hiveui"},
|
||||
Short: "Manage the hive-ui service",
|
||||
}
|
||||
|
||||
var predastoreStartCmd = &cobra.Command{
|
||||
Use: "start",
|
||||
Short: "Start the predastore service",
|
||||
@@ -259,6 +266,12 @@ var viperblockStartCmd = &cobra.Command{
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Resolve sharded WAL setting: default false unless explicitly set to true
|
||||
shardWAL := false
|
||||
if nodeConfig.Viperblock.ShardWAL != nil {
|
||||
shardWAL = *nodeConfig.Viperblock.ShardWAL
|
||||
}
|
||||
|
||||
service, err := service.New("viperblock", &viperblockd.Config{
|
||||
NatsHost: nodeConfig.NATS.Host,
|
||||
PluginPath: pluginPath,
|
||||
@@ -268,6 +281,8 @@ var viperblockStartCmd = &cobra.Command{
|
||||
AccessKey: nodeConfig.Predastore.AccessKey,
|
||||
SecretKey: nodeConfig.Predastore.SecretKey,
|
||||
BaseDir: nodeConfig.Predastore.BaseDir,
|
||||
NodeName: clusterConfig.Node,
|
||||
ShardWAL: shardWAL,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -556,6 +571,73 @@ var awsgwStatusCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var hiveUIStartCmd = &cobra.Command{
|
||||
Use: "start",
|
||||
Short: "Start the hive-ui service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("Starting hive-ui service...")
|
||||
|
||||
port := viper.GetInt("hive-ui-port")
|
||||
host := viper.GetString("hive-ui-host")
|
||||
tlsCert := viper.GetString("hive-ui-tls-cert")
|
||||
tlsKey := viper.GetString("hive-ui-tls-key")
|
||||
|
||||
svc, err := service.New("hive-ui", &hiveui.Config{
|
||||
Port: port,
|
||||
Host: host,
|
||||
TLSCert: tlsCert,
|
||||
TLSKey: tlsKey,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
fmt.Println("Error starting hive-ui service:", err)
|
||||
return
|
||||
}
|
||||
|
||||
svc.Start()
|
||||
fmt.Println("hive-ui service started")
|
||||
},
|
||||
}
|
||||
|
||||
var hiveUIStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the hive-ui service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("Stopping hive-ui service...")
|
||||
|
||||
svc, err := service.New("hive-ui", &hiveui.Config{})
|
||||
|
||||
if err != nil {
|
||||
fmt.Println("Error stopping hive-ui service:", err)
|
||||
return
|
||||
}
|
||||
|
||||
svc.Stop()
|
||||
fmt.Println("hive-ui service stopped")
|
||||
},
|
||||
}
|
||||
|
||||
var hiveUIStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the hive-ui service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
svc, err := service.New("hive-ui", &hiveui.Config{})
|
||||
|
||||
if err != nil {
|
||||
fmt.Println("Error getting hive-ui service status:", err)
|
||||
return
|
||||
}
|
||||
|
||||
status, err := svc.Status()
|
||||
if err != nil {
|
||||
fmt.Println("Error getting hive-ui service status:", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("hive-ui service status:", status)
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
viper.SetEnvPrefix("HIVE") // Prefix for environment variables
|
||||
@@ -712,4 +794,27 @@ func init() {
|
||||
awsgwCmd.AddCommand(awsgwStopCmd)
|
||||
awsgwCmd.AddCommand(awsgwStatusCmd)
|
||||
|
||||
// hive-ui
|
||||
serviceCmd.AddCommand(hiveUICmd)
|
||||
|
||||
hiveUICmd.PersistentFlags().Int("port", 3000, "hive-ui server port")
|
||||
viper.BindEnv("hive-ui-port", "HIVE_UI_PORT")
|
||||
viper.BindPFlag("hive-ui-port", hiveUICmd.PersistentFlags().Lookup("port"))
|
||||
|
||||
hiveUICmd.PersistentFlags().String("host", "0.0.0.0", "hive-ui server host")
|
||||
viper.BindEnv("hive-ui-host", "HIVE_UI_HOST")
|
||||
viper.BindPFlag("hive-ui-host", hiveUICmd.PersistentFlags().Lookup("host"))
|
||||
|
||||
hiveUICmd.PersistentFlags().String("tls-cert", "", "TLS certificate path")
|
||||
viper.BindEnv("hive-ui-tls-cert", "HIVE_UI_TLS_CERT")
|
||||
viper.BindPFlag("hive-ui-tls-cert", hiveUICmd.PersistentFlags().Lookup("tls-cert"))
|
||||
|
||||
hiveUICmd.PersistentFlags().String("tls-key", "", "TLS key path")
|
||||
viper.BindEnv("hive-ui-tls-key", "HIVE_UI_TLS_KEY")
|
||||
viper.BindPFlag("hive-ui-tls-key", hiveUICmd.PersistentFlags().Lookup("tls-key"))
|
||||
|
||||
hiveUICmd.AddCommand(hiveUIStartCmd)
|
||||
hiveUICmd.AddCommand(hiveUIStopCmd)
|
||||
hiveUICmd.AddCommand(hiveUIStatusCmd)
|
||||
|
||||
}
|
||||
|
||||
@@ -5,8 +5,12 @@ node = "{{.Node}}"
|
||||
|
||||
[nodes.{{.Node}}]
|
||||
node = "{{.Node}}"
|
||||
host = "{{.BindIP}}"
|
||||
region = "{{.Region}}"
|
||||
az = "{{.Az}}"
|
||||
{{- if .Services}}
|
||||
services = [{{range $i, $s := .Services}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
|
||||
{{- end}}
|
||||
|
||||
account_id = "{{.AccountID}}"
|
||||
accesskey = "{{.AccessKey}}"
|
||||
@@ -32,9 +36,26 @@ token = "{{.NatsToken}}"
|
||||
|
||||
[nodes.{{.Node}}.nats.sub]
|
||||
|
||||
[nodes.{{.Node}}.viperblock]
|
||||
shardwal = false
|
||||
|
||||
[nodes.{{.Node}}.predastore]
|
||||
host = "{{.BindIP}}:8443"
|
||||
region = "{{.Region}}"
|
||||
bucket = "predastore"
|
||||
accesskey = "{{.AccessKey}}"
|
||||
secretkey = "{{.SecretKey}}"
|
||||
secretkey = "{{.SecretKey}}"
|
||||
{{- if gt .PredastoreNodeID 0}}
|
||||
node_id = {{.PredastoreNodeID}}
|
||||
{{- end}}
|
||||
{{- range .RemoteNodes}}
|
||||
|
||||
[nodes.{{.Name}}]
|
||||
node = "{{.Name}}"
|
||||
host = "{{.Host}}"
|
||||
region = "{{.Region}}"
|
||||
az = "{{.AZ}}"
|
||||
{{- if .Services}}
|
||||
services = [{{range $i, $s := .Services}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
@@ -11,7 +11,7 @@ listen: 0.0.0.0:4222
|
||||
|
||||
{{ if and .ClusterBindIP .ClusterRoutes }}
|
||||
cluster {
|
||||
name: C1
|
||||
name: {{ .ClusterName }}
|
||||
listen: {{.ClusterBindIP}}:4248
|
||||
|
||||
routes = [
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Predastore S3-compatible storage configuration file in TOML format.
|
||||
# Generated for multi-node cluster
|
||||
version = "1.0"
|
||||
region = "{{.Region}}"
|
||||
|
||||
host = "0.0.0.0"
|
||||
port = 8443
|
||||
|
||||
debug = false
|
||||
|
||||
[rs]
|
||||
data = 2
|
||||
parity = 1
|
||||
{{range .Nodes}}
|
||||
[[db]]
|
||||
id = {{.ID}}
|
||||
host = "{{.Host}}"
|
||||
port = 6660
|
||||
path = "distributed/db/node-{{.ID}}/"
|
||||
access_key_id = "{{$.AccessKey}}"
|
||||
secret_access_key = "{{$.SecretKey}}"
|
||||
{{- if eq .ID 1}}
|
||||
leader = true
|
||||
{{- end}}
|
||||
{{end}}
|
||||
{{- range .Nodes}}
|
||||
[[nodes]]
|
||||
id = {{.ID}}
|
||||
host = "{{.Host}}"
|
||||
port = 9991
|
||||
path = "distributed/nodes/node-{{.ID}}/"
|
||||
{{end}}
|
||||
[[buckets]]
|
||||
name = "predastore"
|
||||
region = "{{.Region}}"
|
||||
type = "distributed"
|
||||
public = false
|
||||
encryption = ""
|
||||
|
||||
[[auth]]
|
||||
access_key_id = "{{.AccessKey}}"
|
||||
secret_access_key = "{{.SecretKey}}"
|
||||
policy = [
|
||||
{ bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
|
||||
]
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
Copyright © 2025 Mulga Defense Corporation
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/hive/hive/config"
|
||||
"github.com/pterm/pterm"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var topCmd = &cobra.Command{
|
||||
Use: "top",
|
||||
Short: "Display cluster resource usage",
|
||||
Long: `Display resource usage (CPU, memory) for cluster nodes.`,
|
||||
}
|
||||
|
||||
var topNodesCmd = &cobra.Command{
|
||||
Use: "nodes",
|
||||
Short: "Display resource usage per node",
|
||||
Long: `Display CPU and memory usage per node, plus a summary of available instance types across the cluster.`,
|
||||
Run: runTopNodes,
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(topCmd)
|
||||
topCmd.AddCommand(topNodesCmd)
|
||||
|
||||
topCmd.PersistentFlags().Duration("timeout", 3*time.Second, "Timeout for collecting responses from nodes")
|
||||
}
|
||||
|
||||
func runTopNodes(cmd *cobra.Command, args []string) {
|
||||
cfg, nc, err := loadConfigAndConnect()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer nc.Close()
|
||||
|
||||
timeout, _ := cmd.Flags().GetDuration("timeout")
|
||||
responses, err := collectResponses(nc, "hive.node.status", timeout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
respondedNodes := make(map[string]config.NodeStatusResponse)
|
||||
for _, data := range responses {
|
||||
var resp config.NodeStatusResponse
|
||||
if err := json.Unmarshal(data, &resp); err != nil {
|
||||
continue
|
||||
}
|
||||
respondedNodes[resp.Node] = resp
|
||||
}
|
||||
|
||||
// Node resource table: union of config-known nodes + NATS responders
|
||||
nodeSet := make(map[string]struct{})
|
||||
for name := range cfg.Nodes {
|
||||
nodeSet[name] = struct{}{}
|
||||
}
|
||||
for name := range respondedNodes {
|
||||
nodeSet[name] = struct{}{}
|
||||
}
|
||||
nodeNames := make([]string, 0, len(nodeSet))
|
||||
for name := range nodeSet {
|
||||
nodeNames = append(nodeNames, name)
|
||||
}
|
||||
sort.Strings(nodeNames)
|
||||
|
||||
nodeTable := pterm.TableData{
|
||||
{"NAME", "CPU (used/total)", "MEM (used/total)", "VMs"},
|
||||
}
|
||||
|
||||
// Aggregate instance type capacity across all nodes
|
||||
capacityMap := make(map[string]*aggregatedCap)
|
||||
|
||||
for _, name := range nodeNames {
|
||||
if resp, ok := respondedNodes[name]; ok {
|
||||
nodeTable = append(nodeTable, []string{
|
||||
resp.Node,
|
||||
fmt.Sprintf("%d/%d", resp.AllocVCPU, resp.TotalVCPU),
|
||||
fmt.Sprintf("%s/%s", formatMemGB(resp.AllocMemGB), formatMemGB(resp.TotalMemGB)),
|
||||
fmt.Sprintf("%d", resp.VMCount),
|
||||
})
|
||||
|
||||
for _, cap := range resp.InstanceTypes {
|
||||
if agg, ok := capacityMap[cap.Name]; ok {
|
||||
agg.Available += cap.Available
|
||||
} else {
|
||||
capacityMap[cap.Name] = &aggregatedCap{
|
||||
VCPU: cap.VCPU,
|
||||
MemoryGB: cap.MemoryGB,
|
||||
Available: cap.Available,
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
nodeTable = append(nodeTable, []string{
|
||||
name,
|
||||
"-",
|
||||
"-",
|
||||
"-",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(nodeTable).Render()
|
||||
|
||||
// Instance type capacity summary
|
||||
if len(capacityMap) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
|
||||
capNames := make([]string, 0, len(capacityMap))
|
||||
for name := range capacityMap {
|
||||
capNames = append(capNames, name)
|
||||
}
|
||||
sort.Strings(capNames)
|
||||
|
||||
capTable := pterm.TableData{
|
||||
{"INSTANCE TYPE", "AVAILABLE", "VCPU", "MEMORY"},
|
||||
}
|
||||
|
||||
for _, name := range capNames {
|
||||
agg := capacityMap[name]
|
||||
capTable = append(capTable, []string{
|
||||
name,
|
||||
fmt.Sprintf("%d", agg.Available),
|
||||
fmt.Sprintf("%d", agg.VCPU),
|
||||
formatMemGB(agg.MemoryGB),
|
||||
})
|
||||
}
|
||||
|
||||
pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(capTable).Render()
|
||||
}
|
||||
|
||||
type aggregatedCap struct {
|
||||
VCPU int
|
||||
MemoryGB float64
|
||||
Available int
|
||||
}
|
||||
+1
-1
@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
# AWS Command Implementation Matrix
|
||||
|
||||
## EC2 - Instance Management
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `run-instances` | `--image-id`, `--instance-type`, `--count` (Min/MaxCount), `--key-name`, `--user-data`, `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination) | `--security-group-ids`, `--subnet-id`, `--placement`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--iam-instance-profile`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | `describe-images` (AMI must exist), `create-key-pair` (optional), VPC/SG (optional) | Gateway parses AWS query → NATS `ec2.runinstances` → daemon creates QEMU/KVM VM with viperblock-backed root volume via NBD → cloud-init injects user-data/keys → returns reservation with instance ID | 1. Launch with valid AMI and key pair<br>2. Launch with invalid AMI ID (error)<br>3. Launch with block device mappings (custom volume size)<br>4. Launch multiple instances (MinCount/MaxCount)<br>5. DryRun returns validation-only<br>6. Invalid instance type returns error | **DONE** |
|
||||
| `describe-instances` | `--instance-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes | **DONE** |
|
||||
| `start-instances` | `--instance-ids` | `--dry-run`, `--force` | `run-instances` (instance must exist in stopped state) | Gateway sends NATS `ec2.cmd.{instance-id}` → daemon restarts stopped QEMU process with same config → state transitions stopped→pending→running | 1. Start a stopped instance<br>2. Start already-running instance (error: IncorrectInstanceState)<br>3. Start with invalid instance ID<br>4. Verify volumes re-mount on start | **DONE** |
|
||||
| `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_powerdown` for graceful shutdown → monitors heartbeat until QEMU exits → state transitions running→stopping→stopped | 1. Graceful stop of running instance<br>2. Force stop (kills QEMU process)<br>3. Stop already-stopped instance (error)<br>4. Verify ~30s heartbeat detection | **DONE** |
|
||||
| `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID | **DONE** |
|
||||
| `reboot-instances` | — | `--instance-ids`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_reset` → instance reboots without stopping | 1. Reboot running instance<br>2. Reboot stopped instance (error)<br>3. Verify instance stays in running state after reboot | **NOT STARTED** (parser test exists) |
|
||||
| `describe-instance-types` | `--filters` (capacity filter only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, all other filters | None | Gateway fans out NATS `ec2.DescribeInstanceTypes` to all nodes → each daemon reports supported types (t3.micro/small/medium/large) with vCPU/memory specs → gateway deduplicates and returns | 1. List all instance types<br>2. Filter by specific type<br>3. Filter with `capacity=true` shows available slots<br>4. Verify vCPU/memory specs match hardware | **DONE** |
|
||||
| `modify-instance-attribute` | — | `--instance-id`, `--instance-type`, `--user-data`, `--disable-api-termination`, `--ebs-optimized` | Instance must be stopped | Validate instance stopped → update instance metadata in NATS KV store → changes applied on next start | 1. Change instance type while stopped<br>2. Modify running instance (error)<br>3. Toggle disable-api-termination | **NOT STARTED** |
|
||||
| `get-console-output` | — | `--instance-id`, `--latest` | Instance must exist | QMP command to read serial console ring buffer → return base64-encoded output with timestamp | 1. Get output from running instance<br>2. Get output from stopped instance<br>3. Verify cloud-init boot logs visible | **NOT STARTED** |
|
||||
| `monitor-instances` | — | `--instance-ids` | Instance must exist | Enable basic monitoring (CPU, disk, network) → store metrics in NATS KV → return monitoring state | 1. Enable monitoring<br>2. Verify monitoring state in describe-instances | **NOT STARTED** |
|
||||
| `unmonitor-instances` | — | `--instance-ids` | Instance must exist | Disable detailed monitoring → revert to basic monitoring → return monitoring state | 1. Disable monitoring<br>2. Verify monitoring state reverts in describe-instances | **NOT STARTED** |
|
||||
|
||||
## EC2 - Key Pair Management
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519) | `--key-format` (pem/ppk), `--tag-specifications`, `--dry-run` | None | NATS `ec2.CreateKeyPair` → daemon generates SSH keypair → stores public key in Predastore S3 (`/bucket/ec2/{name}.pub`) → returns private key material (one-time) | 1. Create RSA key pair<br>2. Create ED25519 key pair<br>3. Duplicate key name (error: InvalidKeyPair.Duplicate)<br>4. Verify key material returned only on creation | **DONE** |
|
||||
| `describe-key-pairs` | `--key-names`, `--key-pair-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty | **DONE** |
|
||||
| `delete-key-pair` | `--key-name`, `--key-pair-id` | `--dry-run` | Key must exist | NATS `ec2.DeleteKeyPair` → daemon deletes public key from Predastore S3 → returns success | 1. Delete existing key pair<br>2. Delete non-existent key (idempotent, no error)<br>3. Verify key no longer in describe-key-pairs | **DONE** |
|
||||
| `import-key-pair` | `--key-name`, `--public-key-material` | `--tag-specifications`, `--dry-run` | None | NATS `ec2.ImportKeyPair` → daemon stores provided public key in Predastore S3 → returns key name and fingerprint | 1. Import valid SSH public key<br>2. Import invalid key material (error)<br>3. Import duplicate name (error)<br>4. Verify imported key usable with run-instances | **DONE** |
|
||||
|
||||
## EC2 - AMI Image Management
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `describe-images` | `--image-ids` (format validation only) | `--owners`, `--executable-users`, `--filters`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName) | **DONE** |
|
||||
| `create-image` | — | `--instance-id`, `--name`, `--description`, `--no-reboot`, `--block-device-mappings`, `--tag-specifications` | Instance must exist | Stop instance (unless --no-reboot) → snapshot root volume via viperblock → create AMI metadata in S3 → register as new AMI → restart instance → return ami-ID | 1. Create image from running instance<br>2. Create with --no-reboot<br>3. Invalid instance ID (error)<br>4. Verify new AMI appears in describe-images<br>5. Launch new instance from created AMI | **NOT STARTED** |
|
||||
| `register-image` | — | `--name`, `--description`, `--architecture`, `--root-device-name`, `--virtualization-type`, `--block-device-mappings`, `--boot-mode`, `--ena-support`, `--sriov-net-support`, `--tpm-support`, `--imds-support` | Image data must exist in S3 | Create AMI metadata entry in Predastore → assign ami-ID → set state to available → return ami-ID | 1. Register with valid metadata<br>2. Missing required name (error)<br>3. Verify registered image in describe-images | **NOT STARTED** |
|
||||
| `deregister-image` | — | `--image-id`, `--dry-run` | AMI must exist, no instances using it | Mark AMI metadata as deregistered in S3 → AMI no longer available for new launches → existing instances unaffected | 1. Deregister existing AMI<br>2. Deregister non-existent AMI (error)<br>3. Verify deregistered AMI not in describe-images<br>4. Existing instances from AMI still run | **NOT STARTED** |
|
||||
| `copy-image` | — | `--source-image-id`, `--source-region`, `--name`, `--description`, `--encrypted`, `--kms-key-id`, `--client-token`, `--copy-image-tags`, `--tag-specifications` | Source AMI must exist | Copy AMI data between regions/nodes in Predastore → create new AMI metadata with new ami-ID → return new ami-ID | 1. Copy image within same region<br>2. Copy non-existent image (error)<br>3. Verify copied image independent of source | **NOT STARTED** |
|
||||
| `import-image` | — | `--disk-containers` (Format, Url/S3Bucket+S3Key), `--description`, `--architecture`, `--platform` | S3 bucket with disk image | Download disk image from S3 → convert format (VMDK/VHD/RAW→QCOW2) → create viperblock volume → register as AMI | 1. Import QCOW2 image<br>2. Import RAW image<br>3. Invalid format (error)<br>4. Verify imported image launchable | **NOT STARTED** |
|
||||
| `describe-image-attribute` | — | `--image-id`, `--attribute`, `--dry-run` | AMI must exist | Read specific attribute from AMI metadata in S3 → return attribute value | 1. Get description attribute<br>2. Get launch permission<br>3. Invalid attribute name (error) | **NOT STARTED** |
|
||||
| `modify-image-attribute` | — | `--image-id`, `--launch-permission`, `--description`, `--operation-type`, `--user-ids`, `--user-groups`, `--organization-arns`, `--dry-run` | AMI must exist | Update specific attribute in AMI metadata → persist to S3 | 1. Add launch permission<br>2. Modify description<br>3. Invalid AMI (error) | **NOT STARTED** |
|
||||
| `reset-image-attribute` | — | `--image-id`, `--attribute`, `--dry-run` | AMI must exist | Reset attribute to default value in AMI metadata | 1. Reset launch permission to owner-only<br>2. Invalid AMI (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - Volume (EBS) Management
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata) | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value | **DONE** |
|
||||
| `modify-volume` | `--volume-id`, `--size`, `--volume-type`, `--iops` | `--throughput`, `--dry-run`, `--multi-attach-enabled` | Volume must exist | NATS `ec2.ModifyVolume` → daemon sends resize request to viperblock → NBD does not support live resize, instance must be stopped → returns modification state | 1. Increase volume size<br>2. Modify volume type<br>3. Decrease size (error - not supported)<br>4. Modify attached volume (requires stop/start) | **DONE** |
|
||||
| `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3 only) | `--iops`, `--encrypted`, `--snapshot-id`, `--tag-specifications` | Valid AZ configured via `hive init` | Gateway validates input → NATS `ec2.CreateVolume` → daemon generates vol-ID via viperblock → creates empty volume of specified size → persists config.json to Predastore S3 → returns vol-ID with state=available | 1. Create 80GB gp3 volume<br>2. Boundary sizes (1 GiB min, 16384 GiB max)<br>3. Invalid AZ (error)<br>4. Verify volume in describe-volumes<br>5. Unsupported volume type (error - only gp3)<br>6. Size out of range (error) | **DONE** |
|
||||
| `delete-volume` | `--volume-id` | `--dry-run` | Volume must exist and be detached (state=available) | Gateway validates vol- prefix → NATS `ec2.DeleteVolume` → daemon confirms state=available and no AttachedInstance → NATS `ebs.delete` to viperblockd (stops nbdkit/WAL) → deletes S3 objects under vol-id/, vol-id-efi/, vol-id-cloudinit/ → returns success | 1. Delete detached volume<br>2. Delete attached volume (error: VolumeInUse)<br>3. Delete non-existent volume (error: InvalidVolume.NotFound)<br>4. Verify volume gone from describe-volumes<br>5. Malformed volume ID (error: InvalidVolumeID.Malformed)<br>6. Double delete (idempotent NotFound) | **DONE** |
|
||||
| `attach-volume` | `--volume-id`, `--instance-id`, `--device` (optional, auto-assigns `/dev/sd[f-p]`) | `--dry-run` | Volume must exist (available), instance must exist (running) | Gateway sends to `ec2.cmd.{instanceId}` → daemon validates volume (Predastore) → `ebs.mount` via NATS (viperblock starts NBD server) → QMP `blockdev-add` (nbd-{volId}) → QMP `device_add` (virtio-blk-pci, vdisk-{volId}) → three-phase rollback on failure → update EBSRequests + BlockDeviceMappings → persist to JetStream + Predastore → respond with VolumeAttachment | 1. Attach volume to running instance<br>2. Auto-assign device name<br>3. Attach already-attached volume (VolumeInUse)<br>4. Attach to non-existent instance (InvalidInstanceID.NotFound)<br>5. Attach to stopped instance (IncorrectInstanceState)<br>6. Volume not found (InvalidVolume.NotFound)<br>7. All device slots full (AttachmentLimitExceeded)<br>8. Volume persists across stop/start | **DONE** |
|
||||
| `detach-volume` | `--volume-id`, `--instance-id` (optional, resolved via DescribeVolumes), `--device` (optional cross-check), `--force` | `--dry-run` | Volume must be attached, instance must be running | Gateway resolves InstanceId if omitted (via DescribeVolumes) → sends to `ec2.cmd.{instanceId}` → daemon validates (running, attached, not boot/EFI/CloudInit, device match) → three-phase hot-unplug: QMP `device_del` (force continues on failure) → QMP `blockdev-del` (abort if fails, preserves state to prevent double-attach) → `ebs.unmount` via NATS (best-effort) → remove from EBSRequests + BlockDeviceMappings → update volume metadata to available → persist state → respond with VolumeAttachment (state=detaching) | 1. Detach with explicit InstanceId<br>2. Detach without InstanceId (gateway resolution)<br>3. Detach with correct --device cross-check<br>4. Missing VolumeId (InvalidParameterValue)<br>5. Volume not attached (IncorrectState)<br>6. Nonexistent volume (InvalidVolume.NotFound)<br>7. Nonexistent instance (InvalidInstanceID.NotFound)<br>8. Instance not running (IncorrectInstanceState)<br>9. Device mismatch (InvalidParameterValue)<br>10. Boot volume protection (OperationNotPermitted)<br>11. Force flag (continues past device_del failure)<br>12. Volume reusability (re-attach after detach) | **DONE** |
|
||||
| `describe-volume-status` | `--volume-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes | **DONE** |
|
||||
| `describe-volumes-modifications` | — | `--volume-ids`, `--filters`, `--max-results` | None | Query pending/completed volume modifications → return modification state, progress, original/target size | 1. Check in-progress modification<br>2. Check completed modification<br>3. No modifications returns empty | **NOT STARTED** |
|
||||
|
||||
## EC2 - Snapshot Management
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-snapshot` | `--volume-id`, `--description`, `--tag-specifications` | `--dry-run` | Volume must exist in Predastore | Gateway validates vol- prefix → NATS `ec2.CreateSnapshot` → daemon reads VolumeConfig from Predastore → generates snap-ID via viperblock → stores SnapshotConfig (metadata-only, points to source volume) as completed → returns Snapshot with state=completed, progress=100% | 1. Snapshot from valid volume<br>2. Missing volume ID (InvalidParameterValue)<br>3. Volume not found (InvalidVolume.NotFound)<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Snapshot with description<br>6. Snapshot with tag specifications | **DONE** |
|
||||
| `create-snapshots` | — | `--instance-specification`, `--description`, `--tag-specifications` | Instance must exist, instance-volume attachment tracking | Create snapshots of all volumes attached to instance → return list of snapshot IDs. Blocked: requires instance-volume attachment tracking to discover which volumes to snapshot. | 1. Snapshot all volumes on instance<br>2. Instance with no volumes | **NOT STARTED** |
|
||||
| `delete-snapshot` | `--snapshot-id` | `--dry-run` | Snapshot must exist | Gateway validates snap- prefix → NATS `ec2.DeleteSnapshot` → daemon verifies snapshot exists in Predastore → lists and deletes all objects under snapshot prefix → returns success | 1. Delete existing snapshot<br>2. Delete non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing snapshot ID (InvalidParameterValue)<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
|
||||
| `describe-snapshots` | `--snapshot-ids` | `--owner-ids`, `--filters`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → filters by snapshot ID if specified → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
|
||||
| `copy-snapshot` | `--source-snapshot-id`, `--source-region`, `--description` | `--encrypted`, `--dry-run` | Source snapshot must exist | Gateway validates snap- prefix + source region → NATS `ec2.CopySnapshot` → daemon reads source SnapshotConfig → generates new snap-ID → copies metadata (preserves tags, description override) → stores as completed → returns new snapshot ID | 1. Copy within same region<br>2. Copy non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing source ID (InvalidParameterValue)<br>4. Missing source region (MissingParameter)<br>5. Copy preserves tags<br>6. Copy with description override | **DONE** |
|
||||
|
||||
## EC2 - Tags
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-tags` | `--resources`, `--tags` (Key=,Value=) | `--dry-run` | None | Gateway validates input (resources non-empty, tags non-empty, keys non-empty) → NATS `ec2.CreateTags` → daemon merges tags into per-resource JSON in Predastore S3 (`tags/{resourceId}.json`) → return success | 1. Tag an instance<br>2. Tag multiple resources at once<br>3. Overwrite existing tag value<br>4. Empty resources list (error: MissingParameter)<br>5. Empty tags list (error: MissingParameter)<br>6. Empty tag key (error: InvalidParameterValue) | **DONE** |
|
||||
| `delete-tags` | `--resources`, `--tags` | `--dry-run` | None | Gateway validates input (resources non-empty) → NATS `ec2.DeleteTags` → daemon removes specified tag keys from resource JSON, or all tags if no keys specified → return success | 1. Delete specific tag by key<br>2. Delete all tags (no tags specified)<br>3. Empty resources list (error: MissingParameter) | **DONE** |
|
||||
| `describe-tags` | `--filters` (resource-id, resource-type, key, value) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeTags` → daemon lists all `tags/*.json` from Predastore S3 → applies filters (resource-id, resource-type, key, value) → returns tag list with resource type auto-detected from ID prefix | 1. List all tags<br>2. Filter by resource-id<br>3. Filter by resource-type (instance, volume, image, snapshot, vpc, subnet, security-group, route-table, internet-gateway, egress-only-internet-gateway)<br>4. Filter by key<br>5. Filter by value<br>6. Empty result when no tags exist | **DONE** |
|
||||
|
||||
## EC2 - Regions & Availability Zones
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `describe-regions` | *(returns configured region, endpoint, opt-in status from config — input params ignored)* | `--region-names`, `--filters`, `--all-regions`, `--dry-run` | None | Return configured region from hive init config with Endpoint and OptInStatus → local-only response, no NATS | 1. List all regions<br>2. Filter by region name<br>3. Verify endpoint URL returned<br>4. Verify OptInStatus returned | **DONE** |
|
||||
| `describe-availability-zones` | *(returns configured AZ, region, zone ID, state, opt-in status from config — input params ignored)* | `--zone-names`, `--filters`, `--all-availability-zones` | None | Return configured AZ from hive init config with zone ID, state, group name, network border group → local-only response, no NATS | 1. List all AZs<br>2. Filter by zone name<br>3. Verify zone state is available<br>4. Verify region name matches config | **DONE** |
|
||||
|
||||
## EC2 - Account Attributes
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `describe-account-attributes` | `--attribute-names` | `--dry-run` | None | Gateway parses input → returns static account attributes: supported-platforms=VPC, default-vpc=none, max-instances=100, vpc-max-security-groups-per-interface=5, max-elastic-ips=5, vpc-max-elastic-ips=20 → local-only response, no NATS | 1. List all account attributes<br>2. Filter by attribute name<br>3. Verify all 6 attributes returned with correct values | **DONE** |
|
||||
|
||||
## EC2 - Account Settings (Deferred)
|
||||
|
||||
Persistence layer works (NATS JetStream KV), but stored values are not yet enforced by downstream services. Will be completed post-merge.
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `enable-ebs-encryption-by-default` | — | `--dry-run` | None | Store encryption default flag in JetStream KV → return `EbsEncryptionByDefault=true` | 1. Enable encryption<br>2. Verify via get | **NOT STARTED** (enforcement pending) |
|
||||
| `disable-ebs-encryption-by-default` | — | `--dry-run` | None | Clear encryption default flag in JetStream KV → return `EbsEncryptionByDefault=false` | 1. Disable encryption<br>2. Verify via get | **NOT STARTED** (enforcement pending) |
|
||||
| `get-ebs-encryption-by-default` | — | `--dry-run` | None | Read encryption default flag from JetStream KV → return current state | 1. Get default state<br>2. Verify matches last enable/disable | **NOT STARTED** (enforcement pending) |
|
||||
| `enable-serial-console-access` | — | `--dry-run` | None | Store serial console access flag in JetStream KV → return enabled=true | 1. Enable access<br>2. Verify via get | **NOT STARTED** (enforcement pending) |
|
||||
| `disable-serial-console-access` | — | `--dry-run` | None | Clear serial console access flag in JetStream KV → return enabled=false | 1. Disable access | **NOT STARTED** (enforcement pending) |
|
||||
| `get-serial-console-access-status` | — | `--dry-run` | None | Read serial console access flag from JetStream KV | 1. Get current status | **NOT STARTED** (enforcement pending) |
|
||||
| `enable-snapshot-block-public-access` | — | `--state` | None | Store snapshot block public access state in JetStream KV | 1. Enable with block-all-sharing<br>2. Enable with block-new-sharing | **NOT STARTED** (enforcement pending) |
|
||||
| `disable-snapshot-block-public-access` | — | `--dry-run` | None | Clear snapshot block public access state in JetStream KV | 1. Disable access | **NOT STARTED** (enforcement pending) |
|
||||
| `get-snapshot-block-public-access-state` | — | `--dry-run` | None | Read snapshot block public access state from JetStream KV | 1. Get current state | **NOT STARTED** (enforcement pending) |
|
||||
| `enable-image-block-public-access` | — | `--image-block-public-access-state` | None | Store image block public access state in JetStream KV | 1. Enable with block-new-sharing | **NOT STARTED** (enforcement pending) |
|
||||
| `disable-image-block-public-access` | — | `--dry-run` | None | Clear image block public access state in JetStream KV | 1. Disable access | **NOT STARTED** (enforcement pending) |
|
||||
| `get-image-block-public-access-state` | — | `--dry-run` | None | Read image block public access state from JetStream KV | 1. Get current state | **NOT STARTED** (enforcement pending) |
|
||||
| `modify-instance-metadata-defaults` | — | `--http-tokens`, `--http-put-response-hop-limit`, `--http-endpoint`, `--instance-metadata-tags` | None | Store instance metadata defaults in JetStream KV | 1. Set httpTokens=required<br>2. Set hop limit | **NOT STARTED** (enforcement pending) |
|
||||
| `get-instance-metadata-defaults` | — | `--dry-run` | None | Read instance metadata defaults from JetStream KV | 1. Get current defaults | **NOT STARTED** (enforcement pending) |
|
||||
|
||||
## EC2 - Egress-Only Internet Gateway
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-egress-only-internet-gateway` | — | `--vpc-id`, `--client-token`, `--tag-specifications`, `--dry-run` | VPC must exist | NATS `ec2.CreateEgressOnlyInternetGateway` → daemon generates eigw-ID → creates record in NATS JetStream KV with state=attached → returns EgressOnlyInternetGateway with attachment | 1. Create with valid VpcId<br>2. Missing VpcId (error: MissingParameter)<br>3. Create with inline tags<br>4. Verify ID format eigw-{16hex} | **NOT STARTED** |
|
||||
| `delete-egress-only-internet-gateway` | — | `--egress-only-internet-gateway-id`, `--dry-run` | EIGW must exist | NATS `ec2.DeleteEgressOnlyInternetGateway` → daemon verifies EIGW exists in KV → deletes from KV → returns ReturnCode=true | 1. Delete existing EIGW<br>2. Delete non-existent EIGW (error)<br>3. Missing ID (error: MissingParameter) | **NOT STARTED** |
|
||||
| `describe-egress-only-internet-gateways` | — | `--egress-only-internet-gateway-ids`, `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from KV bucket → filters by ID if specified → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list | **NOT STARTED** |
|
||||
|
||||
## EC2 - Placement Groups
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-placement-group` | — | `--group-name`, `--strategy` (cluster/spread/partition), `--partition-count`, `--spread-level`, `--tag-specifications` | None | NATS `ec2.CreatePlacementGroup` → daemon stores placement group in JetStream KV with strategy, state=available → return group metadata | 1. Create with cluster strategy<br>2. Create with spread strategy<br>3. Create with partition strategy + partition count<br>4. Duplicate name (error)<br>5. Invalid strategy (error) | **NOT STARTED** |
|
||||
| `delete-placement-group` | — | `--group-name`, `--dry-run` | Group must exist, no instances in group | NATS `ec2.DeletePlacementGroup` → daemon verifies group exists and is empty → delete from KV → return success | 1. Delete empty group<br>2. Delete group with instances (error)<br>3. Delete non-existent group (error) | **NOT STARTED** |
|
||||
| `describe-placement-groups` | — | `--group-names`, `--group-ids`, `--filters` | None | NATS `ec2.DescribePlacementGroups` → daemon lists groups from KV → return list with strategy and state | 1. List all groups<br>2. Filter by name<br>3. Filter by strategy | **NOT STARTED** |
|
||||
|
||||
## EC2 - Dedicated Hosts
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `allocate-hosts` | — | `--availability-zone`, `--instance-type`, `--quantity`, `--auto-placement`, `--tag-specifications` | None | NATS `ec2.AllocateHosts` → daemon allocates host resources → stores in JetStream KV with h-ID → return host ID list | 1. Allocate single host<br>2. Allocate multiple hosts<br>3. Missing AZ or instance type (error) | **NOT STARTED** |
|
||||
| `describe-hosts` | — | `--host-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeHosts` → daemon lists hosts from KV → return host list with capacity and instance info | 1. List all hosts<br>2. Filter by host ID<br>3. Filter by instance type | **NOT STARTED** |
|
||||
| `release-hosts` | — | `--host-ids` | Host must exist, no running instances | NATS `ec2.ReleaseHosts` → daemon verifies no instances on host → delete from KV → return success/failure per host | 1. Release empty host<br>2. Release host with instances (error)<br>3. Release non-existent host (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - IPv4 Pools
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-public-ipv4-pool` | — | `--tag-specifications`, `--dry-run` | None | NATS `ec2.CreatePublicIpv4Pool` → daemon creates pool in JetStream KV with ipv4pool-ID → return pool ID | 1. Create pool<br>2. Verify in describe | **NOT STARTED** |
|
||||
| `delete-public-ipv4-pool` | — | `--pool-id`, `--dry-run` | Pool must exist and be empty | NATS `ec2.DeletePublicIpv4Pool` → daemon verifies pool empty → delete from KV → return success | 1. Delete empty pool<br>2. Delete pool with addresses (error) | **NOT STARTED** |
|
||||
| `describe-public-ipv4-pools` | — | `--pool-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribePublicIpv4Pools` → daemon lists pools from KV → return pool list with address counts | 1. List all pools<br>2. Filter by pool ID | **NOT STARTED** |
|
||||
|
||||
## EC2 - DHCP Options
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-dhcp-options` | — | `--dhcp-configurations` (domain-name, domain-name-servers, ntp-servers, netbios-name-servers, netbios-node-type), `--tag-specifications` | None | NATS `ec2.CreateDhcpOptions` → daemon stores DHCP option set in JetStream KV with dopt-ID → return DHCP options with configurations | 1. Create with domain-name + DNS servers<br>2. Create with NTP servers<br>3. Verify in describe | **NOT STARTED** |
|
||||
| `delete-dhcp-options` | — | `--dhcp-options-id`, `--dry-run` | DHCP options must exist and not be associated with a VPC | NATS `ec2.DeleteDhcpOptions` → daemon verifies not associated → delete from KV → return success | 1. Delete unassociated options<br>2. Delete associated options (error)<br>3. Delete non-existent (error) | **NOT STARTED** |
|
||||
| `describe-dhcp-options` | — | `--dhcp-options-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeDhcpOptions` → daemon lists DHCP options from KV → return list with configurations | 1. List all DHCP options<br>2. Filter by ID | **NOT STARTED** |
|
||||
| `associate-dhcp-options` | — | `--dhcp-options-id`, `--vpc-id`, `--dry-run` | DHCP options and VPC must exist | NATS `ec2.AssociateDhcpOptions` → daemon links DHCP options to VPC → update VPC metadata → return success | 1. Associate with VPC<br>2. Missing DHCP options ID (error)<br>3. Missing VPC ID (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - Capacity Reservations
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-capacity-reservation` | — | `--instance-type`, `--instance-platform`, `--availability-zone`, `--instance-count`, `--end-date`, `--end-date-type`, `--instance-match-criteria`, `--tag-specifications` | None | NATS `ec2.CreateCapacityReservation` → daemon stores reservation in JetStream KV with cr-ID → return reservation with state=active | 1. Create with required fields<br>2. Missing instance type (error)<br>3. Verify in describe | **NOT STARTED** |
|
||||
| `cancel-capacity-reservation` | — | `--capacity-reservation-id`, `--dry-run` | Reservation must exist | NATS `ec2.CancelCapacityReservation` → daemon marks reservation as cancelled in KV → return success | 1. Cancel active reservation<br>2. Cancel non-existent (error) | **NOT STARTED** |
|
||||
| `describe-capacity-reservations` | — | `--capacity-reservation-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeCapacityReservations` → daemon lists reservations from KV → return list with state and counts | 1. List all reservations<br>2. Filter by ID<br>3. Filter by instance type | **NOT STARTED** |
|
||||
| `modify-capacity-reservation` | — | `--capacity-reservation-id`, `--instance-count`, `--end-date`, `--end-date-type` | Reservation must exist | NATS `ec2.ModifyCapacityReservation` → daemon updates reservation in KV → return success | 1. Modify instance count<br>2. Modify end date<br>3. Missing reservation ID (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - Elastic IP
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `allocate-address` | — | `--domain` (vpc), `--tag-specifications`, `--dry-run` | None | NATS `ec2.AllocateAddress` → daemon allocates IP from pool → stores in JetStream KV with eipalloc-ID → return AllocationId, PublicIp, Domain | 1. Allocate EIP<br>2. Verify in describe-addresses<br>3. Allocate without --domain uses default | **NOT STARTED** |
|
||||
| `release-address` | — | `--allocation-id`, `--dry-run` | EIP must exist and be disassociated | NATS `ec2.ReleaseAddress` → daemon verifies not associated → release IP back to pool → delete from KV → return success | 1. Release disassociated EIP<br>2. Release associated EIP (error)<br>3. Release non-existent (error) | **NOT STARTED** |
|
||||
| `associate-address` | — | `--allocation-id`, `--instance-id` or `--network-interface-id`, `--dry-run` | EIP and target must exist | NATS `ec2.AssociateAddress` → daemon links EIP to instance/ENI → configure OVS DNAT/SNAT rules → return eipassoc-ID | 1. Associate with instance<br>2. Re-associate (moves EIP)<br>3. Associate non-existent EIP (error) | **NOT STARTED** |
|
||||
| `disassociate-address` | — | `--association-id`, `--dry-run` | Association must exist | NATS `ec2.DisassociateAddress` → daemon unlinks EIP from instance/ENI → remove OVS DNAT/SNAT rules → return success | 1. Disassociate existing<br>2. Disassociate non-existent (error) | **NOT STARTED** |
|
||||
| `describe-addresses` | — | `--allocation-ids`, `--filters`, `--public-ips` | None | NATS `ec2.DescribeAddresses` → daemon lists EIPs from KV → return list with association info | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP | **NOT STARTED** |
|
||||
|
||||
## EC2 - NAT Gateway
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-nat-gateway` | — | `--subnet-id`, `--allocation-id`, `--connectivity-type`, `--tag-specifications` | Subnet must exist, EIP must be allocated (for public type) | NATS `ec2.CreateNatGateway` → daemon creates NAT gateway in JetStream KV with nat-ID → configure OVS SNAT rules → state=pending→available → return NatGateway | 1. Create public NAT gateway<br>2. Create private NAT gateway (no allocation-id)<br>3. Invalid subnet (error) | **NOT STARTED** |
|
||||
| `delete-nat-gateway` | — | `--nat-gateway-id`, `--dry-run` | NAT gateway must exist | NATS `ec2.DeleteNatGateway` → daemon marks as deleting → remove OVS SNAT rules → delete from KV → return NatGatewayId | 1. Delete existing NAT gateway<br>2. Delete non-existent (error) | **NOT STARTED** |
|
||||
| `describe-nat-gateways` | — | `--nat-gateway-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNatGateways` → daemon lists NAT gateways from KV → return list with state, subnet, addresses | 1. List all NAT gateways<br>2. Filter by ID<br>3. Filter by VPC ID | **NOT STARTED** |
|
||||
| `assign-private-nat-gateway-address` | — | `--nat-gateway-id`, `--private-ip-addresses` | NAT gateway must exist (private type) | NATS `ec2.AssignPrivateNatGatewayAddress` → daemon assigns private IPs to NAT gateway → return assigned addresses | 1. Assign address<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
|
||||
| `associate-nat-gateway-address` | — | `--nat-gateway-id`, `--allocation-ids` | NAT gateway and EIPs must exist | NATS `ec2.AssociateNatGatewayAddress` → daemon associates EIPs with NAT gateway → return association info | 1. Associate EIP<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - VPC Networking (Core)
|
||||
|
||||
Requires Open vSwitch (`apt install openvswitch-switch openvswitch-common`). Single AZ for Hive v1.
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-vpc` | — | `--cidr-block`, `--tag-specifications`, `--instance-tenancy` | None | Create OVS bridge for VPC → assign vpc-ID → store VPC metadata in NATS KV → configure CIDR range → return VPC with state=available | 1. Create VPC with /16 CIDR<br>2. Overlapping CIDR (error)<br>3. Invalid CIDR format (error)<br>4. Verify in describe-vpcs | **NOT STARTED** |
|
||||
| `delete-vpc` | — | `--vpc-id` | VPC must be empty (no subnets, gateways, instances) | Verify no dependent resources → delete OVS bridge → remove metadata → return success | 1. Delete empty VPC<br>2. Delete VPC with subnets (error: DependencyViolation)<br>3. Delete non-existent VPC (error) | **NOT STARTED** |
|
||||
| `describe-vpcs` | — | `--vpc-ids`, `--filters`, `--max-results` | None | Query NATS KV for VPC metadata → return VPC list with CIDR, state, tags | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Filter by CIDR block | **NOT STARTED** |
|
||||
| `modify-vpc-attribute` | — | `--vpc-id`, `--enable-dns-support`, `--enable-dns-hostnames` | VPC must exist | NATS `ec2.ModifyVpcAttribute` → daemon updates VPC attribute in KV → return success | 1. Enable DNS support<br>2. Enable DNS hostnames<br>3. Missing VPC ID (error) | **NOT STARTED** |
|
||||
| `associate-vpc-cidr-block` | — | `--vpc-id`, `--cidr-block` | VPC must exist | NATS `ec2.AssociateVpcCidrBlock` → daemon adds secondary CIDR to VPC metadata → return association | 1. Add secondary CIDR<br>2. Overlapping CIDR (error)<br>3. Max CIDR blocks exceeded (error) | **NOT STARTED** |
|
||||
| `disassociate-vpc-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateVpcCidrBlock` → daemon removes secondary CIDR from VPC → return success | 1. Remove secondary CIDR<br>2. Remove primary CIDR (error) | **NOT STARTED** |
|
||||
| `create-subnet` | — | `--vpc-id`, `--cidr-block`, `--availability-zone`, `--tag-specifications` | VPC must exist | Validate CIDR within VPC range → create OVS port group with VLAN tag → assign subnet-ID → store metadata | 1. Create subnet within VPC CIDR<br>2. Subnet CIDR outside VPC range (error)<br>3. Overlapping subnet CIDRs (error) | **NOT STARTED** |
|
||||
| `delete-subnet` | — | `--subnet-id` | Subnet must be empty (no instances) | Verify no instances in subnet → remove OVS port group → delete metadata | 1. Delete empty subnet<br>2. Delete subnet with instances (error) | **NOT STARTED** |
|
||||
| `describe-subnets` | — | `--subnet-ids`, `--filters`, `--max-results` | None | Query NATS KV for subnet metadata → return subnet list | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by AZ | **NOT STARTED** |
|
||||
| `modify-subnet-attribute` | — | `--subnet-id`, `--map-public-ip-on-launch`, `--assign-ipv6-address-on-creation` | Subnet must exist | NATS `ec2.ModifySubnetAttribute` → daemon updates subnet attribute in KV → return success | 1. Enable auto-assign public IP<br>2. Missing subnet ID (error) | **NOT STARTED** |
|
||||
| `associate-subnet-cidr-block` | — | `--subnet-id`, `--ipv6-cidr-block` | Subnet must exist | NATS `ec2.AssociateSubnetCidrBlock` → daemon adds IPv6 CIDR to subnet → return association | 1. Add IPv6 CIDR<br>2. Missing subnet ID (error) | **NOT STARTED** |
|
||||
| `disassociate-subnet-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateSubnetCidrBlock` → daemon removes IPv6 CIDR from subnet → return success | 1. Remove IPv6 CIDR<br>2. Invalid association (error) | **NOT STARTED** |
|
||||
| `create-security-group` | — | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | VPC must exist | Create security group metadata → create default OVS flow rules (deny all inbound, allow all outbound) → assign sg-ID | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error)<br>3. Verify default rules | **NOT STARTED** |
|
||||
| `delete-security-group` | — | `--group-id` | SG must not be in use by instances | Verify no instances reference SG → remove OVS flow rules → delete metadata | 1. Delete unused SG<br>2. Delete SG in use (error) | **NOT STARTED** |
|
||||
| `describe-security-groups` | — | `--group-ids`, `--group-names`, `--filters`, `--max-results` | None | Query NATS KV for SG metadata → return SG list with rules | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name | **NOT STARTED** |
|
||||
| `authorize-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr`, `--source-group` | SG must exist | Add inbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Duplicate rule (idempotent) | **NOT STARTED** |
|
||||
| `authorize-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist | Add outbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR | **NOT STARTED** |
|
||||
| `revoke-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove inbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
|
||||
| `revoke-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove outbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
|
||||
| `create-internet-gateway` | — | `--tag-specifications` | None | Create IGW metadata → assign igw-ID → configure OVS NAT bridge for external access | 1. Create IGW<br>2. Verify in describe-internet-gateways | **NOT STARTED** |
|
||||
| `attach-internet-gateway` | — | `--internet-gateway-id`, `--vpc-id` | IGW and VPC must exist | Link IGW to VPC → configure OVS flows for internet routing → update metadata | 1. Attach IGW to VPC<br>2. Attach already-attached IGW (error)<br>3. Attach to non-existent VPC (error) | **NOT STARTED** |
|
||||
| `detach-internet-gateway` | — | `--internet-gateway-id`, `--vpc-id` | IGW must be attached to VPC | Unlink IGW from VPC → remove OVS internet routing flows | 1. Detach attached IGW<br>2. Detach unattached IGW (error) | **NOT STARTED** |
|
||||
| `delete-internet-gateway` | — | `--internet-gateway-id` | IGW must be detached | Verify IGW detached → remove OVS NAT bridge → delete metadata | 1. Delete detached IGW<br>2. Delete attached IGW (error) | **NOT STARTED** |
|
||||
| `describe-internet-gateways` | — | `--internet-gateway-ids`, `--filters`, `--max-results` | None | Query NATS KV for IGW metadata → return list with attachment info | 1. List all IGWs<br>2. Filter by attachment VPC | **NOT STARTED** |
|
||||
|
||||
## EC2 - Route Tables
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-route-table` | — | `--vpc-id`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateRouteTable` → daemon creates route table in KV with rtb-ID → add default local route for VPC CIDR → return route table | 1. Create route table<br>2. Verify default local route<br>3. Missing VPC ID (error) | **NOT STARTED** |
|
||||
| `delete-route-table` | — | `--route-table-id`, `--dry-run` | Route table must exist and not be main route table | NATS `ec2.DeleteRouteTable` → daemon verifies not main table → delete from KV → return success | 1. Delete non-main route table<br>2. Delete main route table (error)<br>3. Delete non-existent (error) | **NOT STARTED** |
|
||||
| `describe-route-tables` | — | `--route-table-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeRouteTables` → daemon lists route tables from KV → return tables with routes | 1. List all route tables<br>2. Filter by VPC | **NOT STARTED** |
|
||||
| `create-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` or `--egress-only-internet-gateway-id` | Route table and target must exist | NATS `ec2.CreateRoute` → daemon adds route entry → configure OVS flow for destination CIDR → update metadata | 1. Add internet route via IGW<br>2. Add route to NAT gateway<br>3. Add route to EIGW<br>4. Conflicting route (error) | **NOT STARTED** |
|
||||
| `delete-route` | — | `--route-table-id`, `--destination-cidr-block` | Route must exist | NATS `ec2.DeleteRoute` → daemon removes route entry → remove OVS flow → update metadata | 1. Delete existing route<br>2. Delete non-existent route (error)<br>3. Delete local route (error) | **NOT STARTED** |
|
||||
| `replace-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` | Route must exist | NATS `ec2.ReplaceRoute` → daemon replaces route target → update OVS flow → update metadata | 1. Replace IGW route with NAT<br>2. Replace non-existent route (error) | **NOT STARTED** |
|
||||
| `associate-route-table` | — | `--route-table-id`, `--subnet-id` or `--gateway-id` | Route table and target must exist | NATS `ec2.AssociateRouteTable` → daemon links route table to subnet/gateway → return rtbassoc-ID | 1. Associate with subnet<br>2. Missing route table ID (error) | **NOT STARTED** |
|
||||
| `disassociate-route-table` | — | `--association-id` | Association must exist, not main table association | NATS `ec2.DisassociateRouteTable` → daemon unlinks route table → return success | 1. Disassociate existing<br>2. Disassociate main table (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - VPC Peering
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-vpc-peering-connection` | — | `--vpc-id`, `--peer-vpc-id`, `--peer-owner-id`, `--peer-region`, `--tag-specifications` | Both VPCs must exist | NATS `ec2.CreateVpcPeeringConnection` → daemon creates peering in KV with pcx-ID → state=pending-acceptance → return peering connection | 1. Create peering between two VPCs<br>2. Missing VPC ID (error)<br>3. Overlapping CIDRs (error) | **NOT STARTED** |
|
||||
| `accept-vpc-peering-connection` | — | `--vpc-peering-connection-id`, `--dry-run` | Peering must exist in pending-acceptance state | NATS `ec2.AcceptVpcPeeringConnection` → daemon updates state to active → configure OVS cross-VPC flows → return peering | 1. Accept pending peering<br>2. Accept already active (error) | **NOT STARTED** |
|
||||
| `reject-vpc-peering-connection` | — | `--vpc-peering-connection-id`, `--dry-run` | Peering must exist in pending-acceptance state | NATS `ec2.RejectVpcPeeringConnection` → daemon updates state to rejected → return peering | 1. Reject pending peering<br>2. Reject already active (error) | **NOT STARTED** |
|
||||
| `delete-vpc-peering-connection` | — | `--vpc-peering-connection-id`, `--dry-run` | Peering must exist | NATS `ec2.DeleteVpcPeeringConnection` → daemon removes OVS cross-VPC flows → delete from KV → return success | 1. Delete active peering<br>2. Delete non-existent (error) | **NOT STARTED** |
|
||||
| `describe-vpc-peering-connections` | — | `--vpc-peering-connection-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeVpcPeeringConnections` → daemon lists peerings from KV → return list with state | 1. List all peerings<br>2. Filter by ID<br>3. Filter by VPC | **NOT STARTED** |
|
||||
| `modify-vpc-peering-connection-options` | — | `--vpc-peering-connection-id`, `--requester-peering-connection-options`, `--accepter-peering-connection-options` | Peering must be active | NATS `ec2.ModifyVpcPeeringConnectionOptions` → daemon updates peering options in KV → return modified options | 1. Enable DNS resolution<br>2. Missing peering ID (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - VPC Endpoints
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-vpc-endpoint` | — | `--vpc-id`, `--service-name`, `--vpc-endpoint-type` (Gateway/Interface), `--route-table-ids`, `--subnet-ids`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateVpcEndpoint` → daemon creates endpoint in KV with vpce-ID → return endpoint | 1. Create Gateway endpoint for S3<br>2. Create Interface endpoint<br>3. Missing VPC ID (error) | **NOT STARTED** |
|
||||
| `delete-vpc-endpoints` | — | `--vpc-endpoint-ids`, `--dry-run` | Endpoints must exist | NATS `ec2.DeleteVpcEndpoints` → daemon deletes endpoints from KV → return success/failure per endpoint | 1. Delete existing endpoint<br>2. Delete non-existent (error) | **NOT STARTED** |
|
||||
| `describe-vpc-endpoints` | — | `--vpc-endpoint-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeVpcEndpoints` → daemon lists endpoints from KV → return list with state and type | 1. List all endpoints<br>2. Filter by VPC<br>3. Filter by service name | **NOT STARTED** |
|
||||
| `describe-vpc-endpoint-services` | — | `--service-names`, `--filters`, `--max-results` | None | NATS `ec2.DescribeVpcEndpointServices` → daemon returns available services (S3, DynamoDB equivalents) → return service list | 1. List available services<br>2. Filter by service name | **NOT STARTED** |
|
||||
| `modify-vpc-endpoint` | — | `--vpc-endpoint-id`, `--add-route-table-ids`, `--remove-route-table-ids`, `--add-subnet-ids`, `--remove-subnet-ids` | Endpoint must exist | NATS `ec2.ModifyVpcEndpoint` → daemon updates endpoint route tables/subnets in KV → return success | 1. Add route table<br>2. Remove subnet<br>3. Missing endpoint ID (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - VPN Gateway & Customer Gateway
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-customer-gateway` | — | `--type` (ipsec.1), `--bgp-asn`, `--ip-address`, `--tag-specifications` | None | NATS `ec2.CreateCustomerGateway` → daemon stores in JetStream KV with cgw-ID → return customer gateway | 1. Create with BGP ASN and IP<br>2. Missing required fields (error) | **NOT STARTED** |
|
||||
| `delete-customer-gateway` | — | `--customer-gateway-id`, `--dry-run` | Must exist, no VPN connections | NATS `ec2.DeleteCustomerGateway` → daemon verifies no active connections → delete from KV → return success | 1. Delete existing<br>2. Delete with active VPN (error) | **NOT STARTED** |
|
||||
| `describe-customer-gateways` | — | `--customer-gateway-ids`, `--filters` | None | NATS `ec2.DescribeCustomerGateways` → daemon lists from KV → return list with BGP/IP info | 1. List all<br>2. Filter by ID | **NOT STARTED** |
|
||||
| `create-vpn-gateway` | — | `--type` (ipsec.1), `--amazon-side-asn`, `--tag-specifications` | None | NATS `ec2.CreateVpnGateway` → daemon stores in JetStream KV with vgw-ID → return VPN gateway | 1. Create VPN gateway<br>2. Verify in describe | **NOT STARTED** |
|
||||
| `delete-vpn-gateway` | — | `--vpn-gateway-id`, `--dry-run` | Must be detached | NATS `ec2.DeleteVpnGateway` → daemon verifies detached → delete from KV → return success | 1. Delete detached gateway<br>2. Delete attached (error) | **NOT STARTED** |
|
||||
| `attach-vpn-gateway` | — | `--vpn-gateway-id`, `--vpc-id` | Both must exist | NATS `ec2.AttachVpnGateway` → daemon links VPN gateway to VPC → return attachment | 1. Attach to VPC<br>2. Already attached (error) | **NOT STARTED** |
|
||||
| `detach-vpn-gateway` | — | `--vpn-gateway-id`, `--vpc-id` | Must be attached | NATS `ec2.DetachVpnGateway` → daemon unlinks VPN gateway from VPC → return success | 1. Detach from VPC<br>2. Not attached (error) | **NOT STARTED** |
|
||||
| `describe-vpn-gateways` | — | `--vpn-gateway-ids`, `--filters` | None | NATS `ec2.DescribeVpnGateways` → daemon lists from KV → return list with attachment info | 1. List all<br>2. Filter by VPC attachment | **NOT STARTED** |
|
||||
|
||||
## EC2 - VPN Connections
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-vpn-connection` | — | `--type` (ipsec.1), `--customer-gateway-id`, `--vpn-gateway-id`, `--options`, `--tag-specifications` | Customer gateway and VPN gateway must exist | NATS `ec2.CreateVpnConnection` → daemon stores connection in JetStream KV with vpn-ID → return connection with tunnel info | 1. Create IPsec connection<br>2. Missing customer gateway (error)<br>3. Missing VPN gateway (error) | **NOT STARTED** |
|
||||
| `delete-vpn-connection` | — | `--vpn-connection-id`, `--dry-run` | Connection must exist | NATS `ec2.DeleteVpnConnection` → daemon deletes from KV → return success | 1. Delete existing connection<br>2. Delete non-existent (error) | **NOT STARTED** |
|
||||
| `describe-vpn-connections` | — | `--vpn-connection-ids`, `--filters` | None | NATS `ec2.DescribeVpnConnections` → daemon lists from KV → return list with state and tunnel info | 1. List all connections<br>2. Filter by ID<br>3. Filter by VPN gateway | **NOT STARTED** |
|
||||
| `modify-vpn-connection` | — | `--vpn-connection-id`, `--vpn-gateway-id`, `--customer-gateway-id` | Connection must exist | NATS `ec2.ModifyVpnConnection` → daemon updates connection target in KV → return modified connection | 1. Change VPN gateway<br>2. Missing connection ID (error) | **NOT STARTED** |
|
||||
| `modify-vpn-connection-options` | — | `--vpn-connection-id`, `--local-ipv4-network-cidr`, `--remote-ipv4-network-cidr` | Connection must exist | NATS `ec2.ModifyVpnConnectionOptions` → daemon updates tunnel options in KV → return modified connection | 1. Modify CIDR ranges<br>2. Missing connection ID (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - Network ACLs
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-network-acl` | — | `--vpc-id`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateNetworkAcl` → daemon creates NACL in KV with acl-ID → add default deny-all rules → return NACL | 1. Create NACL in VPC<br>2. Missing VPC ID (error)<br>3. Verify default deny rules | **NOT STARTED** |
|
||||
| `delete-network-acl` | — | `--network-acl-id`, `--dry-run` | Must not be default NACL | NATS `ec2.DeleteNetworkAcl` → daemon verifies not default → delete from KV → return success | 1. Delete non-default NACL<br>2. Delete default NACL (error) | **NOT STARTED** |
|
||||
| `describe-network-acls` | — | `--network-acl-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNetworkAcls` → daemon lists NACLs from KV → return list with entries | 1. List all NACLs<br>2. Filter by VPC | **NOT STARTED** |
|
||||
| `create-network-acl-entry` | — | `--network-acl-id`, `--rule-number`, `--protocol`, `--rule-action` (allow/deny), `--cidr-block`, `--ingress`/`--egress`, `--port-range` | NACL must exist | NATS `ec2.CreateNetworkAclEntry` → daemon adds rule entry to NACL in KV → configure OVS flow → return success | 1. Add ingress allow rule<br>2. Add egress deny rule<br>3. ICMP type/code parsing<br>4. Missing NACL ID (error) | **NOT STARTED** |
|
||||
| `delete-network-acl-entry` | — | `--network-acl-id`, `--rule-number`, `--ingress`/`--egress` | Entry must exist | NATS `ec2.DeleteNetworkAclEntry` → daemon removes rule from NACL → remove OVS flow → return success | 1. Delete existing entry<br>2. Delete non-existent entry (error) | **NOT STARTED** |
|
||||
| `replace-network-acl-association` | — | `--association-id`, `--network-acl-id` | Both must exist | NATS `ec2.ReplaceNetworkAclAssociation` → daemon swaps NACL for subnet → return new association ID | 1. Replace association<br>2. Missing association ID (error) | **NOT STARTED** |
|
||||
| `replace-network-acl-entry` | — | `--network-acl-id`, `--rule-number`, `--protocol`, `--rule-action`, `--cidr-block`, `--ingress`/`--egress`, `--port-range` | Entry must exist | NATS `ec2.ReplaceNetworkAclEntry` → daemon replaces rule in KV → update OVS flow → return success | 1. Replace existing entry<br>2. Replace non-existent entry (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - Prefix Lists
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-managed-prefix-list` | — | `--prefix-list-name`, `--address-family` (IPv4/IPv6), `--max-entries`, `--entries`, `--tag-specifications` | None | NATS `ec2.CreateManagedPrefixList` → daemon stores prefix list in JetStream KV with pl-ID → return prefix list with version | 1. Create with IPv4 entries<br>2. Name, address family, max entries required<br>3. Verify in describe | **NOT STARTED** |
|
||||
| `delete-managed-prefix-list` | — | `--prefix-list-id`, `--dry-run` | Must exist, not referenced by security groups/route tables | NATS `ec2.DeleteManagedPrefixList` → daemon verifies not referenced → delete from KV → return success | 1. Delete unreferenced list<br>2. Delete referenced list (error) | **NOT STARTED** |
|
||||
| `describe-managed-prefix-lists` | — | `--prefix-list-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeManagedPrefixLists` → daemon lists from KV → return list with state and version | 1. List all prefix lists<br>2. Filter by ID | **NOT STARTED** |
|
||||
| `modify-managed-prefix-list` | — | `--prefix-list-id`, `--current-version`, `--add-entries`, `--remove-entries`, `--prefix-list-name` | Must exist, current version must match | NATS `ec2.ModifyManagedPrefixList` → daemon validates version → apply changes → increment version → return updated list | 1. Add entries<br>2. Remove entries<br>3. Version mismatch (error) | **NOT STARTED** |
|
||||
| `get-managed-prefix-list-entries` | — | `--prefix-list-id`, `--target-version`, `--max-results` | Prefix list must exist | NATS `ec2.GetManagedPrefixListEntries` → daemon reads entries from KV → return entry list | 1. List entries<br>2. List entries at specific version | **NOT STARTED** |
|
||||
| `get-managed-prefix-list-associations` | — | `--prefix-list-id`, `--max-results` | Prefix list must exist | NATS `ec2.GetManagedPrefixListAssociations` → daemon finds references in SGs/route tables → return association list | 1. List associations<br>2. No associations returns empty | **NOT STARTED** |
|
||||
|
||||
## EC2 - Network Interfaces
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-network-interface` | — | `--subnet-id`, `--description`, `--private-ip-address`, `--groups`, `--tag-specifications` | Subnet must exist | NATS `ec2.CreateNetworkInterface` → daemon creates ENI in KV with eni-ID → assign private IP from subnet pool → return ENI | 1. Create in subnet<br>2. Create with specific private IP<br>3. Missing subnet ID (error) | **NOT STARTED** |
|
||||
| `delete-network-interface` | — | `--network-interface-id`, `--dry-run` | Must not be attached | NATS `ec2.DeleteNetworkInterface` → daemon verifies detached → release IP → delete from KV → return success | 1. Delete detached ENI<br>2. Delete attached ENI (error) | **NOT STARTED** |
|
||||
| `describe-network-interfaces` | — | `--network-interface-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNetworkInterfaces` → daemon lists ENIs from KV → return list with IP/attachment info | 1. List all ENIs<br>2. Filter by subnet<br>3. Filter by attachment | **NOT STARTED** |
|
||||
| `attach-network-interface` | — | `--network-interface-id`, `--instance-id`, `--device-index` | ENI and instance must exist | NATS `ec2.AttachNetworkInterface` → daemon attaches OVS port to instance QEMU process → configure MAC/IP → return eni-attach-ID | 1. Attach ENI to instance<br>2. Already attached (error)<br>3. Missing device index (error) | **NOT STARTED** |
|
||||
| `detach-network-interface` | — | `--attachment-id`, `--force` | ENI must be attached | NATS `ec2.DetachNetworkInterface` → daemon detaches OVS port from instance → return success | 1. Detach ENI<br>2. Force detach<br>3. Not attached (error) | **NOT STARTED** |
|
||||
| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--source-dest-check`, `--groups` | ENI must exist | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon updates ENI attributes in KV → return success | 1. Modify description<br>2. Toggle source/dest check<br>3. Update security groups | **NOT STARTED** |
|
||||
| `assign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` or `--secondary-private-ip-address-count` | ENI must exist | NATS `ec2.AssignPrivateIpAddresses` → daemon allocates IPs from subnet pool → assign to ENI → return assigned IPs | 1. Assign specific IPs<br>2. Assign by count<br>3. Cannot specify both (error) | **NOT STARTED** |
|
||||
| `unassign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` | IPs must be assigned to ENI | NATS `ec2.UnassignPrivateIpAddresses` → daemon releases IPs → return success | 1. Unassign secondary IP<br>2. Unassign primary IP (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - Launch Templates
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-launch-template` | — | `--launch-template-name`, `--launch-template-data` (ImageId, InstanceType, etc.), `--tag-specifications` | None | NATS `ec2.CreateLaunchTemplate` → daemon stores template in JetStream KV with lt-ID → version 1 → return template metadata | 1. Create template with full config<br>2. Duplicate name (error)<br>3. Verify in describe output | **NOT STARTED** |
|
||||
| `create-launch-template-version` | — | `--launch-template-id` or `--launch-template-name`, `--launch-template-data`, `--source-version` | Template must exist | NATS `ec2.CreateLaunchTemplateVersion` → daemon creates new version in KV → return version details | 1. Create version from scratch<br>2. Create version from source version<br>3. Template not found (error) | **NOT STARTED** |
|
||||
| `delete-launch-template` | — | `--launch-template-id` or `--launch-template-name`, `--dry-run` | Template must exist | NATS `ec2.DeleteLaunchTemplate` → daemon deletes template and all versions from KV → return deleted template info | 1. Delete by ID<br>2. Delete by name<br>3. Non-existent template (error) | **NOT STARTED** |
|
||||
| `describe-launch-templates` | — | `--launch-template-ids`, `--launch-template-names`, `--filters` | None | NATS `ec2.DescribeLaunchTemplates` → daemon lists templates from KV → return list | 1. List all templates<br>2. Filter by name<br>3. Filter by ID | **NOT STARTED** |
|
||||
| `describe-launch-template-versions` | — | `--launch-template-id` or `--launch-template-name`, `--versions`, `--min-version`, `--max-version` | Template must exist | NATS `ec2.DescribeLaunchTemplateVersions` → daemon lists versions from KV → return version list with data | 1. List all versions<br>2. List specific version<br>3. Template not found (error) | **NOT STARTED** |
|
||||
|
||||
## EC2 - Misc Operations
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `delete-network-interface-permission` | — | `--network-interface-permission-id`, `--force` | Permission must exist | NATS `ec2.DeleteNetworkInterfacePermission` → daemon deletes permission from KV → return success | 1. Delete existing permission<br>2. Force delete | **NOT STARTED** |
|
||||
| `enable-address-transfer` | — | `--allocation-id`, `--transfer-account-id` | EIP must exist | NATS `ec2.EnableAddressTransfer` → daemon stores transfer in JetStream KV with 7-day expiration → return AllocationId and TransferAccountId | 1. Enable transfer<br>2. Verify transfer stored with expiration | **NOT STARTED** |
|
||||
| `disable-address-transfer` | — | `--allocation-id` | Transfer must exist | NATS `ec2.DisableAddressTransfer` → daemon updates transfer status to disabled in KV → return status | 1. Disable active transfer<br>2. Disable non-existent transfer (error) | **NOT STARTED** |
|
||||
|
||||
## EBS Direct API
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `start-snapshot` | — | `--volume-size`, `--parent-snapshot-id`, `--description`, `--encrypted` | None | Initialize snapshot in viperblock → return snap-ID for block-level writes | 1. Start new snapshot<br>2. Start incremental from parent | **NOT STARTED** |
|
||||
| `put-snapshot-block` | — | `--snapshot-id`, `--block-index`, `--block-data`, `--checksum` | start-snapshot | Write block data to snapshot at specified index in viperblock → verify checksum | 1. Write single block<br>2. Write with bad checksum (error) | **NOT STARTED** |
|
||||
| `get-snapshot-block` | — | `--snapshot-id`, `--block-index` | Snapshot must exist | Read block data from snapshot at specified index → return data with checksum | 1. Read existing block<br>2. Read sparse block (zeros) | **NOT STARTED** |
|
||||
| `complete-snapshot` | — | `--snapshot-id`, `--changed-blocks-count` | start-snapshot, put-snapshot-block | Finalize snapshot → mark as completed → sync to Predastore | 1. Complete valid snapshot<br>2. Complete with wrong block count (error) | **NOT STARTED** |
|
||||
| `list-snapshot-blocks` | — | `--snapshot-id`, `--max-results`, `--next-token` | Snapshot must exist | List all non-empty blocks in snapshot with tokens and sizes | 1. List blocks of populated snapshot<br>2. List blocks of empty snapshot | **NOT STARTED** |
|
||||
| `list-changed-blocks` | — | `--second-snapshot-id`, `--first-snapshot-id`, `--max-results` | Snapshots must exist | Compare two snapshots → return list of differing block indexes | 1. Compare parent and child snapshots<br>2. Compare unrelated snapshots | **NOT STARTED** |
|
||||
|
||||
## S3 (via Predastore)
|
||||
|
||||
Currently S3 operations are handled directly by Predastore. Consider moving control/data plane to Hive format for AWS gateway integration.
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `s3 mb` (CreateBucket) | — | `s3://bucket-name` | None | NATS `s3.createbucket` → Predastore creates bucket → return success | 1. Create valid bucket<br>2. Duplicate bucket name (error)<br>3. Invalid bucket name (error) | **NOT STARTED** (Predastore handles directly) |
|
||||
| `s3 rb` (DeleteBucket) | — | `s3://bucket-name`, `--force` | Bucket must exist | NATS `s3.deletebucket` → verify bucket empty (or force) → Predastore deletes bucket | 1. Delete empty bucket<br>2. Delete non-empty bucket (error)<br>3. Force delete non-empty bucket | **NOT STARTED** |
|
||||
| `s3 ls` (ListBuckets/Objects) | — | `s3://bucket/prefix`, `--recursive` | None | NATS `s3.listobjects` → Predastore lists objects with prefix → return list | 1. List all buckets<br>2. List objects in bucket<br>3. List with prefix filter | **NOT STARTED** |
|
||||
| `s3 cp` (PutObject/GetObject) | — | `source dest`, `--recursive`, `--acl` | Bucket must exist | NATS `s3.putobject`/`s3.getobject` → Predastore stores/retrieves with Reed-Solomon encoding | 1. Upload single file<br>2. Download single file<br>3. Recursive upload/download<br>4. Large file (multipart) | **NOT STARTED** |
|
||||
| `s3 rm` (DeleteObject) | — | `s3://bucket/key`, `--recursive` | Object must exist | NATS `s3.deleteobject` → Predastore deletes object shards | 1. Delete single object<br>2. Recursive delete<br>3. Delete non-existent (idempotent) | **NOT STARTED** |
|
||||
| `s3 sync` | — | `source dest`, `--delete`, `--exclude`, `--include` | Bucket must exist | Compare source and dest → upload changed/new files → optionally delete removed files | 1. Sync local dir to S3<br>2. Sync with --delete<br>3. Sync with exclude filter | **NOT STARTED** |
|
||||
|
||||
## IAM
|
||||
|
||||
| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
|
||||
|---------|-------------------|---------------|---------------|-------------|------------|--------|
|
||||
| `create-user` | — | `--user-name`, `--path`, `--tags` | None | Store user in JetStream KV → generate AIDA-prefixed UserId → return User with ARN | 1. Create user<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing username (error) | **NOT STARTED** |
|
||||
| `get-user` | — | `--user-name` | User must exist | Read user from JetStream KV → return User details | 1. Get existing user<br>2. Get non-existent user (NoSuchEntity) | **NOT STARTED** |
|
||||
| `list-users` | — | `--path-prefix`, `--max-items`, `--marker` | None | List users from JetStream KV → return user list | 1. List all users<br>2. List with path prefix<br>3. Empty list | **NOT STARTED** |
|
||||
| `delete-user` | — | `--user-name` | User must exist, no access keys/policies | Remove user from JetStream KV → return success | 1. Delete user with no keys<br>2. Delete user with keys (error)<br>3. Delete non-existent (NoSuchEntity) | **NOT STARTED** |
|
||||
| `create-access-key` | — | `--user-name` | User must exist | Generate access key ID (AKIA prefix) and secret → store in JetStream KV → return credentials | 1. Create key for user<br>2. Max keys exceeded (error) | **NOT STARTED** (gateway stub exists) |
|
||||
| `delete-access-key` | — | `--access-key-id`, `--user-name` | Key must exist | Remove access key from JetStream KV → return success | 1. Delete existing key<br>2. Delete non-existent key (error) | **NOT STARTED** |
|
||||
| `list-access-keys` | — | `--user-name`, `--max-items` | None | Query JetStream KV for user's access keys → return list | 1. List keys for user<br>2. No keys returns empty | **NOT STARTED** |
|
||||
| `create-policy` | — | `--policy-name`, `--policy-document`, `--path`, `--description`, `--tags` | None | Validate JSON policy document → store in JetStream KV → generate policy ARN → return Policy | 1. Create with valid document<br>2. Invalid JSON (error)<br>3. Duplicate name (EntityAlreadyExists) | **NOT STARTED** |
|
||||
| `delete-policy` | — | `--policy-arn` | Policy must exist, not attached to any user | Remove policy from JetStream KV → return success | 1. Delete unattached policy<br>2. Delete attached policy (error) | **NOT STARTED** |
|
||||
| `attach-user-policy` | — | `--user-name`, `--policy-arn` | User and policy must exist | Link policy to user in JetStream KV → return success | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent user (NoSuchEntity) | **NOT STARTED** |
|
||||
| `detach-user-policy` | — | `--user-name`, `--policy-arn` | User and policy must exist, policy attached | Unlink policy from user in JetStream KV → return success | 1. Detach attached policy<br>2. Not attached (error) | **NOT STARTED** |
|
||||
| `list-attached-user-policies` | — | `--user-name`, `--path-prefix`, `--max-items` | User must exist | List policies attached to user from JetStream KV → return policy list | 1. List attached policies<br>2. No policies returns empty | **NOT STARTED** |
|
||||
@@ -30,9 +30,7 @@ Hive is an AWS-compatible infrastructure platform for bare-metal, edge, and on-p
|
||||
│ Topics: Queue Groups: │
|
||||
│ • ec2.RunInstances • hive-workers (load balanced) │
|
||||
│ • ec2.DescribeInstances • No queue (fan-out to all nodes) │
|
||||
│ • ec2.StartInstances │
|
||||
│ • ec2.StopInstances │
|
||||
│ • ec2.TerminateInstances │
|
||||
│ • ec2.cmd.{instanceID} (per-instance start/stop/terminate) │
|
||||
│ • ebs.mount / ebs.unmount │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
@@ -229,9 +227,7 @@ type ResourceManager struct {
|
||||
|-------|-------------|---------|
|
||||
| `ec2.RunInstances` | `hive-workers` | Launch new instances |
|
||||
| `ec2.DescribeInstances` | None (fan-out) | Query all nodes for instances |
|
||||
| `ec2.StartInstances` | `hive-workers` | Start stopped instances |
|
||||
| `ec2.StopInstances` | `hive-workers` | Stop running instances |
|
||||
| `ec2.TerminateInstances` | `hive-workers` | Terminate instances |
|
||||
| `ec2.cmd.{instanceID}` | None (owner only) | Per-instance commands (start/stop/terminate, attach/detach volume) |
|
||||
| `ec2.CreateKeyPair` | `hive-workers` | Generate SSH keypair |
|
||||
| `ec2.DescribeKeyPairs` | `hive-workers` | List SSH keypairs |
|
||||
| `ec2.DescribeImages` | `hive-workers` | List AMIs |
|
||||
@@ -375,25 +371,3 @@ aws --no-verify-ssl ec2 describe-instances
|
||||
# Terminate instance
|
||||
aws --no-verify-ssl ec2 terminate-instances --instance-ids i-xxxxx
|
||||
```
|
||||
|
||||
|
||||
## Improvements
|
||||
|
||||
Improve `aws ec2 stop-instances` , currently uses a ~30 sec heartbeat to detect if an instance is terminated. When the stop-instance command is sent, run the heartbeat more frequently to detect closer to realtime when the instance has stopped.
|
||||
|
||||
Currently a ~54 second window to detect termination.
|
||||
|
||||
```sh
|
||||
aws ec2 stop-instances --instance-ids i-1e306bb58b99e03bd
|
||||
```
|
||||
|
||||
```sh
|
||||
2026/01/22 20:01:52 INFO QMP event event=POWERDOWN
|
||||
2026/01/22 20:02:17 INFO QMP heartbeat instance=i-1e306bb58b99e03bd
|
||||
2026/01/22 20:02:17 INFO QMP status status="{\"running\":true,\"singlestep\":false,\"status\":\"running\"}"
|
||||
2026/01/22 20:02:47 INFO QMP heartbeat instance=i-1e306bb58b99e03bd
|
||||
2026/01/22 20:02:47 ERROR Failed to send QMP command err="encode error: write unix @->/run/user/1000/qmp-i-1e306bb58b99e03bd.sock: write: broken pipe"
|
||||
2026/01/22 20:02:47 INFO QMP Status - Instance stopped, exiting heartbeat id=i-1e306bb58b99e03bd
|
||||
2026/01/22 20:02:47 INFO Unsubscribing from NATS subject instance=i-1e306bb58b99e03bd
|
||||
2026/01/22 20:02:47 INFO Closing QMP client connection instance=i-1e306bb58b99e03bd
|
||||
```
|
||||
@@ -1,14 +1,16 @@
|
||||
module github.com/mulgadc/hive
|
||||
|
||||
go 1.25.5
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/NYTimes/gziphandler v1.1.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/gofiber/fiber/v2 v2.52.9
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/kdomanski/iso9660 v0.4.0
|
||||
github.com/mulgadc/predastore v1.0.0
|
||||
github.com/mulgadc/viperblock v1.0.2
|
||||
github.com/klauspost/cpuid/v2 v2.3.0
|
||||
github.com/mulgadc/predastore v1.0.1
|
||||
github.com/mulgadc/viperblock v1.0.15
|
||||
github.com/nats-io/nats-server/v2 v2.11.6
|
||||
github.com/nats-io/nats.go v1.43.0
|
||||
github.com/pelletier/go-toml/v2 v2.2.4
|
||||
@@ -68,7 +70,6 @@ require (
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.12.6 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
|
||||
@@ -27,6 +27,8 @@ github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi
|
||||
github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY=
|
||||
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
|
||||
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
|
||||
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
@@ -270,12 +272,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mulgadc/predastore v1.0.0 h1:IZ5V/EL+k+zz0zGoTYp+1BgODb43JxHsb/kvORuydKI=
|
||||
github.com/mulgadc/predastore v1.0.0/go.mod h1:x46A1z091hlZkMt3kkcuLSy2PAzSgROr6ft04qcYOU4=
|
||||
github.com/mulgadc/viperblock v0.1.5 h1:vS7ImGApQvxKuU1+fJNmwOycduwr9fx/xyofxWYKPcI=
|
||||
github.com/mulgadc/viperblock v0.1.5/go.mod h1:JNooDHb6ZuPvNPWaf0gChlft/D8xs5jHJFln+P3B9lQ=
|
||||
github.com/mulgadc/viperblock v1.0.2 h1:l5tQ96EO7rUKbaGtJHY4TJLG/25GSxYsxv/NsSN8+ZA=
|
||||
github.com/mulgadc/viperblock v1.0.2/go.mod h1:SljqW90JjI6RtBlm4Pji59A7/VQXHy8ncSQ0Ht6xKko=
|
||||
github.com/mulgadc/predastore v1.0.1 h1:Qohnnn3a2VxLI41GABHPxcV6cTAv009+mfhzFZ91RTc=
|
||||
github.com/mulgadc/predastore v1.0.1/go.mod h1:/ktr/gfFMECGCKFiz9Ng0DgPii/BhaGNjDxaIUgBEd0=
|
||||
github.com/mulgadc/viperblock v1.0.15 h1:QZ+oV4JE2uPxaK+XtcIr1RelAZHwTW00Ix7/ZNGsXpI=
|
||||
github.com/mulgadc/viperblock v1.0.15/go.mod h1:dV1eRKaIjH8YfuCk2Bd4hxKd3Rx5mWfO6gelJ9SbbeY=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI=
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
# Quick Reference
|
||||
|
||||
Most formatting and common issues are automatically fixed by Biome. Run `pnpm fix` before committing to ensure compliance.
|
||||
|
||||
## Core Principles
|
||||
|
||||
Write code that is **accessible, performant, type-safe, and maintainable**. Focus on clarity and explicit intent over brevity.
|
||||
|
||||
### Type Safety & Explicitness
|
||||
|
||||
- Use explicit types for function parameters and return values when they enhance clarity
|
||||
- Prefer `unknown` over `any` when the type is genuinely unknown
|
||||
- Use const assertions (`as const`) for immutable values and literal types
|
||||
- Leverage TypeScript's type narrowing instead of type assertions
|
||||
- Use meaningful variable names instead of magic numbers - extract constants with descriptive names
|
||||
|
||||
### Modern JavaScript/TypeScript
|
||||
|
||||
- Use arrow functions for callbacks and short functions
|
||||
- Prefer `for...of` loops over `.forEach()` and indexed `for` loops
|
||||
- Use optional chaining (`?.`) and nullish coalescing (`??`) for safer property access
|
||||
- Prefer template literals over string concatenation
|
||||
- Use destructuring for object and array assignments
|
||||
- Use `const` by default, `let` only when reassignment is needed, never `var`
|
||||
- Use interfaces for objects, otherwise use type aliases
|
||||
- Prefer using function declarations over arrow functions for named functions
|
||||
|
||||
### Async & Promises
|
||||
|
||||
- Always `await` promises in async functions - don't forget to use the return value
|
||||
- Use `async/await` syntax instead of promise chains for better readability
|
||||
- Handle errors appropriately in async code with try-catch blocks
|
||||
- Don't use async functions as Promise executors
|
||||
|
||||
### React & JSX
|
||||
|
||||
- Use function components over class components
|
||||
- Call hooks at the top level only, never conditionally
|
||||
- Specify all dependencies in hook dependency arrays correctly
|
||||
- Use the `key` prop for elements in iterables (prefer unique IDs over array indices)
|
||||
- Nest children between opening and closing tags instead of passing as props
|
||||
- Don't define components inside other components
|
||||
- Use semantic HTML and ARIA attributes for accessibility:
|
||||
- Provide meaningful alt text for images
|
||||
- Use proper heading hierarchy
|
||||
- Add labels for form inputs
|
||||
- Include keyboard event handlers alongside mouse events
|
||||
- Use semantic elements (`<button>`, `<nav>`, etc.) instead of divs with roles
|
||||
- Don't use `useMemo` or `useCallback`
|
||||
- Use ref as a prop instead of `React.forwardRef`
|
||||
|
||||
### Error Handling & Debugging
|
||||
|
||||
- Remove `console.log`, `debugger`, and `alert` statements from production code
|
||||
- Throw `Error` objects with descriptive messages, not strings or other values
|
||||
- Use `try-catch` blocks meaningfully - don't catch errors just to rethrow them
|
||||
- Prefer early returns over nested conditionals for error cases
|
||||
|
||||
### Code Organization
|
||||
|
||||
- Keep functions focused and under reasonable cognitive complexity limits
|
||||
- Extract complex conditions into well-named boolean variables
|
||||
- Use early returns to reduce nesting
|
||||
- Prefer simple conditionals over nested ternary operators
|
||||
- Group related code together and separate concerns
|
||||
|
||||
### Security
|
||||
|
||||
- Add `rel="noopener"` when using `target="_blank"` on links
|
||||
- Avoid `dangerouslySetInnerHTML` unless absolutely necessary
|
||||
- Don't use `eval()` or assign directly to `document.cookie`
|
||||
- Validate and sanitize user input
|
||||
|
||||
### Performance
|
||||
|
||||
- Avoid spread syntax in accumulators within loops
|
||||
- Use top-level regex literals instead of creating them in loops
|
||||
- Prefer specific imports over namespace imports
|
||||
- Avoid barrel files (index files that re-export everything)
|
||||
|
||||
## When Biome Can't Help
|
||||
|
||||
Biome's linter will catch most issues automatically. Focus your attention on:
|
||||
|
||||
1. **Business logic correctness** - Biome can't validate your algorithms
|
||||
2. **Meaningful naming** - Use descriptive names for functions, variables, and types
|
||||
3. **Architecture decisions** - Component structure, data flow, and API design
|
||||
4. **Edge cases** - Handle boundary conditions and error states
|
||||
5. **User experience** - Accessibility, performance, and usability considerations
|
||||
6. **Documentation** - Add comments for complex logic, but prefer self-documenting code
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PostToolUse": [
|
||||
{
|
||||
"matcher": "Write|Edit",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "pnpm fix"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
---
|
||||
name: react-best-practices
|
||||
description: React performance optimization guidelines. This skill should be used when writing, reviewing, or refactoring React code to ensure optimal performance patterns. Triggers on tasks involving React components, data fetching, bundle optimization, or performance improvements.
|
||||
---
|
||||
|
||||
# React Best Practices
|
||||
|
||||
Comprehensive performance optimization guide for React applications. Contains 36 rules across 6 categories, prioritized by impact to guide automated refactoring and code generation.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Reference these guidelines when:
|
||||
- Writing new React components
|
||||
- Implementing data fetching
|
||||
- Reviewing code for performance issues
|
||||
- Refactoring existing React code
|
||||
- Optimizing bundle size or load times
|
||||
|
||||
## Rule Categories by Priority
|
||||
|
||||
| Priority | Category | Impact | Prefix |
|
||||
|----------|----------|--------|--------|
|
||||
| 1 | Eliminating Waterfalls | CRITICAL | `async-` |
|
||||
| 2 | Client-Side Data Fetching | MEDIUM-HIGH | `client-` |
|
||||
| 3 | Re-render Optimization | MEDIUM | `rerender-` |
|
||||
| 4 | Rendering Performance | MEDIUM | `rendering-` |
|
||||
| 5 | JavaScript Performance | LOW-MEDIUM | `js-` |
|
||||
| 6 | Advanced Patterns | LOW | `advanced-` |
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### 1. Eliminating Waterfalls (CRITICAL)
|
||||
|
||||
- `async-defer-await` - Move await into branches where actually used
|
||||
- `async-parallel` - Use Promise.all() for independent operations
|
||||
- `async-dependencies` - Use better-all for partial dependencies
|
||||
|
||||
### 2. Client-Side Data Fetching (MEDIUM-HIGH)
|
||||
|
||||
- `client-passive-event-listeners` - Use passive listeners for scroll
|
||||
- `client-localstorage-schema` - Version and minimize localStorage data
|
||||
|
||||
### 3. Re-render Optimization (MEDIUM)
|
||||
|
||||
- `rerender-defer-reads` - Don't subscribe to state only used in callbacks
|
||||
- `rerender-dependencies` - Use primitive dependencies in effects
|
||||
- `rerender-derived-state` - Subscribe to derived booleans, not raw values
|
||||
- `rerender-derived-state-no-effect` - Derive state during render, not effects
|
||||
- `rerender-functional-setstate` - Use functional setState for stable callbacks
|
||||
- `rerender-lazy-state-init` - Pass function to useState for expensive values
|
||||
- `rerender-move-effect-to-event` - Put interaction logic in event handlers
|
||||
- `rerender-transitions` - Use startTransition for non-urgent updates
|
||||
- `rerender-use-ref-transient-values` - Use refs for transient frequent values
|
||||
|
||||
### 4. Rendering Performance (MEDIUM)
|
||||
|
||||
- `rendering-animate-svg-wrapper` - Animate div wrapper, not SVG element
|
||||
- `rendering-content-visibility` - Use content-visibility for long lists
|
||||
- `rendering-hoist-jsx` - Extract static JSX outside components
|
||||
- `rendering-svg-precision` - Reduce SVG coordinate precision
|
||||
- `rendering-activity` - Use Activity component for show/hide
|
||||
- `rendering-conditional-render` - Use ternary, not && for conditionals
|
||||
- `rendering-usetransition-loading` - Prefer useTransition for loading state
|
||||
|
||||
### 5. JavaScript Performance (LOW-MEDIUM)
|
||||
|
||||
- `js-batch-dom-css` - Group CSS changes via classes or cssText
|
||||
- `js-index-maps` - Build Map for repeated lookups
|
||||
- `js-cache-property-access` - Cache object properties in loops
|
||||
- `js-cache-function-results` - Cache function results in module-level Map
|
||||
- `js-cache-storage` - Cache localStorage/sessionStorage reads
|
||||
- `js-combine-iterations` - Combine multiple filter/map into one loop
|
||||
- `js-length-check-first` - Check array length before expensive comparison
|
||||
- `js-early-exit` - Return early from functions
|
||||
- `js-hoist-regexp` - Hoist RegExp creation outside loops
|
||||
- `js-min-max-loop` - Use loop for min/max instead of sort
|
||||
- `js-set-map-lookups` - Use Set/Map for O(1) lookups
|
||||
- `js-tosorted-immutable` - Use toSorted() for immutability
|
||||
|
||||
### 6. Advanced Patterns (LOW)
|
||||
|
||||
- `advanced-event-handler-refs` - Store event handlers in refs
|
||||
- `advanced-init-once` - Initialize app once per app load
|
||||
- `advanced-use-latest` - useLatest for stable callback refs
|
||||
|
||||
## How to Use
|
||||
|
||||
Read individual rule files for detailed explanations and code examples:
|
||||
|
||||
```
|
||||
rules/async-parallel.md
|
||||
rules/bundle-barrel-imports.md
|
||||
```
|
||||
|
||||
Each rule file contains:
|
||||
- Brief explanation of why it matters
|
||||
- Incorrect code example with explanation
|
||||
- Correct code example with explanation
|
||||
- Additional context and references
|
||||
@@ -1,55 +0,0 @@
|
||||
---
|
||||
title: Store Event Handlers in Refs
|
||||
impact: LOW
|
||||
impactDescription: stable subscriptions
|
||||
tags: advanced, hooks, refs, event-handlers, optimization
|
||||
---
|
||||
|
||||
## Store Event Handlers in Refs
|
||||
|
||||
Store callbacks in refs when used in effects that shouldn't re-subscribe on callback changes.
|
||||
|
||||
**Incorrect (re-subscribes on every render):**
|
||||
|
||||
```tsx
|
||||
function useWindowEvent(event: string, handler: (e) => void) {
|
||||
useEffect(() => {
|
||||
window.addEventListener(event, handler)
|
||||
return () => window.removeEventListener(event, handler)
|
||||
}, [event, handler])
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (stable subscription):**
|
||||
|
||||
```tsx
|
||||
function useWindowEvent(event: string, handler: (e) => void) {
|
||||
const handlerRef = useRef(handler)
|
||||
useEffect(() => {
|
||||
handlerRef.current = handler
|
||||
}, [handler])
|
||||
|
||||
useEffect(() => {
|
||||
const listener = (e) => handlerRef.current(e)
|
||||
window.addEventListener(event, listener)
|
||||
return () => window.removeEventListener(event, listener)
|
||||
}, [event])
|
||||
}
|
||||
```
|
||||
|
||||
**Alternative: use `useEffectEvent` if you're on latest React:**
|
||||
|
||||
```tsx
|
||||
import { useEffectEvent } from 'react'
|
||||
|
||||
function useWindowEvent(event: string, handler: (e) => void) {
|
||||
const onEvent = useEffectEvent(handler)
|
||||
|
||||
useEffect(() => {
|
||||
window.addEventListener(event, onEvent)
|
||||
return () => window.removeEventListener(event, onEvent)
|
||||
}, [event])
|
||||
}
|
||||
```
|
||||
|
||||
`useEffectEvent` provides a cleaner API for the same pattern: it creates a stable function reference that always calls the latest version of the handler.
|
||||
@@ -1,42 +0,0 @@
|
||||
---
|
||||
title: Initialize App Once, Not Per Mount
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: avoids duplicate init in development
|
||||
tags: initialization, useEffect, app-startup, side-effects
|
||||
---
|
||||
|
||||
## Initialize App Once, Not Per Mount
|
||||
|
||||
Do not put app-wide initialization that must run once per app load inside `useEffect([])` of a component. Components can remount and effects will re-run. Use a module-level guard or top-level init in the entry module instead.
|
||||
|
||||
**Incorrect (runs twice in dev, re-runs on remount):**
|
||||
|
||||
```tsx
|
||||
function Comp() {
|
||||
useEffect(() => {
|
||||
loadFromStorage()
|
||||
checkAuthToken()
|
||||
}, [])
|
||||
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (once per app load):**
|
||||
|
||||
```tsx
|
||||
let didInit = false
|
||||
|
||||
function Comp() {
|
||||
useEffect(() => {
|
||||
if (didInit) return
|
||||
didInit = true
|
||||
loadFromStorage()
|
||||
checkAuthToken()
|
||||
}, [])
|
||||
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
Reference: [Initializing the application](https://react.dev/learn/you-might-not-need-an-effect#initializing-the-application)
|
||||
@@ -1,39 +0,0 @@
|
||||
---
|
||||
title: useEffectEvent for Stable Callback Refs
|
||||
impact: LOW
|
||||
impactDescription: prevents effect re-runs
|
||||
tags: advanced, hooks, useEffectEvent, refs, optimization
|
||||
---
|
||||
|
||||
## useEffectEvent for Stable Callback Refs
|
||||
|
||||
Access latest values in callbacks without adding them to dependency arrays. Prevents effect re-runs while avoiding stale closures.
|
||||
|
||||
**Incorrect (effect re-runs on every callback change):**
|
||||
|
||||
```tsx
|
||||
function SearchInput({ onSearch }: { onSearch: (q: string) => void }) {
|
||||
const [query, setQuery] = useState('')
|
||||
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => onSearch(query), 300)
|
||||
return () => clearTimeout(timeout)
|
||||
}, [query, onSearch])
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (using React's useEffectEvent):**
|
||||
|
||||
```tsx
|
||||
import { useEffectEvent } from 'react';
|
||||
|
||||
function SearchInput({ onSearch }: { onSearch: (q: string) => void }) {
|
||||
const [query, setQuery] = useState('')
|
||||
const onSearchEvent = useEffectEvent(onSearch)
|
||||
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => onSearchEvent(query), 300)
|
||||
return () => clearTimeout(timeout)
|
||||
}, [query])
|
||||
}
|
||||
```
|
||||
@@ -1,80 +0,0 @@
|
||||
---
|
||||
title: Defer Await Until Needed
|
||||
impact: HIGH
|
||||
impactDescription: avoids blocking unused code paths
|
||||
tags: async, await, conditional, optimization
|
||||
---
|
||||
|
||||
## Defer Await Until Needed
|
||||
|
||||
Move `await` operations into the branches where they're actually used to avoid blocking code paths that don't need them.
|
||||
|
||||
**Incorrect (blocks both branches):**
|
||||
|
||||
```typescript
|
||||
async function handleRequest(userId: string, skipProcessing: boolean) {
|
||||
const userData = await fetchUserData(userId)
|
||||
|
||||
if (skipProcessing) {
|
||||
// Returns immediately but still waited for userData
|
||||
return { skipped: true }
|
||||
}
|
||||
|
||||
// Only this branch uses userData
|
||||
return processUserData(userData)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (only blocks when needed):**
|
||||
|
||||
```typescript
|
||||
async function handleRequest(userId: string, skipProcessing: boolean) {
|
||||
if (skipProcessing) {
|
||||
// Returns immediately without waiting
|
||||
return { skipped: true }
|
||||
}
|
||||
|
||||
// Fetch only when needed
|
||||
const userData = await fetchUserData(userId)
|
||||
return processUserData(userData)
|
||||
}
|
||||
```
|
||||
|
||||
**Another example (early return optimization):**
|
||||
|
||||
```typescript
|
||||
// Incorrect: always fetches permissions
|
||||
async function updateResource(resourceId: string, userId: string) {
|
||||
const permissions = await fetchPermissions(userId)
|
||||
const resource = await getResource(resourceId)
|
||||
|
||||
if (!resource) {
|
||||
return { error: 'Not found' }
|
||||
}
|
||||
|
||||
if (!permissions.canEdit) {
|
||||
return { error: 'Forbidden' }
|
||||
}
|
||||
|
||||
return await updateResourceData(resource, permissions)
|
||||
}
|
||||
|
||||
// Correct: fetches only when needed
|
||||
async function updateResource(resourceId: string, userId: string) {
|
||||
const resource = await getResource(resourceId)
|
||||
|
||||
if (!resource) {
|
||||
return { error: 'Not found' }
|
||||
}
|
||||
|
||||
const permissions = await fetchPermissions(userId)
|
||||
|
||||
if (!permissions.canEdit) {
|
||||
return { error: 'Forbidden' }
|
||||
}
|
||||
|
||||
return await updateResourceData(resource, permissions)
|
||||
}
|
||||
```
|
||||
|
||||
This optimization is especially valuable when the skipped branch is frequently taken, or when the deferred operation is expensive.
|
||||
@@ -1,51 +0,0 @@
|
||||
---
|
||||
title: Dependency-Based Parallelization
|
||||
impact: CRITICAL
|
||||
impactDescription: 2-10× improvement
|
||||
tags: async, parallelization, dependencies, better-all
|
||||
---
|
||||
|
||||
## Dependency-Based Parallelization
|
||||
|
||||
For operations with partial dependencies, use `better-all` to maximize parallelism. It automatically starts each task at the earliest possible moment.
|
||||
|
||||
**Incorrect (profile waits for config unnecessarily):**
|
||||
|
||||
```typescript
|
||||
const [user, config] = await Promise.all([
|
||||
fetchUser(),
|
||||
fetchConfig()
|
||||
])
|
||||
const profile = await fetchProfile(user.id)
|
||||
```
|
||||
|
||||
**Correct (config and profile run in parallel):**
|
||||
|
||||
```typescript
|
||||
import { all } from 'better-all'
|
||||
|
||||
const { user, config, profile } = await all({
|
||||
async user() { return fetchUser() },
|
||||
async config() { return fetchConfig() },
|
||||
async profile() {
|
||||
return fetchProfile((await this.$.user).id)
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
**Alternative without extra dependencies:**
|
||||
|
||||
We can also create all the promises first, and do `Promise.all()` at the end.
|
||||
|
||||
```typescript
|
||||
const userPromise = fetchUser()
|
||||
const profilePromise = userPromise.then(user => fetchProfile(user.id))
|
||||
|
||||
const [user, config, profile] = await Promise.all([
|
||||
userPromise,
|
||||
fetchConfig(),
|
||||
profilePromise
|
||||
])
|
||||
```
|
||||
|
||||
Reference: [https://github.com/shuding/better-all](https://github.com/shuding/better-all)
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
title: Promise.all() for Independent Operations
|
||||
impact: CRITICAL
|
||||
impactDescription: 2-10× improvement
|
||||
tags: async, parallelization, promises, waterfalls
|
||||
---
|
||||
|
||||
## Promise.all() for Independent Operations
|
||||
|
||||
When async operations have no interdependencies, execute them concurrently using `Promise.all()`.
|
||||
|
||||
**Incorrect (sequential execution, 3 round trips):**
|
||||
|
||||
```typescript
|
||||
const user = await fetchUser()
|
||||
const posts = await fetchPosts()
|
||||
const comments = await fetchComments()
|
||||
```
|
||||
|
||||
**Correct (parallel execution, 1 round trip):**
|
||||
|
||||
```typescript
|
||||
const [user, posts, comments] = await Promise.all([
|
||||
fetchUser(),
|
||||
fetchPosts(),
|
||||
fetchComments()
|
||||
])
|
||||
```
|
||||
@@ -1,71 +0,0 @@
|
||||
---
|
||||
title: Version and Minimize localStorage Data
|
||||
impact: MEDIUM
|
||||
impactDescription: prevents schema conflicts, reduces storage size
|
||||
tags: client, localStorage, storage, versioning, data-minimization
|
||||
---
|
||||
|
||||
## Version and Minimize localStorage Data
|
||||
|
||||
Add version prefix to keys and store only needed fields. Prevents schema conflicts and accidental storage of sensitive data.
|
||||
|
||||
**Incorrect:**
|
||||
|
||||
```typescript
|
||||
// No version, stores everything, no error handling
|
||||
localStorage.setItem('userConfig', JSON.stringify(fullUserObject))
|
||||
const data = localStorage.getItem('userConfig')
|
||||
```
|
||||
|
||||
**Correct:**
|
||||
|
||||
```typescript
|
||||
const VERSION = 'v2'
|
||||
|
||||
function saveConfig(config: { theme: string; language: string }) {
|
||||
try {
|
||||
localStorage.setItem(`userConfig:${VERSION}`, JSON.stringify(config))
|
||||
} catch {
|
||||
// Throws in incognito/private browsing, quota exceeded, or disabled
|
||||
}
|
||||
}
|
||||
|
||||
function loadConfig() {
|
||||
try {
|
||||
const data = localStorage.getItem(`userConfig:${VERSION}`)
|
||||
return data ? JSON.parse(data) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// Migration from v1 to v2
|
||||
function migrate() {
|
||||
try {
|
||||
const v1 = localStorage.getItem('userConfig:v1')
|
||||
if (v1) {
|
||||
const old = JSON.parse(v1)
|
||||
saveConfig({ theme: old.darkMode ? 'dark' : 'light', language: old.lang })
|
||||
localStorage.removeItem('userConfig:v1')
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
```
|
||||
|
||||
**Store minimal fields from server responses:**
|
||||
|
||||
```typescript
|
||||
// User object has 20+ fields, only store what UI needs
|
||||
function cachePrefs(user: FullUser) {
|
||||
try {
|
||||
localStorage.setItem('prefs:v1', JSON.stringify({
|
||||
theme: user.preferences.theme,
|
||||
notifications: user.preferences.notifications
|
||||
}))
|
||||
} catch {}
|
||||
}
|
||||
```
|
||||
|
||||
**Always wrap in try-catch:** `getItem()` and `setItem()` throw in incognito/private browsing (Safari, Firefox), when quota exceeded, or when disabled.
|
||||
|
||||
**Benefits:** Schema evolution via versioning, reduced storage size, prevents storing tokens/PII/internal flags.
|
||||
@@ -1,48 +0,0 @@
|
||||
---
|
||||
title: Use Passive Event Listeners for Scrolling Performance
|
||||
impact: MEDIUM
|
||||
impactDescription: eliminates scroll delay caused by event listeners
|
||||
tags: client, event-listeners, scrolling, performance, touch, wheel
|
||||
---
|
||||
|
||||
## Use Passive Event Listeners for Scrolling Performance
|
||||
|
||||
Add `{ passive: true }` to touch and wheel event listeners to enable immediate scrolling. Browsers normally wait for listeners to finish to check if `preventDefault()` is called, causing scroll delay.
|
||||
|
||||
**Incorrect:**
|
||||
|
||||
```typescript
|
||||
useEffect(() => {
|
||||
const handleTouch = (e: TouchEvent) => console.log(e.touches[0].clientX)
|
||||
const handleWheel = (e: WheelEvent) => console.log(e.deltaY)
|
||||
|
||||
document.addEventListener('touchstart', handleTouch)
|
||||
document.addEventListener('wheel', handleWheel)
|
||||
|
||||
return () => {
|
||||
document.removeEventListener('touchstart', handleTouch)
|
||||
document.removeEventListener('wheel', handleWheel)
|
||||
}
|
||||
}, [])
|
||||
```
|
||||
|
||||
**Correct:**
|
||||
|
||||
```typescript
|
||||
useEffect(() => {
|
||||
const handleTouch = (e: TouchEvent) => console.log(e.touches[0].clientX)
|
||||
const handleWheel = (e: WheelEvent) => console.log(e.deltaY)
|
||||
|
||||
document.addEventListener('touchstart', handleTouch, { passive: true })
|
||||
document.addEventListener('wheel', handleWheel, { passive: true })
|
||||
|
||||
return () => {
|
||||
document.removeEventListener('touchstart', handleTouch)
|
||||
document.removeEventListener('wheel', handleWheel)
|
||||
}
|
||||
}, [])
|
||||
```
|
||||
|
||||
**Use passive when:** tracking/analytics, logging, any listener that doesn't call `preventDefault()`.
|
||||
|
||||
**Don't use passive when:** implementing custom swipe gestures, custom zoom controls, or any listener that needs `preventDefault()`.
|
||||
@@ -1,107 +0,0 @@
|
||||
---
|
||||
title: Avoid Layout Thrashing
|
||||
impact: MEDIUM
|
||||
impactDescription: prevents forced synchronous layouts and reduces performance bottlenecks
|
||||
tags: javascript, dom, css, performance, reflow, layout-thrashing
|
||||
---
|
||||
|
||||
## Avoid Layout Thrashing
|
||||
|
||||
Avoid interleaving style writes with layout reads. When you read a layout property (like `offsetWidth`, `getBoundingClientRect()`, or `getComputedStyle()`) between style changes, the browser is forced to trigger a synchronous reflow.
|
||||
|
||||
**This is OK (browser batches style changes):**
|
||||
```typescript
|
||||
function updateElementStyles(element: HTMLElement) {
|
||||
// Each line invalidates style, but browser batches the recalculation
|
||||
element.style.width = '100px'
|
||||
element.style.height = '200px'
|
||||
element.style.backgroundColor = 'blue'
|
||||
element.style.border = '1px solid black'
|
||||
}
|
||||
```
|
||||
|
||||
**Incorrect (interleaved reads and writes force reflows):**
|
||||
```typescript
|
||||
function layoutThrashing(element: HTMLElement) {
|
||||
element.style.width = '100px'
|
||||
const width = element.offsetWidth // Forces reflow
|
||||
element.style.height = '200px'
|
||||
const height = element.offsetHeight // Forces another reflow
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (batch writes, then read once):**
|
||||
```typescript
|
||||
function updateElementStyles(element: HTMLElement) {
|
||||
// Batch all writes together
|
||||
element.style.width = '100px'
|
||||
element.style.height = '200px'
|
||||
element.style.backgroundColor = 'blue'
|
||||
element.style.border = '1px solid black'
|
||||
|
||||
// Read after all writes are done (single reflow)
|
||||
const { width, height } = element.getBoundingClientRect()
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (batch reads, then writes):**
|
||||
```typescript
|
||||
function avoidThrashing(element: HTMLElement) {
|
||||
// Read phase - all layout queries first
|
||||
const rect1 = element.getBoundingClientRect()
|
||||
const offsetWidth = element.offsetWidth
|
||||
const offsetHeight = element.offsetHeight
|
||||
|
||||
// Write phase - all style changes after
|
||||
element.style.width = '100px'
|
||||
element.style.height = '200px'
|
||||
}
|
||||
```
|
||||
|
||||
**Better: use CSS classes**
|
||||
```css
|
||||
.highlighted-box {
|
||||
width: 100px;
|
||||
height: 200px;
|
||||
background-color: blue;
|
||||
border: 1px solid black;
|
||||
}
|
||||
```
|
||||
```typescript
|
||||
function updateElementStyles(element: HTMLElement) {
|
||||
element.classList.add('highlighted-box')
|
||||
|
||||
const { width, height } = element.getBoundingClientRect()
|
||||
}
|
||||
```
|
||||
|
||||
**React example:**
|
||||
```tsx
|
||||
// Incorrect: interleaving style changes with layout queries
|
||||
function Box({ isHighlighted }: { isHighlighted: boolean }) {
|
||||
const ref = useRef<HTMLDivElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (ref.current && isHighlighted) {
|
||||
ref.current.style.width = '100px'
|
||||
const width = ref.current.offsetWidth // Forces layout
|
||||
ref.current.style.height = '200px'
|
||||
}
|
||||
}, [isHighlighted])
|
||||
|
||||
return <div ref={ref}>Content</div>
|
||||
}
|
||||
|
||||
// Correct: toggle class
|
||||
function Box({ isHighlighted }: { isHighlighted: boolean }) {
|
||||
return (
|
||||
<div className={isHighlighted ? 'highlighted-box' : ''}>
|
||||
Content
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
Prefer CSS classes over inline styles when possible. CSS files are cached by the browser, and classes provide better separation of concerns and are easier to maintain.
|
||||
|
||||
See [this gist](https://gist.github.com/paulirish/5d52fb081b3570c81e3a) and [CSS Triggers](https://csstriggers.com/) for more information on layout-forcing operations.
|
||||
@@ -1,80 +0,0 @@
|
||||
---
|
||||
title: Cache Repeated Function Calls
|
||||
impact: MEDIUM
|
||||
impactDescription: avoid redundant computation
|
||||
tags: javascript, cache, memoization, performance
|
||||
---
|
||||
|
||||
## Cache Repeated Function Calls
|
||||
|
||||
Use a module-level Map to cache function results when the same function is called repeatedly with the same inputs during render.
|
||||
|
||||
**Incorrect (redundant computation):**
|
||||
|
||||
```typescript
|
||||
function ProjectList({ projects }: { projects: Project[] }) {
|
||||
return (
|
||||
<div>
|
||||
{projects.map(project => {
|
||||
// slugify() called 100+ times for same project names
|
||||
const slug = slugify(project.name)
|
||||
|
||||
return <ProjectCard key={project.id} slug={slug} />
|
||||
})}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (cached results):**
|
||||
|
||||
```typescript
|
||||
// Module-level cache
|
||||
const slugifyCache = new Map<string, string>()
|
||||
|
||||
function cachedSlugify(text: string): string {
|
||||
if (slugifyCache.has(text)) {
|
||||
return slugifyCache.get(text)!
|
||||
}
|
||||
const result = slugify(text)
|
||||
slugifyCache.set(text, result)
|
||||
return result
|
||||
}
|
||||
|
||||
function ProjectList({ projects }: { projects: Project[] }) {
|
||||
return (
|
||||
<div>
|
||||
{projects.map(project => {
|
||||
// Computed only once per unique project name
|
||||
const slug = cachedSlugify(project.name)
|
||||
|
||||
return <ProjectCard key={project.id} slug={slug} />
|
||||
})}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Simpler pattern for single-value functions:**
|
||||
|
||||
```typescript
|
||||
let isLoggedInCache: boolean | null = null
|
||||
|
||||
function isLoggedIn(): boolean {
|
||||
if (isLoggedInCache !== null) {
|
||||
return isLoggedInCache
|
||||
}
|
||||
|
||||
isLoggedInCache = document.cookie.includes('auth=')
|
||||
return isLoggedInCache
|
||||
}
|
||||
|
||||
// Clear cache when auth changes
|
||||
function onAuthChange() {
|
||||
isLoggedInCache = null
|
||||
}
|
||||
```
|
||||
|
||||
Use a Map (not a hook) so it works everywhere: utilities, event handlers, not just React components.
|
||||
|
||||
Reference: [How we made the Vercel Dashboard twice as fast](https://vercel.com/blog/how-we-made-the-vercel-dashboard-twice-as-fast)
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
title: Cache Property Access in Loops
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: reduces lookups
|
||||
tags: javascript, loops, optimization, caching
|
||||
---
|
||||
|
||||
## Cache Property Access in Loops
|
||||
|
||||
Cache object property lookups in hot paths.
|
||||
|
||||
**Incorrect (3 lookups × N iterations):**
|
||||
|
||||
```typescript
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
process(obj.config.settings.value)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (1 lookup total):**
|
||||
|
||||
```typescript
|
||||
const value = obj.config.settings.value
|
||||
const len = arr.length
|
||||
for (let i = 0; i < len; i++) {
|
||||
process(value)
|
||||
}
|
||||
```
|
||||
@@ -1,70 +0,0 @@
|
||||
---
|
||||
title: Cache Storage API Calls
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: reduces expensive I/O
|
||||
tags: javascript, localStorage, storage, caching, performance
|
||||
---
|
||||
|
||||
## Cache Storage API Calls
|
||||
|
||||
`localStorage`, `sessionStorage`, and `document.cookie` are synchronous and expensive. Cache reads in memory.
|
||||
|
||||
**Incorrect (reads storage on every call):**
|
||||
|
||||
```typescript
|
||||
function getTheme() {
|
||||
return localStorage.getItem('theme') ?? 'light'
|
||||
}
|
||||
// Called 10 times = 10 storage reads
|
||||
```
|
||||
|
||||
**Correct (Map cache):**
|
||||
|
||||
```typescript
|
||||
const storageCache = new Map<string, string | null>()
|
||||
|
||||
function getLocalStorage(key: string) {
|
||||
if (!storageCache.has(key)) {
|
||||
storageCache.set(key, localStorage.getItem(key))
|
||||
}
|
||||
return storageCache.get(key)
|
||||
}
|
||||
|
||||
function setLocalStorage(key: string, value: string) {
|
||||
localStorage.setItem(key, value)
|
||||
storageCache.set(key, value) // keep cache in sync
|
||||
}
|
||||
```
|
||||
|
||||
Use a Map (not a hook) so it works everywhere: utilities, event handlers, not just React components.
|
||||
|
||||
**Cookie caching:**
|
||||
|
||||
```typescript
|
||||
let cookieCache: Record<string, string> | null = null
|
||||
|
||||
function getCookie(name: string) {
|
||||
if (!cookieCache) {
|
||||
cookieCache = Object.fromEntries(
|
||||
document.cookie.split('; ').map(c => c.split('='))
|
||||
)
|
||||
}
|
||||
return cookieCache[name]
|
||||
}
|
||||
```
|
||||
|
||||
**Important (invalidate on external changes):**
|
||||
|
||||
If storage can change externally (another tab, server-set cookies), invalidate cache:
|
||||
|
||||
```typescript
|
||||
window.addEventListener('storage', (e) => {
|
||||
if (e.key) storageCache.delete(e.key)
|
||||
})
|
||||
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
if (document.visibilityState === 'visible') {
|
||||
storageCache.clear()
|
||||
}
|
||||
})
|
||||
```
|
||||
@@ -1,32 +0,0 @@
|
||||
---
|
||||
title: Combine Multiple Array Iterations
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: reduces iterations
|
||||
tags: javascript, arrays, loops, performance
|
||||
---
|
||||
|
||||
## Combine Multiple Array Iterations
|
||||
|
||||
Multiple `.filter()` or `.map()` calls iterate the array multiple times. Combine into one loop.
|
||||
|
||||
**Incorrect (3 iterations):**
|
||||
|
||||
```typescript
|
||||
const admins = users.filter(u => u.isAdmin)
|
||||
const testers = users.filter(u => u.isTester)
|
||||
const inactive = users.filter(u => !u.isActive)
|
||||
```
|
||||
|
||||
**Correct (1 iteration):**
|
||||
|
||||
```typescript
|
||||
const admins: User[] = []
|
||||
const testers: User[] = []
|
||||
const inactive: User[] = []
|
||||
|
||||
for (const user of users) {
|
||||
if (user.isAdmin) admins.push(user)
|
||||
if (user.isTester) testers.push(user)
|
||||
if (!user.isActive) inactive.push(user)
|
||||
}
|
||||
```
|
||||
@@ -1,50 +0,0 @@
|
||||
---
|
||||
title: Early Return from Functions
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: avoids unnecessary computation
|
||||
tags: javascript, functions, optimization, early-return
|
||||
---
|
||||
|
||||
## Early Return from Functions
|
||||
|
||||
Return early when result is determined to skip unnecessary processing.
|
||||
|
||||
**Incorrect (processes all items even after finding answer):**
|
||||
|
||||
```typescript
|
||||
function validateUsers(users: User[]) {
|
||||
let hasError = false
|
||||
let errorMessage = ''
|
||||
|
||||
for (const user of users) {
|
||||
if (!user.email) {
|
||||
hasError = true
|
||||
errorMessage = 'Email required'
|
||||
}
|
||||
if (!user.name) {
|
||||
hasError = true
|
||||
errorMessage = 'Name required'
|
||||
}
|
||||
// Continues checking all users even after error found
|
||||
}
|
||||
|
||||
return hasError ? { valid: false, error: errorMessage } : { valid: true }
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (returns immediately on first error):**
|
||||
|
||||
```typescript
|
||||
function validateUsers(users: User[]) {
|
||||
for (const user of users) {
|
||||
if (!user.email) {
|
||||
return { valid: false, error: 'Email required' }
|
||||
}
|
||||
if (!user.name) {
|
||||
return { valid: false, error: 'Name required' }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
}
|
||||
```
|
||||
@@ -1,45 +0,0 @@
|
||||
---
|
||||
title: Hoist RegExp Creation
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: avoids recreation
|
||||
tags: javascript, regexp, optimization, memoization
|
||||
---
|
||||
|
||||
## Hoist RegExp Creation
|
||||
|
||||
Don't create RegExp inside render. Hoist to module scope or memoize with `useMemo()`.
|
||||
|
||||
**Incorrect (new RegExp every render):**
|
||||
|
||||
```tsx
|
||||
function Highlighter({ text, query }: Props) {
|
||||
const regex = new RegExp(`(${query})`, 'gi')
|
||||
const parts = text.split(regex)
|
||||
return <>{parts.map((part, i) => ...)}</>
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (memoize or hoist):**
|
||||
|
||||
```tsx
|
||||
const EMAIL_REGEX = /^[^\s@]+@[^\s@]+\.[^\s@]+$/
|
||||
|
||||
function Highlighter({ text, query }: Props) {
|
||||
const regex = useMemo(
|
||||
() => new RegExp(`(${escapeRegex(query)})`, 'gi'),
|
||||
[query]
|
||||
)
|
||||
const parts = text.split(regex)
|
||||
return <>{parts.map((part, i) => ...)}</>
|
||||
}
|
||||
```
|
||||
|
||||
**Warning (global regex has mutable state):**
|
||||
|
||||
Global regex (`/g`) has mutable `lastIndex` state:
|
||||
|
||||
```typescript
|
||||
const regex = /foo/g
|
||||
regex.test('foo') // true, lastIndex = 3
|
||||
regex.test('foo') // false, lastIndex = 0
|
||||
```
|
||||
@@ -1,37 +0,0 @@
|
||||
---
|
||||
title: Build Index Maps for Repeated Lookups
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: 1M ops to 2K ops
|
||||
tags: javascript, map, indexing, optimization, performance
|
||||
---
|
||||
|
||||
## Build Index Maps for Repeated Lookups
|
||||
|
||||
Multiple `.find()` calls by the same key should use a Map.
|
||||
|
||||
**Incorrect (O(n) per lookup):**
|
||||
|
||||
```typescript
|
||||
function processOrders(orders: Order[], users: User[]) {
|
||||
return orders.map(order => ({
|
||||
...order,
|
||||
user: users.find(u => u.id === order.userId)
|
||||
}))
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (O(1) per lookup):**
|
||||
|
||||
```typescript
|
||||
function processOrders(orders: Order[], users: User[]) {
|
||||
const userById = new Map(users.map(u => [u.id, u]))
|
||||
|
||||
return orders.map(order => ({
|
||||
...order,
|
||||
user: userById.get(order.userId)
|
||||
}))
|
||||
}
|
||||
```
|
||||
|
||||
Build map once (O(n)), then all lookups are O(1).
|
||||
For 1000 orders × 1000 users: 1M ops → 2K ops.
|
||||
@@ -1,49 +0,0 @@
|
||||
---
|
||||
title: Early Length Check for Array Comparisons
|
||||
impact: MEDIUM-HIGH
|
||||
impactDescription: avoids expensive operations when lengths differ
|
||||
tags: javascript, arrays, performance, optimization, comparison
|
||||
---
|
||||
|
||||
## Early Length Check for Array Comparisons
|
||||
|
||||
When comparing arrays with expensive operations (sorting, deep equality, serialization), check lengths first. If lengths differ, the arrays cannot be equal.
|
||||
|
||||
In real-world applications, this optimization is especially valuable when the comparison runs in hot paths (event handlers, render loops).
|
||||
|
||||
**Incorrect (always runs expensive comparison):**
|
||||
|
||||
```typescript
|
||||
function hasChanges(current: string[], original: string[]) {
|
||||
// Always sorts and joins, even when lengths differ
|
||||
return current.sort().join() !== original.sort().join()
|
||||
}
|
||||
```
|
||||
|
||||
Two O(n log n) sorts run even when `current.length` is 5 and `original.length` is 100. There is also overhead of joining the arrays and comparing the strings.
|
||||
|
||||
**Correct (O(1) length check first):**
|
||||
|
||||
```typescript
|
||||
function hasChanges(current: string[], original: string[]) {
|
||||
// Early return if lengths differ
|
||||
if (current.length !== original.length) {
|
||||
return true
|
||||
}
|
||||
// Only sort when lengths match
|
||||
const currentSorted = current.toSorted()
|
||||
const originalSorted = original.toSorted()
|
||||
for (let i = 0; i < currentSorted.length; i++) {
|
||||
if (currentSorted[i] !== originalSorted[i]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
```
|
||||
|
||||
This new approach is more efficient because:
|
||||
- It avoids the overhead of sorting and joining the arrays when lengths differ
|
||||
- It avoids consuming memory for the joined strings (especially important for large arrays)
|
||||
- It avoids mutating the original arrays
|
||||
- It returns early when a difference is found
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
title: Use Loop for Min/Max Instead of Sort
|
||||
impact: LOW
|
||||
impactDescription: O(n) instead of O(n log n)
|
||||
tags: javascript, arrays, performance, sorting, algorithms
|
||||
---
|
||||
|
||||
## Use Loop for Min/Max Instead of Sort
|
||||
|
||||
Finding the smallest or largest element only requires a single pass through the array. Sorting is wasteful and slower.
|
||||
|
||||
**Incorrect (O(n log n) - sort to find latest):**
|
||||
|
||||
```typescript
|
||||
interface Project {
|
||||
id: string
|
||||
name: string
|
||||
updatedAt: number
|
||||
}
|
||||
|
||||
function getLatestProject(projects: Project[]) {
|
||||
const sorted = [...projects].sort((a, b) => b.updatedAt - a.updatedAt)
|
||||
return sorted[0]
|
||||
}
|
||||
```
|
||||
|
||||
Sorts the entire array just to find the maximum value.
|
||||
|
||||
**Incorrect (O(n log n) - sort for oldest and newest):**
|
||||
|
||||
```typescript
|
||||
function getOldestAndNewest(projects: Project[]) {
|
||||
const sorted = [...projects].sort((a, b) => a.updatedAt - b.updatedAt)
|
||||
return { oldest: sorted[0], newest: sorted[sorted.length - 1] }
|
||||
}
|
||||
```
|
||||
|
||||
Still sorts unnecessarily when only min/max are needed.
|
||||
|
||||
**Correct (O(n) - single loop):**
|
||||
|
||||
```typescript
|
||||
function getLatestProject(projects: Project[]) {
|
||||
if (projects.length === 0) return null
|
||||
|
||||
let latest = projects[0]
|
||||
|
||||
for (let i = 1; i < projects.length; i++) {
|
||||
if (projects[i].updatedAt > latest.updatedAt) {
|
||||
latest = projects[i]
|
||||
}
|
||||
}
|
||||
|
||||
return latest
|
||||
}
|
||||
|
||||
function getOldestAndNewest(projects: Project[]) {
|
||||
if (projects.length === 0) return { oldest: null, newest: null }
|
||||
|
||||
let oldest = projects[0]
|
||||
let newest = projects[0]
|
||||
|
||||
for (let i = 1; i < projects.length; i++) {
|
||||
if (projects[i].updatedAt < oldest.updatedAt) oldest = projects[i]
|
||||
if (projects[i].updatedAt > newest.updatedAt) newest = projects[i]
|
||||
}
|
||||
|
||||
return { oldest, newest }
|
||||
}
|
||||
```
|
||||
|
||||
Single pass through the array, no copying, no sorting.
|
||||
|
||||
**Alternative (Math.min/Math.max for small arrays):**
|
||||
|
||||
```typescript
|
||||
const numbers = [5, 2, 8, 1, 9]
|
||||
const min = Math.min(...numbers)
|
||||
const max = Math.max(...numbers)
|
||||
```
|
||||
|
||||
This works for small arrays, but can be slower or just throw an error for very large arrays due to spread operator limitations. Maximal array length is approximately 124000 in Chrome 143 and 638000 in Safari 18; exact numbers may vary - see [the fiddle](https://jsfiddle.net/qw1jabsx/4/). Use the loop approach for reliability.
|
||||
@@ -1,24 +0,0 @@
|
||||
---
|
||||
title: Use Set/Map for O(1) Lookups
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: O(n) to O(1)
|
||||
tags: javascript, set, map, data-structures, performance
|
||||
---
|
||||
|
||||
## Use Set/Map for O(1) Lookups
|
||||
|
||||
Convert arrays to Set/Map for repeated membership checks.
|
||||
|
||||
**Incorrect (O(n) per check):**
|
||||
|
||||
```typescript
|
||||
const allowedIds = ['a', 'b', 'c', ...]
|
||||
items.filter(item => allowedIds.includes(item.id))
|
||||
```
|
||||
|
||||
**Correct (O(1) per check):**
|
||||
|
||||
```typescript
|
||||
const allowedIds = new Set(['a', 'b', 'c', ...])
|
||||
items.filter(item => allowedIds.has(item.id))
|
||||
```
|
||||
@@ -1,57 +0,0 @@
|
||||
---
|
||||
title: Use toSorted() Instead of sort() for Immutability
|
||||
impact: MEDIUM-HIGH
|
||||
impactDescription: prevents mutation bugs in React state
|
||||
tags: javascript, arrays, immutability, react, state, mutation
|
||||
---
|
||||
|
||||
## Use toSorted() Instead of sort() for Immutability
|
||||
|
||||
`.sort()` mutates the array in place, which can cause bugs with React state and props. Use `.toSorted()` to create a new sorted array without mutation.
|
||||
|
||||
**Incorrect (mutates original array):**
|
||||
|
||||
```typescript
|
||||
function UserList({ users }: { users: User[] }) {
|
||||
// Mutates the users prop array!
|
||||
const sorted = useMemo(
|
||||
() => users.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
[users]
|
||||
)
|
||||
return <div>{sorted.map(renderUser)}</div>
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (creates new array):**
|
||||
|
||||
```typescript
|
||||
function UserList({ users }: { users: User[] }) {
|
||||
// Creates new sorted array, original unchanged
|
||||
const sorted = useMemo(
|
||||
() => users.toSorted((a, b) => a.name.localeCompare(b.name)),
|
||||
[users]
|
||||
)
|
||||
return <div>{sorted.map(renderUser)}</div>
|
||||
}
|
||||
```
|
||||
|
||||
**Why this matters in React:**
|
||||
|
||||
1. Props/state mutations break React's immutability model - React expects props and state to be treated as read-only
|
||||
2. Causes stale closure bugs - Mutating arrays inside closures (callbacks, effects) can lead to unexpected behavior
|
||||
|
||||
**Browser support (fallback for older browsers):**
|
||||
|
||||
`.toSorted()` is available in all modern browsers (Chrome 110+, Safari 16+, Firefox 115+, Node.js 20+). For older environments, use spread operator:
|
||||
|
||||
```typescript
|
||||
// Fallback for older browsers
|
||||
const sorted = [...items].sort((a, b) => a.value - b.value)
|
||||
```
|
||||
|
||||
**Other immutable array methods:**
|
||||
|
||||
- `.toSorted()` - immutable sort
|
||||
- `.toReversed()` - immutable reverse
|
||||
- `.toSpliced()` - immutable splice
|
||||
- `.with()` - immutable element replacement
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
title: Use Activity Component for Show/Hide
|
||||
impact: MEDIUM
|
||||
impactDescription: preserves state/DOM
|
||||
tags: rendering, activity, visibility, state-preservation
|
||||
---
|
||||
|
||||
## Use Activity Component for Show/Hide
|
||||
|
||||
Use React's `<Activity>` to preserve state/DOM for expensive components that frequently toggle visibility.
|
||||
|
||||
**Usage:**
|
||||
|
||||
```tsx
|
||||
import { Activity } from 'react'
|
||||
|
||||
function Dropdown({ isOpen }: Props) {
|
||||
return (
|
||||
<Activity mode={isOpen ? 'visible' : 'hidden'}>
|
||||
<ExpensiveMenu />
|
||||
</Activity>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
Avoids expensive re-renders and state loss.
|
||||
@@ -1,47 +0,0 @@
|
||||
---
|
||||
title: Animate SVG Wrapper Instead of SVG Element
|
||||
impact: LOW
|
||||
impactDescription: enables hardware acceleration
|
||||
tags: rendering, svg, css, animation, performance
|
||||
---
|
||||
|
||||
## Animate SVG Wrapper Instead of SVG Element
|
||||
|
||||
Many browsers don't have hardware acceleration for CSS3 animations on SVG elements. Wrap SVG in a `<div>` and animate the wrapper instead.
|
||||
|
||||
**Incorrect (animating SVG directly - no hardware acceleration):**
|
||||
|
||||
```tsx
|
||||
function LoadingSpinner() {
|
||||
return (
|
||||
<svg
|
||||
className="animate-spin"
|
||||
width="24"
|
||||
height="24"
|
||||
viewBox="0 0 24 24"
|
||||
>
|
||||
<circle cx="12" cy="12" r="10" stroke="currentColor" />
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (animating wrapper div - hardware accelerated):**
|
||||
|
||||
```tsx
|
||||
function LoadingSpinner() {
|
||||
return (
|
||||
<div className="animate-spin">
|
||||
<svg
|
||||
width="24"
|
||||
height="24"
|
||||
viewBox="0 0 24 24"
|
||||
>
|
||||
<circle cx="12" cy="12" r="10" stroke="currentColor" />
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
This applies to all CSS transforms and transitions (`transform`, `opacity`, `translate`, `scale`, `rotate`). The wrapper div allows browsers to use GPU acceleration for smoother animations.
|
||||
@@ -1,40 +0,0 @@
|
||||
---
|
||||
title: Use Explicit Conditional Rendering
|
||||
impact: LOW
|
||||
impactDescription: prevents rendering 0 or NaN
|
||||
tags: rendering, conditional, jsx, falsy-values
|
||||
---
|
||||
|
||||
## Use Explicit Conditional Rendering
|
||||
|
||||
Use explicit ternary operators (`? :`) instead of `&&` for conditional rendering when the condition can be `0`, `NaN`, or other falsy values that render.
|
||||
|
||||
**Incorrect (renders "0" when count is 0):**
|
||||
|
||||
```tsx
|
||||
function Badge({ count }: { count: number }) {
|
||||
return (
|
||||
<div>
|
||||
{count && <span className="badge">{count}</span>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// When count = 0, renders: <div>0</div>
|
||||
// When count = 5, renders: <div><span class="badge">5</span></div>
|
||||
```
|
||||
|
||||
**Correct (renders nothing when count is 0):**
|
||||
|
||||
```tsx
|
||||
function Badge({ count }: { count: number }) {
|
||||
return (
|
||||
<div>
|
||||
{count > 0 ? <span className="badge">{count}</span> : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// When count = 0, renders: <div></div>
|
||||
// When count = 5, renders: <div><span class="badge">5</span></div>
|
||||
```
|
||||
@@ -1,38 +0,0 @@
|
||||
---
|
||||
title: CSS content-visibility for Long Lists
|
||||
impact: HIGH
|
||||
impactDescription: faster initial render
|
||||
tags: rendering, css, content-visibility, long-lists
|
||||
---
|
||||
|
||||
## CSS content-visibility for Long Lists
|
||||
|
||||
Apply `content-visibility: auto` to defer off-screen rendering.
|
||||
|
||||
**CSS:**
|
||||
|
||||
```css
|
||||
.message-item {
|
||||
content-visibility: auto;
|
||||
contain-intrinsic-size: 0 80px;
|
||||
}
|
||||
```
|
||||
|
||||
**Example:**
|
||||
|
||||
```tsx
|
||||
function MessageList({ messages }: { messages: Message[] }) {
|
||||
return (
|
||||
<div className="overflow-y-auto h-screen">
|
||||
{messages.map(msg => (
|
||||
<div key={msg.id} className="message-item">
|
||||
<Avatar user={msg.author} />
|
||||
<div>{msg.content}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
For 1000 messages, browser skips layout/paint for ~990 off-screen items (10× faster initial render).
|
||||
@@ -1,46 +0,0 @@
|
||||
---
|
||||
title: Hoist Static JSX Elements
|
||||
impact: LOW
|
||||
impactDescription: avoids re-creation
|
||||
tags: rendering, jsx, static, optimization
|
||||
---
|
||||
|
||||
## Hoist Static JSX Elements
|
||||
|
||||
Extract static JSX outside components to avoid re-creation.
|
||||
|
||||
**Incorrect (recreates element every render):**
|
||||
|
||||
```tsx
|
||||
function LoadingSkeleton() {
|
||||
return <div className="animate-pulse h-20 bg-gray-200" />
|
||||
}
|
||||
|
||||
function Container() {
|
||||
return (
|
||||
<div>
|
||||
{loading && <LoadingSkeleton />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (reuses same element):**
|
||||
|
||||
```tsx
|
||||
const loadingSkeleton = (
|
||||
<div className="animate-pulse h-20 bg-gray-200" />
|
||||
)
|
||||
|
||||
function Container() {
|
||||
return (
|
||||
<div>
|
||||
{loading && loadingSkeleton}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
This is especially helpful for large and static SVG nodes, which can be expensive to recreate on every render.
|
||||
|
||||
**Note:** If your project has [React Compiler](https://react.dev/learn/react-compiler) enabled, the compiler automatically hoists static JSX elements and optimizes component re-renders, making manual hoisting unnecessary.
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
title: Optimize SVG Precision
|
||||
impact: LOW
|
||||
impactDescription: reduces file size
|
||||
tags: rendering, svg, optimization, svgo
|
||||
---
|
||||
|
||||
## Optimize SVG Precision
|
||||
|
||||
Reduce SVG coordinate precision to decrease file size. The optimal precision depends on the viewBox size, but in general reducing precision should be considered.
|
||||
|
||||
**Incorrect (excessive precision):**
|
||||
|
||||
```svg
|
||||
<path d="M 10.293847 20.847362 L 30.938472 40.192837" />
|
||||
```
|
||||
|
||||
**Correct (1 decimal place):**
|
||||
|
||||
```svg
|
||||
<path d="M 10.3 20.8 L 30.9 40.2" />
|
||||
```
|
||||
|
||||
**Automate with SVGO:**
|
||||
|
||||
```bash
|
||||
npx svgo --precision=1 --multipass icon.svg
|
||||
```
|
||||
@@ -1,75 +0,0 @@
|
||||
---
|
||||
title: Use useTransition Over Manual Loading States
|
||||
impact: LOW
|
||||
impactDescription: reduces re-renders and improves code clarity
|
||||
tags: rendering, transitions, useTransition, loading, state
|
||||
---
|
||||
|
||||
## Use useTransition Over Manual Loading States
|
||||
|
||||
Use `useTransition` instead of manual `useState` for loading states. This provides built-in `isPending` state and automatically manages transitions.
|
||||
|
||||
**Incorrect (manual loading state):**
|
||||
|
||||
```tsx
|
||||
function SearchResults() {
|
||||
const [query, setQuery] = useState('')
|
||||
const [results, setResults] = useState([])
|
||||
const [isLoading, setIsLoading] = useState(false)
|
||||
|
||||
const handleSearch = async (value: string) => {
|
||||
setIsLoading(true)
|
||||
setQuery(value)
|
||||
const data = await fetchResults(value)
|
||||
setResults(data)
|
||||
setIsLoading(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<input onChange={(e) => handleSearch(e.target.value)} />
|
||||
{isLoading && <Spinner />}
|
||||
<ResultsList results={results} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (useTransition with built-in pending state):**
|
||||
|
||||
```tsx
|
||||
import { useTransition, useState } from 'react'
|
||||
|
||||
function SearchResults() {
|
||||
const [query, setQuery] = useState('')
|
||||
const [results, setResults] = useState([])
|
||||
const [isPending, startTransition] = useTransition()
|
||||
|
||||
const handleSearch = (value: string) => {
|
||||
setQuery(value) // Update input immediately
|
||||
|
||||
startTransition(async () => {
|
||||
// Fetch and update results
|
||||
const data = await fetchResults(value)
|
||||
setResults(data)
|
||||
})
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<input onChange={(e) => handleSearch(e.target.value)} />
|
||||
{isPending && <Spinner />}
|
||||
<ResultsList results={results} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
|
||||
- **Automatic pending state**: No need to manually manage `setIsLoading(true/false)`
|
||||
- **Error resilience**: Pending state correctly resets even if the transition throws
|
||||
- **Better responsiveness**: Keeps the UI responsive during updates
|
||||
- **Interrupt handling**: New transitions automatically cancel pending ones
|
||||
|
||||
Reference: [useTransition](https://react.dev/reference/react/useTransition)
|
||||
@@ -1,39 +0,0 @@
|
||||
---
|
||||
title: Defer State Reads to Usage Point
|
||||
impact: MEDIUM
|
||||
impactDescription: avoids unnecessary subscriptions
|
||||
tags: rerender, searchParams, localStorage, optimization
|
||||
---
|
||||
|
||||
## Defer State Reads to Usage Point
|
||||
|
||||
Don't subscribe to dynamic state (searchParams, localStorage) if you only read it inside callbacks.
|
||||
|
||||
**Incorrect (subscribes to all searchParams changes):**
|
||||
|
||||
```tsx
|
||||
function ShareButton({ chatId }: { chatId: string }) {
|
||||
const searchParams = useSearchParams()
|
||||
|
||||
const handleShare = () => {
|
||||
const ref = searchParams.get('ref')
|
||||
shareChat(chatId, { ref })
|
||||
}
|
||||
|
||||
return <button onClick={handleShare}>Share</button>
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (reads on demand, no subscription):**
|
||||
|
||||
```tsx
|
||||
function ShareButton({ chatId }: { chatId: string }) {
|
||||
const handleShare = () => {
|
||||
const params = new URLSearchParams(window.location.search)
|
||||
const ref = params.get('ref')
|
||||
shareChat(chatId, { ref })
|
||||
}
|
||||
|
||||
return <button onClick={handleShare}>Share</button>
|
||||
}
|
||||
```
|
||||
@@ -1,45 +0,0 @@
|
||||
---
|
||||
title: Narrow Effect Dependencies
|
||||
impact: LOW
|
||||
impactDescription: minimizes effect re-runs
|
||||
tags: rerender, useEffect, dependencies, optimization
|
||||
---
|
||||
|
||||
## Narrow Effect Dependencies
|
||||
|
||||
Specify primitive dependencies instead of objects to minimize effect re-runs.
|
||||
|
||||
**Incorrect (re-runs on any user field change):**
|
||||
|
||||
```tsx
|
||||
useEffect(() => {
|
||||
console.log(user.id)
|
||||
}, [user])
|
||||
```
|
||||
|
||||
**Correct (re-runs only when id changes):**
|
||||
|
||||
```tsx
|
||||
useEffect(() => {
|
||||
console.log(user.id)
|
||||
}, [user.id])
|
||||
```
|
||||
|
||||
**For derived state, compute outside effect:**
|
||||
|
||||
```tsx
|
||||
// Incorrect: runs on width=767, 766, 765...
|
||||
useEffect(() => {
|
||||
if (width < 768) {
|
||||
enableMobileMode()
|
||||
}
|
||||
}, [width])
|
||||
|
||||
// Correct: runs only on boolean transition
|
||||
const isMobile = width < 768
|
||||
useEffect(() => {
|
||||
if (isMobile) {
|
||||
enableMobileMode()
|
||||
}
|
||||
}, [isMobile])
|
||||
```
|
||||
@@ -1,40 +0,0 @@
|
||||
---
|
||||
title: Calculate Derived State During Rendering
|
||||
impact: MEDIUM
|
||||
impactDescription: avoids redundant renders and state drift
|
||||
tags: rerender, derived-state, useEffect, state
|
||||
---
|
||||
|
||||
## Calculate Derived State During Rendering
|
||||
|
||||
If a value can be computed from current props/state, do not store it in state or update it in an effect. Derive it during render to avoid extra renders and state drift. Do not set state in effects solely in response to prop changes; prefer derived values or keyed resets instead.
|
||||
|
||||
**Incorrect (redundant state and effect):**
|
||||
|
||||
```tsx
|
||||
function Form() {
|
||||
const [firstName, setFirstName] = useState('First')
|
||||
const [lastName, setLastName] = useState('Last')
|
||||
const [fullName, setFullName] = useState('')
|
||||
|
||||
useEffect(() => {
|
||||
setFullName(firstName + ' ' + lastName)
|
||||
}, [firstName, lastName])
|
||||
|
||||
return <p>{fullName}</p>
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (derive during render):**
|
||||
|
||||
```tsx
|
||||
function Form() {
|
||||
const [firstName, setFirstName] = useState('First')
|
||||
const [lastName, setLastName] = useState('Last')
|
||||
const fullName = firstName + ' ' + lastName
|
||||
|
||||
return <p>{fullName}</p>
|
||||
}
|
||||
```
|
||||
|
||||
References: [You Might Not Need an Effect](https://react.dev/learn/you-might-not-need-an-effect)
|
||||
@@ -1,29 +0,0 @@
|
||||
---
|
||||
title: Subscribe to Derived State
|
||||
impact: MEDIUM
|
||||
impactDescription: reduces re-render frequency
|
||||
tags: rerender, derived-state, media-query, optimization
|
||||
---
|
||||
|
||||
## Subscribe to Derived State
|
||||
|
||||
Subscribe to derived boolean state instead of continuous values to reduce re-render frequency.
|
||||
|
||||
**Incorrect (re-renders on every pixel change):**
|
||||
|
||||
```tsx
|
||||
function Sidebar() {
|
||||
const width = useWindowWidth() // updates continuously
|
||||
const isMobile = width < 768
|
||||
return <nav className={isMobile ? 'mobile' : 'desktop'} />
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (re-renders only when boolean changes):**
|
||||
|
||||
```tsx
|
||||
function Sidebar() {
|
||||
const isMobile = useMediaQuery('(max-width: 767px)')
|
||||
return <nav className={isMobile ? 'mobile' : 'desktop'} />
|
||||
}
|
||||
```
|
||||
@@ -1,74 +0,0 @@
|
||||
---
|
||||
title: Use Functional setState Updates
|
||||
impact: MEDIUM
|
||||
impactDescription: prevents stale closures and unnecessary callback recreations
|
||||
tags: react, hooks, useState, useCallback, callbacks, closures
|
||||
---
|
||||
|
||||
## Use Functional setState Updates
|
||||
|
||||
When updating state based on the current state value, use the functional update form of setState instead of directly referencing the state variable. This prevents stale closures, eliminates unnecessary dependencies, and creates stable callback references.
|
||||
|
||||
**Incorrect (requires state as dependency):**
|
||||
|
||||
```tsx
|
||||
function TodoList() {
|
||||
const [items, setItems] = useState(initialItems)
|
||||
|
||||
// Callback must depend on items, recreated on every items change
|
||||
const addItems = useCallback((newItems: Item[]) => {
|
||||
setItems([...items, ...newItems])
|
||||
}, [items]) // ❌ items dependency causes recreations
|
||||
|
||||
// Risk of stale closure if dependency is forgotten
|
||||
const removeItem = useCallback((id: string) => {
|
||||
setItems(items.filter(item => item.id !== id))
|
||||
}, []) // ❌ Missing items dependency - will use stale items!
|
||||
|
||||
return <ItemsEditor items={items} onAdd={addItems} onRemove={removeItem} />
|
||||
}
|
||||
```
|
||||
|
||||
The first callback is recreated every time `items` changes, which can cause child components to re-render unnecessarily. The second callback has a stale closure bug—it will always reference the initial `items` value.
|
||||
|
||||
**Correct (stable callbacks, no stale closures):**
|
||||
|
||||
```tsx
|
||||
function TodoList() {
|
||||
const [items, setItems] = useState(initialItems)
|
||||
|
||||
// Stable callback, never recreated
|
||||
const addItems = useCallback((newItems: Item[]) => {
|
||||
setItems(curr => [...curr, ...newItems])
|
||||
}, []) // ✅ No dependencies needed
|
||||
|
||||
// Always uses latest state, no stale closure risk
|
||||
const removeItem = useCallback((id: string) => {
|
||||
setItems(curr => curr.filter(item => item.id !== id))
|
||||
}, []) // ✅ Safe and stable
|
||||
|
||||
return <ItemsEditor items={items} onAdd={addItems} onRemove={removeItem} />
|
||||
}
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
|
||||
1. **Stable callback references** - Callbacks don't need to be recreated when state changes
|
||||
2. **No stale closures** - Always operates on the latest state value
|
||||
3. **Fewer dependencies** - Simplifies dependency arrays and reduces memory leaks
|
||||
4. **Prevents bugs** - Eliminates the most common source of React closure bugs
|
||||
|
||||
**When to use functional updates:**
|
||||
|
||||
- Any setState that depends on the current state value
|
||||
- Inside useCallback/useMemo when state is needed
|
||||
- Event handlers that reference state
|
||||
- Async operations that update state
|
||||
|
||||
**When direct updates are fine:**
|
||||
|
||||
- Setting state to a static value: `setCount(0)`
|
||||
- Setting state from props/arguments only: `setName(newName)`
|
||||
- State doesn't depend on previous value
|
||||
|
||||
**Note:** If your project has [React Compiler](https://react.dev/learn/react-compiler) enabled, the compiler can automatically optimize some cases, but functional updates are still recommended for correctness and to prevent stale closure bugs.
|
||||
@@ -1,58 +0,0 @@
|
||||
---
|
||||
title: Use Lazy State Initialization
|
||||
impact: MEDIUM
|
||||
impactDescription: wasted computation on every render
|
||||
tags: react, hooks, useState, performance, initialization
|
||||
---
|
||||
|
||||
## Use Lazy State Initialization
|
||||
|
||||
Pass a function to `useState` for expensive initial values. Without the function form, the initializer runs on every render even though the value is only used once.
|
||||
|
||||
**Incorrect (runs on every render):**
|
||||
|
||||
```tsx
|
||||
function FilteredList({ items }: { items: Item[] }) {
|
||||
// buildSearchIndex() runs on EVERY render, even after initialization
|
||||
const [searchIndex, setSearchIndex] = useState(buildSearchIndex(items))
|
||||
const [query, setQuery] = useState('')
|
||||
|
||||
// When query changes, buildSearchIndex runs again unnecessarily
|
||||
return <SearchResults index={searchIndex} query={query} />
|
||||
}
|
||||
|
||||
function UserProfile() {
|
||||
// JSON.parse runs on every render
|
||||
const [settings, setSettings] = useState(
|
||||
JSON.parse(localStorage.getItem('settings') || '{}')
|
||||
)
|
||||
|
||||
return <SettingsForm settings={settings} onChange={setSettings} />
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (runs only once):**
|
||||
|
||||
```tsx
|
||||
function FilteredList({ items }: { items: Item[] }) {
|
||||
// buildSearchIndex() runs ONLY on initial render
|
||||
const [searchIndex, setSearchIndex] = useState(() => buildSearchIndex(items))
|
||||
const [query, setQuery] = useState('')
|
||||
|
||||
return <SearchResults index={searchIndex} query={query} />
|
||||
}
|
||||
|
||||
function UserProfile() {
|
||||
// JSON.parse runs only on initial render
|
||||
const [settings, setSettings] = useState(() => {
|
||||
const stored = localStorage.getItem('settings')
|
||||
return stored ? JSON.parse(stored) : {}
|
||||
})
|
||||
|
||||
return <SettingsForm settings={settings} onChange={setSettings} />
|
||||
}
|
||||
```
|
||||
|
||||
Use lazy initialization when computing initial values from localStorage/sessionStorage, building data structures (indexes, maps), reading from the DOM, or performing heavy transformations.
|
||||
|
||||
For simple primitives (`useState(0)`), direct references (`useState(props.value)`), or cheap literals (`useState({})`), the function form is unnecessary.
|
||||
@@ -1,45 +0,0 @@
|
||||
---
|
||||
title: Put Interaction Logic in Event Handlers
|
||||
impact: MEDIUM
|
||||
impactDescription: avoids effect re-runs and duplicate side effects
|
||||
tags: rerender, useEffect, events, side-effects, dependencies
|
||||
---
|
||||
|
||||
## Put Interaction Logic in Event Handlers
|
||||
|
||||
If a side effect is triggered by a specific user action (submit, click, drag), run it in that event handler. Do not model the action as state + effect; it makes effects re-run on unrelated changes and can duplicate the action.
|
||||
|
||||
**Incorrect (event modeled as state + effect):**
|
||||
|
||||
```tsx
|
||||
function Form() {
|
||||
const [submitted, setSubmitted] = useState(false)
|
||||
const theme = useContext(ThemeContext)
|
||||
|
||||
useEffect(() => {
|
||||
if (submitted) {
|
||||
post('/api/register')
|
||||
showToast('Registered', theme)
|
||||
}
|
||||
}, [submitted, theme])
|
||||
|
||||
return <button onClick={() => setSubmitted(true)}>Submit</button>
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (do it in the handler):**
|
||||
|
||||
```tsx
|
||||
function Form() {
|
||||
const theme = useContext(ThemeContext)
|
||||
|
||||
function handleSubmit() {
|
||||
post('/api/register')
|
||||
showToast('Registered', theme)
|
||||
}
|
||||
|
||||
return <button onClick={handleSubmit}>Submit</button>
|
||||
}
|
||||
```
|
||||
|
||||
Reference: [Should this code move to an event handler?](https://react.dev/learn/removing-effect-dependencies#should-this-code-move-to-an-event-handler)
|
||||
@@ -1,40 +0,0 @@
|
||||
---
|
||||
title: Use Transitions for Non-Urgent Updates
|
||||
impact: MEDIUM
|
||||
impactDescription: maintains UI responsiveness
|
||||
tags: rerender, transitions, startTransition, performance
|
||||
---
|
||||
|
||||
## Use Transitions for Non-Urgent Updates
|
||||
|
||||
Mark frequent, non-urgent state updates as transitions to maintain UI responsiveness.
|
||||
|
||||
**Incorrect (blocks UI on every scroll):**
|
||||
|
||||
```tsx
|
||||
function ScrollTracker() {
|
||||
const [scrollY, setScrollY] = useState(0)
|
||||
useEffect(() => {
|
||||
const handler = () => setScrollY(window.scrollY)
|
||||
window.addEventListener('scroll', handler, { passive: true })
|
||||
return () => window.removeEventListener('scroll', handler)
|
||||
}, [])
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (non-blocking updates):**
|
||||
|
||||
```tsx
|
||||
import { startTransition } from 'react'
|
||||
|
||||
function ScrollTracker() {
|
||||
const [scrollY, setScrollY] = useState(0)
|
||||
useEffect(() => {
|
||||
const handler = () => {
|
||||
startTransition(() => setScrollY(window.scrollY))
|
||||
}
|
||||
window.addEventListener('scroll', handler, { passive: true })
|
||||
return () => window.removeEventListener('scroll', handler)
|
||||
}, [])
|
||||
}
|
||||
```
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
---
|
||||
title: Use useRef for Transient Values
|
||||
impact: MEDIUM
|
||||
impactDescription: avoids unnecessary re-renders on frequent updates
|
||||
tags: rerender, useref, state, performance
|
||||
---
|
||||
|
||||
## Use useRef for Transient Values
|
||||
|
||||
When a value changes frequently and you don't want a re-render on every update (e.g., mouse trackers, intervals, transient flags), store it in `useRef` instead of `useState`. Keep component state for UI; use refs for temporary DOM-adjacent values. Updating a ref does not trigger a re-render.
|
||||
|
||||
**Incorrect (renders every update):**
|
||||
|
||||
```tsx
|
||||
function Tracker() {
|
||||
const [lastX, setLastX] = useState(0)
|
||||
|
||||
useEffect(() => {
|
||||
const onMove = (e: MouseEvent) => setLastX(e.clientX)
|
||||
window.addEventListener('mousemove', onMove)
|
||||
return () => window.removeEventListener('mousemove', onMove)
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
position: 'fixed',
|
||||
top: 0,
|
||||
left: lastX,
|
||||
width: 8,
|
||||
height: 8,
|
||||
background: 'black',
|
||||
}}
|
||||
/>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (no re-render for tracking):**
|
||||
|
||||
```tsx
|
||||
function Tracker() {
|
||||
const lastXRef = useRef(0)
|
||||
const dotRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const onMove = (e: MouseEvent) => {
|
||||
lastXRef.current = e.clientX
|
||||
const node = dotRef.current
|
||||
if (node) {
|
||||
node.style.transform = `translateX(${e.clientX}px)`
|
||||
}
|
||||
}
|
||||
window.addEventListener('mousemove', onMove)
|
||||
return () => window.removeEventListener('mousemove', onMove)
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={dotRef}
|
||||
style={{
|
||||
position: 'fixed',
|
||||
top: 0,
|
||||
left: 0,
|
||||
width: 8,
|
||||
height: 8,
|
||||
background: 'black',
|
||||
transform: 'translateX(0px)',
|
||||
}}
|
||||
/>
|
||||
)
|
||||
}
|
||||
```
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
name: react-composition-patterns
|
||||
description:
|
||||
React composition patterns that scale. Use when refactoring components with
|
||||
boolean prop proliferation, building flexible component libraries, or
|
||||
designing reusable APIs. Triggers on tasks involving compound components,
|
||||
render props, context providers, or component architecture.
|
||||
---
|
||||
|
||||
# React Composition Patterns
|
||||
|
||||
Composition patterns for building flexible, maintainable React components. Avoid
|
||||
boolean prop proliferation by using compound components, lifting state, and
|
||||
composing internals. These patterns make codebases easier for both humans and AI
|
||||
agents to work with as they scale.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Reference these guidelines when:
|
||||
|
||||
- Refactoring components with many boolean props
|
||||
- Building reusable component libraries
|
||||
- Designing flexible component APIs
|
||||
- Reviewing component architecture
|
||||
- Working with compound components or context providers
|
||||
|
||||
## Rule Categories by Priority
|
||||
|
||||
| Priority | Category | Impact | Prefix |
|
||||
| -------- | ----------------------- | ------ | --------------- |
|
||||
| 1 | Component Architecture | HIGH | `architecture-` |
|
||||
| 2 | State Management | MEDIUM | `state-` |
|
||||
| 3 | Implementation Patterns | MEDIUM | `patterns-` |
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### 1. Component Architecture (HIGH)
|
||||
|
||||
- `architecture-avoid-boolean-props` - Don't add boolean props to customize
|
||||
behavior; use composition
|
||||
- `architecture-compound-components` - Structure complex components with shared
|
||||
context
|
||||
|
||||
### 2. State Management (MEDIUM)
|
||||
|
||||
- `state-decouple-implementation` - Provider is the only place that knows how
|
||||
state is managed
|
||||
- `state-context-interface` - Define generic interface with state, actions, meta
|
||||
for dependency injection
|
||||
- `state-lift-state` - Move state into provider components for sibling access
|
||||
|
||||
### 3. Implementation Patterns (MEDIUM)
|
||||
|
||||
- `patterns-explicit-variants` - Create explicit variant components instead of
|
||||
boolean modes
|
||||
- `patterns-children-over-render-props` - Use children for composition instead
|
||||
of renderX props
|
||||
|
||||
## How to Use
|
||||
|
||||
Read individual rule files for detailed explanations and code examples:
|
||||
|
||||
```
|
||||
rules/architecture-avoid-boolean-props.md
|
||||
rules/state-context-interface.md
|
||||
```
|
||||
|
||||
Each rule file contains:
|
||||
|
||||
- Brief explanation of why it matters
|
||||
- Incorrect code example with explanation
|
||||
- Correct code example with explanation
|
||||
- Additional context and references
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
---
|
||||
title: Avoid Boolean Prop Proliferation
|
||||
impact: CRITICAL
|
||||
impactDescription: prevents unmaintainable component variants
|
||||
tags: composition, props, architecture
|
||||
---
|
||||
|
||||
## Avoid Boolean Prop Proliferation
|
||||
|
||||
Don't add boolean props like `isThread`, `isEditing`, `isDMThread` to customize
|
||||
component behavior. Each boolean doubles possible states and creates
|
||||
unmaintainable conditional logic. Use composition instead.
|
||||
|
||||
**Incorrect (boolean props create exponential complexity):**
|
||||
|
||||
```tsx
|
||||
function Composer({
|
||||
onSubmit,
|
||||
isThread,
|
||||
channelId,
|
||||
isDMThread,
|
||||
dmId,
|
||||
isEditing,
|
||||
isForwarding,
|
||||
}: Props) {
|
||||
return (
|
||||
<form>
|
||||
<Header />
|
||||
<Input />
|
||||
{isDMThread ? (
|
||||
<AlsoSendToDMField id={dmId} />
|
||||
) : isThread ? (
|
||||
<AlsoSendToChannelField id={channelId} />
|
||||
) : null}
|
||||
{isEditing ? (
|
||||
<EditActions />
|
||||
) : isForwarding ? (
|
||||
<ForwardActions />
|
||||
) : (
|
||||
<DefaultActions />
|
||||
)}
|
||||
<Footer onSubmit={onSubmit} />
|
||||
</form>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (composition eliminates conditionals):**
|
||||
|
||||
```tsx
|
||||
// Channel composer
|
||||
function ChannelComposer() {
|
||||
return (
|
||||
<Composer.Frame>
|
||||
<Composer.Header />
|
||||
<Composer.Input />
|
||||
<Composer.Footer>
|
||||
<Composer.Attachments />
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
<Composer.Submit />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
)
|
||||
}
|
||||
|
||||
// Thread composer - adds "also send to channel" field
|
||||
function ThreadComposer({ channelId }: { channelId: string }) {
|
||||
return (
|
||||
<Composer.Frame>
|
||||
<Composer.Header />
|
||||
<Composer.Input />
|
||||
<AlsoSendToChannelField id={channelId} />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
<Composer.Submit />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
)
|
||||
}
|
||||
|
||||
// Edit composer - different footer actions
|
||||
function EditComposer() {
|
||||
return (
|
||||
<Composer.Frame>
|
||||
<Composer.Input />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
<Composer.CancelEdit />
|
||||
<Composer.SaveEdit />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
Each variant is explicit about what it renders. We can share internals without
|
||||
sharing a single monolithic parent.
|
||||
-112
@@ -1,112 +0,0 @@
|
||||
---
|
||||
title: Use Compound Components
|
||||
impact: HIGH
|
||||
impactDescription: enables flexible composition without prop drilling
|
||||
tags: composition, compound-components, architecture
|
||||
---
|
||||
|
||||
## Use Compound Components
|
||||
|
||||
Structure complex components as compound components with a shared context. Each
|
||||
subcomponent accesses shared state via context, not props. Consumers compose the
|
||||
pieces they need.
|
||||
|
||||
**Incorrect (monolithic component with render props):**
|
||||
|
||||
```tsx
|
||||
function Composer({
|
||||
renderHeader,
|
||||
renderFooter,
|
||||
renderActions,
|
||||
showAttachments,
|
||||
showFormatting,
|
||||
showEmojis,
|
||||
}: Props) {
|
||||
return (
|
||||
<form>
|
||||
{renderHeader?.()}
|
||||
<Input />
|
||||
{showAttachments && <Attachments />}
|
||||
{renderFooter ? (
|
||||
renderFooter()
|
||||
) : (
|
||||
<Footer>
|
||||
{showFormatting && <Formatting />}
|
||||
{showEmojis && <Emojis />}
|
||||
{renderActions?.()}
|
||||
</Footer>
|
||||
)}
|
||||
</form>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (compound components with shared context):**
|
||||
|
||||
```tsx
|
||||
const ComposerContext = createContext<ComposerContextValue | null>(null)
|
||||
|
||||
function ComposerProvider({ children, state, actions, meta }: ProviderProps) {
|
||||
return (
|
||||
<ComposerContext value={{ state, actions, meta }}>
|
||||
{children}
|
||||
</ComposerContext>
|
||||
)
|
||||
}
|
||||
|
||||
function ComposerFrame({ children }: { children: React.ReactNode }) {
|
||||
return <form>{children}</form>
|
||||
}
|
||||
|
||||
function ComposerInput() {
|
||||
const {
|
||||
state,
|
||||
actions: { update },
|
||||
meta: { inputRef },
|
||||
} = use(ComposerContext)
|
||||
return (
|
||||
<TextInput
|
||||
ref={inputRef}
|
||||
value={state.input}
|
||||
onChangeText={(text) => update((s) => ({ ...s, input: text }))}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function ComposerSubmit() {
|
||||
const {
|
||||
actions: { submit },
|
||||
} = use(ComposerContext)
|
||||
return <Button onPress={submit}>Send</Button>
|
||||
}
|
||||
|
||||
// Export as compound component
|
||||
const Composer = {
|
||||
Provider: ComposerProvider,
|
||||
Frame: ComposerFrame,
|
||||
Input: ComposerInput,
|
||||
Submit: ComposerSubmit,
|
||||
Header: ComposerHeader,
|
||||
Footer: ComposerFooter,
|
||||
Attachments: ComposerAttachments,
|
||||
Formatting: ComposerFormatting,
|
||||
Emojis: ComposerEmojis,
|
||||
}
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
|
||||
```tsx
|
||||
<Composer.Provider state={state} actions={actions} meta={meta}>
|
||||
<Composer.Frame>
|
||||
<Composer.Header />
|
||||
<Composer.Input />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Submit />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
</Composer.Provider>
|
||||
```
|
||||
|
||||
Consumers explicitly compose exactly what they need. No hidden conditionals. And the state, actions and meta are dependency-injected by a parent provider, allowing multiple usages of the same component structure.
|
||||
-87
@@ -1,87 +0,0 @@
|
||||
---
|
||||
title: Prefer Composing Children Over Render Props
|
||||
impact: MEDIUM
|
||||
impactDescription: cleaner composition, better readability
|
||||
tags: composition, children, render-props
|
||||
---
|
||||
|
||||
## Prefer Children Over Render Props
|
||||
|
||||
Use `children` for composition instead of `renderX` props. Children are more
|
||||
readable, compose naturally, and don't require understanding callback
|
||||
signatures.
|
||||
|
||||
**Incorrect (render props):**
|
||||
|
||||
```tsx
|
||||
function Composer({
|
||||
renderHeader,
|
||||
renderFooter,
|
||||
renderActions,
|
||||
}: {
|
||||
renderHeader?: () => React.ReactNode
|
||||
renderFooter?: () => React.ReactNode
|
||||
renderActions?: () => React.ReactNode
|
||||
}) {
|
||||
return (
|
||||
<form>
|
||||
{renderHeader?.()}
|
||||
<Input />
|
||||
{renderFooter ? renderFooter() : <DefaultFooter />}
|
||||
{renderActions?.()}
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
// Usage is awkward and inflexible
|
||||
return (
|
||||
<Composer
|
||||
renderHeader={() => <CustomHeader />}
|
||||
renderFooter={() => (
|
||||
<>
|
||||
<Formatting />
|
||||
<Emojis />
|
||||
</>
|
||||
)}
|
||||
renderActions={() => <SubmitButton />}
|
||||
/>
|
||||
)
|
||||
```
|
||||
|
||||
**Correct (compound components with children):**
|
||||
|
||||
```tsx
|
||||
function ComposerFrame({ children }: { children: React.ReactNode }) {
|
||||
return <form>{children}</form>
|
||||
}
|
||||
|
||||
function ComposerFooter({ children }: { children: React.ReactNode }) {
|
||||
return <footer className='flex'>{children}</footer>
|
||||
}
|
||||
|
||||
// Usage is flexible
|
||||
return (
|
||||
<Composer.Frame>
|
||||
<CustomHeader />
|
||||
<Composer.Input />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
<SubmitButton />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
)
|
||||
```
|
||||
|
||||
**When render props are appropriate:**
|
||||
|
||||
```tsx
|
||||
// Render props work well when you need to pass data back
|
||||
<List
|
||||
data={items}
|
||||
renderItem={({ item, index }) => <Item item={item} index={index} />}
|
||||
/>
|
||||
```
|
||||
|
||||
Use render props when the parent needs to provide data or state to the child.
|
||||
Use children when composing static structure.
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
---
|
||||
title: Create Explicit Component Variants
|
||||
impact: MEDIUM
|
||||
impactDescription: self-documenting code, no hidden conditionals
|
||||
tags: composition, variants, architecture
|
||||
---
|
||||
|
||||
## Create Explicit Component Variants
|
||||
|
||||
Instead of one component with many boolean props, create explicit variant
|
||||
components. Each variant composes the pieces it needs. The code documents
|
||||
itself.
|
||||
|
||||
**Incorrect (one component, many modes):**
|
||||
|
||||
```tsx
|
||||
// What does this component actually render?
|
||||
<Composer
|
||||
isThread
|
||||
isEditing={false}
|
||||
channelId='abc'
|
||||
showAttachments
|
||||
showFormatting={false}
|
||||
/>
|
||||
```
|
||||
|
||||
**Correct (explicit variants):**
|
||||
|
||||
```tsx
|
||||
// Immediately clear what this renders
|
||||
<ThreadComposer channelId="abc" />
|
||||
|
||||
// Or
|
||||
<EditMessageComposer messageId="xyz" />
|
||||
|
||||
// Or
|
||||
<ForwardMessageComposer messageId="123" />
|
||||
```
|
||||
|
||||
Each implementation is unique, explicit and self-contained. Yet they can each
|
||||
use shared parts.
|
||||
|
||||
**Implementation:**
|
||||
|
||||
```tsx
|
||||
function ThreadComposer({ channelId }: { channelId: string }) {
|
||||
return (
|
||||
<ThreadProvider channelId={channelId}>
|
||||
<Composer.Frame>
|
||||
<Composer.Input />
|
||||
<AlsoSendToChannelField channelId={channelId} />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
<Composer.Submit />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
</ThreadProvider>
|
||||
)
|
||||
}
|
||||
|
||||
function EditMessageComposer({ messageId }: { messageId: string }) {
|
||||
return (
|
||||
<EditMessageProvider messageId={messageId}>
|
||||
<Composer.Frame>
|
||||
<Composer.Input />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
<Composer.CancelEdit />
|
||||
<Composer.SaveEdit />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
</EditMessageProvider>
|
||||
)
|
||||
}
|
||||
|
||||
function ForwardMessageComposer({ messageId }: { messageId: string }) {
|
||||
return (
|
||||
<ForwardMessageProvider messageId={messageId}>
|
||||
<Composer.Frame>
|
||||
<Composer.Input placeholder="Add a message, if you'd like." />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
<Composer.Mentions />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
</ForwardMessageProvider>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
Each variant is explicit about:
|
||||
|
||||
- What provider/state it uses
|
||||
- What UI elements it includes
|
||||
- What actions are available
|
||||
|
||||
No boolean prop combinations to reason about. No impossible states.
|
||||
@@ -1,191 +0,0 @@
|
||||
---
|
||||
title: Define Generic Context Interfaces for Dependency Injection
|
||||
impact: HIGH
|
||||
impactDescription: enables dependency-injectable state across use-cases
|
||||
tags: composition, context, state, typescript, dependency-injection
|
||||
---
|
||||
|
||||
## Define Generic Context Interfaces for Dependency Injection
|
||||
|
||||
Define a **generic interface** for your component context with three parts:
|
||||
`state`, `actions`, and `meta`. This interface is a contract that any provider
|
||||
can implement—enabling the same UI components to work with completely different
|
||||
state implementations.
|
||||
|
||||
**Core principle:** Lift state, compose internals, make state
|
||||
dependency-injectable.
|
||||
|
||||
**Incorrect (UI coupled to specific state implementation):**
|
||||
|
||||
```tsx
|
||||
function ComposerInput() {
|
||||
// Tightly coupled to a specific hook
|
||||
const { input, setInput } = useChannelComposerState()
|
||||
return <TextInput value={input} onChangeText={setInput} />
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (generic interface enables dependency injection):**
|
||||
|
||||
```tsx
|
||||
// Define a GENERIC interface that any provider can implement
|
||||
interface ComposerState {
|
||||
input: string
|
||||
attachments: Attachment[]
|
||||
isSubmitting: boolean
|
||||
}
|
||||
|
||||
interface ComposerActions {
|
||||
update: (updater: (state: ComposerState) => ComposerState) => void
|
||||
submit: () => void
|
||||
}
|
||||
|
||||
interface ComposerMeta {
|
||||
inputRef: React.RefObject<TextInput>
|
||||
}
|
||||
|
||||
interface ComposerContextValue {
|
||||
state: ComposerState
|
||||
actions: ComposerActions
|
||||
meta: ComposerMeta
|
||||
}
|
||||
|
||||
const ComposerContext = createContext<ComposerContextValue | null>(null)
|
||||
```
|
||||
|
||||
**UI components consume the interface, not the implementation:**
|
||||
|
||||
```tsx
|
||||
function ComposerInput() {
|
||||
const {
|
||||
state,
|
||||
actions: { update },
|
||||
meta,
|
||||
} = use(ComposerContext)
|
||||
|
||||
// This component works with ANY provider that implements the interface
|
||||
return (
|
||||
<TextInput
|
||||
ref={meta.inputRef}
|
||||
value={state.input}
|
||||
onChangeText={(text) => update((s) => ({ ...s, input: text }))}
|
||||
/>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Different providers implement the same interface:**
|
||||
|
||||
```tsx
|
||||
// Provider A: Local state for ephemeral forms
|
||||
function ForwardMessageProvider({ children }: { children: React.ReactNode }) {
|
||||
const [state, setState] = useState(initialState)
|
||||
const inputRef = useRef(null)
|
||||
const submit = useForwardMessage()
|
||||
|
||||
return (
|
||||
<ComposerContext
|
||||
value={{
|
||||
state,
|
||||
actions: { update: setState, submit },
|
||||
meta: { inputRef },
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</ComposerContext>
|
||||
)
|
||||
}
|
||||
|
||||
// Provider B: Global synced state for channels
|
||||
function ChannelProvider({ channelId, children }: Props) {
|
||||
const { state, update, submit } = useGlobalChannel(channelId)
|
||||
const inputRef = useRef(null)
|
||||
|
||||
return (
|
||||
<ComposerContext
|
||||
value={{
|
||||
state,
|
||||
actions: { update, submit },
|
||||
meta: { inputRef },
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</ComposerContext>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**The same composed UI works with both:**
|
||||
|
||||
```tsx
|
||||
// Works with ForwardMessageProvider (local state)
|
||||
<ForwardMessageProvider>
|
||||
<Composer.Frame>
|
||||
<Composer.Input />
|
||||
<Composer.Submit />
|
||||
</Composer.Frame>
|
||||
</ForwardMessageProvider>
|
||||
|
||||
// Works with ChannelProvider (global synced state)
|
||||
<ChannelProvider channelId="abc">
|
||||
<Composer.Frame>
|
||||
<Composer.Input />
|
||||
<Composer.Submit />
|
||||
</Composer.Frame>
|
||||
</ChannelProvider>
|
||||
```
|
||||
|
||||
**Custom UI outside the component can access state and actions:**
|
||||
|
||||
The provider boundary is what matters—not the visual nesting. Components that
|
||||
need shared state don't have to be inside the `Composer.Frame`. They just need
|
||||
to be within the provider.
|
||||
|
||||
```tsx
|
||||
function ForwardMessageDialog() {
|
||||
return (
|
||||
<ForwardMessageProvider>
|
||||
<Dialog>
|
||||
{/* The composer UI */}
|
||||
<Composer.Frame>
|
||||
<Composer.Input placeholder="Add a message, if you'd like." />
|
||||
<Composer.Footer>
|
||||
<Composer.Formatting />
|
||||
<Composer.Emojis />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
|
||||
{/* Custom UI OUTSIDE the composer, but INSIDE the provider */}
|
||||
<MessagePreview />
|
||||
|
||||
{/* Actions at the bottom of the dialog */}
|
||||
<DialogActions>
|
||||
<CancelButton />
|
||||
<ForwardButton />
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
</ForwardMessageProvider>
|
||||
)
|
||||
}
|
||||
|
||||
// This button lives OUTSIDE Composer.Frame but can still submit based on its context!
|
||||
function ForwardButton() {
|
||||
const {
|
||||
actions: { submit },
|
||||
} = use(ComposerContext)
|
||||
return <Button onPress={submit}>Forward</Button>
|
||||
}
|
||||
|
||||
// This preview lives OUTSIDE Composer.Frame but can read composer's state!
|
||||
function MessagePreview() {
|
||||
const { state } = use(ComposerContext)
|
||||
return <Preview message={state.input} attachments={state.attachments} />
|
||||
}
|
||||
```
|
||||
|
||||
The `ForwardButton` and `MessagePreview` are not visually inside the composer
|
||||
box, but they can still access its state and actions. This is the power of
|
||||
lifting state into providers.
|
||||
|
||||
The UI is reusable bits you compose together. The state is dependency-injected
|
||||
by the provider. Swap the provider, keep the UI.
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
---
|
||||
title: Decouple State Management from UI
|
||||
impact: MEDIUM
|
||||
impactDescription: enables swapping state implementations without changing UI
|
||||
tags: composition, state, architecture
|
||||
---
|
||||
|
||||
## Decouple State Management from UI
|
||||
|
||||
The provider component should be the only place that knows how state is managed.
|
||||
UI components consume the context interface—they don't know if state comes from
|
||||
useState, Zustand, or a server sync.
|
||||
|
||||
**Incorrect (UI coupled to state implementation):**
|
||||
|
||||
```tsx
|
||||
function ChannelComposer({ channelId }: { channelId: string }) {
|
||||
// UI component knows about global state implementation
|
||||
const state = useGlobalChannelState(channelId)
|
||||
const { submit, updateInput } = useChannelSync(channelId)
|
||||
|
||||
return (
|
||||
<Composer.Frame>
|
||||
<Composer.Input
|
||||
value={state.input}
|
||||
onChange={(text) => sync.updateInput(text)}
|
||||
/>
|
||||
<Composer.Submit onPress={() => sync.submit()} />
|
||||
</Composer.Frame>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (state management isolated in provider):**
|
||||
|
||||
```tsx
|
||||
// Provider handles all state management details
|
||||
function ChannelProvider({
|
||||
channelId,
|
||||
children,
|
||||
}: {
|
||||
channelId: string
|
||||
children: React.ReactNode
|
||||
}) {
|
||||
const { state, update, submit } = useGlobalChannel(channelId)
|
||||
const inputRef = useRef(null)
|
||||
|
||||
return (
|
||||
<Composer.Provider
|
||||
state={state}
|
||||
actions={{ update, submit }}
|
||||
meta={{ inputRef }}
|
||||
>
|
||||
{children}
|
||||
</Composer.Provider>
|
||||
)
|
||||
}
|
||||
|
||||
// UI component only knows about the context interface
|
||||
function ChannelComposer() {
|
||||
return (
|
||||
<Composer.Frame>
|
||||
<Composer.Header />
|
||||
<Composer.Input />
|
||||
<Composer.Footer>
|
||||
<Composer.Submit />
|
||||
</Composer.Footer>
|
||||
</Composer.Frame>
|
||||
)
|
||||
}
|
||||
|
||||
// Usage
|
||||
function Channel({ channelId }: { channelId: string }) {
|
||||
return (
|
||||
<ChannelProvider channelId={channelId}>
|
||||
<ChannelComposer />
|
||||
</ChannelProvider>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Different providers, same UI:**
|
||||
|
||||
```tsx
|
||||
// Local state for ephemeral forms
|
||||
function ForwardMessageProvider({ children }) {
|
||||
const [state, setState] = useState(initialState)
|
||||
const forwardMessage = useForwardMessage()
|
||||
|
||||
return (
|
||||
<Composer.Provider
|
||||
state={state}
|
||||
actions={{ update: setState, submit: forwardMessage }}
|
||||
>
|
||||
{children}
|
||||
</Composer.Provider>
|
||||
)
|
||||
}
|
||||
|
||||
// Global synced state for channels
|
||||
function ChannelProvider({ channelId, children }) {
|
||||
const { state, update, submit } = useGlobalChannel(channelId)
|
||||
|
||||
return (
|
||||
<Composer.Provider state={state} actions={{ update, submit }}>
|
||||
{children}
|
||||
</Composer.Provider>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
The same `Composer.Input` component works with both providers because it only
|
||||
depends on the context interface, not the implementation.
|
||||
@@ -1,125 +0,0 @@
|
||||
---
|
||||
title: Lift State into Provider Components
|
||||
impact: HIGH
|
||||
impactDescription: enables state sharing outside component boundaries
|
||||
tags: composition, state, context, providers
|
||||
---
|
||||
|
||||
## Lift State into Provider Components
|
||||
|
||||
Move state management into dedicated provider components. This allows sibling
|
||||
components outside the main UI to access and modify state without prop drilling
|
||||
or awkward refs.
|
||||
|
||||
**Incorrect (state trapped inside component):**
|
||||
|
||||
```tsx
|
||||
function ForwardMessageComposer() {
|
||||
const [state, setState] = useState(initialState)
|
||||
const forwardMessage = useForwardMessage()
|
||||
|
||||
return (
|
||||
<Composer.Frame>
|
||||
<Composer.Input />
|
||||
<Composer.Footer />
|
||||
</Composer.Frame>
|
||||
)
|
||||
}
|
||||
|
||||
// Problem: How does this button access composer state?
|
||||
function ForwardMessageDialog() {
|
||||
return (
|
||||
<Dialog>
|
||||
<ForwardMessageComposer />
|
||||
<MessagePreview /> {/* Needs composer state */}
|
||||
<DialogActions>
|
||||
<CancelButton />
|
||||
<ForwardButton /> {/* Needs to call submit */}
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Incorrect (useEffect to sync state up):**
|
||||
|
||||
```tsx
|
||||
function ForwardMessageDialog() {
|
||||
const [input, setInput] = useState('')
|
||||
return (
|
||||
<Dialog>
|
||||
<ForwardMessageComposer onInputChange={setInput} />
|
||||
<MessagePreview input={input} />
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
function ForwardMessageComposer({ onInputChange }) {
|
||||
const [state, setState] = useState(initialState)
|
||||
useEffect(() => {
|
||||
onInputChange(state.input) // Sync on every change 😬
|
||||
}, [state.input])
|
||||
}
|
||||
```
|
||||
|
||||
**Incorrect (reading state from ref on submit):**
|
||||
|
||||
```tsx
|
||||
function ForwardMessageDialog() {
|
||||
const stateRef = useRef(null)
|
||||
return (
|
||||
<Dialog>
|
||||
<ForwardMessageComposer stateRef={stateRef} />
|
||||
<ForwardButton onPress={() => submit(stateRef.current)} />
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
**Correct (state lifted to provider):**
|
||||
|
||||
```tsx
|
||||
function ForwardMessageProvider({ children }: { children: React.ReactNode }) {
|
||||
const [state, setState] = useState(initialState)
|
||||
const forwardMessage = useForwardMessage()
|
||||
const inputRef = useRef(null)
|
||||
|
||||
return (
|
||||
<Composer.Provider
|
||||
state={state}
|
||||
actions={{ update: setState, submit: forwardMessage }}
|
||||
meta={{ inputRef }}
|
||||
>
|
||||
{children}
|
||||
</Composer.Provider>
|
||||
)
|
||||
}
|
||||
|
||||
function ForwardMessageDialog() {
|
||||
return (
|
||||
<ForwardMessageProvider>
|
||||
<Dialog>
|
||||
<ForwardMessageComposer />
|
||||
<MessagePreview /> {/* Custom components can access state and actions */}
|
||||
<DialogActions>
|
||||
<CancelButton />
|
||||
<ForwardButton /> {/* Custom components can access state and actions */}
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
</ForwardMessageProvider>
|
||||
)
|
||||
}
|
||||
|
||||
function ForwardButton() {
|
||||
const { actions } = use(Composer.Context)
|
||||
return <Button onPress={actions.submit}>Forward</Button>
|
||||
}
|
||||
```
|
||||
|
||||
The ForwardButton lives outside the Composer.Frame but still has access to the
|
||||
submit action because it's within the provider. Even though it's a one-off
|
||||
component, it can still access the composer's state and actions from outside the
|
||||
UI itself.
|
||||
|
||||
**Key insight:** Components that need shared state don't have to be visually
|
||||
nested inside each other—they just need to be within the same provider.
|
||||
@@ -1,19 +0,0 @@
|
||||
---
|
||||
name: web-design-guidelines
|
||||
description: Review UI code for Web Interface Guidelines compliance. Use when asked to "review my UI", "check accessibility", "audit design", "review UX", or "check my site against best practices".
|
||||
argument-hint: <file-or-pattern>
|
||||
---
|
||||
|
||||
# Web Interface Guidelines
|
||||
|
||||
Review files for compliance with Web Interface Guidelines.
|
||||
|
||||
## Usage
|
||||
|
||||
When a user provides a file or pattern argument:
|
||||
1. Read guidelines from rules/accessibility.md
|
||||
2. Read the specified files
|
||||
3. Apply all rules from the fetched guidelines
|
||||
4. Output findings using the format specified in the guidelines
|
||||
|
||||
If no files specified, ask the user which files to review.
|
||||
@@ -1,179 +0,0 @@
|
||||
---
|
||||
description: Review UI code for Web Interface Guidelines compliance
|
||||
argument-hint: <file-or-pattern>
|
||||
---
|
||||
|
||||
# Web Interface Guidelines
|
||||
|
||||
Review these files for compliance: $ARGUMENTS
|
||||
|
||||
Read files, check against rules below. Output concise but comprehensive—sacrifice grammar for brevity. High signal-to-noise.
|
||||
|
||||
## Rules
|
||||
|
||||
### Accessibility
|
||||
|
||||
- Icon-only buttons need `aria-label`
|
||||
- Form controls need `<label>` or `aria-label`
|
||||
- Interactive elements need keyboard handlers (`onKeyDown`/`onKeyUp`)
|
||||
- `<button>` for actions, `<a>`/`<Link>` for navigation (not `<div onClick>`)
|
||||
- Images need `alt` (or `alt=""` if decorative)
|
||||
- Decorative icons need `aria-hidden="true"`
|
||||
- Async updates (toasts, validation) need `aria-live="polite"`
|
||||
- Use semantic HTML (`<button>`, `<a>`, `<label>`, `<table>`) before ARIA
|
||||
- Headings hierarchical `<h1>`–`<h6>`; include skip link for main content
|
||||
- `scroll-margin-top` on heading anchors
|
||||
|
||||
### Focus States
|
||||
|
||||
- Interactive elements need visible focus: `focus-visible:ring-*` or equivalent
|
||||
- Never `outline-none` / `outline: none` without focus replacement
|
||||
- Use `:focus-visible` over `:focus` (avoid focus ring on click)
|
||||
- Group focus with `:focus-within` for compound controls
|
||||
|
||||
### Forms
|
||||
|
||||
- Inputs need `autocomplete` and meaningful `name`
|
||||
- Use correct `type` (`email`, `tel`, `url`, `number`) and `inputmode`
|
||||
- Never block paste (`onPaste` + `preventDefault`)
|
||||
- Labels clickable (`htmlFor` or wrapping control)
|
||||
- Disable spellcheck on emails, codes, usernames (`spellCheck={false}`)
|
||||
- Checkboxes/radios: label + control share single hit target (no dead zones)
|
||||
- Submit button stays enabled until request starts; spinner during request
|
||||
- Errors inline next to fields; focus first error on submit
|
||||
- Placeholders end with `…` and show example pattern
|
||||
- `autocomplete="off"` on non-auth fields to avoid password manager triggers
|
||||
- Warn before navigation with unsaved changes (`beforeunload` or router guard)
|
||||
|
||||
### Animation
|
||||
|
||||
- Honor `prefers-reduced-motion` (provide reduced variant or disable)
|
||||
- Animate `transform`/`opacity` only (compositor-friendly)
|
||||
- Never `transition: all`—list properties explicitly
|
||||
- Set correct `transform-origin`
|
||||
- SVG: transforms on `<g>` wrapper with `transform-box: fill-box; transform-origin: center`
|
||||
- Animations interruptible—respond to user input mid-animation
|
||||
|
||||
### Typography
|
||||
|
||||
- `…` not `...`
|
||||
- Curly quotes `"` `"` not straight `"`
|
||||
- Non-breaking spaces: `10 MB`, `⌘ K`, brand names
|
||||
- Loading states end with `…`: `"Loading…"`, `"Saving…"`
|
||||
- `font-variant-numeric: tabular-nums` for number columns/comparisons
|
||||
- Use `text-wrap: balance` or `text-pretty` on headings (prevents widows)
|
||||
|
||||
### Content Handling
|
||||
|
||||
- Text containers handle long content: `truncate`, `line-clamp-*`, or `break-words`
|
||||
- Flex children need `min-w-0` to allow text truncation
|
||||
- Handle empty states—don't render broken UI for empty strings/arrays
|
||||
- User-generated content: anticipate short, average, and very long inputs
|
||||
|
||||
### Images
|
||||
|
||||
- `<img>` needs explicit `width` and `height` (prevents CLS)
|
||||
- Below-fold images: `loading="lazy"`
|
||||
- Above-fold critical images: `priority` or `fetchpriority="high"`
|
||||
|
||||
### Performance
|
||||
|
||||
- Large lists (>50 items): virtualize (`virtua`, `content-visibility: auto`)
|
||||
- No layout reads in render (`getBoundingClientRect`, `offsetHeight`, `offsetWidth`, `scrollTop`)
|
||||
- Batch DOM reads/writes; avoid interleaving
|
||||
- Prefer uncontrolled inputs; controlled inputs must be cheap per keystroke
|
||||
- Add `<link rel="preconnect">` for CDN/asset domains
|
||||
- Critical fonts: `<link rel="preload" as="font">` with `font-display: swap`
|
||||
|
||||
### Navigation & State
|
||||
|
||||
- URL reflects state—filters, tabs, pagination, expanded panels in query params
|
||||
- Links use `<a>`/`<Link>` (Cmd/Ctrl+click, middle-click support)
|
||||
- Deep-link all stateful UI (if uses `useState`, consider URL sync via nuqs or similar)
|
||||
- Destructive actions need confirmation modal or undo window—never immediate
|
||||
|
||||
### Touch & Interaction
|
||||
|
||||
- `touch-action: manipulation` (prevents double-tap zoom delay)
|
||||
- `-webkit-tap-highlight-color` set intentionally
|
||||
- `overscroll-behavior: contain` in modals/drawers/sheets
|
||||
- During drag: disable text selection, `inert` on dragged elements
|
||||
- `autoFocus` sparingly—desktop only, single primary input; avoid on mobile
|
||||
|
||||
### Safe Areas & Layout
|
||||
|
||||
- Full-bleed layouts need `env(safe-area-inset-*)` for notches
|
||||
- Avoid unwanted scrollbars: `overflow-x-hidden` on containers, fix content overflow
|
||||
- Flex/grid over JS measurement for layout
|
||||
|
||||
### Dark Mode & Theming
|
||||
|
||||
- `color-scheme: dark` on `<html>` for dark themes (fixes scrollbar, inputs)
|
||||
- `<meta name="theme-color">` matches page background
|
||||
- Native `<select>`: explicit `background-color` and `color` (Windows dark mode)
|
||||
|
||||
### Locale & i18n
|
||||
|
||||
- Dates/times: use `Intl.DateTimeFormat` not hardcoded formats
|
||||
- Numbers/currency: use `Intl.NumberFormat` not hardcoded formats
|
||||
- Detect language via `Accept-Language` / `navigator.languages`, not IP
|
||||
|
||||
### Hydration Safety
|
||||
|
||||
- Inputs with `value` need `onChange` (or use `defaultValue` for uncontrolled)
|
||||
- Date/time rendering: guard against hydration mismatch (server vs client)
|
||||
- `suppressHydrationWarning` only where truly needed
|
||||
|
||||
### Hover & Interactive States
|
||||
|
||||
- Buttons/links need `hover:` state (visual feedback)
|
||||
- Interactive states increase contrast: hover/active/focus more prominent than rest
|
||||
|
||||
### Content & Copy
|
||||
|
||||
- Active voice: "Install the CLI" not "The CLI will be installed"
|
||||
- Title Case for headings/buttons (Chicago style)
|
||||
- Numerals for counts: "8 deployments" not "eight"
|
||||
- Specific button labels: "Save API Key" not "Continue"
|
||||
- Error messages include fix/next step, not just problem
|
||||
- Second person; avoid first person
|
||||
- `&` over "and" where space-constrained
|
||||
|
||||
### Anti-patterns (flag these)
|
||||
|
||||
- `user-scalable=no` or `maximum-scale=1` disabling zoom
|
||||
- `onPaste` with `preventDefault`
|
||||
- `transition: all`
|
||||
- `outline-none` without focus-visible replacement
|
||||
- Inline `onClick` navigation without `<a>`
|
||||
- `<div>` or `<span>` with click handlers (should be `<button>`)
|
||||
- Images without dimensions
|
||||
- Large arrays `.map()` without virtualization
|
||||
- Form inputs without labels
|
||||
- Icon buttons without `aria-label`
|
||||
- Hardcoded date/number formats (use `Intl.*`)
|
||||
- `autoFocus` without clear justification
|
||||
|
||||
## Output Format
|
||||
|
||||
Group by file. Use `file:line` format (VS Code clickable). Terse findings.
|
||||
|
||||
```text
|
||||
## src/Button.tsx
|
||||
|
||||
src/Button.tsx:42 - icon button missing aria-label
|
||||
src/Button.tsx:18 - input lacks label
|
||||
src/Button.tsx:55 - animation missing prefers-reduced-motion
|
||||
src/Button.tsx:67 - transition: all → list properties
|
||||
|
||||
## src/Modal.tsx
|
||||
|
||||
src/Modal.tsx:12 - missing overscroll-behavior: contain
|
||||
src/Modal.tsx:34 - "..." → "…"
|
||||
|
||||
## src/Card.tsx
|
||||
|
||||
✓ pass
|
||||
```
|
||||
|
||||
State issue + location. Skip explanation unless fix non-obvious. No preamble.
|
||||
@@ -1,37 +0,0 @@
|
||||
GO_PROJECT_NAME := webserver
|
||||
|
||||
build:
|
||||
@echo "\n....Building frontend...."
|
||||
cd frontend && pnpm build && cd ..
|
||||
|
||||
@echo "\n....Building $(GO_PROJECT_NAME)...."
|
||||
cd webserver && GOWORK=off go build -ldflags "-s -w" -o ../bin/$(GO_PROJECT_NAME) .
|
||||
|
||||
run:
|
||||
$(MAKE) build
|
||||
@echo "\n....Running $(GO_PROJECT_NAME)...."
|
||||
./bin/$(GO_PROJECT_NAME)
|
||||
|
||||
clean:
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
|
||||
test:
|
||||
@echo "\n....Running tests for $(GO_PROJECT_NAME)...."
|
||||
cd $(GO_PROJECT_NAME) && LOG_IGNORE=1 go test -v -timeout 120s ./...
|
||||
|
||||
security:
|
||||
@echo "\n....Running security checks for $(GO_PROJECT_NAME)...."
|
||||
|
||||
cd $(GO_PROJECT_NAME) && go tool govulncheck ./... > ../tests/govulncheck-report.txt || true
|
||||
@echo "Govulncheck report saved to tests/govulncheck-report.txt"
|
||||
|
||||
cd $(GO_PROJECT_NAME) && go tool gosec -exclude-generated ./... > ../tests/gosec-report.txt || true
|
||||
@echo "Gosec report saved to tests/gosec-report.txt"
|
||||
|
||||
cd $(GO_PROJECT_NAME) && go tool staticcheck ./... > ../tests/staticcheck-report.txt || true
|
||||
@echo "Staticcheck report saved to tests/staticcheck-report.txt"
|
||||
|
||||
cd $(GO_PROJECT_NAME) && go vet ./... 2>&1 | tee ../tests/govet-report.txt || true
|
||||
@echo "Go vet report saved to tests/govet-report.txt"
|
||||
|
||||
.PHONY: build run clean test security
|
||||
@@ -1,178 +0,0 @@
|
||||
import {
|
||||
type _InstanceType,
|
||||
CreateKeyPairCommand,
|
||||
DeleteKeyPairCommand,
|
||||
ImportKeyPairCommand,
|
||||
RebootInstancesCommand,
|
||||
RunInstancesCommand,
|
||||
StartInstancesCommand,
|
||||
StopInstancesCommand,
|
||||
TerminateInstancesCommand,
|
||||
} from "@aws-sdk/client-ec2"
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query"
|
||||
|
||||
import { getEc2Client } from "@/lib/awsClient"
|
||||
import {
|
||||
ec2InstancesQueryOptions,
|
||||
ec2KeyPairsQueryOptions,
|
||||
} from "@/queries/ec2"
|
||||
import type {
|
||||
CreateInstanceParams,
|
||||
CreateKeyPairData,
|
||||
ImportKeyPairData,
|
||||
} from "@/types/ec2"
|
||||
|
||||
const WHITESPACE_REGEX = /\s+/
|
||||
|
||||
export function useStartInstance() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (instanceId: string) => {
|
||||
const command = new StartInstancesCommand({
|
||||
InstanceIds: [instanceId],
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["ec2", "instances"] })
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useStopInstance() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (instanceId: string) => {
|
||||
const command = new StopInstancesCommand({
|
||||
InstanceIds: [instanceId],
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["ec2", "instances"] })
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useTerminateInstance() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (instanceId: string) => {
|
||||
const command = new TerminateInstancesCommand({
|
||||
InstanceIds: [instanceId],
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["ec2", "instances"] })
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useRebootInstance() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (instanceId: string) => {
|
||||
const command = new RebootInstancesCommand({
|
||||
InstanceIds: [instanceId],
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["ec2", "instances"] })
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useCreateInstance() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (params: CreateInstanceParams) => {
|
||||
const command = new RunInstancesCommand({
|
||||
ImageId: params.imageId,
|
||||
InstanceType: params.instanceType as _InstanceType,
|
||||
KeyName: params.keyName,
|
||||
MinCount: params.count,
|
||||
MaxCount: params.count,
|
||||
SecurityGroupIds: params.securityGroupIds,
|
||||
SubnetId: params.subnetId,
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: async (data) => {
|
||||
await queryClient.cancelQueries(ec2InstancesQueryOptions)
|
||||
|
||||
const previousData = queryClient.getQueryData(
|
||||
ec2InstancesQueryOptions.queryKey,
|
||||
)
|
||||
|
||||
if (previousData && data.Instances) {
|
||||
const newReservation = {
|
||||
Instances: data.Instances,
|
||||
OwnerId: data.OwnerId,
|
||||
RequesterId: data.RequesterId,
|
||||
ReservationId: data.ReservationId,
|
||||
Groups: [],
|
||||
}
|
||||
|
||||
queryClient.setQueryData(ec2InstancesQueryOptions.queryKey, {
|
||||
...previousData,
|
||||
Reservations: [...(previousData.Reservations || []), newReservation],
|
||||
})
|
||||
}
|
||||
|
||||
queryClient.invalidateQueries({ queryKey: ["ec2", "instances"] })
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useCreateKeyPair() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (params: CreateKeyPairData) => {
|
||||
const command = new CreateKeyPairCommand({
|
||||
KeyName: params.keyName,
|
||||
KeyType: "rsa",
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries(ec2KeyPairsQueryOptions)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useImportKeyPair() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (params: ImportKeyPairData) => {
|
||||
// remove optional comment from ssh key as it breaks the import
|
||||
const keyParts = params.publicKeyMaterial.trim().split(WHITESPACE_REGEX)
|
||||
const cleanedKey = keyParts.slice(0, 2).join(" ")
|
||||
|
||||
const command = new ImportKeyPairCommand({
|
||||
KeyName: params.keyName,
|
||||
PublicKeyMaterial: new TextEncoder().encode(cleanedKey),
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries(ec2KeyPairsQueryOptions)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useDeleteKeyPair() {
|
||||
const queryClient = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: async (keyPairId: string) => {
|
||||
const command = new DeleteKeyPairCommand({
|
||||
KeyPairId: keyPairId,
|
||||
})
|
||||
return await getEc2Client().send(command)
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries(ec2KeyPairsQueryOptions)
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
import { DeleteObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query"
|
||||
|
||||
import { getS3Client } from "@/lib/awsClient"
|
||||
|
||||
interface UploadObjectParams {
|
||||
bucket: string
|
||||
key: string
|
||||
file: File
|
||||
}
|
||||
|
||||
export function useUploadObject() {
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
return useMutation({
|
||||
mutationFn: async ({ bucket, key, file }: UploadObjectParams) => {
|
||||
const arrayBuffer = await file.arrayBuffer()
|
||||
const body = new Uint8Array(arrayBuffer)
|
||||
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: file.type,
|
||||
})
|
||||
|
||||
return await getS3Client().send(command)
|
||||
},
|
||||
onSuccess: (_, variables) => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ["s3", "buckets", variables.bucket, "objects"],
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function useDeleteObject() {
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
return useMutation({
|
||||
mutationFn: async ({ bucket, key }: { bucket: string; key: string }) => {
|
||||
const command = new DeleteObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
})
|
||||
|
||||
return await getS3Client().send(command)
|
||||
},
|
||||
onSuccess: (_, variables) => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ["s3", "buckets", variables.bucket, "objects"],
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
import { useSuspenseQuery } from "@tanstack/react-query"
|
||||
import { createFileRoute } from "@tanstack/react-router"
|
||||
|
||||
import { BackLink } from "@/components/back-link"
|
||||
import { DetailCard } from "@/components/detail-card"
|
||||
import { DetailRow } from "@/components/detail-row"
|
||||
import { PageHeading } from "@/components/page-heading"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { formatDateTime } from "@/lib/utils"
|
||||
import { ec2VolumeQueryOptions } from "@/queries/ec2"
|
||||
|
||||
export const Route = createFileRoute(
|
||||
"/_auth/ec2/(volumes)/describe-volumes/$id",
|
||||
)({
|
||||
loader: async ({ context, params }) => {
|
||||
await context.queryClient.ensureQueryData(ec2VolumeQueryOptions(params.id))
|
||||
},
|
||||
head: ({ params }) => ({
|
||||
meta: [
|
||||
{
|
||||
title: `${params.id} | EC2 | Mulga`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
component: VolumeDetail,
|
||||
})
|
||||
|
||||
function VolumeDetail() {
|
||||
const { id } = Route.useParams()
|
||||
const { data } = useSuspenseQuery(ec2VolumeQueryOptions(id))
|
||||
const volume = data.Volumes?.[0]
|
||||
|
||||
if (!volume?.VolumeId) {
|
||||
return (
|
||||
<>
|
||||
<BackLink to="/ec2/describe-volumes">Back to volumes</BackLink>
|
||||
<p className="text-muted-foreground">Volume not found.</p>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
const createTime = formatDateTime(volume.CreateTime)
|
||||
|
||||
return (
|
||||
<>
|
||||
<BackLink to="/ec2/describe-volumes">Back to volumes</BackLink>
|
||||
|
||||
<div className="space-y-6">
|
||||
<PageHeading
|
||||
actions={
|
||||
volume.State ? (
|
||||
<Badge
|
||||
variant={volume.State === "available" ? "default" : "outline"}
|
||||
>
|
||||
{volume.State}
|
||||
</Badge>
|
||||
) : undefined
|
||||
}
|
||||
subtitle="EC2 Volume Details"
|
||||
title={volume.VolumeId}
|
||||
/>
|
||||
|
||||
<DetailCard>
|
||||
<DetailCard.Header>Volume Information</DetailCard.Header>
|
||||
<DetailCard.Content>
|
||||
<DetailRow label="Volume ID" value={volume.VolumeId} />
|
||||
<DetailRow label="Size" value={`${volume.Size} GiB`} />
|
||||
<DetailRow label="Volume Type" value={volume.VolumeType} />
|
||||
<DetailRow label="State" value={volume.State} />
|
||||
<DetailRow
|
||||
label="Availability Zone"
|
||||
value={volume.AvailabilityZone}
|
||||
/>
|
||||
<DetailRow label="Create Time" value={createTime} />
|
||||
<DetailRow
|
||||
label="Encrypted"
|
||||
value={volume.Encrypted ? "Yes" : "No"}
|
||||
/>
|
||||
<DetailRow label="KMS Key ID" value={volume.KmsKeyId} />
|
||||
</DetailCard.Content>
|
||||
</DetailCard>
|
||||
|
||||
{volume.Attachments && volume.Attachments.length > 0 && (
|
||||
<DetailCard>
|
||||
<DetailCard.Header>Attachments</DetailCard.Header>
|
||||
<DetailCard.Content>
|
||||
{volume.Attachments.map((attachment) => (
|
||||
<div
|
||||
className="space-y-2 border-b pb-2 last:border-0 last:pb-0"
|
||||
key={attachment.InstanceId}
|
||||
>
|
||||
<DetailRow
|
||||
label="Instance ID"
|
||||
value={attachment.InstanceId}
|
||||
/>
|
||||
<DetailRow label="Device" value={attachment.Device} />
|
||||
<DetailRow label="Status" value={attachment.State} />
|
||||
<DetailRow
|
||||
label="Attach Time"
|
||||
value={formatDateTime(attachment.AttachTime)}
|
||||
/>
|
||||
<DetailRow
|
||||
label="Delete on Termination"
|
||||
value={attachment.DeleteOnTermination ? "Yes" : "No"}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</DetailCard.Content>
|
||||
</DetailCard>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
import type { _Object } from "@aws-sdk/client-s3"
|
||||
import { GetObjectCommand } from "@aws-sdk/client-s3"
|
||||
import { Download, File, Trash2 } from "lucide-react"
|
||||
import { useState } from "react"
|
||||
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogAction,
|
||||
AlertDialogCancel,
|
||||
AlertDialogContent,
|
||||
AlertDialogDescription,
|
||||
AlertDialogFooter,
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
} from "@/components/ui/alert-dialog"
|
||||
import { getS3Client } from "@/lib/awsClient"
|
||||
import { formatSize } from "@/lib/utils"
|
||||
import { useDeleteObject } from "@/mutations/s3"
|
||||
|
||||
interface ObjectListItemProps {
|
||||
object: _Object
|
||||
bucketName: string
|
||||
displayName: string
|
||||
fullKey: string
|
||||
}
|
||||
|
||||
export function ObjectListItem({
|
||||
object,
|
||||
bucketName,
|
||||
displayName,
|
||||
fullKey,
|
||||
}: ObjectListItemProps) {
|
||||
const [showDeleteDialog, setShowDeleteDialog] = useState(false)
|
||||
const deleteMutation = useDeleteObject()
|
||||
|
||||
async function downloadObject() {
|
||||
try {
|
||||
const command = new GetObjectCommand({
|
||||
Bucket: bucketName,
|
||||
Key: fullKey,
|
||||
})
|
||||
const response = await getS3Client().send(command)
|
||||
|
||||
if (response.Body) {
|
||||
const blob = await response.Body.transformToByteArray()
|
||||
const url = URL.createObjectURL(new Blob([blob]))
|
||||
|
||||
const link = document.createElement("a")
|
||||
link.href = url
|
||||
link.download = displayName
|
||||
document.body.appendChild(link)
|
||||
link.click()
|
||||
document.body.removeChild(link)
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
} catch {
|
||||
throw new Error("Failed to download object")
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDelete() {
|
||||
await deleteMutation.mutateAsync({
|
||||
bucket: bucketName,
|
||||
key: fullKey,
|
||||
})
|
||||
setShowDeleteDialog(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className="flex items-center justify-between rounded-lg border bg-card p-4"
|
||||
key={object.Key}
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
<File className="size-5 text-muted-foreground" />
|
||||
<div>
|
||||
<h3 className="font-medium">{displayName}</h3>
|
||||
{object.LastModified && (
|
||||
<p className="text-muted-foreground text-sm">
|
||||
Last Modified: {object.LastModified.toLocaleString()}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="text-muted-foreground text-sm">
|
||||
{formatSize(object.Size || 0)}
|
||||
</div>
|
||||
<button
|
||||
className="rounded-md p-2 transition-colors hover:bg-accent"
|
||||
onClick={downloadObject}
|
||||
type="button"
|
||||
>
|
||||
<Download className="size-4" />
|
||||
</button>
|
||||
<button
|
||||
className="rounded-md p-2 text-destructive transition-colors hover:bg-accent"
|
||||
onClick={() => setShowDeleteDialog(true)}
|
||||
type="button"
|
||||
>
|
||||
<Trash2 className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<AlertDialog onOpenChange={setShowDeleteDialog} open={showDeleteDialog}>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Delete Object</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
Are you sure you want to delete <strong>{displayName}</strong>?
|
||||
This action cannot be undone.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction
|
||||
className="bg-destructive hover:bg-destructive/90"
|
||||
onClick={handleDelete}
|
||||
>
|
||||
{deleteMutation.isPending ? "Deleting…" : "Delete"}
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
module github.com/mulgadc/hive-ui/webserver
|
||||
|
||||
go 1.25.5
|
||||
|
||||
require github.com/NYTimes/gziphandler v1.1.1
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.121.2 // indirect
|
||||
cloud.google.com/go/auth v0.16.5 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.8.0 // indirect
|
||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
|
||||
github.com/anthropics/anthropic-sdk-go v1.19.0 // indirect
|
||||
github.com/ccojocar/zxcvbn-go v1.0.4 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gookit/color v1.6.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/openai/openai-go/v3 v3.8.1 // indirect
|
||||
github.com/securego/gosec/v2 v2.22.11 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
|
||||
go.opentelemetry.io/otel v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.37.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
|
||||
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
|
||||
golang.org/x/vuln v1.1.4 // indirect
|
||||
google.golang.org/genai v1.37.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
honnef.co/go/tools v0.6.1 // indirect
|
||||
)
|
||||
|
||||
tool (
|
||||
github.com/securego/gosec/v2/cmd/gosec
|
||||
golang.org/x/vuln/cmd/govulncheck
|
||||
honnef.co/go/tools/cmd/staticcheck
|
||||
)
|
||||
@@ -1,147 +0,0 @@
|
||||
cloud.google.com/go v0.121.2 h1:v2qQpN6Dx9x2NmwrqlesOt3Ys4ol5/lFZ6Mg1B7OJCg=
|
||||
cloud.google.com/go v0.121.2/go.mod h1:nRFlrHq39MNVWu+zESP2PosMWA0ryJw8KUBZ2iZpxbw=
|
||||
cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI=
|
||||
cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ=
|
||||
cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA=
|
||||
cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw=
|
||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
|
||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
|
||||
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
|
||||
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
|
||||
github.com/anthropics/anthropic-sdk-go v1.19.0 h1:mO6E+ffSzLRvR/YUH9KJC0uGw0uV8GjISIuzem//3KE=
|
||||
github.com/anthropics/anthropic-sdk-go v1.19.0/go.mod h1:WTz31rIUHUHqai2UslPpw5CwXrQP3geYBioRV4WOLvE=
|
||||
github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc=
|
||||
github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
|
||||
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY=
|
||||
github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
|
||||
github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
|
||||
github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
|
||||
github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
|
||||
github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
|
||||
github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
|
||||
github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
|
||||
github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
|
||||
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
|
||||
github.com/openai/openai-go/v3 v3.8.1 h1:b+YWsmwqXnbpSHWQEntZAkKciBZ5CJXwL68j+l59UDg=
|
||||
github.com/openai/openai-go/v3 v3.8.1/go.mod h1:UOpNxkqC9OdNXNUfpNByKOtB4jAL0EssQXq5p8gO0Xs=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/securego/gosec/v2 v2.22.11 h1:tW+weM/hCM/GX3iaCV91d5I6hqaRT2TPsFM1+USPXwg=
|
||||
github.com/securego/gosec/v2 v2.22.11/go.mod h1:KE4MW/eH0GLWztkbt4/7XpyH0zJBBnu7sYB4l6Wn7Mw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
|
||||
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
|
||||
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
|
||||
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
|
||||
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo=
|
||||
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
|
||||
golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
|
||||
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
|
||||
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8=
|
||||
golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I=
|
||||
golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genai v1.37.0 h1:dgp71k1wQ+/+APdZrN3LFgAGnVnr5IdTF1Oj0Dg+BQc=
|
||||
google.golang.org/genai v1.37.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI=
|
||||
honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4=
|
||||
@@ -1,106 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/NYTimes/gziphandler"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// setup logger
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
slog.SetDefault(logger)
|
||||
|
||||
// Get the project root (current directory when run via make)
|
||||
projectRoot, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to get project root:", err)
|
||||
}
|
||||
|
||||
// Check if dist directory exists
|
||||
distDir := filepath.Join(projectRoot, "frontend", "dist")
|
||||
if _, err := os.Stat(distDir); os.IsNotExist(err) {
|
||||
log.Fatalf("Frontend dist directory not found: %s", distDir)
|
||||
}
|
||||
|
||||
// Serve static files
|
||||
fs := http.FileServer(http.Dir(distDir))
|
||||
|
||||
// SPA handler: try to serve the file, fallback to index.html
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Check if the requested path is a file that exists
|
||||
requestedPath := filepath.Join(distDir, r.URL.Path)
|
||||
info, err := os.Stat(requestedPath)
|
||||
if err == nil && !info.IsDir() && filepath.Ext(requestedPath) != ".html" {
|
||||
// File exists, serve it and set cache headers. vite outputs unique hashes for files
|
||||
// so we can cache them forever. cache would only hit if it's not modified since last request
|
||||
w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
|
||||
fs.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// File doesn't exist, serve index.html for SPA routing
|
||||
// set cache headers to no-cache so browser always fetches the latest version
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
http.ServeFile(w, r, filepath.Join(distDir, "index.html"))
|
||||
})
|
||||
|
||||
// Wrap handler with security headers and gzip compression
|
||||
http.Handle("/", securityHeadersMiddleware(gzipMiddleware(handler)))
|
||||
|
||||
certFile := filepath.Join(projectRoot, "certs", "server.crt")
|
||||
keyFile := filepath.Join(projectRoot, "certs", "server.key")
|
||||
|
||||
// Check if certificates exist
|
||||
if _, err := os.Stat(certFile); os.IsNotExist(err) {
|
||||
log.Fatalf("Certificate file not found: %s", certFile)
|
||||
}
|
||||
if _, err := os.Stat(keyFile); os.IsNotExist(err) {
|
||||
log.Fatalf("Key file not found: %s", keyFile)
|
||||
}
|
||||
|
||||
port := ":3000"
|
||||
|
||||
server := &http.Server{
|
||||
Addr: port,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 15 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
slog.Info("Starting server with HTTPS", "port", port)
|
||||
log.Fatal(server.ListenAndServeTLS(certFile, keyFile))
|
||||
}
|
||||
|
||||
func securityHeadersMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self'; font-src 'self' data:; connect-src 'self' https://localhost:9999 https://localhost:8443; object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'; upgrade-insecure-requests;")
|
||||
w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=(), browsing-topics=()")
|
||||
w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func gzipMiddleware(next http.Handler) http.Handler {
|
||||
g, err := gziphandler.GzipHandlerWithOpts(gziphandler.ContentTypes([]string{
|
||||
"text/html",
|
||||
"text/css",
|
||||
"application/javascript",
|
||||
"text/javascript",
|
||||
"application/json",
|
||||
"image/svg+xml",
|
||||
"text/plain",
|
||||
}))
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create gzip middleware:", err)
|
||||
}
|
||||
return g(next)
|
||||
}
|
||||
+124
-10
@@ -8,16 +8,28 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
toml "github.com/pelletier/go-toml/v2"
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
// RemoteNode holds basic info about a remote cluster node for config generation.
|
||||
type RemoteNode struct {
|
||||
Name string
|
||||
Host string
|
||||
Region string
|
||||
AZ string
|
||||
Services []string
|
||||
}
|
||||
|
||||
type ConfigSettings struct {
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
@@ -35,6 +47,22 @@ type ConfigSettings struct {
|
||||
// Cluster settings
|
||||
ClusterBindIP string
|
||||
ClusterRoutes []string
|
||||
ClusterName string
|
||||
|
||||
// Predastore multi-node
|
||||
PredastoreNodeID int
|
||||
|
||||
// Node capabilities
|
||||
Services []string
|
||||
|
||||
// Other nodes in the cluster (for config source of truth)
|
||||
RemoteNodes []RemoteNode
|
||||
}
|
||||
|
||||
// PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
|
||||
type PredastoreNodeConfig struct {
|
||||
ID int
|
||||
Host string
|
||||
}
|
||||
|
||||
type ConfigFile struct {
|
||||
@@ -78,7 +106,7 @@ func GenerateConfigFile(configPath string, configTemplate string, configSettings
|
||||
return nil
|
||||
}
|
||||
|
||||
func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath string) {
|
||||
func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (caCertPath string) {
|
||||
// Certificate paths
|
||||
caCertPath = filepath.Join(configDir, "ca.pem")
|
||||
caKeyPath := filepath.Join(configDir, "ca.key")
|
||||
@@ -102,8 +130,8 @@ func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath stri
|
||||
fmt.Printf(" CA Certificate: %s\n", caCertPath)
|
||||
fmt.Printf(" CA Key: %s\n", caKeyPath)
|
||||
|
||||
// Step 2: Generate server certificate signed by CA
|
||||
if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath); err != nil {
|
||||
// Step 2: Generate server certificate signed by CA (with bind IP in SANs)
|
||||
if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -123,6 +151,23 @@ func GenerateCertificatesIfNeeded(configDir string, force bool) (caCertPath stri
|
||||
return caCertPath
|
||||
}
|
||||
|
||||
// GenerateServerCertOnly generates a server certificate signed by an existing CA.
|
||||
// Used by joining nodes that receive the CA from the leader.
|
||||
func GenerateServerCertOnly(configDir string, bindIP string) error {
|
||||
caCertPath := filepath.Join(configDir, "ca.pem")
|
||||
caKeyPath := filepath.Join(configDir, "ca.key")
|
||||
serverCertPath := filepath.Join(configDir, "server.pem")
|
||||
serverKeyPath := filepath.Join(configDir, "server.key")
|
||||
|
||||
// Verify CA files exist
|
||||
if !FileExists(caCertPath) || !FileExists(caKeyPath) {
|
||||
return fmt.Errorf("CA files not found in %s", configDir)
|
||||
}
|
||||
|
||||
// Generate server cert signed by CA with this node's bind IP
|
||||
return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP)
|
||||
}
|
||||
|
||||
func CreateServiceDirectories(hiveRoot string) {
|
||||
|
||||
// Create additional directories
|
||||
@@ -143,7 +188,7 @@ func CreateServiceDirectories(hiveRoot string) {
|
||||
|
||||
// Check if directory exists
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: Could not create %s: %v\n", dir, err)
|
||||
}
|
||||
}
|
||||
@@ -303,8 +348,9 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GenerateSignedCert generates a server certificate signed by the CA
|
||||
func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error {
|
||||
// GenerateSignedCert generates a server certificate signed by the CA.
|
||||
// extraIPs are additional IP addresses to include in the certificate's SANs.
|
||||
func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error {
|
||||
// Load CA certificate
|
||||
caCertPEM, err := os.ReadFile(caCertPath)
|
||||
if err != nil {
|
||||
@@ -357,6 +403,16 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error {
|
||||
return fmt.Errorf("failed to generate serial number: %w", err)
|
||||
}
|
||||
|
||||
// Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node)
|
||||
ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
|
||||
for _, ip := range extraIPs {
|
||||
if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" {
|
||||
if parsed := net.ParseIP(ip); parsed != nil {
|
||||
ipAddresses = append(ipAddresses, parsed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
serverTemplate := x509.Certificate{
|
||||
SerialNumber: serialNumber,
|
||||
Subject: pkix.Name{
|
||||
@@ -369,7 +425,7 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string) error {
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
DNSNames: []string{"localhost"},
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
|
||||
IPAddresses: ipAddresses,
|
||||
}
|
||||
|
||||
// Sign the server certificate with the CA
|
||||
@@ -476,8 +532,9 @@ func GenerateSelfSignedCert(certPath, keyPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// setupAWSCredentials updates ~/.aws/credentials and ~/.aws/config
|
||||
func SetupAWSCredentials(accessKey, secretKey, region, certPath string) error {
|
||||
// SetupAWSCredentials updates ~/.aws/credentials and ~/.aws/config.
|
||||
// bindIP is the IP the AWS gateway listens on. If empty or "0.0.0.0", defaults to "localhost".
|
||||
func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string) error {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -519,9 +576,14 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath string) error {
|
||||
configSection = "profile " + profileName
|
||||
}
|
||||
|
||||
endpointHost := bindIP
|
||||
if endpointHost == "" || endpointHost == "0.0.0.0" {
|
||||
endpointHost = "localhost"
|
||||
}
|
||||
|
||||
if err := UpdateAWSINIFile(configPath, configSection, map[string]string{
|
||||
"region": region,
|
||||
"endpoint_url": "https://localhost:9999",
|
||||
"endpoint_url": fmt.Sprintf("https://%s:9999", endpointHost),
|
||||
"ca_bundle": certPath,
|
||||
"output": "json",
|
||||
}); err != nil {
|
||||
@@ -535,3 +597,55 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GenerateMultiNodePredastoreConfig produces a complete predastore.toml for a
|
||||
// multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
|
||||
// and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
|
||||
func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region string) (string, error) {
|
||||
if len(nodes) < 3 {
|
||||
return "", fmt.Errorf("multi-node predastore requires at least 3 nodes, got %d", len(nodes))
|
||||
}
|
||||
|
||||
data := struct {
|
||||
Nodes []PredastoreNodeConfig
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Region string
|
||||
}{nodes, accessKey, secretKey, region}
|
||||
|
||||
tmpl, err := template.New("predastore-multinode").Parse(templateStr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse predastore template: %w", err)
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
if err := tmpl.Execute(&b, data); err != nil {
|
||||
return "", fmt.Errorf("failed to execute predastore template: %w", err)
|
||||
}
|
||||
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// FindNodeIDByIP returns the node ID for the given IP in the node list,
|
||||
// or 0 if the IP is not found.
|
||||
func FindNodeIDByIP(nodes []PredastoreNodeConfig, ip string) int {
|
||||
for _, n := range nodes {
|
||||
if n.Host == ip {
|
||||
return n.ID
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ParsePredastoreNodeIDFromConfig parses a predastore.toml string and returns
|
||||
// the node ID whose host matches the given IP, or 0 if not found.
|
||||
func ParsePredastoreNodeIDFromConfig(tomlContent string, ip string) int {
|
||||
var cfg struct {
|
||||
DB []PredastoreNodeConfig `toml:"db"`
|
||||
}
|
||||
if err := toml.Unmarshal([]byte(tomlContent), &cfg); err != nil {
|
||||
slog.Warn("Failed to parse predastore.toml content", "error", err)
|
||||
return 0
|
||||
}
|
||||
return FindNodeIDByIP(cfg.DB, ip)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,621 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// --- Key / Token generation ---
|
||||
|
||||
func TestGenerateAWSAccessKey_Format(t *testing.T) {
|
||||
key := GenerateAWSAccessKey()
|
||||
assert.Len(t, key, 20)
|
||||
assert.True(t, strings.HasPrefix(key, "AKIA"))
|
||||
for _, c := range key[4:] {
|
||||
assert.True(t, (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'),
|
||||
"unexpected character %c in access key suffix", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAWSAccessKey_Uniqueness(t *testing.T) {
|
||||
assert.NotEqual(t, GenerateAWSAccessKey(), GenerateAWSAccessKey())
|
||||
}
|
||||
|
||||
func TestGenerateAWSSecretKey_Format(t *testing.T) {
|
||||
key := GenerateAWSSecretKey()
|
||||
assert.Len(t, key, 40)
|
||||
_, err := base64.StdEncoding.DecodeString(key)
|
||||
assert.NoError(t, err, "secret key should be valid base64")
|
||||
}
|
||||
|
||||
func TestGenerateAWSSecretKey_Uniqueness(t *testing.T) {
|
||||
assert.NotEqual(t, GenerateAWSSecretKey(), GenerateAWSSecretKey())
|
||||
}
|
||||
|
||||
func TestGenerateAccountID_Format(t *testing.T) {
|
||||
id := GenerateAccountID()
|
||||
assert.Len(t, id, 12)
|
||||
for _, c := range id {
|
||||
assert.True(t, c >= '0' && c <= '9', "expected digit, got %c", c)
|
||||
}
|
||||
// First digit should not be 0
|
||||
assert.NotEqual(t, byte('0'), id[0], "12-digit account ID should not start with 0")
|
||||
}
|
||||
|
||||
func TestGenerateAccountID_Uniqueness(t *testing.T) {
|
||||
assert.NotEqual(t, GenerateAccountID(), GenerateAccountID())
|
||||
}
|
||||
|
||||
func TestGenerateNATSToken_Format(t *testing.T) {
|
||||
token := GenerateNATSToken()
|
||||
assert.True(t, strings.HasPrefix(token, "nats_"))
|
||||
assert.Len(t, token, 37) // 5 prefix + 32 random
|
||||
// URL-safe base64: no '+' or '/'
|
||||
assert.NotContains(t, token, "+")
|
||||
assert.NotContains(t, token, "/")
|
||||
}
|
||||
|
||||
func TestGenerateNATSToken_Uniqueness(t *testing.T) {
|
||||
assert.NotEqual(t, GenerateNATSToken(), GenerateNATSToken())
|
||||
}
|
||||
|
||||
// --- Config file generation ---
|
||||
|
||||
func TestGenerateConfigFile_Success(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.conf")
|
||||
|
||||
tmpl := "region = {{.Region}}\nnode = {{.Node}}"
|
||||
settings := ConfigSettings{Region: "us-east-1", Node: "node1"}
|
||||
|
||||
err := GenerateConfigFile(path, tmpl, settings)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(data), "region = us-east-1")
|
||||
assert.Contains(t, string(data), "node = node1")
|
||||
|
||||
info, _ := os.Stat(path)
|
||||
assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
|
||||
}
|
||||
|
||||
func TestGenerateConfigFile_InvalidTemplate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.conf")
|
||||
err := GenerateConfigFile(path, "{{.Unclosed", ConfigSettings{})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to parse template")
|
||||
}
|
||||
|
||||
func TestGenerateConfigFile_InvalidPath(t *testing.T) {
|
||||
err := GenerateConfigFile("/nonexistent/dir/file.conf", "ok", ConfigSettings{})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to create config file")
|
||||
}
|
||||
|
||||
func TestGenerateConfigFile_OverwritesExisting(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "overwrite.conf")
|
||||
|
||||
require.NoError(t, GenerateConfigFile(path, "old={{.Region}}", ConfigSettings{Region: "old"}))
|
||||
require.NoError(t, GenerateConfigFile(path, "new={{.Region}}", ConfigSettings{Region: "new"}))
|
||||
|
||||
data, _ := os.ReadFile(path)
|
||||
assert.Contains(t, string(data), "new=new")
|
||||
assert.NotContains(t, string(data), "old")
|
||||
}
|
||||
|
||||
func TestGenerateConfigFiles_AllSucceed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configs := []ConfigFile{
|
||||
{Name: "a", Path: filepath.Join(dir, "a.conf"), Template: "a={{.Region}}"},
|
||||
{Name: "b", Path: filepath.Join(dir, "b.conf"), Template: "b={{.Node}}"},
|
||||
}
|
||||
err := GenerateConfigFiles(configs, ConfigSettings{Region: "us-west-2", Node: "n1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, cfg := range configs {
|
||||
assert.True(t, FileExists(cfg.Path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateConfigFiles_StopsOnFirstError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configs := []ConfigFile{
|
||||
{Name: "ok", Path: filepath.Join(dir, "ok.conf"), Template: "ok"},
|
||||
{Name: "bad", Path: "/nonexistent/dir/bad.conf", Template: "bad"},
|
||||
{Name: "never", Path: filepath.Join(dir, "never.conf"), Template: "never"},
|
||||
}
|
||||
err := GenerateConfigFiles(configs, ConfigSettings{})
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "bad")
|
||||
assert.False(t, FileExists(filepath.Join(dir, "never.conf")))
|
||||
}
|
||||
|
||||
// --- FileExists ---
|
||||
|
||||
func TestFileExists(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
existing := filepath.Join(dir, "exists.txt")
|
||||
require.NoError(t, os.WriteFile(existing, []byte("hi"), 0644))
|
||||
|
||||
assert.True(t, FileExists(existing))
|
||||
assert.False(t, FileExists(filepath.Join(dir, "nope.txt")))
|
||||
assert.True(t, FileExists(dir)) // directory also returns true
|
||||
}
|
||||
|
||||
// --- UpdateAWSINIFile ---
|
||||
|
||||
func TestUpdateAWSINIFile_CreateNew(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "credentials")
|
||||
|
||||
err := UpdateAWSINIFile(path, "hive", map[string]string{
|
||||
"aws_access_key_id": "AKIATEST",
|
||||
"aws_secret_access_key": "secrettest",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
data, _ := os.ReadFile(path)
|
||||
content := string(data)
|
||||
assert.Contains(t, content, "[hive]")
|
||||
assert.Contains(t, content, "AKIATEST")
|
||||
assert.Contains(t, content, "secrettest")
|
||||
}
|
||||
|
||||
func TestUpdateAWSINIFile_UpdateExisting(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "credentials")
|
||||
|
||||
// Create with initial value
|
||||
require.NoError(t, UpdateAWSINIFile(path, "hive", map[string]string{"key": "old"}))
|
||||
// Update
|
||||
require.NoError(t, UpdateAWSINIFile(path, "hive", map[string]string{"key": "new"}))
|
||||
|
||||
data, _ := os.ReadFile(path)
|
||||
content := string(data)
|
||||
assert.Contains(t, content, "new")
|
||||
}
|
||||
|
||||
func TestUpdateAWSINIFile_AddNewSection(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "credentials")
|
||||
|
||||
require.NoError(t, UpdateAWSINIFile(path, "default", map[string]string{"key": "default-val"}))
|
||||
require.NoError(t, UpdateAWSINIFile(path, "hive", map[string]string{"key": "hive-val"}))
|
||||
|
||||
data, _ := os.ReadFile(path)
|
||||
content := string(data)
|
||||
assert.Contains(t, content, "[default]")
|
||||
assert.Contains(t, content, "[hive]")
|
||||
assert.Contains(t, content, "default-val")
|
||||
assert.Contains(t, content, "hive-val")
|
||||
}
|
||||
|
||||
// --- SetupAWSCredentials ---
|
||||
|
||||
func TestSetupAWSCredentials_CreatesFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("HOME", dir)
|
||||
|
||||
err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/path/to/ca.pem", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
credData, _ := os.ReadFile(filepath.Join(dir, ".aws", "credentials"))
|
||||
configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
|
||||
|
||||
assert.Contains(t, string(credData), "AKIATEST123")
|
||||
assert.Contains(t, string(credData), "secret123")
|
||||
assert.Contains(t, string(configData), "us-east-1")
|
||||
assert.Contains(t, string(configData), "https://localhost:9999")
|
||||
assert.Contains(t, string(configData), "/path/to/ca.pem")
|
||||
}
|
||||
|
||||
func TestSetupAWSCredentials_PreservesExistingProfiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("HOME", dir)
|
||||
|
||||
awsDir := filepath.Join(dir, ".aws")
|
||||
require.NoError(t, os.MkdirAll(awsDir, 0700))
|
||||
require.NoError(t, UpdateAWSINIFile(filepath.Join(awsDir, "credentials"), "default", map[string]string{
|
||||
"aws_access_key_id": "EXISTING_KEY",
|
||||
}))
|
||||
|
||||
err := SetupAWSCredentials("NEWAKEY", "NEWSECRET", "us-west-2", "/ca.pem", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
data, _ := os.ReadFile(filepath.Join(awsDir, "credentials"))
|
||||
content := string(data)
|
||||
assert.Contains(t, content, "EXISTING_KEY")
|
||||
assert.Contains(t, content, "NEWAKEY")
|
||||
}
|
||||
|
||||
func TestSetupAWSCredentials_UsesBindIP(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("HOME", dir)
|
||||
|
||||
err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "10.11.12.1")
|
||||
require.NoError(t, err)
|
||||
|
||||
configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
|
||||
assert.Contains(t, string(configData), "https://10.11.12.1:9999")
|
||||
assert.NotContains(t, string(configData), "localhost")
|
||||
}
|
||||
|
||||
func TestSetupAWSCredentials_FallsBackToLocalhostForWildcard(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("HOME", dir)
|
||||
|
||||
err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "0.0.0.0")
|
||||
require.NoError(t, err)
|
||||
|
||||
configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
|
||||
assert.Contains(t, string(configData), "https://localhost:9999")
|
||||
}
|
||||
|
||||
// --- Certificate generation ---
|
||||
|
||||
func TestGenerateCACert_CreatesValidCA(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "ca.pem")
|
||||
keyPath := filepath.Join(dir, "ca.key")
|
||||
|
||||
err := GenerateCACert(certPath, keyPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Parse certificate
|
||||
certPEM, _ := os.ReadFile(certPath)
|
||||
block, _ := pem.Decode(certPEM)
|
||||
require.NotNil(t, block)
|
||||
assert.Equal(t, "CERTIFICATE", block.Type)
|
||||
|
||||
cert, err := x509.ParseCertificate(block.Bytes)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, cert.IsCA)
|
||||
assert.Equal(t, "Hive Local CA", cert.Subject.CommonName)
|
||||
assert.NotZero(t, cert.KeyUsage&x509.KeyUsageCertSign)
|
||||
|
||||
// Parse key
|
||||
keyPEM, _ := os.ReadFile(keyPath)
|
||||
keyBlock, _ := pem.Decode(keyPEM)
|
||||
require.NotNil(t, keyBlock)
|
||||
assert.Equal(t, "PRIVATE KEY", keyBlock.Type)
|
||||
|
||||
// Verify key file permissions
|
||||
info, _ := os.Stat(keyPath)
|
||||
assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
|
||||
}
|
||||
|
||||
// TestGenerateSignedCert uses subtests sharing a single CA to avoid repeated 4096-bit key generation.
|
||||
func TestGenerateSignedCert(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Generate CA once for all subtests (~0.7s instead of ~0.7s x 3)
|
||||
caDir := t.TempDir()
|
||||
caCertPath := filepath.Join(caDir, "ca.pem")
|
||||
caKeyPath := filepath.Join(caDir, "ca.key")
|
||||
require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
|
||||
|
||||
t.Run("CreatesValidCert", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "server.pem")
|
||||
keyPath := filepath.Join(dir, "server.key")
|
||||
|
||||
require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath))
|
||||
|
||||
certPEM, _ := os.ReadFile(certPath)
|
||||
block, _ := pem.Decode(certPEM)
|
||||
require.NotNil(t, block)
|
||||
|
||||
cert, err := x509.ParseCertificate(block.Bytes)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, cert.IsCA)
|
||||
assert.Equal(t, "localhost", cert.Subject.CommonName)
|
||||
assert.Contains(t, cert.DNSNames, "localhost")
|
||||
|
||||
hasLoopback := false
|
||||
for _, ip := range cert.IPAddresses {
|
||||
if ip.Equal(net.ParseIP("127.0.0.1")) {
|
||||
hasLoopback = true
|
||||
}
|
||||
}
|
||||
assert.True(t, hasLoopback)
|
||||
|
||||
// Verify against CA
|
||||
caCertPEM, _ := os.ReadFile(caCertPath)
|
||||
caBlock, _ := pem.Decode(caCertPEM)
|
||||
caCert, _ := x509.ParseCertificate(caBlock.Bytes)
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(caCert)
|
||||
_, err = cert.Verify(x509.VerifyOptions{Roots: pool})
|
||||
assert.NoError(t, err)
|
||||
|
||||
info, _ := os.Stat(keyPath)
|
||||
assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
|
||||
})
|
||||
|
||||
t.Run("ExtraIPs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "server.pem")
|
||||
keyPath := filepath.Join(dir, "server.key")
|
||||
|
||||
require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath, "192.168.1.100"))
|
||||
|
||||
certPEM, _ := os.ReadFile(certPath)
|
||||
block, _ := pem.Decode(certPEM)
|
||||
cert, _ := x509.ParseCertificate(block.Bytes)
|
||||
|
||||
hasExtraIP := false
|
||||
for _, ip := range cert.IPAddresses {
|
||||
if ip.Equal(net.ParseIP("192.168.1.100")) {
|
||||
hasExtraIP = true
|
||||
}
|
||||
}
|
||||
assert.True(t, hasExtraIP, "cert should contain extra IP 192.168.1.100")
|
||||
})
|
||||
|
||||
t.Run("SkipsDuplicateAndSpecialIPs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "server.pem")
|
||||
keyPath := filepath.Join(dir, "server.key")
|
||||
|
||||
require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath, "127.0.0.1", "::1", "0.0.0.0", ""))
|
||||
|
||||
certPEM, _ := os.ReadFile(certPath)
|
||||
block, _ := pem.Decode(certPEM)
|
||||
cert, _ := x509.ParseCertificate(block.Bytes)
|
||||
|
||||
assert.Len(t, cert.IPAddresses, 2)
|
||||
})
|
||||
|
||||
t.Run("InvalidCACert", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
badCACert := filepath.Join(dir, "bad-ca.pem")
|
||||
require.NoError(t, os.WriteFile(badCACert, []byte("not-a-cert"), 0600))
|
||||
|
||||
err := GenerateSignedCert(filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"), badCACert, caKeyPath)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGenerateSelfSignedCert_CreatesValidCert(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "self.pem")
|
||||
keyPath := filepath.Join(dir, "self.key")
|
||||
|
||||
require.NoError(t, GenerateSelfSignedCert(certPath, keyPath))
|
||||
|
||||
certPEM, _ := os.ReadFile(certPath)
|
||||
block, _ := pem.Decode(certPEM)
|
||||
require.NotNil(t, block)
|
||||
|
||||
cert, err := x509.ParseCertificate(block.Bytes)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, cert.DNSNames, "localhost")
|
||||
|
||||
info, _ := os.Stat(keyPath)
|
||||
assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
|
||||
}
|
||||
|
||||
// --- Certificate orchestrator ---
|
||||
|
||||
// TestGenerateCertificatesIfNeeded uses subtests to share the initial generation.
|
||||
func TestGenerateCertificatesIfNeeded(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
|
||||
// First call generates all certs
|
||||
caCertPath := GenerateCertificatesIfNeeded(dir, false, "")
|
||||
assert.Equal(t, filepath.Join(dir, "ca.pem"), caCertPath)
|
||||
assert.True(t, FileExists(filepath.Join(dir, "ca.pem")))
|
||||
assert.True(t, FileExists(filepath.Join(dir, "ca.key")))
|
||||
assert.True(t, FileExists(filepath.Join(dir, "server.pem")))
|
||||
assert.True(t, FileExists(filepath.Join(dir, "server.key")))
|
||||
|
||||
t.Run("SkipsWhenAllExist", func(t *testing.T) {
|
||||
caInfo, _ := os.Stat(filepath.Join(dir, "ca.pem"))
|
||||
origModTime := caInfo.ModTime()
|
||||
|
||||
GenerateCertificatesIfNeeded(dir, false, "")
|
||||
caInfo2, _ := os.Stat(filepath.Join(dir, "ca.pem"))
|
||||
assert.Equal(t, origModTime, caInfo2.ModTime())
|
||||
})
|
||||
|
||||
t.Run("ForceRegenerates", func(t *testing.T) {
|
||||
origCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))
|
||||
|
||||
GenerateCertificatesIfNeeded(dir, true, "")
|
||||
newCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))
|
||||
assert.NotEqual(t, origCA, newCA)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGenerateServerCertOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Generate CA once, reuse for subtests
|
||||
caDir := t.TempDir()
|
||||
require.NoError(t, GenerateCACert(filepath.Join(caDir, "ca.pem"), filepath.Join(caDir, "ca.key")))
|
||||
|
||||
t.Run("Success", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Copy CA files into test dir
|
||||
caCert, _ := os.ReadFile(filepath.Join(caDir, "ca.pem"))
|
||||
caKey, _ := os.ReadFile(filepath.Join(caDir, "ca.key"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), caCert, 0600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.key"), caKey, 0600))
|
||||
|
||||
err := GenerateServerCertOnly(dir, "10.0.0.5")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, FileExists(filepath.Join(dir, "server.pem")))
|
||||
assert.True(t, FileExists(filepath.Join(dir, "server.key")))
|
||||
|
||||
certPEM, _ := os.ReadFile(filepath.Join(dir, "server.pem"))
|
||||
block, _ := pem.Decode(certPEM)
|
||||
cert, _ := x509.ParseCertificate(block.Bytes)
|
||||
|
||||
hasBindIP := false
|
||||
for _, ip := range cert.IPAddresses {
|
||||
if ip.Equal(net.ParseIP("10.0.0.5")) {
|
||||
hasBindIP = true
|
||||
}
|
||||
}
|
||||
assert.True(t, hasBindIP)
|
||||
})
|
||||
|
||||
t.Run("MissingCA", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
err := GenerateServerCertOnly(dir, "10.0.0.5")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "CA files not found")
|
||||
})
|
||||
}
|
||||
|
||||
// --- Directory creation ---
|
||||
|
||||
func TestCreateServiceDirectories_CreatesAll(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
CreateServiceDirectories(dir)
|
||||
|
||||
expected := []string{"images", "amis", "volumes", "state", "logs", "nats", "predastore", "viperblock", "hive"}
|
||||
for _, name := range expected {
|
||||
path := filepath.Join(dir, name)
|
||||
info, err := os.Stat(path)
|
||||
assert.NoError(t, err, "directory %s should exist", name)
|
||||
if err == nil {
|
||||
assert.True(t, info.IsDir())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateServiceDirectories_Idempotent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
CreateServiceDirectories(dir)
|
||||
// Should not error on second call
|
||||
CreateServiceDirectories(dir)
|
||||
|
||||
info, err := os.Stat(filepath.Join(dir, "images"))
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
}
|
||||
|
||||
// --- Predastore multi-node config ---
|
||||
|
||||
func TestGenerateMultiNodePredastoreConfig_Success(t *testing.T) {
|
||||
tmpl := `{{range .Nodes}}[[db]]
|
||||
id = {{.ID}}
|
||||
host = "{{.Host}}"
|
||||
{{end}}`
|
||||
nodes := []PredastoreNodeConfig{
|
||||
{ID: 1, Host: "10.0.0.1"},
|
||||
{ID: 2, Host: "10.0.0.2"},
|
||||
{ID: 3, Host: "10.0.0.3"},
|
||||
}
|
||||
|
||||
result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, result, `host = "10.0.0.1"`)
|
||||
assert.Contains(t, result, `host = "10.0.0.3"`)
|
||||
}
|
||||
|
||||
func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
|
||||
tmpl := "{{range .Nodes}}{{.ID}}{{end}}"
|
||||
|
||||
_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
|
||||
{ID: 1, Host: "10.0.0.1"},
|
||||
{ID: 2, Host: "10.0.0.2"},
|
||||
}, "AK", "SK", "us-east-1")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "at least 3 nodes")
|
||||
}
|
||||
|
||||
func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
|
||||
_, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []PredastoreNodeConfig{
|
||||
{ID: 1, Host: "a"}, {ID: 2, Host: "b"}, {ID: 3, Host: "c"},
|
||||
}, "AK", "SK", "us-east-1")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to parse")
|
||||
}
|
||||
|
||||
// --- FindNodeIDByIP ---
|
||||
|
||||
func TestFindNodeIDByIP(t *testing.T) {
|
||||
nodes := []PredastoreNodeConfig{
|
||||
{ID: 1, Host: "10.0.0.1"},
|
||||
{ID: 2, Host: "10.0.0.2"},
|
||||
{ID: 3, Host: "10.0.0.3"},
|
||||
}
|
||||
|
||||
assert.Equal(t, 2, FindNodeIDByIP(nodes, "10.0.0.2"))
|
||||
assert.Equal(t, 0, FindNodeIDByIP(nodes, "10.0.0.99"))
|
||||
assert.Equal(t, 0, FindNodeIDByIP(nil, "10.0.0.1"))
|
||||
}
|
||||
|
||||
// --- ParsePredastoreNodeIDFromConfig ---
|
||||
|
||||
func TestParsePredastoreNodeIDFromConfig(t *testing.T) {
|
||||
tomlContent := `
|
||||
[[db]]
|
||||
id = 1
|
||||
host = "10.0.0.1"
|
||||
|
||||
[[db]]
|
||||
id = 2
|
||||
host = "10.0.0.2"
|
||||
|
||||
[[db]]
|
||||
id = 3
|
||||
host = "10.0.0.3"
|
||||
`
|
||||
assert.Equal(t, 2, ParsePredastoreNodeIDFromConfig(tomlContent, "10.0.0.2"))
|
||||
assert.Equal(t, 0, ParsePredastoreNodeIDFromConfig(tomlContent, "10.0.0.99"))
|
||||
assert.Equal(t, 0, ParsePredastoreNodeIDFromConfig("invalid toml {{{", "10.0.0.1"))
|
||||
assert.Equal(t, 0, ParsePredastoreNodeIDFromConfig("", "10.0.0.1"))
|
||||
}
|
||||
|
||||
// --- Integration: Full config generation flow ---
|
||||
|
||||
func TestGenerateConfigFile_HiveTomlTemplate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hive.toml")
|
||||
|
||||
tmpl := `version = "1.0"
|
||||
epoch = 1
|
||||
node = "{{.Node}}"
|
||||
|
||||
[nodes.{{.Node}}]
|
||||
region = "{{.Region}}"
|
||||
accesskey = "{{.AccessKey}}"
|
||||
secretkey = "{{.SecretKey}}"
|
||||
`
|
||||
|
||||
settings := ConfigSettings{
|
||||
Node: "node1",
|
||||
Region: "us-east-1",
|
||||
AccessKey: "AKIATEST",
|
||||
SecretKey: "SECRET",
|
||||
}
|
||||
|
||||
require.NoError(t, GenerateConfigFile(path, tmpl, settings))
|
||||
|
||||
data, _ := os.ReadFile(path)
|
||||
content := string(data)
|
||||
assert.Contains(t, content, `node = "node1"`)
|
||||
assert.Contains(t, content, `region = "us-east-1"`)
|
||||
assert.Contains(t, content, fmt.Sprintf(`accesskey = "%s"`, settings.AccessKey))
|
||||
}
|
||||
@@ -62,10 +62,17 @@ func setStructFields(v reflect.Value, params map[string]string, prefix string) e
|
||||
// Check for locationName tag (AWS SDK uses this for query parameter names)
|
||||
locationName := fieldType.Tag.Get("locationName")
|
||||
|
||||
// Try both the field name and locationName (if different)
|
||||
// Try the field name, locationName, and title-cased locationName.
|
||||
// AWS query params use title case (e.g. "ResourceId") but some SDK
|
||||
// structs use camelCase locationName (e.g. "resourceId" in DeleteTagsInput
|
||||
// vs "ResourceId" in CreateTagsInput).
|
||||
queryKeys := []string{prefix + fieldName}
|
||||
if locationName != "" && locationName != fieldName {
|
||||
queryKeys = append(queryKeys, prefix+locationName)
|
||||
titled := strings.ToUpper(locationName[:1]) + locationName[1:]
|
||||
if titled != fieldName && titled != locationName {
|
||||
queryKeys = append(queryKeys, prefix+titled)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is a simple field (string, int, bool, etc.)
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
package awssmithy
|
||||
|
||||
type FilterList struct {
|
||||
Name string
|
||||
Values []string
|
||||
}
|
||||
|
||||
type DescribeInstancesRequest struct {
|
||||
InstanceIds []string `xml:"InstanceId"`
|
||||
DryRun bool `xml:"DryRun"`
|
||||
Filters []FilterList `xml:"Filter"`
|
||||
|
||||
NextToken string `xml:"nextToken"`
|
||||
|
||||
MaxResults int `xml:"maxResults"`
|
||||
}
|
||||
+144
-18
@@ -21,15 +21,17 @@ type ClusterConfig struct {
|
||||
// Config holds all configuration for the application
|
||||
type Config struct {
|
||||
// Node config
|
||||
Node string `mapstructure:"node"`
|
||||
Host string `mapstructure:"host"` // Unique hostname or IP of this node
|
||||
Region string `mapstructure:"region"`
|
||||
AZ string `mapstructure:"az"`
|
||||
DataDir string `mapstructure:"data_dir"`
|
||||
Node string `mapstructure:"node"`
|
||||
Host string `mapstructure:"host"` // Unique hostname or IP of this node
|
||||
Region string `mapstructure:"region"`
|
||||
AZ string `mapstructure:"az"`
|
||||
DataDir string `mapstructure:"data_dir"`
|
||||
Services []string `mapstructure:"services"` // Which services this node runs locally
|
||||
|
||||
Daemon DaemonConfig `mapstructure:"daemon"`
|
||||
NATS NATSConfig `mapstructure:"nats"`
|
||||
Predastore PredastoreConfig `mapstructure:"predastore"`
|
||||
Viperblock ViperblockConfig `mapstructure:"viperblock"`
|
||||
AWSGW AWSGWConfig `mapstructure:"awsgw"`
|
||||
|
||||
// Authentication
|
||||
@@ -40,6 +42,50 @@ type Config struct {
|
||||
WalDir string `mapstructure:"wal_dir"`
|
||||
}
|
||||
|
||||
// NodeBaseDir returns the BaseDir for the current node, or "" if the config
|
||||
// is nil, the node name is unset, or the node is not found in the Nodes map.
|
||||
func (cc *ClusterConfig) NodeBaseDir() string {
|
||||
if cc == nil || cc.Node == "" {
|
||||
slog.Warn("NodeBaseDir: no config or node name set, using global PID path")
|
||||
return ""
|
||||
}
|
||||
node, ok := cc.Nodes[cc.Node]
|
||||
if !ok {
|
||||
slog.Error("NodeBaseDir: node not found in config", "node", cc.Node)
|
||||
return ""
|
||||
}
|
||||
if node.BaseDir == "" {
|
||||
slog.Warn("NodeBaseDir: BaseDir is empty for node, using global PID path", "node", cc.Node)
|
||||
}
|
||||
return node.BaseDir
|
||||
}
|
||||
|
||||
// AllServices is the default service list when Services is empty (backward compat).
|
||||
var AllServices = []string{"nats", "predastore", "viperblock", "daemon", "awsgw", "ui"}
|
||||
|
||||
// HasService reports whether the node runs the named service.
|
||||
// An empty Services list means all services (backward compat).
|
||||
func (c Config) HasService(name string) bool {
|
||||
services := c.Services
|
||||
if len(services) == 0 {
|
||||
services = AllServices
|
||||
}
|
||||
for _, s := range services {
|
||||
if s == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetServices returns the configured service list, defaulting to AllServices.
|
||||
func (c Config) GetServices() []string {
|
||||
if len(c.Services) == 0 {
|
||||
return AllServices
|
||||
}
|
||||
return c.Services
|
||||
}
|
||||
|
||||
type AWSGWConfig struct {
|
||||
Host string `mapstructure:"host"`
|
||||
TLSKey string `mapstructure:"tlskey"`
|
||||
@@ -50,6 +96,10 @@ type AWSGWConfig struct {
|
||||
ExpectedNodes int `mapstructure:"expected_nodes"` // TODO: Replace with root cluster config
|
||||
}
|
||||
|
||||
type ViperblockConfig struct {
|
||||
ShardWAL *bool `mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
|
||||
}
|
||||
|
||||
type PredastoreConfig struct {
|
||||
Host string `mapstructure:"host"`
|
||||
Bucket string `mapstructure:"bucket"`
|
||||
@@ -57,6 +107,7 @@ type PredastoreConfig struct {
|
||||
AccessKey string `mapstructure:"accesskey"`
|
||||
SecretKey string `mapstructure:"secretkey"`
|
||||
BaseDir string `mapstructure:"base_dir"`
|
||||
NodeID int `mapstructure:"node_id"`
|
||||
}
|
||||
|
||||
// DaemonConfig holds the daemon configuration
|
||||
@@ -89,12 +140,14 @@ type EBSRequests struct {
|
||||
}
|
||||
|
||||
type EBSRequest struct {
|
||||
Name string
|
||||
VolType string
|
||||
Boot bool
|
||||
EFI bool
|
||||
CloudInit bool
|
||||
NBDURI string // NBD URI - socket path (nbd:unix:/path.sock) or TCP (nbd://host:port)
|
||||
Name string
|
||||
VolType string
|
||||
Boot bool
|
||||
EFI bool
|
||||
CloudInit bool
|
||||
DeleteOnTermination bool
|
||||
NBDURI string // NBD URI - socket path (nbd:unix:/path.sock) or TCP (nbd://host:port)
|
||||
DeviceName string // Device name (e.g. /dev/sdf) for hot-plugged volumes
|
||||
}
|
||||
|
||||
// NBDTransport defines the transport type for NBD connections
|
||||
@@ -119,6 +172,16 @@ type EBSUnMountResponse struct {
|
||||
Error string
|
||||
}
|
||||
|
||||
type EBSSyncRequest struct {
|
||||
Volume string
|
||||
}
|
||||
|
||||
type EBSSyncResponse struct {
|
||||
Volume string
|
||||
Synced bool
|
||||
Error string
|
||||
}
|
||||
|
||||
type EBSDeleteRequest struct {
|
||||
Volume string
|
||||
}
|
||||
@@ -129,6 +192,17 @@ type EBSDeleteResponse struct {
|
||||
Error string
|
||||
}
|
||||
|
||||
type EBSSnapshotRequest struct {
|
||||
Volume string
|
||||
SnapshotID string
|
||||
}
|
||||
|
||||
type EBSSnapshotResponse struct {
|
||||
SnapshotID string
|
||||
Success bool
|
||||
Error string
|
||||
}
|
||||
|
||||
// EC2, TODO: Move to vm.go or more applicable place
|
||||
type EC2StartInstancesRequest struct {
|
||||
InstanceID string
|
||||
@@ -149,7 +223,9 @@ func (ec2StartInstanceResponse EC2StartInstancesResponse) Respond(msg *nats.Msg)
|
||||
return
|
||||
}
|
||||
|
||||
msg.Respond(response)
|
||||
if err := msg.Respond(response); err != nil {
|
||||
slog.Error("Failed to respond to NATS request", "err", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -175,14 +251,63 @@ type NodeJoinResponse struct {
|
||||
SharedData *SharedClusterData `json:"shared_data,omitempty"`
|
||||
ConfigHash string `json:"config_hash,omitempty"`
|
||||
JoiningNode string `json:"joining_node,omitempty"` // The node name that is joining
|
||||
// CA distribution for per-node certificate generation
|
||||
CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate
|
||||
CAKey string `json:"ca_key,omitempty"` // PEM-encoded CA private key
|
||||
// Predastore config distribution for multi-node clusters
|
||||
PredastoreConfig string `json:"predastore_config,omitempty"` // Full predastore.toml content
|
||||
}
|
||||
|
||||
type NodeHealthResponse struct {
|
||||
Node string `json:"node"`
|
||||
Status string `json:"status"`
|
||||
ConfigHash string `json:"config_hash"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
Node string `json:"node"`
|
||||
Status string `json:"status"`
|
||||
ConfigHash string `json:"config_hash"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
Services []string `json:"services"`
|
||||
ServiceHealth map[string]string `json:"service_health,omitempty"`
|
||||
}
|
||||
|
||||
// NodeStatusResponse is returned by the hive.node.status NATS topic (fan-out).
|
||||
type NodeStatusResponse struct {
|
||||
Node string `json:"node"`
|
||||
Status string `json:"status"`
|
||||
Host string `json:"host"`
|
||||
Region string `json:"region"`
|
||||
AZ string `json:"az"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
Services []string `json:"services"`
|
||||
TotalVCPU int `json:"total_vcpu"`
|
||||
TotalMemGB float64 `json:"total_mem_gb"`
|
||||
AllocVCPU int `json:"alloc_vcpu"`
|
||||
AllocMemGB float64 `json:"alloc_mem_gb"`
|
||||
VMCount int `json:"vm_count"`
|
||||
InstanceTypes []InstanceTypeCap `json:"instance_types"`
|
||||
}
|
||||
|
||||
// InstanceTypeCap describes available capacity for one instance type on a node.
|
||||
type InstanceTypeCap struct {
|
||||
Name string `json:"name"`
|
||||
VCPU int `json:"vcpu"`
|
||||
MemoryGB float64 `json:"memory_gb"`
|
||||
Available int `json:"available"`
|
||||
}
|
||||
|
||||
// VMInfo describes a single VM for the cluster stats CLI.
|
||||
type VMInfo struct {
|
||||
InstanceID string `json:"instance_id"`
|
||||
Status string `json:"status"`
|
||||
InstanceType string `json:"instance_type"`
|
||||
VCPU int `json:"vcpu"`
|
||||
MemoryGB float64 `json:"memory_gb"`
|
||||
LaunchTime int64 `json:"launch_time"`
|
||||
}
|
||||
|
||||
// NodeVMsResponse is returned by the hive.node.vms NATS topic (fan-out).
|
||||
type NodeVMsResponse struct {
|
||||
Node string `json:"node"`
|
||||
Host string `json:"host"`
|
||||
VMs []VMInfo `json:"vms"`
|
||||
}
|
||||
|
||||
// LoadConfig loads the configuration from file and environment variables
|
||||
@@ -201,7 +326,8 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
return nil, fmt.Errorf("error reading config file: %w", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "Using config file: %s\n", viper.ConfigFileUsed())
|
||||
//fmt.Fprintf(os.Stderr, "Using config file: %s\n", viper.ConfigFileUsed())
|
||||
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "Config file not found: %s, using environment variables and defaults\n", configPath)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func resetViper(t *testing.T) {
|
||||
t.Cleanup(func() { viper.Reset() })
|
||||
}
|
||||
|
||||
func TestLoadConfig_ValidTOMLFile(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hive.toml")
|
||||
|
||||
toml := `
|
||||
version = "1.0"
|
||||
epoch = 1
|
||||
node = "node1"
|
||||
|
||||
[nodes.node1]
|
||||
node = "node1"
|
||||
region = "us-east-1"
|
||||
az = "us-east-1a"
|
||||
accesskey = "AKIATEST"
|
||||
secretkey = "SECRET"
|
||||
|
||||
[nodes.node1.daemon]
|
||||
host = "127.0.0.1:8080"
|
||||
|
||||
[nodes.node1.nats]
|
||||
host = "127.0.0.1:4222"
|
||||
|
||||
[nodes.node1.nats.acl]
|
||||
token = "nats_testtoken"
|
||||
|
||||
[nodes.node1.predastore]
|
||||
host = "127.0.0.1:8443"
|
||||
bucket = "predastore"
|
||||
region = "us-east-1"
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
|
||||
assert.Equal(t, uint64(1), cfg.Epoch)
|
||||
assert.Equal(t, "node1", cfg.Node)
|
||||
assert.Equal(t, "1.0", cfg.Version)
|
||||
|
||||
node, ok := cfg.Nodes["node1"]
|
||||
require.True(t, ok, "node1 should exist in Nodes map")
|
||||
assert.Equal(t, "us-east-1", node.Region)
|
||||
assert.Equal(t, "us-east-1a", node.AZ)
|
||||
assert.Equal(t, "AKIATEST", node.AccessKey)
|
||||
assert.Equal(t, "127.0.0.1:8080", node.Daemon.Host)
|
||||
assert.Equal(t, "127.0.0.1:4222", node.NATS.Host)
|
||||
assert.Equal(t, "nats_testtoken", node.NATS.ACL.Token)
|
||||
assert.Equal(t, "127.0.0.1:8443", node.Predastore.Host)
|
||||
assert.Equal(t, "predastore", node.Predastore.Bucket)
|
||||
}
|
||||
|
||||
func TestLoadConfig_MultipleNodes(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hive.toml")
|
||||
|
||||
toml := `
|
||||
epoch = 2
|
||||
node = "leader"
|
||||
version = "2.0"
|
||||
|
||||
[nodes.leader]
|
||||
region = "us-east-1"
|
||||
|
||||
[nodes.follower]
|
||||
region = "us-west-2"
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, cfg.Nodes, 2)
|
||||
assert.Equal(t, "us-east-1", cfg.Nodes["leader"].Region)
|
||||
assert.Equal(t, "us-west-2", cfg.Nodes["follower"].Region)
|
||||
}
|
||||
|
||||
func TestLoadConfig_EmptyConfigPath(t *testing.T) {
|
||||
resetViper(t)
|
||||
cfg, err := LoadConfig("")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
// All zero values
|
||||
assert.Equal(t, uint64(0), cfg.Epoch)
|
||||
assert.Empty(t, cfg.Node)
|
||||
}
|
||||
|
||||
func TestLoadConfig_NonexistentFile(t *testing.T) {
|
||||
resetViper(t)
|
||||
cfg, err := LoadConfig("/tmp/nonexistent-hive-config-test-12345.toml")
|
||||
// Not an error - falls through to defaults
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
}
|
||||
|
||||
func TestLoadConfig_MalformedTOML(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.toml")
|
||||
require.NoError(t, os.WriteFile(path, []byte("this is not valid toml {{{"), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, cfg)
|
||||
assert.Contains(t, err.Error(), "error reading config file")
|
||||
}
|
||||
|
||||
func TestLoadConfig_PartialConfig(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "partial.toml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(`node = "partial-node"
|
||||
epoch = 5
|
||||
`), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "partial-node", cfg.Node)
|
||||
assert.Equal(t, uint64(5), cfg.Epoch)
|
||||
assert.Empty(t, cfg.Version)
|
||||
assert.Nil(t, cfg.Nodes)
|
||||
}
|
||||
|
||||
func TestLoadConfig_EnvVarOverrideWithFile(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hive.toml")
|
||||
|
||||
// Viper's AutomaticEnv only works for keys Viper already knows about
|
||||
// (from a config file or explicit BindEnv). Provide a minimal config
|
||||
// so Viper registers the "epoch" key, then override via env.
|
||||
require.NoError(t, os.WriteFile(path, []byte(`epoch = 1
|
||||
node = "file-node"
|
||||
`), 0600))
|
||||
|
||||
t.Setenv("HIVE_NODE", "env-node")
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
// Env vars override file values for keys Viper knows about
|
||||
assert.Equal(t, "env-node", cfg.Node)
|
||||
}
|
||||
|
||||
func TestLoadConfig_NestedStructParsing(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "nested.toml")
|
||||
|
||||
toml := `
|
||||
node = "n1"
|
||||
|
||||
[nodes.n1]
|
||||
region = "ap-southeast-2"
|
||||
|
||||
[nodes.n1.daemon]
|
||||
host = "0.0.0.0:8080"
|
||||
tlskey = "server.key"
|
||||
tlscert = "server.pem"
|
||||
|
||||
[nodes.n1.nats]
|
||||
host = "0.0.0.0:4222"
|
||||
|
||||
[nodes.n1.nats.acl]
|
||||
token = "secret-token"
|
||||
|
||||
[nodes.n1.nats.sub]
|
||||
subject = "test-subject"
|
||||
|
||||
[nodes.n1.predastore]
|
||||
host = "0.0.0.0:8443"
|
||||
bucket = "mybucket"
|
||||
region = "ap-southeast-2"
|
||||
accesskey = "AK"
|
||||
secretkey = "SK"
|
||||
base_dir = "/data"
|
||||
node_id = 1
|
||||
|
||||
[nodes.n1.awsgw]
|
||||
host = "0.0.0.0:9999"
|
||||
debug = true
|
||||
expected_nodes = 3
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
n := cfg.Nodes["n1"]
|
||||
assert.Equal(t, "0.0.0.0:8080", n.Daemon.Host)
|
||||
assert.Equal(t, "server.key", n.Daemon.TLSKey)
|
||||
assert.Equal(t, "server.pem", n.Daemon.TLSCert)
|
||||
assert.Equal(t, "0.0.0.0:4222", n.NATS.Host)
|
||||
assert.Equal(t, "secret-token", n.NATS.ACL.Token)
|
||||
assert.Equal(t, "test-subject", n.NATS.Sub.Subject)
|
||||
assert.Equal(t, "0.0.0.0:8443", n.Predastore.Host)
|
||||
assert.Equal(t, "mybucket", n.Predastore.Bucket)
|
||||
assert.Equal(t, "AK", n.Predastore.AccessKey)
|
||||
assert.Equal(t, "/data", n.Predastore.BaseDir)
|
||||
assert.Equal(t, 1, n.Predastore.NodeID)
|
||||
assert.Equal(t, "0.0.0.0:9999", n.AWSGW.Host)
|
||||
assert.True(t, n.AWSGW.Debug)
|
||||
assert.Equal(t, 3, n.AWSGW.ExpectedNodes)
|
||||
}
|
||||
|
||||
// Tests for HasService / GetServices
|
||||
|
||||
func TestHasService_ExplicitList(t *testing.T) {
|
||||
c := Config{Services: []string{"nats", "daemon"}}
|
||||
|
||||
assert.True(t, c.HasService("nats"))
|
||||
assert.True(t, c.HasService("daemon"))
|
||||
assert.False(t, c.HasService("predastore"))
|
||||
assert.False(t, c.HasService("viperblock"))
|
||||
assert.False(t, c.HasService("ui"))
|
||||
}
|
||||
|
||||
func TestHasService_EmptyListBackwardCompat(t *testing.T) {
|
||||
c := Config{} // no Services set
|
||||
|
||||
// Empty list means all services
|
||||
for _, svc := range AllServices {
|
||||
assert.True(t, c.HasService(svc), "expected %s to be available with empty list", svc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasService_UnknownService(t *testing.T) {
|
||||
c := Config{Services: []string{"nats"}}
|
||||
assert.False(t, c.HasService("unknown"))
|
||||
}
|
||||
|
||||
func TestGetServices_DefaultsToAll(t *testing.T) {
|
||||
c := Config{}
|
||||
services := c.GetServices()
|
||||
assert.Equal(t, AllServices, services)
|
||||
}
|
||||
|
||||
func TestGetServices_ExplicitList(t *testing.T) {
|
||||
c := Config{Services: []string{"nats", "predastore"}}
|
||||
services := c.GetServices()
|
||||
assert.Equal(t, []string{"nats", "predastore"}, services)
|
||||
}
|
||||
|
||||
// Tests for NodeBaseDir
|
||||
|
||||
func TestNodeBaseDir_HappyPath(t *testing.T) {
|
||||
cc := &ClusterConfig{
|
||||
Node: "node1",
|
||||
Nodes: map[string]Config{
|
||||
"node1": {BaseDir: "/data/node1"},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, "/data/node1", cc.NodeBaseDir())
|
||||
}
|
||||
|
||||
func TestNodeBaseDir_NilConfig(t *testing.T) {
|
||||
var cc *ClusterConfig
|
||||
assert.Equal(t, "", cc.NodeBaseDir())
|
||||
}
|
||||
|
||||
func TestNodeBaseDir_EmptyNode(t *testing.T) {
|
||||
cc := &ClusterConfig{
|
||||
Node: "",
|
||||
Nodes: map[string]Config{
|
||||
"node1": {BaseDir: "/data/node1"},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, "", cc.NodeBaseDir())
|
||||
}
|
||||
|
||||
func TestNodeBaseDir_NodeNotInMap(t *testing.T) {
|
||||
cc := &ClusterConfig{
|
||||
Node: "missing",
|
||||
Nodes: map[string]Config{
|
||||
"node1": {BaseDir: "/data/node1"},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, "", cc.NodeBaseDir())
|
||||
}
|
||||
|
||||
func TestNodeBaseDir_EmptyBaseDir(t *testing.T) {
|
||||
cc := &ClusterConfig{
|
||||
Node: "node1",
|
||||
Nodes: map[string]Config{
|
||||
"node1": {BaseDir: ""},
|
||||
},
|
||||
}
|
||||
assert.Equal(t, "", cc.NodeBaseDir())
|
||||
}
|
||||
|
||||
// Tests for type constants
|
||||
|
||||
func TestNBDTransportConstants(t *testing.T) {
|
||||
assert.Equal(t, NBDTransport("socket"), NBDTransportSocket)
|
||||
assert.Equal(t, NBDTransport("tcp"), NBDTransportTCP)
|
||||
}
|
||||
|
||||
// Tests for ViperblockConfig
|
||||
|
||||
func TestLoadConfig_ViperblockShardWAL_Explicit(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hive.toml")
|
||||
|
||||
toml := `
|
||||
node = "n1"
|
||||
|
||||
[nodes.n1]
|
||||
region = "us-east-1"
|
||||
|
||||
[nodes.n1.viperblock]
|
||||
shardwal = false
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
n := cfg.Nodes["n1"]
|
||||
require.NotNil(t, n.Viperblock.ShardWAL, "ShardWAL should be set when explicitly configured")
|
||||
assert.False(t, *n.Viperblock.ShardWAL)
|
||||
}
|
||||
|
||||
func TestLoadConfig_ViperblockShardWAL_DefaultNil(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hive.toml")
|
||||
|
||||
toml := `
|
||||
node = "n1"
|
||||
|
||||
[nodes.n1]
|
||||
region = "us-east-1"
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
n := cfg.Nodes["n1"]
|
||||
assert.Nil(t, n.Viperblock.ShardWAL, "ShardWAL should be nil when not configured (defaults to false in service)")
|
||||
}
|
||||
|
||||
func TestLoadConfig_ViperblockShardWAL_True(t *testing.T) {
|
||||
resetViper(t)
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "hive.toml")
|
||||
|
||||
toml := `
|
||||
node = "n1"
|
||||
|
||||
[nodes.n1]
|
||||
region = "us-east-1"
|
||||
|
||||
[nodes.n1.viperblock]
|
||||
shardwal = true
|
||||
`
|
||||
require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
|
||||
|
||||
cfg, err := LoadConfig(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
n := cfg.Nodes["n1"]
|
||||
require.NotNil(t, n.Viperblock.ShardWAL)
|
||||
assert.True(t, *n.Viperblock.ShardWAL)
|
||||
}
|
||||
@@ -418,3 +418,20 @@ When adding new tests:
|
||||
- Add comments explaining Linux-only requirements
|
||||
- Use build tags where appropriate
|
||||
- Provide alternative test paths for macOS
|
||||
|
||||
## Local Docker E2E Tests
|
||||
|
||||
For more advanced testing (e.g., if GitHub Actions is failing), run the Docker E2E tests locally before committing. These mirror the CI pipeline exactly.
|
||||
|
||||
```bash
|
||||
make test-docker-single # Single-node E2E only
|
||||
make test-docker-multi # Multi-node E2E only
|
||||
make test-docker # Both suites
|
||||
```
|
||||
|
||||
Requirements:
|
||||
- Docker with `--privileged` support
|
||||
- `/dev/kvm` available (KVM-capable host)
|
||||
- Sibling repos (`viperblock/`, `predastore/`) cloned alongside `hive/`
|
||||
|
||||
The Docker build copies local source (including `go.work` if present), so uncommitted changes in sibling repos are picked up automatically.
|
||||
|
||||
+1040
-1481
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+2614
-173
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,274 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/hive/hive/vm"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRestartsInWindow = 3
|
||||
restartWindow = 10 * time.Minute
|
||||
restartBackoffBase = 5 * time.Second
|
||||
restartBackoffMax = 2 * time.Minute
|
||||
)
|
||||
|
||||
// classifyCrashReason extracts a human-readable crash reason from the error
|
||||
// returned by cmd.Wait(). Uses exec.ExitError + syscall.WaitStatus to
|
||||
// distinguish OOM kills (SIGKILL), segfaults (SIGSEGV), etc.
|
||||
func classifyCrashReason(waitErr error) string {
|
||||
if waitErr == nil {
|
||||
return "clean-exit"
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(waitErr, &exitErr) {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
status, ok := exitErr.Sys().(syscall.WaitStatus)
|
||||
if !ok {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
if status.Signaled() {
|
||||
switch status.Signal() {
|
||||
case syscall.SIGKILL:
|
||||
return "oom-killed"
|
||||
case syscall.SIGSEGV:
|
||||
return "segfault"
|
||||
case syscall.SIGABRT:
|
||||
return "abort"
|
||||
default:
|
||||
return fmt.Sprintf("signal-%d", status.Signal())
|
||||
}
|
||||
}
|
||||
|
||||
if status.Exited() {
|
||||
return fmt.Sprintf("exit-%d", status.ExitStatus())
|
||||
}
|
||||
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
// handleInstanceCrash is called by the QEMU launch goroutine after cmd.Wait()
|
||||
// returns during runtime (after startup was confirmed successful). It detects
|
||||
// the crash reason, transitions the instance to error state, cleans up resources,
|
||||
// and schedules a restart if policy allows.
|
||||
func (d *Daemon) handleInstanceCrash(instance *vm.VM, waitErr error) {
|
||||
// Guard: if instance is not running, this was an expected exit
|
||||
// (stopInstance/terminateInstance set status before QEMU exits)
|
||||
d.Instances.Mu.Lock()
|
||||
status := instance.Status
|
||||
d.Instances.Mu.Unlock()
|
||||
|
||||
if status != vm.StateRunning {
|
||||
slog.Debug("QEMU exited but instance not in running state, skipping crash handler",
|
||||
"instance", instance.ID, "status", status)
|
||||
return
|
||||
}
|
||||
|
||||
// Guard: coordinated shutdown in progress
|
||||
if d.shuttingDown.Load() {
|
||||
slog.Debug("QEMU exited during coordinated shutdown, skipping crash handler",
|
||||
"instance", instance.ID)
|
||||
return
|
||||
}
|
||||
|
||||
reason := classifyCrashReason(waitErr)
|
||||
slog.Error("VM process crashed", "instance", instance.ID, "reason", reason, "err", waitErr)
|
||||
|
||||
// Transition to error state
|
||||
if err := d.TransitionState(instance, vm.StateError); err != nil {
|
||||
slog.Error("Failed to transition crashed instance to error state",
|
||||
"instance", instance.ID, "err", err)
|
||||
}
|
||||
|
||||
// Update health tracking
|
||||
now := time.Now()
|
||||
d.Instances.Mu.Lock()
|
||||
instance.Health.CrashCount++
|
||||
instance.Health.LastCrashTime = now
|
||||
instance.Health.LastCrashReason = reason
|
||||
if instance.Health.FirstCrashTime.IsZero() {
|
||||
instance.Health.FirstCrashTime = now
|
||||
}
|
||||
instance.Running = false
|
||||
instance.PID = 0
|
||||
d.Instances.Mu.Unlock()
|
||||
|
||||
// Deallocate resources to fix phantom reservation
|
||||
instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
|
||||
if ok && instanceType != nil {
|
||||
slog.Info("Deallocating resources for crashed instance",
|
||||
"instance", instance.ID, "type", instance.InstanceType)
|
||||
d.resourceMgr.deallocate(instanceType)
|
||||
}
|
||||
|
||||
// Clean up stale QMP socket so QEMU can rebind on restart
|
||||
if instance.Config.QMPSocket != "" {
|
||||
_ = os.Remove(instance.Config.QMPSocket)
|
||||
}
|
||||
|
||||
// Unmount EBS volumes (same pattern as stopInstance)
|
||||
d.unmountInstanceVolumes(instance)
|
||||
|
||||
// Persist state
|
||||
if err := d.WriteState(); err != nil {
|
||||
slog.Error("Failed to persist state after crash handling",
|
||||
"instance", instance.ID, "err", err)
|
||||
}
|
||||
|
||||
d.maybeRestartInstance(instance)
|
||||
}
|
||||
|
||||
// unmountInstanceVolumes sends NATS unmount requests for all volumes attached
|
||||
// to the instance and updates their state to "available".
|
||||
func (d *Daemon) unmountInstanceVolumes(instance *vm.VM) {
|
||||
instance.EBSRequests.Mu.Lock()
|
||||
defer instance.EBSRequests.Mu.Unlock()
|
||||
|
||||
for _, ebsRequest := range instance.EBSRequests.Requests {
|
||||
ebsUnMountRequest, err := json.Marshal(ebsRequest)
|
||||
if err != nil {
|
||||
slog.Error("Failed to marshal volume payload for crash cleanup",
|
||||
"err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
msg, err := d.natsConn.Request(d.ebsTopic("unmount"), ebsUnMountRequest, 30*time.Second)
|
||||
if err != nil {
|
||||
slog.Error("Failed to unmount volume after crash",
|
||||
"name", ebsRequest.Name, "instance", instance.ID, "err", err)
|
||||
} else {
|
||||
slog.Info("Unmounted volume after crash",
|
||||
"instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data))
|
||||
}
|
||||
|
||||
// Update user-visible volume state to "available"
|
||||
if !ebsRequest.EFI && !ebsRequest.CloudInit {
|
||||
if err := d.volumeService.UpdateVolumeState(ebsRequest.Name, "available", "", ""); err != nil {
|
||||
slog.Error("Failed to update volume state to available after crash",
|
||||
"volumeId", ebsRequest.Name, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maybeRestartInstance checks restart policy and schedules a restart if allowed.
|
||||
func (d *Daemon) maybeRestartInstance(instance *vm.VM) {
|
||||
if d.shuttingDown.Load() {
|
||||
slog.Info("Skipping restart during shutdown", "instance", instance.ID)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
d.Instances.Mu.Lock()
|
||||
health := &instance.Health
|
||||
|
||||
// If crashes are outside the restart window, reset counters
|
||||
if !health.FirstCrashTime.IsZero() && now.Sub(health.FirstCrashTime) > restartWindow {
|
||||
slog.Info("Crash window expired, resetting counters", "instance", instance.ID)
|
||||
health.CrashCount = 1
|
||||
health.FirstCrashTime = now
|
||||
health.RestartCount = 0
|
||||
}
|
||||
|
||||
// Check if we've exceeded the max restarts in the window
|
||||
if health.CrashCount > maxRestartsInWindow {
|
||||
d.Instances.Mu.Unlock()
|
||||
slog.Error("Instance exceeded max restarts in window, leaving in error state",
|
||||
"instance", instance.ID,
|
||||
"crashes", health.CrashCount,
|
||||
"window", restartWindow,
|
||||
"max", maxRestartsInWindow)
|
||||
return
|
||||
}
|
||||
|
||||
restartCount := health.RestartCount
|
||||
d.Instances.Mu.Unlock()
|
||||
|
||||
// Check resource availability
|
||||
instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
|
||||
if !ok || instanceType == nil {
|
||||
slog.Error("Unknown instance type, cannot restart",
|
||||
"instance", instance.ID, "type", instance.InstanceType)
|
||||
return
|
||||
}
|
||||
|
||||
if d.resourceMgr.canAllocate(instanceType, 1) < 1 {
|
||||
slog.Error("Insufficient resources to restart instance",
|
||||
"instance", instance.ID, "type", instance.InstanceType)
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate exponential backoff
|
||||
delay := restartBackoffBase
|
||||
for i := 0; i < restartCount; i++ {
|
||||
delay *= 2
|
||||
if delay > restartBackoffMax {
|
||||
delay = restartBackoffMax
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
slog.Info("Scheduling instance restart",
|
||||
"instance", instance.ID,
|
||||
"delay", delay,
|
||||
"restartCount", restartCount+1)
|
||||
|
||||
time.AfterFunc(delay, func() {
|
||||
d.restartCrashedInstance(instance)
|
||||
})
|
||||
}
|
||||
|
||||
// restartCrashedInstance re-verifies the instance is still in error state
|
||||
// and relaunches it via LaunchInstance.
|
||||
func (d *Daemon) restartCrashedInstance(instance *vm.VM) {
|
||||
d.Instances.Mu.Lock()
|
||||
if instance.Status != vm.StateError {
|
||||
d.Instances.Mu.Unlock()
|
||||
slog.Info("Instance no longer in error state, skipping restart",
|
||||
"instance", instance.ID, "status", instance.Status)
|
||||
return
|
||||
}
|
||||
|
||||
if d.shuttingDown.Load() {
|
||||
d.Instances.Mu.Unlock()
|
||||
slog.Info("Daemon shutting down, skipping restart", "instance", instance.ID)
|
||||
return
|
||||
}
|
||||
|
||||
instance.Health.RestartCount++
|
||||
d.Instances.Mu.Unlock()
|
||||
|
||||
slog.Info("Restarting crashed instance",
|
||||
"instance", instance.ID,
|
||||
"restartCount", instance.Health.RestartCount)
|
||||
|
||||
// Transition Error → Pending (valid now that we added it to the transitions map)
|
||||
if err := d.TransitionState(instance, vm.StatePending); err != nil {
|
||||
slog.Error("Failed to transition instance to pending for restart",
|
||||
"instance", instance.ID, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
// LaunchInstance handles the full relaunch flow:
|
||||
// check PID → MountVolumes → StartInstance → CreateQMPClient → NATS subscribe → Running
|
||||
if err := d.LaunchInstance(instance); err != nil {
|
||||
slog.Error("Failed to restart crashed instance",
|
||||
"instance", instance.ID, "err", err)
|
||||
if err := d.TransitionState(instance, vm.StateError); err != nil {
|
||||
slog.Error("Failed to transition instance back to error after restart failure",
|
||||
"instance", instance.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/hive/hive/config"
|
||||
"github.com/mulgadc/hive/hive/vm"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestClassifyCrashReason_CleanExit(t *testing.T) {
|
||||
assert.Equal(t, "clean-exit", classifyCrashReason(nil))
|
||||
}
|
||||
|
||||
func TestClassifyCrashReason_OOMKill(t *testing.T) {
|
||||
// Start a process and kill it with SIGKILL to get a real ExitError
|
||||
cmd := exec.Command("sleep", "60")
|
||||
require.NoError(t, cmd.Start())
|
||||
|
||||
require.NoError(t, cmd.Process.Kill()) // sends SIGKILL
|
||||
waitErr := cmd.Wait()
|
||||
require.Error(t, waitErr)
|
||||
|
||||
assert.Equal(t, "oom-killed", classifyCrashReason(waitErr))
|
||||
}
|
||||
|
||||
func TestClassifyCrashReason_ExitCode(t *testing.T) {
|
||||
// Run a command that exits with a non-zero code
|
||||
cmd := exec.Command("sh", "-c", "exit 42")
|
||||
waitErr := cmd.Run()
|
||||
require.Error(t, waitErr)
|
||||
|
||||
assert.Equal(t, "exit-42", classifyCrashReason(waitErr))
|
||||
}
|
||||
|
||||
func TestClassifyCrashReason_Unknown(t *testing.T) {
|
||||
// A non-ExitError error
|
||||
err := fmt.Errorf("some random error")
|
||||
assert.Equal(t, "unknown", classifyCrashReason(err))
|
||||
}
|
||||
|
||||
func TestHandleInstanceCrash_SkipsNonRunning(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
d := &Daemon{
|
||||
natsConn: nc,
|
||||
resourceMgr: NewResourceManager(),
|
||||
Instances: vm.Instances{VMS: make(map[string]*vm.VM)},
|
||||
}
|
||||
|
||||
instance := &vm.VM{
|
||||
ID: "i-test-stopped",
|
||||
Status: vm.StateStopped,
|
||||
}
|
||||
d.Instances.VMS[instance.ID] = instance
|
||||
|
||||
// Should return without action (no panic, no state change)
|
||||
d.handleInstanceCrash(instance, fmt.Errorf("test error"))
|
||||
assert.Equal(t, vm.StateStopped, instance.Status)
|
||||
}
|
||||
|
||||
func TestHandleInstanceCrash_SkipsShuttingDown(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
d := &Daemon{
|
||||
natsConn: nc,
|
||||
resourceMgr: NewResourceManager(),
|
||||
Instances: vm.Instances{VMS: make(map[string]*vm.VM)},
|
||||
}
|
||||
d.shuttingDown.Store(true)
|
||||
|
||||
instance := &vm.VM{
|
||||
ID: "i-test-shutdown",
|
||||
Status: vm.StateRunning,
|
||||
}
|
||||
d.Instances.VMS[instance.ID] = instance
|
||||
|
||||
d.handleInstanceCrash(instance, fmt.Errorf("test error"))
|
||||
|
||||
// Status unchanged because shuttingDown is true
|
||||
assert.Equal(t, vm.StateRunning, instance.Status)
|
||||
}
|
||||
|
||||
func TestMaybeRestart_ExceedsMaxInWindow(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
d := &Daemon{
|
||||
node: "test-node",
|
||||
natsConn: nc,
|
||||
config: &config.Config{
|
||||
Services: []string{"daemon"},
|
||||
},
|
||||
resourceMgr: NewResourceManager(),
|
||||
Instances: vm.Instances{VMS: make(map[string]*vm.VM)},
|
||||
}
|
||||
|
||||
// Find an allocatable instance type
|
||||
var allocType string
|
||||
for typeName := range d.resourceMgr.instanceTypes {
|
||||
allocType = typeName
|
||||
break
|
||||
}
|
||||
require.NotEmpty(t, allocType)
|
||||
|
||||
instance := &vm.VM{
|
||||
ID: "i-test-maxcrash",
|
||||
Status: vm.StateError,
|
||||
InstanceType: allocType,
|
||||
Health: vm.InstanceHealthState{
|
||||
CrashCount: maxRestartsInWindow + 1, // exceeds limit
|
||||
FirstCrashTime: time.Now(),
|
||||
RestartCount: maxRestartsInWindow,
|
||||
},
|
||||
}
|
||||
d.Instances.VMS[instance.ID] = instance
|
||||
|
||||
// Should not schedule a restart (no panic, instance stays in error)
|
||||
d.maybeRestartInstance(instance)
|
||||
|
||||
// Instance should remain in error state
|
||||
assert.Equal(t, vm.StateError, instance.Status)
|
||||
}
|
||||
|
||||
func TestMaybeRestart_ResetsAfterWindow(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
d := &Daemon{
|
||||
node: "test-node",
|
||||
natsConn: nc,
|
||||
config: &config.Config{
|
||||
Services: []string{"daemon"},
|
||||
},
|
||||
resourceMgr: NewResourceManager(),
|
||||
Instances: vm.Instances{VMS: make(map[string]*vm.VM)},
|
||||
}
|
||||
|
||||
// Find an allocatable instance type
|
||||
var allocType string
|
||||
for typeName := range d.resourceMgr.instanceTypes {
|
||||
allocType = typeName
|
||||
break
|
||||
}
|
||||
require.NotEmpty(t, allocType)
|
||||
|
||||
instance := &vm.VM{
|
||||
ID: "i-test-windowreset",
|
||||
Status: vm.StateError,
|
||||
InstanceType: allocType,
|
||||
Health: vm.InstanceHealthState{
|
||||
CrashCount: 5,
|
||||
FirstCrashTime: time.Now().Add(-restartWindow - time.Minute), // outside window
|
||||
RestartCount: 5,
|
||||
},
|
||||
}
|
||||
d.Instances.VMS[instance.ID] = instance
|
||||
|
||||
// maybeRestartInstance should reset counters and schedule a restart
|
||||
// (restart will fail because there's no real QEMU, but counters should reset)
|
||||
d.maybeRestartInstance(instance)
|
||||
|
||||
// Counters should be reset
|
||||
assert.Equal(t, 1, instance.Health.CrashCount)
|
||||
assert.Equal(t, 0, instance.Health.RestartCount)
|
||||
}
|
||||
|
||||
func TestRestartBackoff_Exponential(t *testing.T) {
|
||||
// Verify the backoff calculation logic
|
||||
expected := []time.Duration{
|
||||
5 * time.Second, // restart 0
|
||||
10 * time.Second, // restart 1
|
||||
20 * time.Second, // restart 2
|
||||
40 * time.Second, // restart 3
|
||||
80 * time.Second, // restart 4
|
||||
120 * time.Second, // restart 5 (capped at 2min)
|
||||
120 * time.Second, // restart 6 (capped at 2min)
|
||||
}
|
||||
|
||||
for i, want := range expected {
|
||||
delay := restartBackoffBase
|
||||
for j := 0; j < i; j++ {
|
||||
delay *= 2
|
||||
if delay > restartBackoffMax {
|
||||
delay = restartBackoffMax
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.Equal(t, want, delay, "Backoff mismatch at restart count %d", i)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
const heartbeatInterval = 10 * time.Second
|
||||
|
||||
// startHeartbeat launches a goroutine that publishes this daemon's heartbeat
|
||||
// to the cluster-state KV store every heartbeatInterval. It fires immediately
|
||||
// on start, then repeats on a ticker. The goroutine exits when d.ctx is cancelled.
|
||||
func (d *Daemon) startHeartbeat() {
|
||||
if d.jsManager == nil {
|
||||
slog.Warn("JetStream not initialized, skipping heartbeat")
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
// Fire immediately on startup
|
||||
d.publishHeartbeat()
|
||||
|
||||
ticker := time.NewTicker(heartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
slog.Info("Heartbeat goroutine stopping")
|
||||
return
|
||||
case <-ticker.C:
|
||||
d.publishHeartbeat()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
slog.Info("Heartbeat started", "interval", heartbeatInterval)
|
||||
}
|
||||
|
||||
// publishHeartbeat builds and writes a heartbeat entry to KV.
|
||||
func (d *Daemon) publishHeartbeat() {
|
||||
h := d.buildHeartbeat()
|
||||
if err := d.jsManager.WriteHeartbeat(h); err != nil {
|
||||
slog.Warn("Failed to publish heartbeat", "error", err)
|
||||
} else {
|
||||
slog.Debug("Heartbeat published", "node", h.Node, "vms", h.VMCount)
|
||||
}
|
||||
}
|
||||
|
||||
// buildHeartbeat constructs a Heartbeat from the daemon's current state.
|
||||
func (d *Daemon) buildHeartbeat() *Heartbeat {
|
||||
totalVCPU, totalMem, allocVCPU, allocMem, _ := d.resourceMgr.GetResourceStats()
|
||||
|
||||
d.Instances.Mu.Lock()
|
||||
vmCount := len(d.Instances.VMS)
|
||||
d.Instances.Mu.Unlock()
|
||||
|
||||
return &Heartbeat{
|
||||
Node: d.node,
|
||||
Epoch: d.clusterConfig.Epoch,
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
Services: d.config.GetServices(),
|
||||
VMCount: vmCount,
|
||||
AllocatedVCPU: allocVCPU,
|
||||
AvailableVCPU: totalVCPU - allocVCPU,
|
||||
AllocatedMem: allocMem,
|
||||
AvailableMem: totalMem - allocMem,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/hive/hive/config"
|
||||
"github.com/mulgadc/hive/hive/vm"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestBuildHeartbeat verifies that buildHeartbeat populates the struct from daemon state.
|
||||
func TestBuildHeartbeat(t *testing.T) {
|
||||
d := &Daemon{
|
||||
node: "test-node",
|
||||
clusterConfig: &config.ClusterConfig{
|
||||
Epoch: 5,
|
||||
},
|
||||
config: &config.Config{
|
||||
Services: []string{"daemon", "nats", "viperblock"},
|
||||
},
|
||||
resourceMgr: NewResourceManager(),
|
||||
Instances: vm.Instances{VMS: make(map[string]*vm.VM)},
|
||||
}
|
||||
|
||||
h := d.buildHeartbeat()
|
||||
|
||||
assert.Equal(t, "test-node", h.Node)
|
||||
assert.Equal(t, uint64(5), h.Epoch)
|
||||
assert.NotEmpty(t, h.Timestamp)
|
||||
assert.Equal(t, []string{"daemon", "nats", "viperblock"}, h.Services)
|
||||
assert.Equal(t, 0, h.VMCount)
|
||||
assert.Equal(t, 0, h.AllocatedVCPU)
|
||||
assert.Greater(t, h.AvailableVCPU, 0)
|
||||
assert.Greater(t, h.AvailableMem, 0.0)
|
||||
}
|
||||
|
||||
// TestHeartbeatReflectsAllocation verifies that allocating resources changes the heartbeat values.
|
||||
func TestHeartbeatReflectsAllocation(t *testing.T) {
|
||||
d := &Daemon{
|
||||
node: "alloc-node",
|
||||
clusterConfig: &config.ClusterConfig{
|
||||
Epoch: 1,
|
||||
},
|
||||
config: &config.Config{
|
||||
Services: []string{"daemon"},
|
||||
},
|
||||
resourceMgr: NewResourceManager(),
|
||||
Instances: vm.Instances{VMS: make(map[string]*vm.VM)},
|
||||
}
|
||||
|
||||
// Take a heartbeat before allocation
|
||||
before := d.buildHeartbeat()
|
||||
|
||||
// Find an instance type we can allocate
|
||||
var allocType string
|
||||
for typeName := range d.resourceMgr.instanceTypes {
|
||||
if d.resourceMgr.canAllocate(d.resourceMgr.instanceTypes[typeName], 1) >= 1 {
|
||||
allocType = typeName
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, allocType, "Should have at least one allocatable instance type")
|
||||
|
||||
// Allocate resources
|
||||
err := d.resourceMgr.allocate(d.resourceMgr.instanceTypes[allocType])
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add a VM to the instance map
|
||||
d.Instances.Mu.Lock()
|
||||
d.Instances.VMS["i-test-001"] = &vm.VM{
|
||||
ID: "i-test-001",
|
||||
Status: vm.StateRunning,
|
||||
InstanceType: allocType,
|
||||
}
|
||||
d.Instances.Mu.Unlock()
|
||||
|
||||
// Take a heartbeat after allocation
|
||||
after := d.buildHeartbeat()
|
||||
|
||||
assert.Equal(t, 1, after.VMCount, "Should reflect 1 VM")
|
||||
assert.Greater(t, after.AllocatedVCPU, before.AllocatedVCPU, "AllocatedVCPU should increase")
|
||||
assert.Less(t, after.AvailableVCPU, before.AvailableVCPU, "AvailableVCPU should decrease")
|
||||
}
|
||||
|
||||
// TestHeartbeatKVRoundTrip verifies that heartbeat can be written to and read from KV.
|
||||
func TestHeartbeatKVRoundTrip(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitClusterStateBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
h := &Heartbeat{
|
||||
Node: "kv-test-node",
|
||||
Epoch: 3,
|
||||
Timestamp: "2025-01-01T00:00:00Z",
|
||||
Services: []string{"daemon", "nats"},
|
||||
VMCount: 2,
|
||||
AllocatedVCPU: 4,
|
||||
AvailableVCPU: 12,
|
||||
AllocatedMem: 8.0,
|
||||
AvailableMem: 24.0,
|
||||
}
|
||||
|
||||
err = jsm.WriteHeartbeat(h)
|
||||
require.NoError(t, err)
|
||||
|
||||
loaded, err := jsm.ReadHeartbeat("kv-test-node")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, loaded)
|
||||
|
||||
assert.Equal(t, h.Node, loaded.Node)
|
||||
assert.Equal(t, h.Epoch, loaded.Epoch)
|
||||
assert.Equal(t, h.Timestamp, loaded.Timestamp)
|
||||
assert.Equal(t, h.Services, loaded.Services)
|
||||
assert.Equal(t, h.VMCount, loaded.VMCount)
|
||||
assert.Equal(t, h.AllocatedVCPU, loaded.AllocatedVCPU)
|
||||
assert.Equal(t, h.AvailableVCPU, loaded.AvailableVCPU)
|
||||
assert.Equal(t, h.AllocatedMem, loaded.AllocatedMem)
|
||||
assert.Equal(t, h.AvailableMem, loaded.AvailableMem)
|
||||
}
|
||||
+312
-10
@@ -4,6 +4,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/hive/hive/vm"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -12,15 +14,20 @@ import (
|
||||
const (
|
||||
// InstanceStateBucket is the name of the KV bucket for storing instance state
|
||||
InstanceStateBucket = "hive-instance-state"
|
||||
// InstanceStatePrefix is the key prefix for instance state entries
|
||||
// ClusterStateBucket is the name of the KV bucket for cluster state (heartbeats, shutdown markers, service maps)
|
||||
ClusterStateBucket = "hive-cluster-state"
|
||||
// InstanceStatePrefix is the key prefix for per-node instance state entries
|
||||
InstanceStatePrefix = "node."
|
||||
// StoppedInstancePrefix is the key prefix for stopped instances in shared KV
|
||||
StoppedInstancePrefix = "instance."
|
||||
)
|
||||
|
||||
// JetStreamManager manages JetStream KV store operations for instance state
|
||||
type JetStreamManager struct {
|
||||
js nats.JetStreamContext
|
||||
kv nats.KeyValue
|
||||
replicas int
|
||||
js nats.JetStreamContext
|
||||
kv nats.KeyValue // hive-instance-state
|
||||
clusterKV nats.KeyValue // hive-cluster-state
|
||||
replicas int
|
||||
}
|
||||
|
||||
// NewJetStreamManager creates a new JetStreamManager from a NATS connection.
|
||||
@@ -44,7 +51,7 @@ func (m *JetStreamManager) InitKVBucket() error {
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrBucketNotFound) {
|
||||
// Bucket doesn't exist, create it
|
||||
slog.Info("Creating JetStream KV bucket", "bucket", InstanceStateBucket, "replicas", m.replicas)
|
||||
slog.Debug("Creating JetStream KV bucket", "bucket", InstanceStateBucket, "replicas", m.replicas)
|
||||
kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: InstanceStateBucket,
|
||||
Description: "Hive instance state storage",
|
||||
@@ -58,22 +65,200 @@ func (m *JetStreamManager) InitKVBucket() error {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
slog.Info("Connected to existing JetStream KV bucket", "bucket", InstanceStateBucket)
|
||||
slog.Debug("Connected to existing JetStream KV bucket", "bucket", InstanceStateBucket)
|
||||
}
|
||||
|
||||
m.kv = kv
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteState writes the instance state to the KV store for the given node
|
||||
// InitClusterStateBucket initializes the cluster-state KV bucket, creating it if it doesn't exist.
|
||||
func (m *JetStreamManager) InitClusterStateBucket() error {
|
||||
kv, err := m.js.KeyValue(ClusterStateBucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrBucketNotFound) {
|
||||
slog.Debug("Creating JetStream KV bucket", "bucket", ClusterStateBucket, "replicas", m.replicas)
|
||||
kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: ClusterStateBucket,
|
||||
Description: "Hive cluster state (heartbeats, shutdown markers, service maps)",
|
||||
History: 1,
|
||||
Replicas: m.replicas,
|
||||
TTL: 1 * time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
slog.Debug("Connected to existing JetStream KV bucket", "bucket", ClusterStateBucket)
|
||||
}
|
||||
|
||||
m.clusterKV = kv
|
||||
return nil
|
||||
}
|
||||
|
||||
// Heartbeat represents a daemon's periodic health status published to cluster KV.
|
||||
type Heartbeat struct {
|
||||
Node string `json:"node"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Services []string `json:"services"`
|
||||
VMCount int `json:"vm_count"`
|
||||
AllocatedVCPU int `json:"allocated_vcpu"`
|
||||
AvailableVCPU int `json:"available_vcpu"`
|
||||
AllocatedMem float64 `json:"allocated_mem_gb"`
|
||||
AvailableMem float64 `json:"available_mem_gb"`
|
||||
}
|
||||
|
||||
// WriteHeartbeat writes a heartbeat entry for the given node to the cluster-state KV.
|
||||
func (m *JetStreamManager) WriteHeartbeat(h *Heartbeat) error {
|
||||
if m.clusterKV == nil {
|
||||
return errors.New("cluster state KV not initialized")
|
||||
}
|
||||
data, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = m.clusterKV.Put("heartbeat."+h.Node, data)
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadHeartbeat reads the heartbeat entry for the given node from the cluster-state KV.
|
||||
func (m *JetStreamManager) ReadHeartbeat(nodeID string) (*Heartbeat, error) {
|
||||
if m.clusterKV == nil {
|
||||
return nil, errors.New("cluster state KV not initialized")
|
||||
}
|
||||
entry, err := m.clusterKV.Get("heartbeat." + nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var h Heartbeat
|
||||
if err := json.Unmarshal(entry.Value(), &h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &h, nil
|
||||
}
|
||||
|
||||
// ClusterShutdownState tracks the coordinated cluster shutdown progress in KV.
|
||||
type ClusterShutdownState struct {
|
||||
Initiator string `json:"initiator"`
|
||||
Phase string `json:"phase"`
|
||||
Started string `json:"started"`
|
||||
Timeout string `json:"timeout"`
|
||||
Force bool `json:"force"`
|
||||
NodesTotal int `json:"nodes_total"`
|
||||
NodesAcked map[string]string `json:"nodes_acked"`
|
||||
}
|
||||
|
||||
// WriteClusterShutdown writes the cluster shutdown state to KV.
|
||||
func (m *JetStreamManager) WriteClusterShutdown(state *ClusterShutdownState) error {
|
||||
if m.clusterKV == nil {
|
||||
return errors.New("cluster state KV not initialized")
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = m.clusterKV.Put("cluster.shutdown", data)
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadClusterShutdown reads the cluster shutdown state from KV.
|
||||
func (m *JetStreamManager) ReadClusterShutdown() (*ClusterShutdownState, error) {
|
||||
if m.clusterKV == nil {
|
||||
return nil, errors.New("cluster state KV not initialized")
|
||||
}
|
||||
entry, err := m.clusterKV.Get("cluster.shutdown")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var state ClusterShutdownState
|
||||
if err := json.Unmarshal(entry.Value(), &state); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &state, nil
|
||||
}
|
||||
|
||||
// DeleteClusterShutdown removes the cluster shutdown state from KV.
|
||||
func (m *JetStreamManager) DeleteClusterShutdown() error {
|
||||
if m.clusterKV == nil {
|
||||
return errors.New("cluster state KV not initialized")
|
||||
}
|
||||
err := m.clusterKV.Delete("cluster.shutdown")
|
||||
if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteShutdownMarker writes a shutdown marker for the given node to the cluster-state KV.
|
||||
func (m *JetStreamManager) WriteShutdownMarker(nodeID string) error {
|
||||
if m.clusterKV == nil {
|
||||
return errors.New("cluster state KV not initialized")
|
||||
}
|
||||
data, _ := json.Marshal(map[string]any{
|
||||
"node": nodeID,
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
_, err := m.clusterKV.Put("shutdown."+nodeID, data)
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadShutdownMarker checks if a clean shutdown marker exists for the given node.
|
||||
func (m *JetStreamManager) ReadShutdownMarker(nodeID string) (bool, error) {
|
||||
if m.clusterKV == nil {
|
||||
return false, errors.New("cluster state KV not initialized")
|
||||
}
|
||||
_, err := m.clusterKV.Get("shutdown." + nodeID)
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// DeleteShutdownMarker removes the shutdown marker for the given node.
|
||||
func (m *JetStreamManager) DeleteShutdownMarker(nodeID string) error {
|
||||
if m.clusterKV == nil {
|
||||
return errors.New("cluster state KV not initialized")
|
||||
}
|
||||
err := m.clusterKV.Delete("shutdown." + nodeID)
|
||||
if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteServiceManifest writes the service manifest for the given node to the cluster-state KV.
|
||||
func (m *JetStreamManager) WriteServiceManifest(nodeID string, services []string, natsHost, predastoreHost string) error {
|
||||
if m.clusterKV == nil {
|
||||
return errors.New("cluster state KV not initialized")
|
||||
}
|
||||
data, _ := json.Marshal(map[string]any{
|
||||
"node": nodeID,
|
||||
"services": services,
|
||||
"nats_host": natsHost,
|
||||
"predastore_host": predastoreHost,
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
_, err := m.clusterKV.Put("node."+nodeID+".services", data)
|
||||
return err
|
||||
}
|
||||
|
||||
// WriteState writes the instance state to the KV store for the given node.
|
||||
// It acquires instances.Mu internally.
|
||||
func (m *JetStreamManager) WriteState(nodeID string, instances *vm.Instances) error {
|
||||
instances.Mu.Lock()
|
||||
defer instances.Mu.Unlock()
|
||||
|
||||
if m.kv == nil {
|
||||
return errors.New("KV bucket not initialized")
|
||||
}
|
||||
|
||||
instances.Mu.Lock()
|
||||
defer instances.Mu.Unlock()
|
||||
|
||||
// Create a struct without the mutex to avoid copying the lock
|
||||
state := struct {
|
||||
VMS map[string]*vm.VM `json:"vms"`
|
||||
@@ -176,5 +361,122 @@ func (m *JetStreamManager) UpdateReplicas(newReplicas int) error {
|
||||
|
||||
m.replicas = newReplicas
|
||||
slog.Info("Updated JetStream KV bucket replicas", "bucket", InstanceStateBucket, "oldReplicas", currentReplicas, "newReplicas", newReplicas)
|
||||
|
||||
// Also update the cluster-state bucket if it exists
|
||||
if m.clusterKV != nil {
|
||||
clusterStreamName := "KV_" + ClusterStateBucket
|
||||
clusterInfo, err := m.js.StreamInfo(clusterStreamName)
|
||||
if err == nil && clusterInfo.Config.Replicas < newReplicas {
|
||||
clusterInfo.Config.Replicas = newReplicas
|
||||
if _, err := m.js.UpdateStream(&clusterInfo.Config); err != nil {
|
||||
slog.Warn("Failed to update cluster-state bucket replicas", "error", err)
|
||||
} else {
|
||||
slog.Info("Updated cluster-state bucket replicas", "bucket", ClusterStateBucket, "newReplicas", newReplicas)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteStoppedInstance writes a stopped instance to the shared KV store.
|
||||
func (m *JetStreamManager) WriteStoppedInstance(instanceID string, instance *vm.VM) error {
|
||||
if m.kv == nil {
|
||||
return errors.New("KV bucket not initialized")
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(instance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
key := StoppedInstancePrefix + instanceID
|
||||
_, err = m.kv.Put(key, jsonData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Debug("Wrote stopped instance to JetStream KV", "key", key, "instanceId", instanceID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadStoppedInstance loads a stopped instance from the shared KV store.
|
||||
// Returns nil, nil if the key does not exist.
|
||||
func (m *JetStreamManager) LoadStoppedInstance(instanceID string) (*vm.VM, error) {
|
||||
if m.kv == nil {
|
||||
return nil, errors.New("KV bucket not initialized")
|
||||
}
|
||||
|
||||
key := StoppedInstancePrefix + instanceID
|
||||
entry, err := m.kv.Get(key)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var instance vm.VM
|
||||
if err := json.Unmarshal(entry.Value(), &instance); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &instance, nil
|
||||
}
|
||||
|
||||
// DeleteStoppedInstance removes a stopped instance from the shared KV store.
|
||||
// It is idempotent — deleting a non-existent key is not an error.
|
||||
func (m *JetStreamManager) DeleteStoppedInstance(instanceID string) error {
|
||||
if m.kv == nil {
|
||||
return errors.New("KV bucket not initialized")
|
||||
}
|
||||
|
||||
key := StoppedInstancePrefix + instanceID
|
||||
err := m.kv.Delete(key)
|
||||
if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Debug("Deleted stopped instance from JetStream KV", "key", key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListStoppedInstances returns all stopped instances from the shared KV store.
|
||||
func (m *JetStreamManager) ListStoppedInstances() ([]*vm.VM, error) {
|
||||
if m.kv == nil {
|
||||
return nil, errors.New("KV bucket not initialized")
|
||||
}
|
||||
|
||||
keys, err := m.kv.Keys()
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var instances []*vm.VM
|
||||
for _, key := range keys {
|
||||
if !strings.HasPrefix(key, StoppedInstancePrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
entry, err := m.kv.Get(key)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var instance vm.VM
|
||||
if err := json.Unmarshal(entry.Value(), &instance); err != nil {
|
||||
slog.Error("Failed to unmarshal stopped instance", "key", key, "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
instances = append(instances, &instance)
|
||||
}
|
||||
|
||||
return instances, nil
|
||||
}
|
||||
|
||||
+217
-74
@@ -1,58 +1,17 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/hive/hive/vm"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// startTestNATSServerWithJetStream starts an embedded NATS server with JetStream enabled for testing
|
||||
func startTestNATSServerWithJetStream(t *testing.T) (*server.Server, string) {
|
||||
// Create a temporary directory for JetStream storage
|
||||
tmpDir, err := os.MkdirTemp("", "nats-js-test-*")
|
||||
require.NoError(t, err, "Failed to create temp directory for JetStream")
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.RemoveAll(tmpDir)
|
||||
})
|
||||
|
||||
opts := &server.Options{
|
||||
Host: "127.0.0.1",
|
||||
Port: -1, // Let NATS auto-allocate an available port
|
||||
JetStream: true,
|
||||
StoreDir: tmpDir,
|
||||
NoLog: true,
|
||||
NoSigs: true,
|
||||
}
|
||||
|
||||
ns, err := server.NewServer(opts)
|
||||
require.NoError(t, err, "Failed to create NATS server with JetStream")
|
||||
|
||||
// Start server in a goroutine
|
||||
go ns.Start()
|
||||
|
||||
// Wait for server to be ready
|
||||
if !ns.ReadyForConnections(5 * time.Second) {
|
||||
t.Fatal("NATS server with JetStream failed to start")
|
||||
}
|
||||
|
||||
// Get the actual URL that was assigned
|
||||
url := ns.ClientURL()
|
||||
t.Logf("Test NATS server with JetStream started at: %s", url)
|
||||
|
||||
return ns, url
|
||||
}
|
||||
|
||||
// TestJetStreamManager_WriteAndLoadState tests round-trip write and load of instance state
|
||||
func TestJetStreamManager_WriteAndLoadState(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
// Connect to NATS
|
||||
nc, err := nats.Connect(natsURL)
|
||||
@@ -73,12 +32,12 @@ func TestJetStreamManager_WriteAndLoadState(t *testing.T) {
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-test-001": {
|
||||
ID: "i-test-001",
|
||||
Status: "running",
|
||||
Status: vm.StateRunning,
|
||||
InstanceType: "t3.micro",
|
||||
},
|
||||
"i-test-002": {
|
||||
ID: "i-test-002",
|
||||
Status: "stopped",
|
||||
Status: vm.StateStopped,
|
||||
InstanceType: "t3.small",
|
||||
},
|
||||
},
|
||||
@@ -97,16 +56,15 @@ func TestJetStreamManager_WriteAndLoadState(t *testing.T) {
|
||||
assert.Len(t, loadedInstances.VMS, 2, "Should have 2 instances")
|
||||
assert.NotNil(t, loadedInstances.VMS["i-test-001"], "Should have i-test-001")
|
||||
assert.NotNil(t, loadedInstances.VMS["i-test-002"], "Should have i-test-002")
|
||||
assert.Equal(t, "running", loadedInstances.VMS["i-test-001"].Status)
|
||||
assert.Equal(t, "stopped", loadedInstances.VMS["i-test-002"].Status)
|
||||
assert.Equal(t, vm.StateRunning, loadedInstances.VMS["i-test-001"].Status)
|
||||
assert.Equal(t, vm.StateStopped, loadedInstances.VMS["i-test-002"].Status)
|
||||
assert.Equal(t, "t3.micro", loadedInstances.VMS["i-test-001"].InstanceType)
|
||||
assert.Equal(t, "t3.small", loadedInstances.VMS["i-test-002"].InstanceType)
|
||||
}
|
||||
|
||||
// TestJetStreamManager_LoadState_KeyNotFound tests that LoadState returns empty state when key doesn't exist
|
||||
func TestJetStreamManager_LoadState_KeyNotFound(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -127,8 +85,7 @@ func TestJetStreamManager_LoadState_KeyNotFound(t *testing.T) {
|
||||
|
||||
// TestJetStreamManager_BucketCreation tests that InitKVBucket creates the bucket when it doesn't exist
|
||||
func TestJetStreamManager_BucketCreation(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -147,8 +104,7 @@ func TestJetStreamManager_BucketCreation(t *testing.T) {
|
||||
|
||||
// TestJetStreamManager_BucketReconnection tests that InitKVBucket connects to existing bucket
|
||||
func TestJetStreamManager_BucketReconnection(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
// First connection - create the bucket
|
||||
nc1, err := nats.Connect(natsURL)
|
||||
@@ -165,7 +121,7 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) {
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-persist": {
|
||||
ID: "i-persist",
|
||||
Status: "running",
|
||||
Status: vm.StateRunning,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -190,13 +146,12 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, loadedInstances.VMS, "Should have persisted instances")
|
||||
assert.NotNil(t, loadedInstances.VMS["i-persist"], "Should have i-persist")
|
||||
assert.Equal(t, "running", loadedInstances.VMS["i-persist"].Status)
|
||||
assert.Equal(t, vm.StateRunning, loadedInstances.VMS["i-persist"].Status)
|
||||
}
|
||||
|
||||
// TestJetStreamManager_DeleteState tests deleting state from the KV store
|
||||
func TestJetStreamManager_DeleteState(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -214,7 +169,7 @@ func TestJetStreamManager_DeleteState(t *testing.T) {
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-delete-me": {
|
||||
ID: "i-delete-me",
|
||||
Status: "running",
|
||||
Status: vm.StateRunning,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -238,8 +193,7 @@ func TestJetStreamManager_DeleteState(t *testing.T) {
|
||||
|
||||
// TestJetStreamManager_DeleteState_NonExistent tests deleting state that doesn't exist
|
||||
func TestJetStreamManager_DeleteState_NonExistent(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -258,8 +212,7 @@ func TestJetStreamManager_DeleteState_NonExistent(t *testing.T) {
|
||||
|
||||
// TestJetStreamManager_WriteState_UpdateExisting tests that writing state updates existing entry
|
||||
func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -278,7 +231,7 @@ func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-initial": {
|
||||
ID: "i-initial",
|
||||
Status: "running",
|
||||
Status: vm.StateRunning,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -290,11 +243,11 @@ func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-initial": {
|
||||
ID: "i-initial",
|
||||
Status: "stopped", // Changed status
|
||||
Status: vm.StateStopped, // Changed status
|
||||
},
|
||||
"i-new": { // Added new instance
|
||||
ID: "i-new",
|
||||
Status: "running",
|
||||
Status: vm.StateRunning,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -305,14 +258,13 @@ func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
|
||||
loadedInstances, err := jsm.LoadState(testNodeID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, loadedInstances.VMS, 2, "Should have 2 instances")
|
||||
assert.Equal(t, "stopped", loadedInstances.VMS["i-initial"].Status, "Status should be updated")
|
||||
assert.Equal(t, vm.StateStopped, loadedInstances.VMS["i-initial"].Status, "Status should be updated")
|
||||
assert.NotNil(t, loadedInstances.VMS["i-new"], "Should have new instance")
|
||||
}
|
||||
|
||||
// TestJetStreamManager_MultipleNodes tests storing state for multiple nodes
|
||||
func TestJetStreamManager_MultipleNodes(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -327,7 +279,7 @@ func TestJetStreamManager_MultipleNodes(t *testing.T) {
|
||||
// Write state for node-1
|
||||
node1Instances := &vm.Instances{
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-node1-001": {ID: "i-node1-001", Status: "running"},
|
||||
"i-node1-001": {ID: "i-node1-001", Status: vm.StateRunning},
|
||||
},
|
||||
}
|
||||
err = jsm.WriteState("node-1", node1Instances)
|
||||
@@ -336,8 +288,8 @@ func TestJetStreamManager_MultipleNodes(t *testing.T) {
|
||||
// Write state for node-2
|
||||
node2Instances := &vm.Instances{
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-node2-001": {ID: "i-node2-001", Status: "stopped"},
|
||||
"i-node2-002": {ID: "i-node2-002", Status: "running"},
|
||||
"i-node2-001": {ID: "i-node2-001", Status: vm.StateStopped},
|
||||
"i-node2-002": {ID: "i-node2-002", Status: vm.StateRunning},
|
||||
},
|
||||
}
|
||||
err = jsm.WriteState("node-2", node2Instances)
|
||||
@@ -363,8 +315,7 @@ func TestJetStreamManager_MultipleNodes(t *testing.T) {
|
||||
|
||||
// TestJetStreamManager_KVNotInitialized tests error handling when KV is not initialized
|
||||
func TestJetStreamManager_KVNotInitialized(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -387,8 +338,7 @@ func TestJetStreamManager_KVNotInitialized(t *testing.T) {
|
||||
|
||||
// TestJetStreamManager_UpdateReplicas tests updating replica count for the KV bucket
|
||||
func TestJetStreamManager_UpdateReplicas(t *testing.T) {
|
||||
ns, natsURL := startTestNATSServerWithJetStream(t)
|
||||
defer ns.Shutdown()
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
@@ -428,3 +378,196 @@ func TestJetStreamManager_UpdateReplicas_NoInit(t *testing.T) {
|
||||
err := jsm.UpdateReplicas(3)
|
||||
assert.Error(t, err, "UpdateReplicas should error when JetStream not initialized")
|
||||
}
|
||||
|
||||
// --- Stopped instance KV tests ---
|
||||
|
||||
// TestJetStreamManager_WriteAndLoadStoppedInstance tests round-trip write and load of a stopped instance
|
||||
func TestJetStreamManager_WriteAndLoadStoppedInstance(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
testVM := &vm.VM{
|
||||
ID: "i-stopped-001",
|
||||
Status: vm.StateStopped,
|
||||
InstanceType: "t3.micro",
|
||||
LastNode: "node-1",
|
||||
}
|
||||
|
||||
// Write stopped instance
|
||||
err = jsm.WriteStoppedInstance(testVM.ID, testVM)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Load stopped instance
|
||||
loaded, err := jsm.LoadStoppedInstance(testVM.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, loaded)
|
||||
assert.Equal(t, testVM.ID, loaded.ID)
|
||||
assert.Equal(t, vm.StateStopped, loaded.Status)
|
||||
assert.Equal(t, "t3.micro", loaded.InstanceType)
|
||||
assert.Equal(t, "node-1", loaded.LastNode)
|
||||
|
||||
// Cleanup
|
||||
_ = jsm.DeleteStoppedInstance(testVM.ID)
|
||||
}
|
||||
|
||||
// TestJetStreamManager_LoadStoppedInstance_NotFound tests that LoadStoppedInstance returns nil for missing key
|
||||
func TestJetStreamManager_LoadStoppedInstance_NotFound(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
loaded, err := jsm.LoadStoppedInstance("i-nonexistent")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, loaded)
|
||||
}
|
||||
|
||||
// TestJetStreamManager_DeleteStoppedInstance tests deleting a stopped instance (including non-existent)
|
||||
func TestJetStreamManager_DeleteStoppedInstance(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
testVM := &vm.VM{
|
||||
ID: "i-stopped-del",
|
||||
Status: vm.StateStopped,
|
||||
}
|
||||
|
||||
// Write then delete
|
||||
err = jsm.WriteStoppedInstance(testVM.ID, testVM)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm.DeleteStoppedInstance(testVM.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify it's gone
|
||||
loaded, err := jsm.LoadStoppedInstance(testVM.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, loaded)
|
||||
|
||||
// Delete non-existent should not error
|
||||
err = jsm.DeleteStoppedInstance("i-does-not-exist")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestJetStreamManager_ListStoppedInstances tests listing multiple stopped instances
|
||||
func TestJetStreamManager_ListStoppedInstances(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write multiple stopped instances
|
||||
vms := []*vm.VM{
|
||||
{ID: "i-list-001", Status: vm.StateStopped, InstanceType: "t3.micro"},
|
||||
{ID: "i-list-002", Status: vm.StateStopped, InstanceType: "t3.small"},
|
||||
{ID: "i-list-003", Status: vm.StateStopped, InstanceType: "t3.medium"},
|
||||
}
|
||||
|
||||
for _, v := range vms {
|
||||
err = jsm.WriteStoppedInstance(v.ID, v)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// List stopped instances
|
||||
instances, err := jsm.ListStoppedInstances()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should contain at least the 3 we wrote (may also contain instances from other tests)
|
||||
instanceIDs := make(map[string]bool)
|
||||
for _, inst := range instances {
|
||||
instanceIDs[inst.ID] = true
|
||||
}
|
||||
assert.True(t, instanceIDs["i-list-001"])
|
||||
assert.True(t, instanceIDs["i-list-002"])
|
||||
assert.True(t, instanceIDs["i-list-003"])
|
||||
|
||||
// Cleanup
|
||||
for _, v := range vms {
|
||||
_ = jsm.DeleteStoppedInstance(v.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJetStreamManager_StoppedInstances_NoInterference tests that stopped instances don't interfere with per-node state
|
||||
func TestJetStreamManager_StoppedInstances_NoInterference(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write per-node state
|
||||
nodeInstances := &vm.Instances{
|
||||
VMS: map[string]*vm.VM{
|
||||
"i-running-001": {ID: "i-running-001", Status: vm.StateRunning},
|
||||
},
|
||||
}
|
||||
err = jsm.WriteState("interference-test-node", nodeInstances)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write stopped instance
|
||||
stoppedVM := &vm.VM{ID: "i-stopped-interf", Status: vm.StateStopped}
|
||||
err = jsm.WriteStoppedInstance(stoppedVM.ID, stoppedVM)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify per-node state is unaffected
|
||||
loaded, err := jsm.LoadState("interference-test-node")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, loaded.VMS, 1)
|
||||
assert.NotNil(t, loaded.VMS["i-running-001"])
|
||||
|
||||
// Verify stopped instance loads independently
|
||||
loadedStopped, err := jsm.LoadStoppedInstance(stoppedVM.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, loadedStopped)
|
||||
assert.Equal(t, "i-stopped-interf", loadedStopped.ID)
|
||||
|
||||
// Cleanup
|
||||
_ = jsm.DeleteStoppedInstance(stoppedVM.ID)
|
||||
_ = jsm.DeleteState("interference-test-node")
|
||||
}
|
||||
|
||||
// TestJetStreamManager_StoppedInstance_KVNotInitialized tests error handling when KV is not initialized
|
||||
func TestJetStreamManager_StoppedInstance_KVNotInitialized(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
// Don't call InitKVBucket
|
||||
|
||||
err = jsm.WriteStoppedInstance("i-test", &vm.VM{})
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = jsm.LoadStoppedInstance("i-test")
|
||||
assert.Error(t, err)
|
||||
|
||||
err = jsm.DeleteStoppedInstance("i-test")
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = jsm.ListStoppedInstances()
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
// Shared NATS servers for all daemon tests — started once in TestMain.
|
||||
var (
|
||||
sharedNATSServer *server.Server
|
||||
sharedNATSURL string
|
||||
sharedJSServer *server.Server
|
||||
sharedJSNATSURL string
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Start a plain NATS server (no JetStream)
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1",
|
||||
Port: -1,
|
||||
NoLog: true,
|
||||
NoSigs: true,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create shared NATS server: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
go ns.Start()
|
||||
if !ns.ReadyForConnections(5 * time.Second) {
|
||||
fmt.Fprintln(os.Stderr, "Shared NATS server failed to start")
|
||||
os.Exit(1)
|
||||
}
|
||||
sharedNATSServer = ns
|
||||
sharedNATSURL = ns.ClientURL()
|
||||
|
||||
// Start a JetStream-enabled NATS server
|
||||
jsTmpDir, err := os.MkdirTemp("", "nats-js-shared-*")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create JetStream temp dir: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
jsNS, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1",
|
||||
Port: -1,
|
||||
JetStream: true,
|
||||
StoreDir: jsTmpDir,
|
||||
NoLog: true,
|
||||
NoSigs: true,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create shared JetStream NATS server: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
go jsNS.Start()
|
||||
if !jsNS.ReadyForConnections(5 * time.Second) {
|
||||
fmt.Fprintln(os.Stderr, "Shared JetStream NATS server failed to start")
|
||||
os.Exit(1)
|
||||
}
|
||||
sharedJSServer = jsNS
|
||||
sharedJSNATSURL = jsNS.ClientURL()
|
||||
|
||||
code := m.Run()
|
||||
|
||||
ns.Shutdown()
|
||||
jsNS.Shutdown()
|
||||
os.RemoveAll(jsTmpDir)
|
||||
|
||||
os.Exit(code)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user