release: update go.mod for v0.2.1

Sharded WAL causes incomplete write errors in E2E tests. Disable by
default in hive.toml template, service.go fallback, and viperblock
constructor. Can be re-enabled via hive.toml shardwal=true.

.github/workflows/e2e.yml

@@ -0,0 +1,80 @@
+name: Hive E2E Test
+
+on:
+  push:
+    branches:
+      - main
+      - e2e
+    paths:
+      - "**/*.go"
+      - go.mod
+      - go.sum
+      - Makefile
+      - tests/e2e/**
+      - scripts/**
+      - .github/workflows/e2e.yml
+  workflow_dispatch:
+
+jobs:
+  build_e2e_image:
+    name: Build E2E Image
+    runs-on: self-hosted
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+        with:
+          path: hive
+
+      - name: Clone Dependencies
+        run: ./hive/scripts/clone-deps.sh
+
+      - name: Build E2E Image
+        run: |
+          docker build \
+            -t hive-e2e:latest \
+            -f hive/tests/e2e/Dockerfile.e2e \
+            .
+
+  hive_e2e_test:
+    name: Hive Single-Node E2E Test
+    runs-on: self-hosted
+    needs: build_e2e_image
+
+    steps:
+      - name: Run E2E Tests
+        run: |
+          docker run --privileged --rm \
+            -v /dev/kvm:/dev/kvm \
+            --name hive-e2e-test \
+            hive-e2e:latest
+
+  hive_multinode_e2e_test:
+    name: Hive Multi-Node E2E Test
+    runs-on: self-hosted
+    needs: build_e2e_image
+
+    steps:
+      - name: Run Multi-Node E2E Tests
+        run: |
+          docker run --privileged --rm \
+            -v /dev/kvm:/dev/kvm \
+            --cap-add=NET_ADMIN \
+            --name hive-multinode-e2e-test \
+            hive-e2e:latest \
+            ./tests/e2e/run-multinode-e2e.sh
+
+  cleanup:
+    name: Cleanup
+    runs-on: self-hosted
+    needs: [hive_e2e_test, hive_multinode_e2e_test]
+    if: always()
+
+    steps:
+      - name: Remove Stale Containers
+        run: |
+          docker rm -f hive-e2e-test 2>/dev/null || true
+          docker rm -f hive-multinode-e2e-test 2>/dev/null || true
+
+      - name: Prune Docker Images
+        run: docker image prune -f

.github/workflows/hive.yml

@@ -1,23 +1,103 @@
-name: Build and Test Hive
+name: Hive Build and Test
 
 on:
   push:
-    branches: ["main"]
-    paths: ["hive/**", ".github/workflows/**"]
+    branches:
+      - main
+      - dev
+    paths:
+      - "**/*.go"
+      - go.mod
+      - go.sum
+      - Makefile
+      - .github/workflows/hive.yml
+  pull_request:
+    paths:
+      - "**/*.go"
+      - go.mod
+      - go.sum
+      - Makefile
+      - .github/workflows/hive.yml
+  workflow_dispatch:
 
 jobs:
-  hive_test_and_push:
-    runs-on: ubuntu-latest
+  build_and_test:
+    name: Hive Build and Test
+    runs-on: ubuntu-24.04
 
     steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
+      - name: Checkout Hive
+        uses: actions/checkout@v6
+        with:
+          path: hive
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/viperblock
+          ref: dev
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/predastore
+          ref: dev
+          path: predastore
+
+      - name: Setup Go Workspace
+        run: go work init ./hive ./viperblock ./predastore
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
-          go-version-file: "go.mod"
-          cache-dependency-path: "go.sum"
+          cache-dependency-path: "hive/go.sum"
+          go-version-file: "hive/go.mod"
+
+      - name: Build Hive
+        run: make -C hive build
 
       - name: Run Tests
-        run: make test
+        run: make -C hive test
+
+  lint_and_security:
+    name: Hive Lint and Security
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout Hive
+        uses: actions/checkout@v6
+        with:
+          path: hive
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/viperblock
+          ref: dev
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/predastore
+          ref: dev
+          path: predastore
+
+      - name: Setup Go Workspace
+        run: go work init ./hive ./viperblock ./predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: "hive/go.sum"
+          go-version-file: "hive/go.mod"
+
+      - name: Check Format
+        run: make -C hive check-format
+
+      - name: Go Vet
+        run: make -C hive vet
+
+      - name: Security Checks
+        run: make -C hive security-check

.github/workflows/ui.yml

@@ -0,0 +1,50 @@
+name: UI Build and Test
+
+on:
+  push:
+    branches:
+      - main
+      - ui
+    paths:
+      - hive/services/hiveui/frontend/**
+      - .github/workflows/ui.yml
+  pull_request:
+    paths:
+      - hive/services/hiveui/frontend/**
+      - .github/workflows/ui.yml
+  workflow_dispatch:
+
+jobs:
+  ui_build_and_test:
+    name: UI Build and Test
+    runs-on: ubuntu-24.04
+    defaults:
+      run:
+        working-directory: hive/services/hiveui/frontend
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: hive/services/hiveui/frontend
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          package_json_file: hive/services/hiveui/frontend/package.json
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          cache: "pnpm"
+          cache-dependency-path: hive/services/hiveui/frontend/pnpm-lock.yaml
+          node-version-file: hive/services/hiveui/frontend/.nvmrc
+
+      - name: Install Dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run Biome Check
+        run: pnpm check
+
+      - name: Build Frontend
+        run: pnpm build

.gitignore

@@ -1,7 +1,41 @@
-.vscode
+# Terraform/OpenTofu
+.terraform/
+.terraform.lock.hcl
+*.tfstate
+*.tfstate.*
+*.tfvars
+!example.tfvars
+scripts/iac/proxmox/clusters/
+
+# Go workspace
 go.work
 go.work.sum
-bin/hive
+
+# Build outputs
+bin/
 tmp/
+*.test
+*.out
+
+# Config
 config/dev.yaml
+
+# OS files
 .DS_Store
+Thumbs.db
+
+# Environment
+.env
+
+# Test reports
+*-report.txt
+
+# Internal docs (moved to private mulga repo)
+docs/INTERNAL.md
+docs/PRIVATE.md
+docs/DEVNOTES.md
+docs/HIVE_DEVELOPMENT_PLAN.md
+docs/PLAN.md
+docs/TODO.md
+docs/development/
+docs/internal/

CLAUDE.md

@@ -1,214 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-This is the Hive platform - an AWS-compatible infrastructure stack for bare-metal, edge, and on-premise deployments. Hive orchestrates multiple independent components that work together to provide AWS-compatible services.
-
-## Component Repositories
-
-The Hive platform consists of these independent repositories:
-
-- **[viperblock](../viperblock/)** - High-performance block storage service (EBS alternative) with WAL-backed storage
-- **[predastore](../predastore/)** - S3-compatible object storage service with Reed-Solomon erasure coding
-- **nbdkit/** - Network Block Device (NBD) server for exporting block devices (upstream project)
-- **hive/** (this repo) - VM orchestration service and platform coordinator
-
-Each component can be used independently or as part of the integrated Hive stack. See component-specific `CLAUDE.md` files for detailed development guidance:
-- `../viperblock/CLAUDE.md` - Viperblock development guide
-- `../predastore/CLAUDE.md` - Predastore development guide
-
-## Cross-Repository Development Setup
-
-For platform development, clone all repositories to the same parent directory:
-```bash
-mkdir mulga
-cd mulga
-git clone <hive-repo-url> hive
-git clone <viperblock-repo-url> viperblock
-git clone <predastore-repo-url> predastore
-```
-
-The `hive/go.mod` uses local replace directives for cross-component development:
-```go
-replace github.com/mulgadc/viperblock => ../viperblock
-replace github.com/mulgadc/predastore => ../predastore
-```
-
-## Build Commands
-
-### Hive (Platform Orchestrator)
-```bash
-make build          # Build the hive binary
-make test           # Run Go tests
-make bench          # Run benchmarks
-make clean          # Clean build artifacts
-```
-
-### **MANDATORY: Unit Testing Requirements**
-
-**⚠️ CRITICAL: Always run unit tests before any commit or push operation.**
-
-```bash
-# REQUIRED before any git commit or push
-make test           # Must pass with zero failures
-
-# Example workflow:
-make test                    # Verify all tests pass
-git add .                    # Stage changes
-git commit -m "..."          # Only after tests pass
-git push origin main         # Only after tests pass
-```
-
-**Testing Policy:**
-- **All unit tests MUST pass** before committing changes
-- **No exceptions** - failing tests block commits
-- Tests must complete without errors or panics
-- Use `make test` command to run the full test suite
-- If tests fail, fix issues before proceeding with git operations
-
-### Component Build Commands
-For detailed build instructions, see component-specific CLAUDE.md files:
-
-**Viperblock** (`../viperblock/CLAUDE.md`):
-```bash
-cd ../viperblock && make build    # Build sfs, vblock binaries + NBD plugin
-```
-
-**Predastore** (`../predastore/CLAUDE.md`):
-```bash
-cd ../predastore && make build    # Build s3d binary
-```
-
-**NBDkit** (system dependency):
-```bash
-cd nbdkit && autoreconf -i && ./configure && make && make install
-```
-
-## Architecture Notes
-
-### Message-Driven Microservices Architecture
-
-Hive uses a **scalable message-driven architecture** that separates API handling from request processing:
-
-```
-AWS SDK (custom endpoint) → Hive API Gateway → NATS Topics → Specialized Daemons
-                                    ↑                              ↓
-                         AWS-compatible response ←  NATS Topics  ←  Processing Results
-```
-
-#### Core Components:
-
-1. **Hive API Gateway Service** - TLS endpoint that handles AWS SDK requests
-2. **Specialized Daemon Services** - Horizontally scalable workers for each AWS service type
-3. **NATS Message Broker** - Topic-based routing with queue groups for load balancing
-
-#### Current Implementation (daemon.go):
-- **Gateway Logic**: `StartAWSGateway()` handles AWS API calls on port 9999
-- **NATS Integration**: Uses request-response pattern: `d.natsConn.Request("ec2.runinstances", data, 30s)`
-- **Queue Groups**: Load balancing via `"hive-workers"` queue group
-- **Service Coordination**: All inter-service communication via NATS topics
-
-### Inter-component Dependencies
-- **Service Startup Order**: NATS → Predastore → Viperblock → NBDkit → Gateway → Daemons
-- **Message Flow**: Gateway translates AWS calls to NATS messages, daemons process and respond
-- **Scaling**: Multiple daemon instances can subscribe to same topics for horizontal scaling
-- All Go projects use local module replacements for development
-
-### Storage Backend Integration
-- **Viperblock Integration**: EBS volumes mounted via NATS messaging (`ebs.mount` topic)
-- **Predastore Integration**: S3 operations proxied through dedicated S3 daemons
-- **NBDkit Integration**: Block device access for VM storage
-- **Volume Lifecycle**: Coordinated through NATS between EC2 and EBS daemons
-
-### Key Design Patterns
-- **Message-Driven**: All service communication via NATS topics (ec2.*, ebs.*, s3.*, vpc.*)
-- **Queue Groups**: Load balancing and fault tolerance via NATS queue groups
-- **Request-Response**: 30-second timeout pattern for AWS API compatibility
-- **Service Specialization**: Dedicated daemon types for each AWS service
-- **Horizontal Scaling**: Multiple daemon instances handle same topic types
-
-### NATS Topic Structure
-
-The system uses structured NATS topics for service communication:
-
-```bash
-# EC2 Service Topics:
-ec2.runinstances              # Launch new instances
-ec2.describeinstances         # Query instance status
-ec2.startinstances           # Start stopped instances
-ec2.stopinstances            # Stop running instances
-ec2.terminateinstances       # Terminate instances
-
-# EBS Service Topics:
-ebs.createvolume             # Create new EBS volume
-ebs.attachvolume             # Attach volume to instance
-ebs.detachvolume             # Detach volume
-ebs.describevolumes          # List volumes
-ebs.mount                    # Mount volume (internal)
-ebs.unmount                  # Unmount volume (internal)
-
-# S3 Service Topics:
-s3.createbucket              # Create S3 bucket
-s3.putobject                 # Store object
-s3.getobject                 # Retrieve object
-s3.listbuckets               # List buckets
-
-# VPC Service Topics:
-vpc.createvpc                # Create VPC
-vpc.createsubnet             # Create subnet
-vpc.describevpcs             # List VPCs
-
-# Health and Discovery:
-health.ec2.daemon.{id}       # Daemon health heartbeats
-discovery.services           # Service registration
-```
-
-### Go Module Structure
-All Go projects use:
-- Go 1.23+ with module mode
-- Local replace directives for cross-component development
-- Standard build flags: `-ldflags "-s -w"` for optimized binaries
-- Test environment variable: `LOG_IGNORE=1` to suppress logs during testing
-
-### Development Workflow
-
-#### Quick Setup (Recommended)
-```bash
-# 1. Clone dependencies (if not already done)
-./scripts/clone-deps.sh
-
-# 2. Setup development environment
-./scripts/dev-setup.sh
-
-# 3. Start all services
-./scripts/start-dev.sh
-
-# 4. Stop all services when done
-./scripts/stop-dev.sh
-```
-
-#### Manual Development Process
-1. **Cross-repo Setup**: Clone all repositories to same parent directory
-2. **Service Dependencies**: Start services in order: NATS → Predastore → Viperblock → NBDkit → Gateway → Daemons
-3. **Build Order**: Build components: predastore → viperblock → hive
-4. **Testing**: Use `make test` in each component to verify changes
-5. **Message-Driven Development**: Test NATS message flows between gateway and daemons
-6. **Scaling**: Test with multiple daemon instances subscribing to same topics
-
-#### Development Tools
-- **Hot Reloading**: Use `air` for automatic restarts during development
-- **TLS Certificates**: Auto-generated self-signed certificates for HTTPS endpoints
-- **Log Monitoring**: Centralized logs in `data/logs/` directory
-- **AWS CLI Testing**: Test endpoints with `aws --endpoint-url https://localhost:9999`
-
-### Development Roadmap
-For major feature development and architectural changes, refer to `HIVE_DEVELOPMENT_PLAN.md` which contains:
-- Comprehensive roadmap for transforming Hive from proof-of-concept to beta release
-- Detailed implementation phases with specific tasks and file structures
-- AWS API compatibility implementation guidelines
-- Development environment automation plans
-- Testing and validation frameworks
-
-When implementing new AWS services or enhancing existing functionality, follow the structured approach outlined in the development plan.

DESIGN.md

@@ -1,74 +0,0 @@
-# Design ideas
-
-### hivectl
-
-Basic commands to setup a compute instance, block storage (viperblock), object storage (predastore)
-
-`hivectl configure --endpoint-url https://localhost:8443/`
-
-`hivectl auth --access_key X --secret_key Y --profile hivelocal --endpoint-url https://localhost:8443/`
-
-Writes to ~/.aws/credentials
-
-`hivectl vol add --size 128M --no-verify-ssl --endpoint-url https://localhost:8443/ --`
-
-Authenticates using ~/.aws/credentials - Backend creates a unique volume id
-
-Connects to S3, creates the /vol-uniquestr/, uploads `config.json`
-
-```
-{
-  "VolumeName": "vol-uniquestr",
-  "VolumeSize": 134217728,
-  "BlockSize": 4096,
-  "ObjBlockSize": 4194304,
-  "SeqNum": 0,
-  "ObjectNum": 0,
-  "WALNum": 0,
-  "BlockToObjectWALNum": 0,
-  "Version": 1
-}
-```
-
-`hivectl ec2 import-key-pair --key-name my-key-pair --public-key-material fileb://~/.ssh/id_rsa.pub`
-
-This will upload the users SSH public key to their S3 bucket, /bucket/ec2/my-key-pair.pub
-
-`hivectl ec2 run-instances --image-id ami-X --instance-type t3.micro --key-name my-key-pair`
-
-This will clone an existing image-id (vol-ID, e.g debian snapshot), copy to the users S3 bucket, and start the new volume from the fresh copy.
-
-Note when starting an image additional volumes are required.
-
-```
-:~/qemu-host$ truncate -s 64m varstore.img
-:~/qemu-host$ truncate -s 64m efi.img
-:~/qemu-host$ dd if=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd of=efi.img conv=notrunc
-```
-
-Sample launch
-
-Require to start NBD (EFI)
-
-./server/nbdkit -p 10810 --pidfile /tmp/vb-vol-1-efi.pid ./plugins/golang/examples/ramdisk/nbdkit-goramdisk-plugin.so -v -f size=67108864 volume=vol-1-efi bucket=predastore region=ap-southeast-2 access_key="AKIAIOSFODNN7EXAMPLE" secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" base_dir="/tmp/vb" host="https://127.0.0.1:8443"
-
-Require to start Debian volume
-
-./server/nbdkit -p 10809 --pidfile /tmp/vb-vol-1-root.pid ./plugins/golang/examples/ramdisk/nbdkit-goramdisk-plugin.so -v -f size=4294967296 volume=vol-1-root bucket=predastore region=ap-southeast-2 access_key="AKIAIOSFODNN7EXAMPLE" secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" base_dir="/tmp/vb" host="https://127.0.0.1:8443"
-
-```
-:~/qemu-host$ ~/qemu/build/aarch64-softmmu/qemu-system-aarch64 -M virt  \
-      -machine virtualization=true -machine virt,gic-version=3  \
-      -cpu max,pauth-impdef=on -smp 2 -m 4096           \
-      -drive if=pflash,format=raw,file=efi.img,readonly=on      \
-      -drive if=pflash,format=raw,file=varstore.img         \
-      -drive if=virtio,format=qcow2,file=disk.img           \
-      -device virtio-scsi-pci,id=scsi0              \
-      -object rng-random,filename=/dev/urandom,id=rng0      \
-      -device virtio-rng-pci,rng=rng0               \
-      -device virtio-net-pci,netdev=net0                \
-      -netdev user,id=net0,hostfwd=tcp::8022-:22            \
-      -nographic                            \
-      -drive if=none,id=cd,file=debian-12.2.0-arm64-netinst.iso \
-      -device scsi-cd,drive=cd
-```

Makefile

@@ -1,23 +1,67 @@
 GO_PROJECT_NAME := hive
+SHELL := /bin/bash
+
+# Detect architecture for cross-platform support
+ARCH := $(shell uname -m)
+ifeq ($(ARCH),x86_64)
+  GO_ARCH := amd64
+  AWS_ARCH := x86_64
+  # On x86, we can run ARM VMs via emulation
+  QEMU_PACKAGES := qemu-system-x86 qemu-system-arm
+else ifeq ($(ARCH),aarch64)
+  GO_ARCH := arm64
+  AWS_ARCH := aarch64
+  # On ARM, we can run x86 VMs via emulation
+  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
+else ifeq ($(ARCH),arm64)
+  GO_ARCH := arm64
+  AWS_ARCH := aarch64
+  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
+else
+  $(error Unsupported architecture: $(ARCH). Only x86_64 and aarch64/arm64 are supported.)
+endif
+
+# Ask Go whether workspace mode is active
+IN_WORKSPACE := $(shell go env GOWORK)
+
+# Use -mod=mod unless Go reports an active workspace path
+ifeq ($(IN_WORKSPACE),)
+  GO_BUILD_MOD := -mod=mod
+else ifeq ($(IN_WORKSPACE),off)
+  GO_BUILD_MOD := -mod=mod
+else
+  GO_BUILD_MOD :=
+endif
 
 build:
 	$(MAKE) go_build
 
+# Build hive-ui frontend (requires pnpm)
+build-ui:
+	@echo -e "\n....Building hive-ui frontend...."
+	cd hive/services/hiveui/frontend && pnpm build
+
 # GO commands
 go_build:
-	@echo "\n....Building $(GO_PROJECT_NAME)"
-	go build -ldflags "-s -w" -o ./bin/$(GO_PROJECT_NAME) cmd/hive/main.go
+	@echo -e "\n....Building $(GO_PROJECT_NAME)"
+	go build $(GO_BUILD_MOD) -ldflags "-s -w" -o ./bin/$(GO_PROJECT_NAME) cmd/hive/main.go
 
 go_run:
-	@echo "\n....Running $(GO_PROJECT_NAME)...."
+	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
 	$(GOPATH)/bin/$(GO_PROJECT_NAME)
 
+# Preflight — runs the same checks as GitHub Actions (format + lint + security + tests).
+# Use this before committing to catch CI failures locally.
+preflight: check-format vet security-check test
+	@echo -e "\n ✅ Preflight passed — safe to commit."
+
+# Run unit tests
 test:
-	@echo "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -v -timeout 300s ./...
+	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
+	LOG_IGNORE=1 go test -v -timeout 120s ./...
 
 bench:
-	@echo "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
+	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	$(MAKE) easyjson
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
 
@@ -26,6 +70,108 @@ run:
 	$(MAKE) go_run
 
 clean:
-	rm ./bin/$(GO_PROJECT_NAME)
+	rm -f ./bin/$(GO_PROJECT_NAME)
+	rm -rf hive/services/hiveui/frontend/dist
 
-.PHONY: go_build go_run build run test clean
+install-system:
+	@echo -e "\n....Installing system dependencies for $(ARCH)...."
+	@echo "QEMU packages: $(QEMU_PACKAGES)"
+	apt-get update && sudo apt-get install -y \
+		nbdkit nbdkit-plugin-dev pkg-config $(QEMU_PACKAGES) qemu-utils qemu-kvm \
+		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
+		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file
+
+install-go:
+	@echo -e "\n....Installing Go 1.26.0 for $(ARCH) ($(GO_ARCH))...."
+	@if [ ! -d "/usr/local/go" ]; then \
+		curl -L https://go.dev/dl/go1.26.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+	else \
+		echo "Go already installed in /usr/local/go"; \
+	fi
+	@echo "Go version: $$(go version)"
+
+install-aws:
+	@echo -e "\n....Installing AWS CLI v2 for $(ARCH) ($(AWS_ARCH))...."
+	@if ! command -v aws >/dev/null 2>&1; then \
+		curl "https://awscli.amazonaws.com/awscli-exe-linux-$(AWS_ARCH).zip" -o "awscliv2.zip"; \
+		unzip -o awscliv2.zip; \
+		./aws/install; \
+		rm -rf awscliv2.zip aws/; \
+	else \
+		echo "AWS CLI already installed"; \
+	fi
+
+quickinstall: install-system install-go install-aws
+	@echo -e "\n✅ Quickinstall complete for $(ARCH)."
+	@echo "   Please ensure /usr/local/go/bin is in your PATH."
+	@echo "   Installed: Go ($(GO_ARCH)), AWS CLI ($(AWS_ARCH)), QEMU ($(QEMU_PACKAGES))"
+
+# Format all Go files in place
+format:
+	gofmt -w .
+
+# Check that all Go files are formatted (CI-compatible, fails on diff)
+check-format:
+	@echo "Checking gofmt..."
+	@UNFORMATTED=$$(gofmt -l .); \
+	if [ -n "$$UNFORMATTED" ]; then \
+		echo "Files not formatted:"; \
+		echo "$$UNFORMATTED"; \
+		echo "Run 'make format' to fix."; \
+		exit 1; \
+	fi
+	@echo "  gofmt ok"
+
+# Go vet (fails on issues, matches CI)
+vet:
+	@echo "Running go vet..."
+	go vet ./...
+	@echo "  go vet ok"
+
+# Security checks — each tool fails the build on findings (matches CI).
+# Reports are also saved to tests/ for review.
+security-check:
+	@echo -e "\n....Running security checks for $(GO_PROJECT_NAME)...."
+	set -o pipefail && go tool govulncheck ./... 2>&1 | tee tests/govulncheck-report.txt
+	@echo "  govulncheck ok"
+	set -o pipefail && go tool gosec -exclude=G204,G304,G402 -exclude-generated -exclude-dir=cmd ./... 2>&1 | tee tests/gosec-report.txt
+	@echo "  gosec ok"
+	set -o pipefail && go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./... 2>&1 | tee tests/staticcheck-report.txt
+	@echo "  staticcheck ok"
+
+# Docker E2E tests (mirrors GitHub Actions e2e.yml)
+# Usage: make test-docker              # both suites
+#        make test-docker-single       # single-node only
+#        make test-docker-multi        # multi-node only
+PARENT_DIR := $(shell cd .. && pwd)
+E2E_IMAGE := hive-e2e:latest
+
+test-docker-build:
+	@echo -e "\n....Building E2E Docker image...."
+	@for dep in viperblock predastore; do \
+		if [ ! -d "$(PARENT_DIR)/$$dep" ]; then \
+			echo "Missing sibling repo $$dep — running clone-deps.sh"; \
+			./scripts/clone-deps.sh; \
+			break; \
+		fi; \
+	done
+	docker build -t $(E2E_IMAGE) -f tests/e2e/Dockerfile.e2e $(PARENT_DIR)
+
+test-docker-single: test-docker-build
+	@echo -e "\n....Running Single-Node E2E Tests...."
+	docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
+
+test-docker-multi: test-docker-build
+	@echo -e "\n....Running Multi-Node E2E Tests...."
+	docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
+
+test-docker: test-docker-build
+	@echo -e "\n....Running Single-Node E2E Tests...."
+	docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
+	@echo -e "\n....Running Multi-Node E2E Tests...."
+	docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
+
+.PHONY: build build-ui go_build go_run preflight test bench run clean \
+	install-system install-go install-aws quickinstall \
+	format check-format vet security-check \
+	test-docker-build test-docker-single test-docker-multi test-docker

README.md

@@ -60,7 +60,7 @@ Get started with Hive development in minutes:
 
 ```bash
 # Clone the repository
-git clone https://github.com/mulgadc/hive.git hive
+git clone https://github.com/mulgadc/hive.git
 cd hive
 
 # Setup dependencies and development environment
@@ -79,7 +79,7 @@ Hive coordinates these independent components:
 
 Each component can be developed independently. See component-specific documentation for focused development guides.
 
-## Detailed Installation method
+## Detailed Installation Method
 
 See [Installation Guide](INSTALL.md) on how to setup a Hive node to get started.
 
@@ -118,7 +118,7 @@ While Hive is architected and implemented by experienced engineers, we leverage
 
 **Development Artifacts:**
 
-- `CLAUDE.md` - Instructions for Claude Code when working on this codebase
+- `.claude/CLAUDE.md` - Instructions for Claude Code when working on this codebase
 - `HIVE_DEVELOPMENT_PLAN.md` - Comprehensive roadmap and implementation strategy
 - Component-specific guidance in `viperblock/CLAUDE.md` and `predastore/CLAUDE.md`
 
@@ -126,4 +126,4 @@ This hybrid approach ensures Hive benefits from both proven engineering expertis
 
 ## License
 
-Hive is open source under the Apache 2.0 License. You're free to use, modify, and deploy it—anywhere you need reliable infrastructure without depending on centralized cloud platforms
+Hive is open source under the Apache 2.0 License. You're free to use, modify, and deploy it—anywhere you need reliable infrastructure without depending on centralized cloud platforms.

TESTING_COMPLETE.md

@@ -1,509 +0,0 @@
-# Testing Implementation Summary
-
-## ✅ Completed Work
-
-### 1. Daemon Test Suite (`hive/daemon/daemon_test.go`)
-
-Created comprehensive unit tests for the daemon's EC2 launch functionality:
-
-**Test Components:**
-- `startTestNATSServer()` - Embedded NATS server with auto port allocation
-- `createTestDaemon()` - Test daemon factory with automatic cleanup
-- `createValidRunInstancesInput()` - Test fixture for valid EC2 requests
-
-**Test Suites:**
-- ✅ `TestHandleEC2Launch_MessageParsing` - NATS message handling (5 test cases)
-- ⏸️ `TestHandleEC2Launch_ResourceManagement` - Skipped (requires full infrastructure)
-- ✅ `TestDaemon_Initialization` - Daemon creation and configuration
-- ✅ `TestResourceManager` - Resource allocation logic
-
-**Test Results:**
-```bash
-$ go test -v ./hive/daemon/
-PASS: TestHandleEC2Launch_MessageParsing (0.17s)
-  ✓ Valid_RunInstancesInput
-  ✓ Invalid_Instance_Type
-  ✓ Missing_Required_ImageId
-  ✓ Invalid_MinCount_(zero)
-  ✓ Malformed_JSON
-SKIP: TestHandleEC2Launch_ResourceManagement
-PASS: TestDaemon_Initialization (0.00s)
-PASS: TestResourceManager (0.00s)
-ok      github.com/mulgadc/hive/hive/daemon     0.592s
-```
-
-### 2. Utils Test Suite (`hive/utils/utils_test.go`)
-
-Added comprehensive tests for previously untested utility functions:
-
-**New Tests:**
-- ✅ `TestUnmarshalJsonPayload` - JSON unmarshaling (4 test cases)
-- ✅ `TestMarshalJsonPayload` - JSON marshaling (2 test cases)
-- ✅ `TestGenerateErrorPayload` - Error generation (3 test cases)
-- ✅ `TestValidateErrorPayload` - Error validation (4 test cases)
-- ✅ `TestMarshalToXML` - XML marshaling (3 test cases)
-- ✅ `TestKillProcess` - Process termination
-- ✅ `TestStopProcess` - Service stop with PID cleanup
-
-**Test Results:**
-```bash
-$ go test -v ./hive/utils/
-PASS: TestGeneratePidFile (0.00s)
-PASS: TestGenerateSocketFile (0.00s)
-PASS: TestExecProcessAndKill (1.71s)
-PASS: TestUnmarshalJsonPayload (0.00s)
-PASS: TestMarshalJsonPayload (0.00s)
-PASS: TestGenerateErrorPayload (0.00s)
-PASS: TestValidateErrorPayload (0.00s)
-PASS: TestMarshalToXML (0.00s)
-PASS: TestKillProcess (4.51s)
-PASS: TestStopProcess (4.01s)
-ok      github.com/mulgadc/hive/hive/utils     10.844s
-```
-
-### 3. Bug Fixes
-
-Fixed critical bugs that prevented tests from passing:
-
-#### Bug 1: Pointer-to-Pointer in UnmarshalJsonPayload
-**Location:** `hive/utils/utils.go:283`
-
-**Before:**
-```go
-err := decoder.Decode(&input)  // input already a pointer, creates **Type
-```
-
-**After:**
-```go
-err := decoder.Decode(input)  // Correct: input is already a pointer
-```
-
-#### Bug 2: Nil Pointer in handleEC2Launch
-**Location:** `hive/daemon/daemon.go:638-645`
-
-**Before:**
-```go
-var runInstancesInput *ec2.RunInstancesInput  // nil pointer
-errResp = utils.UnmarshalJsonPayload(runInstancesInput, msg.Data)  // crashes
-```
-
-**After:**
-```go
-runInstancesInput := &ec2.RunInstancesInput{}  // Initialize before use
-errResp = utils.UnmarshalJsonPayload(runInstancesInput, msg.Data)
-```
-
-#### Bug 3: Uninitialized Slice Elements in launchEC2Instance
-**Location:** `hive/daemon/daemon.go:852-856`
-
-**Before:**
-```go
-reservation.Instances = make([]*ec2.Instance, 1)
-reservation.Instances[0].SetInstanceId(instance.ID)  // nil pointer dereference
-```
-
-**After:**
-```go
-reservation.Instances = make([]*ec2.Instance, 1)
-reservation.Instances[0] = &ec2.Instance{
-    State: &ec2.InstanceState{},
-}
-reservation.Instances[0].SetInstanceId(instance.ID)
-```
-
-#### Bug 4: Same Issue in MarshalJsonPayload
-**Location:** `hive/utils/utils.go:299`
-
-**Before:**
-```go
-err := decoder.Decode(&input)
-```
-
-**After:**
-```go
-err := decoder.Decode(input)
-```
-
-### 4. Documentation
-
-Created comprehensive testing documentation:
-
-**Files Created:**
-1. `hive/daemon/TESTING.md` (423 lines)
-   - Test structure and components
-   - macOS limitations and workarounds
-   - Test execution instructions
-   - Mock service strategies
-   - Debugging guide
-
-2. `hive/daemon/TEST_SUMMARY.md` (467 lines)
-   - Test architecture highlights
-   - Known issues and limitations
-   - Recommendations for different platforms
-   - Next steps and roadmap
-   - Current test results
-
-3. `TESTING_COMPLETE.md` (this file)
-   - Summary of all completed work
-   - Bug fixes
-   - Test results
-   - Future enhancements
-
-## 📊 Test Coverage
-
-### Overall Test Results
-
-```bash
-$ make test
-
-✓ hive/awsec2query        - All tests passing
-✓ hive/awserrors          - All tests passing
-✓ hive/daemon             - 3 passing, 1 skipped (infrastructure required)
-✓ hive/gateway/ec2/instance - All tests passing
-✓ hive/service            - All tests passing
-✓ hive/services/predastore - All tests passing (integration skipped)
-✓ hive/services/viperblockd - All tests passing
-✓ hive/utils              - All tests passing (10 tests, 26 cases)
-✓ hive/vm                 - All tests passing
-
-OVERALL: PASS
-```
-
-### Code Coverage by Package
-
-| Package | Tests | Coverage | Notes |
-|---------|-------|----------|-------|
-| daemon | 4 | ~60% | Message handling, resource mgmt, initialization |
-| utils | 10 | ~85% | All major functions tested |
-| gateway/ec2/instance | 2 | ~75% | RunInstances validation |
-| vm | 2 | ~40% | Config generation, ID creation |
-| services/* | 8 | ~70% | Service lifecycle, NATS integration |
-
-## 🎯 Test Framework Features
-
-### Daemon Tests
-
-**Strengths:**
-- ✅ Isolated NATS server per test
-- ✅ Automatic resource cleanup
-- ✅ Table-driven test design
-- ✅ Comprehensive error scenarios
-- ✅ AWS SDK integration testing
-
-**Limitations:**
-- ⚠️ Requires mocks for viperblock/nbdkit on macOS
-- ⚠️ Full VM launch needs Linux + infrastructure
-- ⚠️ Cloud-init volume creation not tested
-
-### Utils Tests
-
-**Strengths:**
-- ✅ Comprehensive coverage of JSON/XML marshaling
-- ✅ Process management testing
-- ✅ Error handling validation
-- ✅ Platform-aware (macOS vs Linux)
-
-**Coverage:**
-- ✅ PID file operations
-- ✅ Socket file generation
-- ✅ Process lifecycle (start/stop/kill)
-- ✅ JSON payload marshaling/unmarshaling
-- ✅ XML marshaling
-- ✅ Error payload generation/validation
-
-## 🐛 Bugs Fixed
-
-### Critical (Blocking Tests)
-
-1. **Pointer-to-pointer bugs** (2 instances)
-   - Impact: Caused all JSON unmarshaling to fail
-   - Fix: Remove unnecessary `&` operator
-   - Files: `utils.go:283, 299`
-
-2. **Nil pointer dereference in handleEC2Launch**
-   - Impact: All EC2 launch requests crashed
-   - Fix: Initialize struct before passing to unmarshal
-   - File: `daemon.go:638`
-
-3. **Uninitialized slice elements**
-   - Impact: Crash when setting instance fields
-   - Fix: Initialize slice elements before use
-   - File: `daemon.go:852-856`
-
-### Total Lines Changed
-- **Modified:** ~15 lines across 3 files
-- **Added:** ~550 lines of tests
-- **Documentation:** ~1,200 lines
-
-## 📝 Test Scenarios Covered
-
-### Daemon Tests
-
-**Message Parsing:**
-- ✅ Valid RunInstancesInput with all fields
-- ✅ Invalid instance type rejection
-- ✅ Missing required ImageId
-- ✅ Invalid MinCount (zero)
-- ✅ Malformed JSON handling
-
-**Resource Management:**
-- ✅ Instance type lookup
-- ✅ Resource allocation/deallocation
-- ✅ Capacity checking
-- ✅ System resource detection
-
-### Utils Tests
-
-**JSON Operations:**
-- ✅ Valid JSON unmarshaling
-- ✅ Malformed JSON error handling
-- ✅ Unknown field rejection (DisallowUnknownFields)
-- ✅ Empty JSON handling
-
-**Error Handling:**
-- ✅ Error payload generation
-- ✅ Error payload validation
-- ✅ Success vs error discrimination
-
-**XML Operations:**
-- ✅ Struct marshaling
-- ✅ Pointer marshaling
-- ✅ Invalid type error handling
-
-**Process Management:**
-- ✅ Process creation and termination
-- ✅ PID file read/write/remove
-- ✅ Signal handling
-- ✅ Non-existent process error handling
-
-## 🚀 Running Tests
-
-### Quick Test
-
-```bash
-# Run all tests
-make test
-
-# Run specific package
-go test -v ./hive/daemon/
-go test -v ./hive/utils/
-
-# Run with race detection
-go test -race ./...
-
-# Run with coverage
-go test -cover ./...
-```
-
-### Platform-Specific
-
-**macOS (Unit Tests Only):**
-```bash
-# All passing unit tests
-go test -v ./hive/daemon/
-go test -v ./hive/utils/
-```
-
-**Linux (Full Integration):**
-```bash
-# Requires: predastore, viperblock, nbdkit, QEMU
-# Remove SKIP_INTEGRATION flag
-unset SKIP_INTEGRATION
-go test -v ./hive/daemon/
-```
-
-**Docker (Recommended for Full Tests):**
-```bash
-docker-compose up test-env
-docker exec -it hive-test go test -v ./...
-```
-
-## 🔍 What's Tested vs Not Tested
-
-### ✅ Fully Tested
-
-- NATS message parsing and routing
-- JSON marshaling/unmarshaling
-- Error generation and validation
-- Resource manager logic
-- PID file operations
-- Process lifecycle management
-- XML marshaling
-- Input validation
-- Instance type lookup
-
-### ⏸️ Partially Tested (Mocks Needed for macOS)
-
-- EC2 launch flow (stops at viperblock creation)
-- Volume mounting (requires nbdkit)
-- Cloud-init generation (requires S3 backend)
-
-### ❌ Not Yet Tested (Requires Full Infrastructure)
-
-- Complete VM launch
-- QEMU process management
-- QMP socket communication
-- NBD volume attachment
-- EBS volume lifecycle
-- VM state transitions
-- Concurrent instance launches
-- Resource exhaustion scenarios
-
-## 🎯 Future Enhancements
-
-### Short Term (Next Sprint)
-
-1. **Mock Services**
-   ```go
-   // Mock EBS mount service for macOS
-   func startMockEBSService(t *testing.T, nc *nats.Conn) {
-       nc.Subscribe("ebs.mount", func(msg *nats.Msg) {
-           // Return mock NBD URI
-       })
-   }
-   ```
-
-2. **Integration Tests**
-   - Add `//go:build linux` tags
-   - Full VM launch tests
-   - Volume lifecycle tests
-
-3. **Benchmark Tests**
-   ```go
-   func BenchmarkEC2Launch(b *testing.B) {
-       // Measure launch performance
-   }
-   ```
-
-### Medium Term
-
-1. **CI/CD Integration**
-   - GitHub Actions workflow
-   - Docker-based test environment
-   - Coverage reporting (codecov.io)
-
-2. **Load Testing**
-   - Concurrent launch stress tests
-   - Resource exhaustion tests
-   - NATS throughput tests
-
-3. **Mock Framework**
-   - Reusable mock services
-   - Test helpers package
-   - Platform-aware test skipping
-
-### Long Term
-
-1. **E2E Test Suite**
-   - Complete AWS SDK compatibility tests
-   - Multi-instance orchestration
-   - Failure recovery scenarios
-
-2. **Performance Suite**
-   - Latency measurements
-   - Throughput benchmarks
-   - Resource utilization profiling
-
-3. **Chaos Engineering**
-   - Network partition tests
-   - Service failure simulation
-   - Recovery time validation
-
-## 📚 Documentation Structure
-
-```
-hive/
-├── hive/daemon/
-│   ├── daemon.go                 [FIXED: 3 bugs]
-│   ├── daemon_test.go            [NEW: 4 test suites]
-│   ├── TESTING.md                [NEW: 423 lines]
-│   └── TEST_SUMMARY.md           [NEW: 467 lines]
-├── hive/utils/
-│   ├── utils.go                  [FIXED: 2 bugs]
-│   └── utils_test.go             [ENHANCED: +10 tests]
-└── TESTING_COMPLETE.md           [NEW: This file]
-```
-
-## 🎉 Success Metrics
-
-### Before
-- ❌ daemon tests: 0 tests, crashes on startup
-- ❌ utils tests: 3 tests, missing coverage for 13+ functions
-- ❌ Critical bugs: 4 blocking issues
-- ❌ Documentation: None
-
-### After
-- ✅ daemon tests: 4 test suites, 5 test cases passing
-- ✅ utils tests: 10 test suites, 26 test cases passing
-- ✅ Critical bugs: All fixed
-- ✅ Documentation: 1,200+ lines
-- ✅ Total test cases: 50+ across all packages
-- ✅ Overall: `make test` passes 100%
-
-## 💡 Key Learnings
-
-### Test Design
-
-1. **Embedded NATS server is ideal for testing**
-   - No external dependencies
-   - Port auto-allocation prevents conflicts
-   - Fast startup/teardown
-
-2. **Table-driven tests scale well**
-   - Easy to add new cases
-   - Clear test structure
-   - Minimal code duplication
-
-3. **Platform differences matter**
-   - macOS vs Linux signal handling
-   - Process cleanup timing
-   - File system behavior
-
-### Bug Patterns
-
-1. **Pointer confusion is common**
-   - Double-check `&` usage
-   - Verify struct initialization
-   - Test nil pointer paths
-
-2. **AWS SDK requires explicit initialization**
-   - Can't use SetX() on nil pointers
-   - Must initialize nested structs
-   - Slice make() != slice element initialization
-
-3. **JSON unmarshaling needs initialized target**
-   - Can't unmarshal into nil pointer
-   - DisallowUnknownFields is strict
-   - Empty {} is valid but may be treated as error
-
-## 🔗 Related Files
-
-- `CLAUDE.md` - Project overview and development guide
-- `HIVE_DEVELOPMENT_PLAN.md` - Roadmap and feature planning
-- `hive/daemon/TESTING.md` - Detailed testing guide
-- `hive/daemon/TEST_SUMMARY.md` - Test architecture and status
-- `hive/utils/utils_test.go` - Utils test implementation
-- `hive/daemon/daemon_test.go` - Daemon test implementation
-
-## ✅ Acceptance Criteria Met
-
-- [x] Created `daemon_test.go` with NATS server setup
-- [x] Implemented `handleEC2Launch` message tests
-- [x] Created sample RunInstancesInput test cases
-- [x] Validated expected arguments and error handling
-- [x] Documented macOS limitations (viperblock, nbdkit)
-- [x] Provided suggestions for scaffolding
-- [x] Fixed existing bugs (pointer-to-pointer issues)
-- [x] Created comprehensive utils tests
-- [x] All tests passing with `make test`
-- [x] Detailed documentation created
-
-## 🏁 Conclusion
-
-The test framework is now production-ready for unit testing on both macOS and Linux. The daemon EC2 launch functionality has comprehensive message handling tests, and the utils package is thoroughly tested. All critical bugs have been fixed, and the codebase is ready for further development.
-
-**Next Steps:**
-1. Implement mock services for full macOS testing
-2. Add Linux-specific integration tests
-3. Set up CI/CD pipeline with Docker
-4. Expand coverage to volume lifecycle operations

TODO.md

@@ -1,88 +0,0 @@
-# Hive Development Roadmap
-
-This roadmap has been integrated into the comprehensive [HIVE_DEVELOPMENT_PLAN.md](HIVE_DEVELOPMENT_PLAN.md).
-
-## Update Nov 2025
-
-* Implement multi-tenant support
-* Move config settings from ~/hive/*.toml, to using Nats Jetstream for core config which can be synced between nodes.
-* Implement a lightweight IAM using Nats Jetstream, vs current config files for auth settings.
-* Implement Reed Solomon Encoding for Predastore (S3) objects and a WAL implementation, for storing multiple objects in a single shard (e.g 4MB), with the WAL referencing the location of each object (e.g 4kb min)
-    * Implement basic KY lookup, object key sha512 (bucket/key), location to shard on S3 (obj.0000124.bin), read WAL (e.g first 4096 bytes) to determine location of the object. e.g key-1234 => obj.0000124.bin, wal header (4096 bytes) > key-1234 == offset location (seek) 16384, len 32768.
-    * Read multiple nodes (e.g 5 predastore instances, k = 3 (data shards), n = 5 (total shards), n - k = 2 (parity shards) )
-* Complete core scaffolding AWS SDK/API requirements (ec2 describe-instances, run-instances, etc)
-* Implement UEFI support for image downloads and `qemu` exec in `vm.go`
-* Confirm Alpine Linux, fails import image AMI > (run-instance) ""Failed to read block from AMI source" err="request out of range" - Block size correct?
-* Improve shutdown gracefully, `./scripts/stop-dev.sh` waits 60 seconds, while qemu/nbd could still be shutting down.
-* Add delete-volume support via EBS (s3 vol-*) for terminated instance
-* Add default LRU cache support for viperblock, depending on the instance type / volume size and system memory available.
-
-## Original TODO Items → Development Plan Integration
-
-### ✅ **Completed Integration**
-
-All original TODO items have been incorporated into the structured development phases:
-
-| Original TODO | Status | Integrated Into | Phase |
-|---------------|--------|-----------------|-------|
-| #1: Binary compile and install.sh | ✅ Planned | Phase 7: Task 7.1 (Binary Compilation) | Phase 7 |
-| #2: Move daemon.go to services/hived/ | ✅ Planned | Phase 3: Task 3.2 (Specialized Services) | Phase 3 |
-| #3: VPC with openvs-switch as `vpcd` | ✅ Planned | Phase 5: Task 5.1 (VPC with Open vSwitch) | Phase 5 |
-| #4: AWS HTTP gateway (`awsd`) | ✅ Planned | Phase 3: Task 3.1 (AWS Gateway Service) | Phase 3 |
-| #5: AWS SDK v2 input/output | ✅ Planned | Phase 2: Task 2.1 & Phase 3: Task 3.1 | Phase 2-3 |
-| #6: `hive.service` for systemd | ✅ Planned | Phase 7: Task 7.2 (System Service Integration) | Phase 7 |
-| #7: Gossip and RAFT protocols | ✅ Planned | Phase 0: Task 0.1 (Service Registry) | Phase 0 |
-| #8: etcd/KV for configuration sync | ✅ Planned | Phase 0: Task 0.1 (Distributed Config) | Phase 0 |
-| #9: Smithy model code generation | ✅ Planned | Phase 2: Task 2.1 (Smithy Code Generation) | Phase 2 |
-
-## Development Phase Overview
-
-**Phase 0**: Distributed Systems Foundation (1-2 weeks)
-- Gossip, RAFT, and distributed configuration (#7, #8)
-
-**Phase 1**: Development Environment Automation (2-3 weeks)
-- Multi-service orchestration and hot reloading
-
-**Phase 2**: AWS API Model Implementation (3-4 weeks)
-- Smithy-based code generation (#9)
-- AWS SDK v2 integration (#5)
-
-**Phase 3**: Scalable Gateway and Daemon Architecture (2-3 weeks)
-- AWS gateway service `awsd` (#4)
-- Service refactoring to `hived` (#2)
-- VPC daemon `vpcd` foundation (#3)
-
-**Phase 4**: Service Integration and Orchestration (3-4 weeks)
-- NATS clustering and cross-service coordination
-
-**Phase 5**: Infrastructure Services (4-5 weeks)
-- VPC networking with Open vSwitch (#3)
-- Advanced VM features
-
-**Phase 6**: Testing and Validation Framework (2-3 weeks)
-- AWS CLI compatibility testing
-
-**Phase 7**: Production Deployment and Packaging (2-3 weeks)
-- Binary compilation and distribution (#1)
-- Systemd service integration (#6)
-- Production configuration management
-
-## Service Architecture
-
-The development plan implements these services:
-
-- **`awsd`** - AWS API Gateway (TODO #4)
-- **`hived`** - Main Hive compute daemon (TODO #2)
-- **`vpcd`** - VPC networking daemon (TODO #3)
-- **Predastore** - S3 service (existing)
-- **Viperblock** - EBS service (existing)
-
-## Getting Started
-
-For development setup:
-```bash
-./scripts/dev-setup.sh     # Setup complete development environment
-./scripts/start-dev.sh     # Start all services
-```
-
-For detailed implementation guidance, see [HIVE_DEVELOPMENT_PLAN.md](HIVE_DEVELOPMENT_PLAN.md).

cmd/hive/cmd/cluster.go

@@ -0,0 +1,195 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/hive/hive/daemon"
+	"github.com/nats-io/nats.go"
+	"github.com/spf13/cobra"
+)
+
+// runClusterShutdown orchestrates a phased, coordinated shutdown of the cluster.
+func runClusterShutdown(cmd *cobra.Command, args []string) {
+	force, _ := cmd.Flags().GetBool("force")
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	nodeCount := len(cfg.Nodes)
+	phases := []string{"gate", "drain", "storage", "persist", "infra"}
+
+	if dryRun {
+		fmt.Println("Cluster Shutdown Plan (dry-run)")
+		fmt.Printf("  Nodes: %d\n", nodeCount)
+		for name, nodeCfg := range cfg.Nodes {
+			fmt.Printf("    - %s (services: %s)\n", name, strings.Join(nodeCfg.GetServices(), ", "))
+		}
+		fmt.Printf("  Phases: %s\n", strings.Join(phases, " -> "))
+		fmt.Printf("  Timeout per phase: %s\n", timeout)
+		fmt.Printf("  Force: %v\n", force)
+		fmt.Println("\nPhase details:")
+		fmt.Println("  1. GATE    - Stop API gateway and UI, reject new work")
+		fmt.Println("  2. DRAIN   - Gracefully stop all VMs, persist state")
+		fmt.Println("  3. STORAGE - Stop viperblock, cleanup nbdkit")
+		fmt.Println("  4. PERSIST - Stop predastore")
+		fmt.Println("  5. INFRA   - Stop NATS, exit daemons")
+		return
+	}
+
+	fmt.Printf("Starting coordinated cluster shutdown (%d nodes)\n", nodeCount)
+	fmt.Printf("Phases: %s\n", strings.Join(phases, " -> "))
+	fmt.Printf("Timeout per phase: %s\n\n", timeout)
+
+	// Write cluster shutdown marker to KV
+	jsm, err := daemon.NewJetStreamManager(nc, 1)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: failed to create JetStream manager: %v\n", err)
+	} else {
+		if err := jsm.InitClusterStateBucket(); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: failed to init cluster state bucket: %v\n", err)
+		} else {
+			state := &daemon.ClusterShutdownState{
+				Initiator:  cfg.Node,
+				Phase:      "starting",
+				Started:    time.Now().UTC().Format(time.RFC3339),
+				Timeout:    timeout.String(),
+				Force:      force,
+				NodesTotal: nodeCount,
+				NodesAcked: make(map[string]string),
+			}
+			if err := jsm.WriteClusterShutdown(state); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: failed to write shutdown state: %v\n", err)
+			}
+		}
+	}
+
+	start := time.Now()
+
+	// Execute phases sequentially (except INFRA which is fire-and-forget)
+	for _, phase := range phases {
+		topic := "hive.cluster.shutdown." + phase
+		req := daemon.ShutdownRequest{
+			Phase:   phase,
+			Force:   force,
+			Timeout: int(timeout.Seconds()),
+		}
+
+		reqData, err := json.Marshal(req)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error marshaling request: %v\n", err)
+			os.Exit(1)
+		}
+
+		if phase == "infra" {
+			// INFRA is fire-and-forget — NATS is going down, no ACKs possible
+			fmt.Printf("[INFRA] Sending final shutdown to all nodes...\n")
+			if err := nc.Publish(topic, reqData); err != nil {
+				fmt.Fprintf(os.Stderr, "Error publishing infra shutdown: %v\n", err)
+			}
+			nc.Flush()
+			// Wait briefly for messages to propagate
+			time.Sleep(2 * time.Second)
+			fmt.Printf("[INFRA] Complete\n")
+			break
+		}
+
+		// For DRAIN phase, subscribe to progress updates
+		var progressSub *nats.Subscription
+		if phase == "drain" {
+			progressSub, err = nc.Subscribe("hive.cluster.shutdown.progress", func(msg *nats.Msg) {
+				var progress daemon.ShutdownProgress
+				if err := json.Unmarshal(msg.Data, &progress); err != nil {
+					return
+				}
+				fmt.Printf("  [%s] %s: %d/%d VMs remaining\n", strings.ToUpper(phase), progress.Node, progress.Remaining, progress.Total)
+			})
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: failed to subscribe to progress: %v\n", err)
+			}
+		}
+
+		// Collect ACKs from all nodes
+		phaseStart := time.Now()
+		fmt.Printf("[%s] Sending to %d node(s)...\n", strings.ToUpper(phase), nodeCount)
+
+		acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, timeout)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[%s] Error: %v\n", strings.ToUpper(phase), err)
+			if !force {
+				fmt.Fprintf(os.Stderr, "Aborting shutdown. Use --force to continue despite errors.\n")
+				os.Exit(1)
+			}
+		}
+
+		// Unsubscribe from progress
+		if progressSub != nil {
+			progressSub.Unsubscribe()
+		}
+
+		// Print results
+		for _, ack := range acks {
+			if ack.Error != "" {
+				fmt.Printf("  [%s] %s: ERROR - %s\n", strings.ToUpper(phase), ack.Node, ack.Error)
+			} else if len(ack.Stopped) > 0 {
+				fmt.Printf("  [%s] %s: stopped %s\n", strings.ToUpper(phase), ack.Node, strings.Join(ack.Stopped, ", "))
+			} else {
+				fmt.Printf("  [%s] %s: OK\n", strings.ToUpper(phase), ack.Node)
+			}
+		}
+
+		ackedCount := len(acks)
+		if ackedCount < nodeCount && !force {
+			fmt.Fprintf(os.Stderr, "[%s] Only %d/%d nodes responded. Use --force to continue.\n",
+				strings.ToUpper(phase), ackedCount, nodeCount)
+			os.Exit(1)
+		}
+
+		fmt.Printf("[%s] Complete (%d/%d nodes, %s)\n\n", strings.ToUpper(phase), ackedCount, nodeCount, time.Since(phaseStart).Round(time.Millisecond))
+	}
+
+	fmt.Printf("Cluster shutdown complete (%s)\n", time.Since(start).Round(time.Millisecond))
+}
+
+// collectShutdownACKs publishes a shutdown request and collects ACKs from nodes.
+func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, timeout time.Duration) ([]daemon.ShutdownACK, error) {
+	inbox := nats.NewInbox()
+	sub, err := nc.SubscribeSync(inbox)
+	if err != nil {
+		return nil, fmt.Errorf("failed to subscribe to inbox: %w", err)
+	}
+	defer sub.Unsubscribe()
+
+	if err := nc.PublishRequest(topic, inbox, reqData); err != nil {
+		return nil, fmt.Errorf("failed to publish request: %w", err)
+	}
+	nc.Flush()
+
+	var acks []daemon.ShutdownACK
+	deadline := time.Now().Add(timeout)
+	for len(acks) < nodeCount {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		msg, err := sub.NextMsg(remaining)
+		if err != nil {
+			break
+		}
+		var ack daemon.ShutdownACK
+		if err := json.Unmarshal(msg.Data, &ack); err != nil {
+			continue
+		}
+		acks = append(acks, ack)
+	}
+	return acks, nil
+}

cmd/hive/cmd/get.go

@@ -0,0 +1,281 @@
+/*
+Copyright © 2025 Mulga Defense Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var getCmd = &cobra.Command{
+	Use:   "get",
+	Short: "Display cluster resources",
+	Long:  `Display cluster resources such as nodes and VMs.`,
+}
+
+var getNodesCmd = &cobra.Command{
+	Use:   "nodes",
+	Short: "Display cluster nodes",
+	Long:  `Display all physical nodes in the cluster with status, IP, region, and services.`,
+	Run:   runGetNodes,
+}
+
+var getVMsCmd = &cobra.Command{
+	Use:     "vms",
+	Aliases: []string{"instances"},
+	Short:   "Display VMs across the cluster",
+	Long:    `Display all VMs running across the cluster with instance type, resources, and placement.`,
+	Run:     runGetVMs,
+}
+
+func init() {
+	rootCmd.AddCommand(getCmd)
+	getCmd.AddCommand(getNodesCmd)
+	getCmd.AddCommand(getVMsCmd)
+
+	getCmd.PersistentFlags().Duration("timeout", 3*time.Second, "Timeout for collecting responses from nodes")
+}
+
+// loadConfigAndConnect loads the cluster config and connects to NATS.
+func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) {
+	cfgPath := viper.GetString("config")
+	if cfgPath == "" {
+		homeDir, _ := os.UserHomeDir()
+		cfgPath = fmt.Sprintf("%s/hive/config/hive.toml", homeDir)
+	}
+
+	cfg, err := config.LoadConfig(cfgPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to load config: %w", err)
+	}
+
+	nodeConfig := cfg.Nodes[cfg.Node]
+	nc, err := utils.ConnectNATS(nodeConfig.NATS.Host, nodeConfig.NATS.ACL.Token)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to connect to NATS: %w", err)
+	}
+
+	return cfg, nc, nil
+}
+
+// collectResponses publishes to a fan-out topic and collects all responses within the timeout.
+func collectResponses(nc *nats.Conn, topic string, timeout time.Duration) ([][]byte, error) {
+	inbox := nats.NewInbox()
+	sub, err := nc.SubscribeSync(inbox)
+	if err != nil {
+		return nil, fmt.Errorf("failed to subscribe to inbox: %w", err)
+	}
+	defer sub.Unsubscribe()
+
+	if err := nc.PublishRequest(topic, inbox, nil); err != nil {
+		return nil, fmt.Errorf("failed to publish request: %w", err)
+	}
+	nc.Flush()
+
+	var responses [][]byte
+	deadline := time.Now().Add(timeout)
+	for {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		msg, err := sub.NextMsg(remaining)
+		if err != nil {
+			break // timeout or other error — done collecting
+		}
+		responses = append(responses, msg.Data)
+	}
+	return responses, nil
+}
+
+func formatUptime(seconds int64) string {
+	if seconds <= 0 {
+		return "-"
+	}
+	d := time.Duration(seconds) * time.Second
+	days := int(d.Hours()) / 24
+	hours := int(d.Hours()) % 24
+	mins := int(d.Minutes()) % 60
+
+	if days > 0 {
+		return fmt.Sprintf("%dd%dh", days, hours)
+	}
+	if hours > 0 {
+		return fmt.Sprintf("%dh%dm", hours, mins)
+	}
+	return fmt.Sprintf("%dm", mins)
+}
+
+func formatMemGB(gb float64) string {
+	if gb >= 1.0 {
+		return fmt.Sprintf("%.1fGi", gb)
+	}
+	return fmt.Sprintf("%dMi", int(gb*1024))
+}
+
+func runGetNodes(cmd *cobra.Command, args []string) {
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	responses, err := collectResponses(nc, "hive.node.status", timeout)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Parse responses into a map by node name
+	respondedNodes := make(map[string]config.NodeStatusResponse)
+	for _, data := range responses {
+		var resp config.NodeStatusResponse
+		if err := json.Unmarshal(data, &resp); err != nil {
+			continue
+		}
+		respondedNodes[resp.Node] = resp
+	}
+
+	// Build table: union of config-known nodes + NATS responders
+	tableData := pterm.TableData{
+		{"NAME", "STATUS", "IP", "REGION", "AZ", "UPTIME", "VMs", "SERVICES"},
+	}
+
+	// Collect all node names: config + responded (union)
+	nodeSet := make(map[string]struct{})
+	for name := range cfg.Nodes {
+		nodeSet[name] = struct{}{}
+	}
+	for name := range respondedNodes {
+		nodeSet[name] = struct{}{}
+	}
+	nodeNames := make([]string, 0, len(nodeSet))
+	for name := range nodeSet {
+		nodeNames = append(nodeNames, name)
+	}
+	sort.Strings(nodeNames)
+
+	for _, name := range nodeNames {
+		nodeCfg := cfg.Nodes[name]
+		if resp, ok := respondedNodes[name]; ok {
+			tableData = append(tableData, []string{
+				resp.Node,
+				resp.Status,
+				resp.Host,
+				resp.Region,
+				resp.AZ,
+				formatUptime(resp.Uptime),
+				fmt.Sprintf("%d", resp.VMCount),
+				strings.Join(resp.Services, ","),
+			})
+		} else {
+			tableData = append(tableData, []string{
+				name,
+				"NotReady",
+				nodeCfg.Host,
+				nodeCfg.Region,
+				nodeCfg.AZ,
+				"-",
+				"-",
+				strings.Join(nodeCfg.GetServices(), ","),
+			})
+		}
+	}
+
+	pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Render()
+}
+
+func runGetVMs(cmd *cobra.Command, args []string) {
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	responses, err := collectResponses(nc, "hive.node.vms", timeout)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	type vmRow struct {
+		config.VMInfo
+		Node string
+		Host string
+	}
+
+	var allVMs []vmRow
+	for _, data := range responses {
+		var resp config.NodeVMsResponse
+		if err := json.Unmarshal(data, &resp); err != nil {
+			continue
+		}
+		for _, v := range resp.VMs {
+			allVMs = append(allVMs, vmRow{VMInfo: v, Node: resp.Node, Host: resp.Host})
+		}
+	}
+
+	if len(allVMs) == 0 {
+		fmt.Println("No VMs found.")
+		return
+	}
+
+	// Sort by node then instance ID
+	sort.Slice(allVMs, func(i, j int) bool {
+		if allVMs[i].Node != allVMs[j].Node {
+			return allVMs[i].Node < allVMs[j].Node
+		}
+		return allVMs[i].InstanceID < allVMs[j].InstanceID
+	})
+
+	tableData := pterm.TableData{
+		{"INSTANCE", "STATUS", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
+	}
+
+	for _, v := range allVMs {
+		age := "-"
+		if v.LaunchTime > 0 {
+			age = formatUptime(time.Now().Unix() - v.LaunchTime)
+		}
+		tableData = append(tableData, []string{
+			v.InstanceID,
+			v.Status,
+			v.InstanceType,
+			fmt.Sprintf("%d", v.VCPU),
+			formatMemGB(v.MemoryGB),
+			v.Node,
+			v.Host,
+			age,
+		})
+	}
+
+	pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Render()
+}

cmd/hive/cmd/root.go

@@ -25,8 +25,9 @@ import (
 )
 
 var (
-	cfgFile   string
-	appConfig *config.Config
+	cfgFile string
+	//lint:ignore U1000 fixing config loading later
+	appConfig *config.ClusterConfig
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -34,16 +35,16 @@ var rootCmd = &cobra.Command{
 	Use:   "hive",
 	Short: "Hive - Open source AWS-compatible platform for secure edge deployments",
 	Long: `
-__/\\\________/\\\__/\\\\\\\\\\\__/\\\________/\\\__/\\\\\\\\\\\\\\\_        
- _\/\\\_______\/\\\_\/////\\\///__\/\\\_______\/\\\_\/\\\///////////__       
-  _\/\\\_______\/\\\_____\/\\\_____\//\\\______/\\\__\/\\\_____________      
-   _\/\\\\\\\\\\\\\\\_____\/\\\______\//\\\____/\\\___\/\\\\\\\\\\\_____     
-    _\/\\\/////////\\\_____\/\\\_______\//\\\__/\\\____\/\\\///////______    
-     _\/\\\_______\/\\\_____\/\\\________\//\\\/\\\_____\/\\\_____________   
-      _\/\\\_______\/\\\_____\/\\\_________\//\\\\\______\/\\\_____________  
-       _\/\\\_______\/\\\__/\\\\\\\\\\\______\//\\\_______\/\\\\\\\\\\\\\\\_ 
+__/\\\________/\\\__/\\\\\\\\\\\__/\\\________/\\\__/\\\\\\\\\\\\\\\_
+ _\/\\\_______\/\\\_\/////\\\///__\/\\\_______\/\\\_\/\\\///////////__
+  _\/\\\_______\/\\\_____\/\\\_____\//\\\______/\\\__\/\\\_____________
+   _\/\\\\\\\\\\\\\\\_____\/\\\______\//\\\____/\\\___\/\\\\\\\\\\\_____
+    _\/\\\/////////\\\_____\/\\\_______\//\\\__/\\\____\/\\\///////______
+     _\/\\\_______\/\\\_____\/\\\________\//\\\/\\\_____\/\\\_____________
+      _\/\\\_______\/\\\_____\/\\\_________\//\\\\\______\/\\\_____________
+       _\/\\\_______\/\\\__/\\\\\\\\\\\______\//\\\_______\/\\\\\\\\\\\\\\\_
         _\///________\///__\///////////________\///________\///////////////__
-		
+
 Hive – Open source AWS-compatible platform for secure edge deployments.
 Run EC2, VPC, S3, and EBS-like services on bare metal with full control.
 Built for environments where running in the cloud isn’t an option.

cmd/hive/cmd/service.go

@@ -23,9 +23,11 @@ import (
 
 	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/service"
+	"github.com/mulgadc/hive/hive/services/hiveui"
 	"github.com/mulgadc/hive/hive/services/nats"
 	"github.com/mulgadc/hive/hive/services/predastore"
 	"github.com/mulgadc/hive/hive/services/viperblockd"
+	"github.com/mulgadc/predastore/s3"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -60,6 +62,12 @@ var awsgwCmd = &cobra.Command{
 	Short: "Manage the awsgw (AWS gateway) service",
 }
 
+var hiveUICmd = &cobra.Command{
+	Use:     "hive-ui",
+	Aliases: []string{"ui", "hiveui"},
+	Short:   "Manage the hive-ui service",
+}
+
 var predastoreStartCmd = &cobra.Command{
 	Use:   "start",
 	Short: "Start the predastore service",
@@ -100,6 +108,19 @@ var predastoreStartCmd = &cobra.Command{
 			return
 		}
 
+		backendType := viper.GetString("backend")
+		var backend s3.BackendType
+
+		if backendType == "distributed" {
+			backend = s3.BackendDistributed
+		} else {
+			backend = s3.BackendFilesystem
+		}
+
+		nodeID := viper.GetInt("node-id")
+		pprofEnabled := viper.GetBool("pprof")
+		pprofOutput := viper.GetString("pprof-output")
+
 		service, err := service.New("predastore", &predastore.Config{
 			Port:       port,
 			Host:       host,
@@ -108,6 +129,12 @@ var predastoreStartCmd = &cobra.Command{
 			Debug:      debug,
 			TlsCert:    tlsCert,
 			TlsKey:     tlsKey,
+
+			Backend: backend,
+			NodeID:  nodeID,
+
+			PprofEnabled:    pprofEnabled,
+			PprofOutputPath: pprofOutput,
 		})
 
 		if err != nil {
@@ -167,7 +194,8 @@ var viperblockStartCmd = &cobra.Command{
 		fmt.Println("Loading config from:", cfgFile)
 
 		// TODO: Support ENV vars, CLI, otherwise revert to config.LoadConfig()
-		appConfig, err := config.LoadConfig(cfgFile)
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		if err != nil {
 			fmt.Println("Error loading config file:", err)
@@ -178,48 +206,51 @@ var viperblockStartCmd = &cobra.Command{
 
 		if natsHost != "" {
 			fmt.Println("Overwriting natsHost:", natsHost)
-			appConfig.NATS.Host = natsHost
+			nodeConfig.NATS.Host = natsHost
 		}
 
 		s3Host := viper.GetString("s3-host")
 
 		if s3Host != "" {
 			fmt.Println("Overwriting s3host:", s3Host)
-			appConfig.Predastore.Host = s3Host
+			nodeConfig.Predastore.Host = s3Host
 		}
 
 		s3Bucket := viper.GetString("s3-bucket")
 
 		if s3Bucket != "" {
 			fmt.Println("Overwriting s3bucket:", s3Bucket)
-			appConfig.Predastore.Bucket = s3Bucket
+			nodeConfig.Predastore.Bucket = s3Bucket
 		}
 
 		s3Region := viper.GetString("s3-region")
 
 		if s3Region != "" {
 			fmt.Println("Overwriting s3Region:", s3Region)
-			appConfig.Predastore.Region = s3Region
+			nodeConfig.Predastore.Region = s3Region
 		}
 
 		accessKey := viper.GetString("access-key")
 		if accessKey != "" {
 			fmt.Println("Overwriting access-key:", accessKey)
-			appConfig.Predastore.AccessKey = accessKey
+			nodeConfig.Predastore.AccessKey = accessKey
 		}
 
 		secretKey := viper.GetString("secret-key")
 		if secretKey != "" {
 			fmt.Println("Overwriting secret-key:", secretKey)
-			appConfig.Predastore.SecretKey = secretKey
+			nodeConfig.Predastore.SecretKey = secretKey
 		}
 
 		baseDir := viper.GetString("base-dir")
 		if baseDir != "" {
 			fmt.Println("Overwriting base-dir:", baseDir)
-			appConfig.Predastore.BaseDir = baseDir
+			nodeConfig.Predastore.BaseDir = baseDir
 		}
 
+		// Apply changes back to cluster config
+		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
+
 		pluginPath := viper.GetString("plugin-path")
 
 		if pluginPath == "" {
@@ -235,15 +266,23 @@ var viperblockStartCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
+		// Resolve sharded WAL setting: default false unless explicitly set to true
+		shardWAL := false
+		if nodeConfig.Viperblock.ShardWAL != nil {
+			shardWAL = *nodeConfig.Viperblock.ShardWAL
+		}
+
 		service, err := service.New("viperblock", &viperblockd.Config{
-			NatsHost:   appConfig.NATS.Host,
+			NatsHost:   nodeConfig.NATS.Host,
 			PluginPath: pluginPath,
-			S3Host:     appConfig.Predastore.Host,
-			Bucket:     appConfig.Predastore.Bucket,
-			Region:     appConfig.Predastore.Region,
-			AccessKey:  appConfig.Predastore.AccessKey,
-			SecretKey:  appConfig.Predastore.SecretKey,
-			BaseDir:    appConfig.Predastore.BaseDir,
+			S3Host:     nodeConfig.Predastore.Host,
+			Bucket:     nodeConfig.Predastore.Bucket,
+			Region:     nodeConfig.Predastore.Region,
+			AccessKey:  nodeConfig.Predastore.AccessKey,
+			SecretKey:  nodeConfig.Predastore.SecretKey,
+			BaseDir:    nodeConfig.Predastore.BaseDir,
+			NodeName:   clusterConfig.Node,
+			ShardWAL:   shardWAL,
 		})
 
 		if err != nil {
@@ -303,12 +342,15 @@ var natsStartCmd = &cobra.Command{
 		dataDir := viper.GetString("data-dir")
 		jetStream := viper.GetBool("jetstream")
 
+		cfgFile := viper.GetString("config")
+
 		service, err := service.New("nats", &nats.Config{
-			Port:      port,
-			Host:      host,
-			Debug:     debug,
-			DataDir:   dataDir,
-			JetStream: jetStream,
+			ConfigFile: cfgFile,
+			Port:       port,
+			Host:       host,
+			Debug:      debug,
+			DataDir:    dataDir,
+			JetStream:  jetStream,
 		})
 
 		if err != nil {
@@ -363,7 +405,8 @@ var hiveStartCmd = &cobra.Command{
 		}
 
 		// TODO: Support ENV vars, CLI, otherwise revert to config.LoadConfig()
-		appConfig, err := config.LoadConfig(cfgFile)
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		if err != nil {
 			fmt.Println("Error loading config file:", err)
@@ -375,7 +418,7 @@ var hiveStartCmd = &cobra.Command{
 
 		if baseDir != "" {
 			fmt.Println("Overwriting base-dir to:", baseDir)
-			appConfig.BaseDir = baseDir
+			nodeConfig.BaseDir = baseDir
 		}
 
 		// Overwrite defaults (CLI first, config second, env third)
@@ -383,17 +426,25 @@ var hiveStartCmd = &cobra.Command{
 
 		if walDir != "" {
 			fmt.Println("Overwriting wal-dir to:", walDir)
-			appConfig.WalDir = walDir
+			nodeConfig.WalDir = walDir
 		}
 
-		service, err := service.New("hive", appConfig)
+		// Apply changes back to cluster config
+		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
+
+		svc, err := service.New("hive", clusterConfig)
 
 		if err != nil {
 			fmt.Println("Error starting hive service:", err)
 			return
 		}
 
-		service.Start()
+		// Set config path for cluster manager
+		if hiveSvc, ok := svc.(interface{ SetConfigPath(string) }); ok {
+			hiveSvc.SetConfigPath(cfgFile)
+		}
+
+		svc.Start()
 		fmt.Println("HIVE service started")
 	},
 }
@@ -404,7 +455,7 @@ var hiveStopCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Stopping hive service...")
 
-		service, err := service.New("hive", &config.Config{})
+		service, err := service.New("hive", &config.ClusterConfig{})
 
 		if err != nil {
 			fmt.Println("Error stopping hive service:", err)
@@ -443,7 +494,8 @@ var awsgwStartCmd = &cobra.Command{
 		fmt.Println("Loading config from:", cfgFile)
 
 		// TODO: Support ENV vars, CLI, otherwise revert to config.LoadConfig()
-		appConfig, err := config.LoadConfig(cfgFile)
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		if err != nil {
 			fmt.Println("Error loading config file:", err)
@@ -454,30 +506,33 @@ var awsgwStartCmd = &cobra.Command{
 		awsgwHost := viper.GetString("host")
 		if awsgwHost != "" {
 			fmt.Println("Overwriting awsgw host to:", awsgwHost)
-			appConfig.AWSGW.Host = awsgwHost
+			//nodeConfig.AWSGW.Host = awsgwHost
 		}
 
 		awsgwTlsCert := viper.GetString("tls-cert")
 		if awsgwTlsCert != "" {
 			fmt.Println("Overwriting awsgw tls-cert to:", awsgwTlsCert)
-			appConfig.AWSGW.TLSCert = awsgwTlsCert
+			nodeConfig.AWSGW.TLSCert = awsgwTlsCert
 		}
 
 		awsgwTlsKey := viper.GetString("tls-key")
 
 		if awsgwTlsKey != "" {
 			fmt.Println("Overwriting awsgw tls-key to:", awsgwTlsKey)
-			appConfig.AWSGW.TLSKey = awsgwTlsKey
+			nodeConfig.AWSGW.TLSKey = awsgwTlsKey
 		}
 
 		baseDir := viper.GetString("base-dir")
 
 		if awsgwTlsKey != "" {
 			fmt.Println("Overwriting awsgw base-dir to:", baseDir)
-			appConfig.BaseDir = baseDir
+			nodeConfig.BaseDir = baseDir
 		}
 
-		service, err := service.New("awsgw", appConfig)
+		// Apply changes back to cluster config
+		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
+
+		service, err := service.New("awsgw", clusterConfig)
 
 		if err != nil {
 			fmt.Println("Error starting awsgw service:", err)
@@ -495,7 +550,7 @@ var awsgwStopCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Stopping awsgw service...")
 
-		service, err := service.New("awsgw", &config.Config{})
+		service, err := service.New("awsgw", &config.ClusterConfig{})
 
 		if err != nil {
 			fmt.Println("Error stopping awsgw service:", err)
@@ -516,6 +571,73 @@ var awsgwStatusCmd = &cobra.Command{
 	},
 }
 
+var hiveUIStartCmd = &cobra.Command{
+	Use:   "start",
+	Short: "Start the hive-ui service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Starting hive-ui service...")
+
+		port := viper.GetInt("hive-ui-port")
+		host := viper.GetString("hive-ui-host")
+		tlsCert := viper.GetString("hive-ui-tls-cert")
+		tlsKey := viper.GetString("hive-ui-tls-key")
+
+		svc, err := service.New("hive-ui", &hiveui.Config{
+			Port:    port,
+			Host:    host,
+			TLSCert: tlsCert,
+			TLSKey:  tlsKey,
+		})
+
+		if err != nil {
+			fmt.Println("Error starting hive-ui service:", err)
+			return
+		}
+
+		svc.Start()
+		fmt.Println("hive-ui service started")
+	},
+}
+
+var hiveUIStopCmd = &cobra.Command{
+	Use:   "stop",
+	Short: "Stop the hive-ui service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Stopping hive-ui service...")
+
+		svc, err := service.New("hive-ui", &hiveui.Config{})
+
+		if err != nil {
+			fmt.Println("Error stopping hive-ui service:", err)
+			return
+		}
+
+		svc.Stop()
+		fmt.Println("hive-ui service stopped")
+	},
+}
+
+var hiveUIStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Get status of the hive-ui service",
+	Run: func(cmd *cobra.Command, args []string) {
+		svc, err := service.New("hive-ui", &hiveui.Config{})
+
+		if err != nil {
+			fmt.Println("Error getting hive-ui service status:", err)
+			return
+		}
+
+		status, err := svc.Status()
+		if err != nil {
+			fmt.Println("Error getting hive-ui service status:", err)
+			return
+		}
+
+		fmt.Println("hive-ui service status:", status)
+	},
+}
+
 func init() {
 
 	viper.SetEnvPrefix("HIVE") // Prefix for environment variables
@@ -562,6 +684,26 @@ func init() {
 	viper.BindEnv("tls-key", "HIVE_PREDASTORE_TLS_KEY")
 	viper.BindPFlag("tls-key", predastoreCmd.PersistentFlags().Lookup("tls-key"))
 
+	// Predastore Backend
+	predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend")
+	viper.BindEnv("backend", "HIVE_PREDASTORE_BACKEND")
+	viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend"))
+
+	// Predastore Node ID
+	predastoreCmd.PersistentFlags().Int("node-id", 0, "Predastore (S3) node ID")
+	viper.BindEnv("node-id", "HIVE_PREDASTORE_NODE_ID")
+	viper.BindPFlag("node-id", predastoreCmd.PersistentFlags().Lookup("node-id"))
+
+	// Predastore CPU Profiling
+	predastoreCmd.PersistentFlags().Bool("pprof", false, "Enable CPU profiling (also via PPROF_ENABLED=1)")
+	viper.BindEnv("pprof", "PPROF_ENABLED")
+	viper.BindPFlag("pprof", predastoreCmd.PersistentFlags().Lookup("pprof"))
+
+	// Predastore CPU Profile Output Path
+	predastoreCmd.PersistentFlags().String("pprof-output", "/tmp/predastore-cpu.prof", "CPU profile output path")
+	viper.BindEnv("pprof-output", "PPROF_OUTPUT")
+	viper.BindPFlag("pprof-output", predastoreCmd.PersistentFlags().Lookup("pprof-output"))
+
 	predastoreCmd.AddCommand(predastoreStartCmd)
 	predastoreCmd.AddCommand(predastoreStopCmd)
 	predastoreCmd.AddCommand(predastoreStatusCmd)
@@ -652,4 +794,27 @@ func init() {
 	awsgwCmd.AddCommand(awsgwStopCmd)
 	awsgwCmd.AddCommand(awsgwStatusCmd)
 
+	// hive-ui
+	serviceCmd.AddCommand(hiveUICmd)
+
+	hiveUICmd.PersistentFlags().Int("port", 3000, "hive-ui server port")
+	viper.BindEnv("hive-ui-port", "HIVE_UI_PORT")
+	viper.BindPFlag("hive-ui-port", hiveUICmd.PersistentFlags().Lookup("port"))
+
+	hiveUICmd.PersistentFlags().String("host", "0.0.0.0", "hive-ui server host")
+	viper.BindEnv("hive-ui-host", "HIVE_UI_HOST")
+	viper.BindPFlag("hive-ui-host", hiveUICmd.PersistentFlags().Lookup("host"))
+
+	hiveUICmd.PersistentFlags().String("tls-cert", "", "TLS certificate path")
+	viper.BindEnv("hive-ui-tls-cert", "HIVE_UI_TLS_CERT")
+	viper.BindPFlag("hive-ui-tls-cert", hiveUICmd.PersistentFlags().Lookup("tls-cert"))
+
+	hiveUICmd.PersistentFlags().String("tls-key", "", "TLS key path")
+	viper.BindEnv("hive-ui-tls-key", "HIVE_UI_TLS_KEY")
+	viper.BindPFlag("hive-ui-tls-key", hiveUICmd.PersistentFlags().Lookup("tls-key"))
+
+	hiveUICmd.AddCommand(hiveUIStartCmd)
+	hiveUICmd.AddCommand(hiveUIStopCmd)
+	hiveUICmd.AddCommand(hiveUIStatusCmd)
+
 }

cmd/hive/cmd/templates/hive.toml

@@ -1,34 +1,61 @@
 # Domain configuration file in TOML format.
 version = "1.0"
+epoch = 1
+node = "{{.Node}}"
+
+[nodes.{{.Node}}]
+node = "{{.Node}}"
+host = "{{.BindIP}}"
 region = "{{.Region}}"
+az = "{{.Az}}"
+{{- if .Services}}
+services = [{{range $i, $s := .Services}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
+{{- end}}
+
 account_id = "{{.AccountID}}"
+accesskey = "{{.AccessKey}}"
+secretkey = "{{.SecretKey}}"
 
-access_key = "{{.AccessKey}}"
-secret_key = "{{.SecretKey}}"
-
-[daemon]
-host = "0.0.0.0:4432"
+[nodes.{{.Node}}.daemon]
+host = "{{.BindIP}}:{{.Port}}"
 tlskey = "config/server.key"
 tlscert = "config/server.pem"
 
-[awsgw]
-host = "0.0.0.0:9999"
+[nodes.{{.Node}}.awsgw]
+host = "{{.BindIP}}:9999"
 tlskey = "config/server.key"
 tlscert = "config/server.pem"
 debug = true
 config = "awsgw/awsgw.toml"
 
-[nats]
-host = "0.0.0.0:4222"
+[nodes.{{.Node}}.nats]
+host = "{{.BindIP}}:4222"
 
-[nats.acl]
+[nodes.{{.Node}}.nats.acl]
 token = "{{.NatsToken}}"
 
-[nats.sub]
+[nodes.{{.Node}}.nats.sub]
 
-[predastore]
-host = "0.0.0.0:8443"
+[nodes.{{.Node}}.viperblock]
+shardwal = false
+
+[nodes.{{.Node}}.predastore]
+host = "{{.BindIP}}:8443"
 region = "{{.Region}}"
 bucket = "predastore"
-access_key = "{{.AccessKey}}"
-secret_key = "{{.SecretKey}}"
+accesskey = "{{.AccessKey}}"
+secretkey = "{{.SecretKey}}"
+{{- if gt .PredastoreNodeID 0}}
+node_id = {{.PredastoreNodeID}}
+{{- end}}
+{{- range .RemoteNodes}}
+
+[nodes.{{.Name}}]
+node = "{{.Name}}"
+host = "{{.Host}}"
+region = "{{.Region}}"
+az = "{{.AZ}}"
+{{- if .Services}}
+services = [{{range $i, $s := .Services}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
+{{- end}}
+{{- end}}

cmd/hive/cmd/templates/nats.conf

@@ -2,17 +2,37 @@
 # Generated by Hive admin init
 
 # Server configuration
-port: 4222
-server_name: hive-nats
+{{ if .BindIP }}
+listen: {{.BindIP}}:4222
+{{ else }}
+listen: 0.0.0.0:4222
+{{ end }}
+
+
+{{ if and .ClusterBindIP .ClusterRoutes }}
+cluster {
+  name: {{ .ClusterName }}
+  listen: {{.ClusterBindIP}}:4248
+
+  routes = [
+{{- range .ClusterRoutes }}
+    "nats://{{ . }}",
+{{- end }}
+  ]
+}
+{{ end }}
+
+
+server_name: hive-nats-{{.Node}}
 
 # Authorization
 authorization {
-  token: "{{.NatsToken}}"
+#  token: "{{.NatsToken}}"
 }
 
-# JetStream for persistent messaging
+# JetStream for persistent messaging and KV store
 jetstream {
-  store_dir: "{{.DataDir}}/nats/storage"
+  store_dir: "{{.DataDir}}/nats/"
   max_memory_store: 1GB
   max_file_store: 10GB
 }
@@ -29,4 +49,9 @@ max_payload: 1MB
 max_pending: 64MB
 
 # Monitoring
+{{ if .BindIP }}
+http: {{.BindIP}}:8222
+{{ else }}
 http_port: 8222
+{{ end }}
+

cmd/hive/cmd/templates/predastore-multinode.toml

@@ -0,0 +1,45 @@
+# Predastore S3-compatible storage configuration file in TOML format.
+# Generated for multi-node cluster
+version = "1.0"
+region = "{{.Region}}"
+
+host = "0.0.0.0"
+port = 8443
+
+debug = false
+
+[rs]
+data = 2
+parity = 1
+{{range .Nodes}}
+[[db]]
+id = {{.ID}}
+host = "{{.Host}}"
+port = 6660
+path = "distributed/db/node-{{.ID}}/"
+access_key_id = "{{$.AccessKey}}"
+secret_access_key = "{{$.SecretKey}}"
+{{- if eq .ID 1}}
+leader = true
+{{- end}}
+{{end}}
+{{- range .Nodes}}
+[[nodes]]
+id = {{.ID}}
+host = "{{.Host}}"
+port = 9991
+path = "distributed/nodes/node-{{.ID}}/"
+{{end}}
+[[buckets]]
+name = "predastore"
+region = "{{.Region}}"
+type = "distributed"
+public = false
+encryption = ""
+
+[[auth]]
+access_key_id = "{{.AccessKey}}"
+secret_access_key = "{{.SecretKey}}"
+policy = [
+  { bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
+]

cmd/hive/cmd/templates/predastore.toml

@@ -1,4 +1,5 @@
 # Predastore S3-compatible storage configuration file in TOML format.
+# MULTI NODE used for testing
 version = "1.0"
 region = "{{.Region}}"
 
@@ -12,11 +13,64 @@ debug = false
 # Optionally configure base-path if full directory path not defined for "fs" buckets
 # base_path = "/home/predastore/data/"
 
+# Specify reed solomon encoding - 
+# RS(2,1), 2 data, 1 parity. Minimum
+# RS(3,2), 3 data, 2 parity. Recommended for production
+[rs]
+data = 2
+parity = 1
+
+[[db]]
+id = 1
+host = "127.0.0.1"
+port = 6660
+#raft_port = 7660       # Optional: Raft consensus port (default: port + 1000)
+#advertise_host = ""    # Optional: Address to advertise to other Raft nodes
+path = "distributed/db/node-1/"
+access_key_id = "{{.AccessKey}}"
+secret_access_key = "{{.SecretKey}}"
+leader = true      # Bootstrap leader hint
+
+[[db]]
+id = 2
+host = "127.0.0.1"
+port = 6661
+path = "distributed/db/node-2/"
+access_key_id = "{{.AccessKey}}"
+secret_access_key = "{{.SecretKey}}"
+
+[[db]]
+id = 3
+host = "127.0.0.1"
+port = 6662
+path = "distributed/db/node-3/"
+access_key_id = "{{.AccessKey}}"
+secret_access_key = "{{.SecretKey}}"
+
+# Shard nodes
+# Using distributed hash ring for object > shard placement.
+[[nodes]]
+id = 1
+host = "0.0.0.0"
+port = 9991
+path = "distributed/nodes/node-1/"
+
+[[nodes]]
+id = 2
+host = "0.0.0.0"
+port = 9992
+path = "distributed/nodes/node-2/"
+
+[[nodes]]
+id = 3
+host = "0.0.0.0"
+port = 9993
+path = "distributed/nodes/node-3/"
+
 [[buckets]]
 name = "predastore"
-region = "{{.Region}}"
-type = "fs"
-pathname = "data/predastore"
+region = "ap-southeast-2"
+type = "distributed"
 public = false
 encryption = ""
 
@@ -25,4 +79,4 @@ access_key_id = "{{.AccessKey}}"
 secret_access_key = "{{.SecretKey}}"
 policy = [
   { bucket = "predastore", actions = ["s3:ListBucket",  "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
-]
+]

cmd/hive/cmd/top.go

@@ -0,0 +1,161 @@
+/*
+Copyright © 2025 Mulga Defense Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/mulgadc/hive/hive/config"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+)
+
+var topCmd = &cobra.Command{
+	Use:   "top",
+	Short: "Display cluster resource usage",
+	Long:  `Display resource usage (CPU, memory) for cluster nodes.`,
+}
+
+var topNodesCmd = &cobra.Command{
+	Use:   "nodes",
+	Short: "Display resource usage per node",
+	Long:  `Display CPU and memory usage per node, plus a summary of available instance types across the cluster.`,
+	Run:   runTopNodes,
+}
+
+func init() {
+	rootCmd.AddCommand(topCmd)
+	topCmd.AddCommand(topNodesCmd)
+
+	topCmd.PersistentFlags().Duration("timeout", 3*time.Second, "Timeout for collecting responses from nodes")
+}
+
+func runTopNodes(cmd *cobra.Command, args []string) {
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	responses, err := collectResponses(nc, "hive.node.status", timeout)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	respondedNodes := make(map[string]config.NodeStatusResponse)
+	for _, data := range responses {
+		var resp config.NodeStatusResponse
+		if err := json.Unmarshal(data, &resp); err != nil {
+			continue
+		}
+		respondedNodes[resp.Node] = resp
+	}
+
+	// Node resource table: union of config-known nodes + NATS responders
+	nodeSet := make(map[string]struct{})
+	for name := range cfg.Nodes {
+		nodeSet[name] = struct{}{}
+	}
+	for name := range respondedNodes {
+		nodeSet[name] = struct{}{}
+	}
+	nodeNames := make([]string, 0, len(nodeSet))
+	for name := range nodeSet {
+		nodeNames = append(nodeNames, name)
+	}
+	sort.Strings(nodeNames)
+
+	nodeTable := pterm.TableData{
+		{"NAME", "CPU (used/total)", "MEM (used/total)", "VMs"},
+	}
+
+	// Aggregate instance type capacity across all nodes
+	capacityMap := make(map[string]*aggregatedCap)
+
+	for _, name := range nodeNames {
+		if resp, ok := respondedNodes[name]; ok {
+			nodeTable = append(nodeTable, []string{
+				resp.Node,
+				fmt.Sprintf("%d/%d", resp.AllocVCPU, resp.TotalVCPU),
+				fmt.Sprintf("%s/%s", formatMemGB(resp.AllocMemGB), formatMemGB(resp.TotalMemGB)),
+				fmt.Sprintf("%d", resp.VMCount),
+			})
+
+			for _, cap := range resp.InstanceTypes {
+				if agg, ok := capacityMap[cap.Name]; ok {
+					agg.Available += cap.Available
+				} else {
+					capacityMap[cap.Name] = &aggregatedCap{
+						VCPU:      cap.VCPU,
+						MemoryGB:  cap.MemoryGB,
+						Available: cap.Available,
+					}
+				}
+			}
+		} else {
+			nodeTable = append(nodeTable, []string{
+				name,
+				"-",
+				"-",
+				"-",
+			})
+		}
+	}
+
+	pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(nodeTable).Render()
+
+	// Instance type capacity summary
+	if len(capacityMap) == 0 {
+		return
+	}
+
+	fmt.Println()
+
+	capNames := make([]string, 0, len(capacityMap))
+	for name := range capacityMap {
+		capNames = append(capNames, name)
+	}
+	sort.Strings(capNames)
+
+	capTable := pterm.TableData{
+		{"INSTANCE TYPE", "AVAILABLE", "VCPU", "MEMORY"},
+	}
+
+	for _, name := range capNames {
+		agg := capacityMap[name]
+		capTable = append(capTable, []string{
+			name,
+			fmt.Sprintf("%d", agg.Available),
+			fmt.Sprintf("%d", agg.VCPU),
+			formatMemGB(agg.MemoryGB),
+		})
+	}
+
+	pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(capTable).Render()
+}
+
+type aggregatedCap struct {
+	VCPU      int
+	MemoryGB  float64
+	Available int
+}

cmd/hive/main.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

config/predastore.toml

@@ -0,0 +1,80 @@
+# Predastore S3-compatible storage configuration file in TOML format.
+# MULTI NODE used for testing
+version = "1.0"
+region = "ap-southeast-2"
+
+# Optionally configure port and host, or specify via daemon CLI options
+host = "0.0.0.0"
+port = 8443
+
+# Enable verbose debugging
+debug = false
+
+# Optionally configure base-path if full directory path not defined for "fs" buckets
+# base_path = "/home/predastore/data/"
+
+# Specify reed solomon encoding
+[rs]
+data = 2
+parity = 1
+
+[[db]]
+id = 1
+host = "127.0.0.1"
+port = 6660
+#raft_port = 7660       # Optional: Raft consensus port (default: port + 1000)
+#advertise_host = ""    # Optional: Address to advertise to other Raft nodes
+path = "distributed/db/node-1/"
+access_key_id = "X"
+secret_access_key = "X"
+leader = true      # Bootstrap leader hint
+
+[[db]]
+id = 2
+host = "127.0.0.1"
+port = 6661
+path = "distributed/db/node-2/"
+access_key_id = "X"
+secret_access_key = "X"
+
+[[db]]
+id = 3
+host = "127.0.0.1"
+port = 6662
+path = "distributed/db/node-3/"
+access_key_id = "X"
+secret_access_key = "X"
+
+[[nodes]]
+id = 1
+host = "0.0.0.0"
+port = 9991
+path = "distributed/nodes/node-1/"
+
+[[nodes]]
+id = 2
+host = "0.0.0.0"
+port = 9992
+path = "distributed/nodes/node-2/"
+
+[[nodes]]
+id = 3
+host = "0.0.0.0"
+port = 9993
+path = "distributed/nodes/node-3/"
+
+[[buckets]]
+name = "predastore"
+region = "ap-southeast-2"
+type = "distributed"
+public = false
+encryption = ""
+
+[[auth]]
+access_key_id = "X"
+secret_access_key = "X"
+policy = [
+  { bucket = "predastore", actions = ["s3:ListBucket",  "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
+]
+
+

docs/COMMANDS.md

@@ -0,0 +1,358 @@
+# AWS Command Implementation Matrix
+
+## EC2 - Instance Management
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `run-instances` | `--image-id`, `--instance-type`, `--count` (Min/MaxCount), `--key-name`, `--user-data`, `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination) | `--security-group-ids`, `--subnet-id`, `--placement`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--iam-instance-profile`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | `describe-images` (AMI must exist), `create-key-pair` (optional), VPC/SG (optional) | Gateway parses AWS query → NATS `ec2.runinstances` → daemon creates QEMU/KVM VM with viperblock-backed root volume via NBD → cloud-init injects user-data/keys → returns reservation with instance ID | 1. Launch with valid AMI and key pair<br>2. Launch with invalid AMI ID (error)<br>3. Launch with block device mappings (custom volume size)<br>4. Launch multiple instances (MinCount/MaxCount)<br>5. DryRun returns validation-only<br>6. Invalid instance type returns error | **DONE** |
+| `describe-instances` | `--instance-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes | **DONE** |
+| `start-instances` | `--instance-ids` | `--dry-run`, `--force` | `run-instances` (instance must exist in stopped state) | Gateway sends NATS `ec2.cmd.{instance-id}` → daemon restarts stopped QEMU process with same config → state transitions stopped→pending→running | 1. Start a stopped instance<br>2. Start already-running instance (error: IncorrectInstanceState)<br>3. Start with invalid instance ID<br>4. Verify volumes re-mount on start | **DONE** |
+| `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_powerdown` for graceful shutdown → monitors heartbeat until QEMU exits → state transitions running→stopping→stopped | 1. Graceful stop of running instance<br>2. Force stop (kills QEMU process)<br>3. Stop already-stopped instance (error)<br>4. Verify ~30s heartbeat detection | **DONE** |
+| `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID | **DONE** |
+| `reboot-instances` | — | `--instance-ids`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_reset` → instance reboots without stopping | 1. Reboot running instance<br>2. Reboot stopped instance (error)<br>3. Verify instance stays in running state after reboot | **NOT STARTED** (parser test exists) |
+| `describe-instance-types` | `--filters` (capacity filter only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, all other filters | None | Gateway fans out NATS `ec2.DescribeInstanceTypes` to all nodes → each daemon reports supported types (t3.micro/small/medium/large) with vCPU/memory specs → gateway deduplicates and returns | 1. List all instance types<br>2. Filter by specific type<br>3. Filter with `capacity=true` shows available slots<br>4. Verify vCPU/memory specs match hardware | **DONE** |
+| `modify-instance-attribute` | — | `--instance-id`, `--instance-type`, `--user-data`, `--disable-api-termination`, `--ebs-optimized` | Instance must be stopped | Validate instance stopped → update instance metadata in NATS KV store → changes applied on next start | 1. Change instance type while stopped<br>2. Modify running instance (error)<br>3. Toggle disable-api-termination | **NOT STARTED** |
+| `get-console-output` | — | `--instance-id`, `--latest` | Instance must exist | QMP command to read serial console ring buffer → return base64-encoded output with timestamp | 1. Get output from running instance<br>2. Get output from stopped instance<br>3. Verify cloud-init boot logs visible | **NOT STARTED** |
+| `monitor-instances` | — | `--instance-ids` | Instance must exist | Enable basic monitoring (CPU, disk, network) → store metrics in NATS KV → return monitoring state | 1. Enable monitoring<br>2. Verify monitoring state in describe-instances | **NOT STARTED** |
+| `unmonitor-instances` | — | `--instance-ids` | Instance must exist | Disable detailed monitoring → revert to basic monitoring → return monitoring state | 1. Disable monitoring<br>2. Verify monitoring state reverts in describe-instances | **NOT STARTED** |
+
+## EC2 - Key Pair Management
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519) | `--key-format` (pem/ppk), `--tag-specifications`, `--dry-run` | None | NATS `ec2.CreateKeyPair` → daemon generates SSH keypair → stores public key in Predastore S3 (`/bucket/ec2/{name}.pub`) → returns private key material (one-time) | 1. Create RSA key pair<br>2. Create ED25519 key pair<br>3. Duplicate key name (error: InvalidKeyPair.Duplicate)<br>4. Verify key material returned only on creation | **DONE** |
+| `describe-key-pairs` | `--key-names`, `--key-pair-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty | **DONE** |
+| `delete-key-pair` | `--key-name`, `--key-pair-id` | `--dry-run` | Key must exist | NATS `ec2.DeleteKeyPair` → daemon deletes public key from Predastore S3 → returns success | 1. Delete existing key pair<br>2. Delete non-existent key (idempotent, no error)<br>3. Verify key no longer in describe-key-pairs | **DONE** |
+| `import-key-pair` | `--key-name`, `--public-key-material` | `--tag-specifications`, `--dry-run` | None | NATS `ec2.ImportKeyPair` → daemon stores provided public key in Predastore S3 → returns key name and fingerprint | 1. Import valid SSH public key<br>2. Import invalid key material (error)<br>3. Import duplicate name (error)<br>4. Verify imported key usable with run-instances | **DONE** |
+
+## EC2 - AMI Image Management
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `describe-images` | `--image-ids` (format validation only) | `--owners`, `--executable-users`, `--filters`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName) | **DONE** |
+| `create-image` | — | `--instance-id`, `--name`, `--description`, `--no-reboot`, `--block-device-mappings`, `--tag-specifications` | Instance must exist | Stop instance (unless --no-reboot) → snapshot root volume via viperblock → create AMI metadata in S3 → register as new AMI → restart instance → return ami-ID | 1. Create image from running instance<br>2. Create with --no-reboot<br>3. Invalid instance ID (error)<br>4. Verify new AMI appears in describe-images<br>5. Launch new instance from created AMI | **NOT STARTED** |
+| `register-image` | — | `--name`, `--description`, `--architecture`, `--root-device-name`, `--virtualization-type`, `--block-device-mappings`, `--boot-mode`, `--ena-support`, `--sriov-net-support`, `--tpm-support`, `--imds-support` | Image data must exist in S3 | Create AMI metadata entry in Predastore → assign ami-ID → set state to available → return ami-ID | 1. Register with valid metadata<br>2. Missing required name (error)<br>3. Verify registered image in describe-images | **NOT STARTED** |
+| `deregister-image` | — | `--image-id`, `--dry-run` | AMI must exist, no instances using it | Mark AMI metadata as deregistered in S3 → AMI no longer available for new launches → existing instances unaffected | 1. Deregister existing AMI<br>2. Deregister non-existent AMI (error)<br>3. Verify deregistered AMI not in describe-images<br>4. Existing instances from AMI still run | **NOT STARTED** |
+| `copy-image` | — | `--source-image-id`, `--source-region`, `--name`, `--description`, `--encrypted`, `--kms-key-id`, `--client-token`, `--copy-image-tags`, `--tag-specifications` | Source AMI must exist | Copy AMI data between regions/nodes in Predastore → create new AMI metadata with new ami-ID → return new ami-ID | 1. Copy image within same region<br>2. Copy non-existent image (error)<br>3. Verify copied image independent of source | **NOT STARTED** |
+| `import-image` | — | `--disk-containers` (Format, Url/S3Bucket+S3Key), `--description`, `--architecture`, `--platform` | S3 bucket with disk image | Download disk image from S3 → convert format (VMDK/VHD/RAW→QCOW2) → create viperblock volume → register as AMI | 1. Import QCOW2 image<br>2. Import RAW image<br>3. Invalid format (error)<br>4. Verify imported image launchable | **NOT STARTED** |
+| `describe-image-attribute` | — | `--image-id`, `--attribute`, `--dry-run` | AMI must exist | Read specific attribute from AMI metadata in S3 → return attribute value | 1. Get description attribute<br>2. Get launch permission<br>3. Invalid attribute name (error) | **NOT STARTED** |
+| `modify-image-attribute` | — | `--image-id`, `--launch-permission`, `--description`, `--operation-type`, `--user-ids`, `--user-groups`, `--organization-arns`, `--dry-run` | AMI must exist | Update specific attribute in AMI metadata → persist to S3 | 1. Add launch permission<br>2. Modify description<br>3. Invalid AMI (error) | **NOT STARTED** |
+| `reset-image-attribute` | — | `--image-id`, `--attribute`, `--dry-run` | AMI must exist | Reset attribute to default value in AMI metadata | 1. Reset launch permission to owner-only<br>2. Invalid AMI (error) | **NOT STARTED** |
+
+## EC2 - Volume (EBS) Management
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata) | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value | **DONE** |
+| `modify-volume` | `--volume-id`, `--size`, `--volume-type`, `--iops` | `--throughput`, `--dry-run`, `--multi-attach-enabled` | Volume must exist | NATS `ec2.ModifyVolume` → daemon sends resize request to viperblock → NBD does not support live resize, instance must be stopped → returns modification state | 1. Increase volume size<br>2. Modify volume type<br>3. Decrease size (error - not supported)<br>4. Modify attached volume (requires stop/start) | **DONE** |
+| `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3 only) | `--iops`, `--encrypted`, `--snapshot-id`, `--tag-specifications` | Valid AZ configured via `hive init` | Gateway validates input → NATS `ec2.CreateVolume` → daemon generates vol-ID via viperblock → creates empty volume of specified size → persists config.json to Predastore S3 → returns vol-ID with state=available | 1. Create 80GB gp3 volume<br>2. Boundary sizes (1 GiB min, 16384 GiB max)<br>3. Invalid AZ (error)<br>4. Verify volume in describe-volumes<br>5. Unsupported volume type (error - only gp3)<br>6. Size out of range (error) | **DONE** |
+| `delete-volume` | `--volume-id` | `--dry-run` | Volume must exist and be detached (state=available) | Gateway validates vol- prefix → NATS `ec2.DeleteVolume` → daemon confirms state=available and no AttachedInstance → NATS `ebs.delete` to viperblockd (stops nbdkit/WAL) → deletes S3 objects under vol-id/, vol-id-efi/, vol-id-cloudinit/ → returns success | 1. Delete detached volume<br>2. Delete attached volume (error: VolumeInUse)<br>3. Delete non-existent volume (error: InvalidVolume.NotFound)<br>4. Verify volume gone from describe-volumes<br>5. Malformed volume ID (error: InvalidVolumeID.Malformed)<br>6. Double delete (idempotent NotFound) | **DONE** |
+| `attach-volume` | `--volume-id`, `--instance-id`, `--device` (optional, auto-assigns `/dev/sd[f-p]`) | `--dry-run` | Volume must exist (available), instance must exist (running) | Gateway sends to `ec2.cmd.{instanceId}` → daemon validates volume (Predastore) → `ebs.mount` via NATS (viperblock starts NBD server) → QMP `blockdev-add` (nbd-{volId}) → QMP `device_add` (virtio-blk-pci, vdisk-{volId}) → three-phase rollback on failure → update EBSRequests + BlockDeviceMappings → persist to JetStream + Predastore → respond with VolumeAttachment | 1. Attach volume to running instance<br>2. Auto-assign device name<br>3. Attach already-attached volume (VolumeInUse)<br>4. Attach to non-existent instance (InvalidInstanceID.NotFound)<br>5. Attach to stopped instance (IncorrectInstanceState)<br>6. Volume not found (InvalidVolume.NotFound)<br>7. All device slots full (AttachmentLimitExceeded)<br>8. Volume persists across stop/start | **DONE** |
+| `detach-volume` | `--volume-id`, `--instance-id` (optional, resolved via DescribeVolumes), `--device` (optional cross-check), `--force` | `--dry-run` | Volume must be attached, instance must be running | Gateway resolves InstanceId if omitted (via DescribeVolumes) → sends to `ec2.cmd.{instanceId}` → daemon validates (running, attached, not boot/EFI/CloudInit, device match) → three-phase hot-unplug: QMP `device_del` (force continues on failure) → QMP `blockdev-del` (abort if fails, preserves state to prevent double-attach) → `ebs.unmount` via NATS (best-effort) → remove from EBSRequests + BlockDeviceMappings → update volume metadata to available → persist state → respond with VolumeAttachment (state=detaching) | 1. Detach with explicit InstanceId<br>2. Detach without InstanceId (gateway resolution)<br>3. Detach with correct --device cross-check<br>4. Missing VolumeId (InvalidParameterValue)<br>5. Volume not attached (IncorrectState)<br>6. Nonexistent volume (InvalidVolume.NotFound)<br>7. Nonexistent instance (InvalidInstanceID.NotFound)<br>8. Instance not running (IncorrectInstanceState)<br>9. Device mismatch (InvalidParameterValue)<br>10. Boot volume protection (OperationNotPermitted)<br>11. Force flag (continues past device_del failure)<br>12. Volume reusability (re-attach after detach) | **DONE** |
+| `describe-volume-status` | `--volume-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes | **DONE** |
+| `describe-volumes-modifications` | — | `--volume-ids`, `--filters`, `--max-results` | None | Query pending/completed volume modifications → return modification state, progress, original/target size | 1. Check in-progress modification<br>2. Check completed modification<br>3. No modifications returns empty | **NOT STARTED** |
+
+## EC2 - Snapshot Management
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-snapshot` | `--volume-id`, `--description`, `--tag-specifications` | `--dry-run` | Volume must exist in Predastore | Gateway validates vol- prefix → NATS `ec2.CreateSnapshot` → daemon reads VolumeConfig from Predastore → generates snap-ID via viperblock → stores SnapshotConfig (metadata-only, points to source volume) as completed → returns Snapshot with state=completed, progress=100% | 1. Snapshot from valid volume<br>2. Missing volume ID (InvalidParameterValue)<br>3. Volume not found (InvalidVolume.NotFound)<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Snapshot with description<br>6. Snapshot with tag specifications | **DONE** |
+| `create-snapshots` | — | `--instance-specification`, `--description`, `--tag-specifications` | Instance must exist, instance-volume attachment tracking | Create snapshots of all volumes attached to instance → return list of snapshot IDs. Blocked: requires instance-volume attachment tracking to discover which volumes to snapshot. | 1. Snapshot all volumes on instance<br>2. Instance with no volumes | **NOT STARTED** |
+| `delete-snapshot` | `--snapshot-id` | `--dry-run` | Snapshot must exist | Gateway validates snap- prefix → NATS `ec2.DeleteSnapshot` → daemon verifies snapshot exists in Predastore → lists and deletes all objects under snapshot prefix → returns success | 1. Delete existing snapshot<br>2. Delete non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing snapshot ID (InvalidParameterValue)<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
+| `describe-snapshots` | `--snapshot-ids` | `--owner-ids`, `--filters`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → filters by snapshot ID if specified → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
+| `copy-snapshot` | `--source-snapshot-id`, `--source-region`, `--description` | `--encrypted`, `--dry-run` | Source snapshot must exist | Gateway validates snap- prefix + source region → NATS `ec2.CopySnapshot` → daemon reads source SnapshotConfig → generates new snap-ID → copies metadata (preserves tags, description override) → stores as completed → returns new snapshot ID | 1. Copy within same region<br>2. Copy non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing source ID (InvalidParameterValue)<br>4. Missing source region (MissingParameter)<br>5. Copy preserves tags<br>6. Copy with description override | **DONE** |
+
+## EC2 - Tags
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-tags` | `--resources`, `--tags` (Key=,Value=) | `--dry-run` | None | Gateway validates input (resources non-empty, tags non-empty, keys non-empty) → NATS `ec2.CreateTags` → daemon merges tags into per-resource JSON in Predastore S3 (`tags/{resourceId}.json`) → return success | 1. Tag an instance<br>2. Tag multiple resources at once<br>3. Overwrite existing tag value<br>4. Empty resources list (error: MissingParameter)<br>5. Empty tags list (error: MissingParameter)<br>6. Empty tag key (error: InvalidParameterValue) | **DONE** |
+| `delete-tags` | `--resources`, `--tags` | `--dry-run` | None | Gateway validates input (resources non-empty) → NATS `ec2.DeleteTags` → daemon removes specified tag keys from resource JSON, or all tags if no keys specified → return success | 1. Delete specific tag by key<br>2. Delete all tags (no tags specified)<br>3. Empty resources list (error: MissingParameter) | **DONE** |
+| `describe-tags` | `--filters` (resource-id, resource-type, key, value) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeTags` → daemon lists all `tags/*.json` from Predastore S3 → applies filters (resource-id, resource-type, key, value) → returns tag list with resource type auto-detected from ID prefix | 1. List all tags<br>2. Filter by resource-id<br>3. Filter by resource-type (instance, volume, image, snapshot, vpc, subnet, security-group, route-table, internet-gateway, egress-only-internet-gateway)<br>4. Filter by key<br>5. Filter by value<br>6. Empty result when no tags exist | **DONE** |
+
+## EC2 - Regions & Availability Zones
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `describe-regions` | *(returns configured region, endpoint, opt-in status from config — input params ignored)* | `--region-names`, `--filters`, `--all-regions`, `--dry-run` | None | Return configured region from hive init config with Endpoint and OptInStatus → local-only response, no NATS | 1. List all regions<br>2. Filter by region name<br>3. Verify endpoint URL returned<br>4. Verify OptInStatus returned | **DONE** |
+| `describe-availability-zones` | *(returns configured AZ, region, zone ID, state, opt-in status from config — input params ignored)* | `--zone-names`, `--filters`, `--all-availability-zones` | None | Return configured AZ from hive init config with zone ID, state, group name, network border group → local-only response, no NATS | 1. List all AZs<br>2. Filter by zone name<br>3. Verify zone state is available<br>4. Verify region name matches config | **DONE** |
+
+## EC2 - Account Attributes
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `describe-account-attributes` | `--attribute-names` | `--dry-run` | None | Gateway parses input → returns static account attributes: supported-platforms=VPC, default-vpc=none, max-instances=100, vpc-max-security-groups-per-interface=5, max-elastic-ips=5, vpc-max-elastic-ips=20 → local-only response, no NATS | 1. List all account attributes<br>2. Filter by attribute name<br>3. Verify all 6 attributes returned with correct values | **DONE** |
+
+## EC2 - Account Settings (Deferred)
+
+Persistence layer works (NATS JetStream KV), but stored values are not yet enforced by downstream services. Will be completed post-merge.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `enable-ebs-encryption-by-default` | — | `--dry-run` | None | Store encryption default flag in JetStream KV → return `EbsEncryptionByDefault=true` | 1. Enable encryption<br>2. Verify via get | **NOT STARTED** (enforcement pending) |
+| `disable-ebs-encryption-by-default` | — | `--dry-run` | None | Clear encryption default flag in JetStream KV → return `EbsEncryptionByDefault=false` | 1. Disable encryption<br>2. Verify via get | **NOT STARTED** (enforcement pending) |
+| `get-ebs-encryption-by-default` | — | `--dry-run` | None | Read encryption default flag from JetStream KV → return current state | 1. Get default state<br>2. Verify matches last enable/disable | **NOT STARTED** (enforcement pending) |
+| `enable-serial-console-access` | — | `--dry-run` | None | Store serial console access flag in JetStream KV → return enabled=true | 1. Enable access<br>2. Verify via get | **NOT STARTED** (enforcement pending) |
+| `disable-serial-console-access` | — | `--dry-run` | None | Clear serial console access flag in JetStream KV → return enabled=false | 1. Disable access | **NOT STARTED** (enforcement pending) |
+| `get-serial-console-access-status` | — | `--dry-run` | None | Read serial console access flag from JetStream KV | 1. Get current status | **NOT STARTED** (enforcement pending) |
+| `enable-snapshot-block-public-access` | — | `--state` | None | Store snapshot block public access state in JetStream KV | 1. Enable with block-all-sharing<br>2. Enable with block-new-sharing | **NOT STARTED** (enforcement pending) |
+| `disable-snapshot-block-public-access` | — | `--dry-run` | None | Clear snapshot block public access state in JetStream KV | 1. Disable access | **NOT STARTED** (enforcement pending) |
+| `get-snapshot-block-public-access-state` | — | `--dry-run` | None | Read snapshot block public access state from JetStream KV | 1. Get current state | **NOT STARTED** (enforcement pending) |
+| `enable-image-block-public-access` | — | `--image-block-public-access-state` | None | Store image block public access state in JetStream KV | 1. Enable with block-new-sharing | **NOT STARTED** (enforcement pending) |
+| `disable-image-block-public-access` | — | `--dry-run` | None | Clear image block public access state in JetStream KV | 1. Disable access | **NOT STARTED** (enforcement pending) |
+| `get-image-block-public-access-state` | — | `--dry-run` | None | Read image block public access state from JetStream KV | 1. Get current state | **NOT STARTED** (enforcement pending) |
+| `modify-instance-metadata-defaults` | — | `--http-tokens`, `--http-put-response-hop-limit`, `--http-endpoint`, `--instance-metadata-tags` | None | Store instance metadata defaults in JetStream KV | 1. Set httpTokens=required<br>2. Set hop limit | **NOT STARTED** (enforcement pending) |
+| `get-instance-metadata-defaults` | — | `--dry-run` | None | Read instance metadata defaults from JetStream KV | 1. Get current defaults | **NOT STARTED** (enforcement pending) |
+
+## EC2 - Egress-Only Internet Gateway
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-egress-only-internet-gateway` | — | `--vpc-id`, `--client-token`, `--tag-specifications`, `--dry-run` | VPC must exist | NATS `ec2.CreateEgressOnlyInternetGateway` → daemon generates eigw-ID → creates record in NATS JetStream KV with state=attached → returns EgressOnlyInternetGateway with attachment | 1. Create with valid VpcId<br>2. Missing VpcId (error: MissingParameter)<br>3. Create with inline tags<br>4. Verify ID format eigw-{16hex} | **NOT STARTED** |
+| `delete-egress-only-internet-gateway` | — | `--egress-only-internet-gateway-id`, `--dry-run` | EIGW must exist | NATS `ec2.DeleteEgressOnlyInternetGateway` → daemon verifies EIGW exists in KV → deletes from KV → returns ReturnCode=true | 1. Delete existing EIGW<br>2. Delete non-existent EIGW (error)<br>3. Missing ID (error: MissingParameter) | **NOT STARTED** |
+| `describe-egress-only-internet-gateways` | — | `--egress-only-internet-gateway-ids`, `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from KV bucket → filters by ID if specified → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list | **NOT STARTED** |
+
+## EC2 - Placement Groups
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-placement-group` | — | `--group-name`, `--strategy` (cluster/spread/partition), `--partition-count`, `--spread-level`, `--tag-specifications` | None | NATS `ec2.CreatePlacementGroup` → daemon stores placement group in JetStream KV with strategy, state=available → return group metadata | 1. Create with cluster strategy<br>2. Create with spread strategy<br>3. Create with partition strategy + partition count<br>4. Duplicate name (error)<br>5. Invalid strategy (error) | **NOT STARTED** |
+| `delete-placement-group` | — | `--group-name`, `--dry-run` | Group must exist, no instances in group | NATS `ec2.DeletePlacementGroup` → daemon verifies group exists and is empty → delete from KV → return success | 1. Delete empty group<br>2. Delete group with instances (error)<br>3. Delete non-existent group (error) | **NOT STARTED** |
+| `describe-placement-groups` | — | `--group-names`, `--group-ids`, `--filters` | None | NATS `ec2.DescribePlacementGroups` → daemon lists groups from KV → return list with strategy and state | 1. List all groups<br>2. Filter by name<br>3. Filter by strategy | **NOT STARTED** |
+
+## EC2 - Dedicated Hosts
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `allocate-hosts` | — | `--availability-zone`, `--instance-type`, `--quantity`, `--auto-placement`, `--tag-specifications` | None | NATS `ec2.AllocateHosts` → daemon allocates host resources → stores in JetStream KV with h-ID → return host ID list | 1. Allocate single host<br>2. Allocate multiple hosts<br>3. Missing AZ or instance type (error) | **NOT STARTED** |
+| `describe-hosts` | — | `--host-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeHosts` → daemon lists hosts from KV → return host list with capacity and instance info | 1. List all hosts<br>2. Filter by host ID<br>3. Filter by instance type | **NOT STARTED** |
+| `release-hosts` | — | `--host-ids` | Host must exist, no running instances | NATS `ec2.ReleaseHosts` → daemon verifies no instances on host → delete from KV → return success/failure per host | 1. Release empty host<br>2. Release host with instances (error)<br>3. Release non-existent host (error) | **NOT STARTED** |
+
+## EC2 - IPv4 Pools
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-public-ipv4-pool` | — | `--tag-specifications`, `--dry-run` | None | NATS `ec2.CreatePublicIpv4Pool` → daemon creates pool in JetStream KV with ipv4pool-ID → return pool ID | 1. Create pool<br>2. Verify in describe | **NOT STARTED** |
+| `delete-public-ipv4-pool` | — | `--pool-id`, `--dry-run` | Pool must exist and be empty | NATS `ec2.DeletePublicIpv4Pool` → daemon verifies pool empty → delete from KV → return success | 1. Delete empty pool<br>2. Delete pool with addresses (error) | **NOT STARTED** |
+| `describe-public-ipv4-pools` | — | `--pool-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribePublicIpv4Pools` → daemon lists pools from KV → return pool list with address counts | 1. List all pools<br>2. Filter by pool ID | **NOT STARTED** |
+
+## EC2 - DHCP Options
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-dhcp-options` | — | `--dhcp-configurations` (domain-name, domain-name-servers, ntp-servers, netbios-name-servers, netbios-node-type), `--tag-specifications` | None | NATS `ec2.CreateDhcpOptions` → daemon stores DHCP option set in JetStream KV with dopt-ID → return DHCP options with configurations | 1. Create with domain-name + DNS servers<br>2. Create with NTP servers<br>3. Verify in describe | **NOT STARTED** |
+| `delete-dhcp-options` | — | `--dhcp-options-id`, `--dry-run` | DHCP options must exist and not be associated with a VPC | NATS `ec2.DeleteDhcpOptions` → daemon verifies not associated → delete from KV → return success | 1. Delete unassociated options<br>2. Delete associated options (error)<br>3. Delete non-existent (error) | **NOT STARTED** |
+| `describe-dhcp-options` | — | `--dhcp-options-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeDhcpOptions` → daemon lists DHCP options from KV → return list with configurations | 1. List all DHCP options<br>2. Filter by ID | **NOT STARTED** |
+| `associate-dhcp-options` | — | `--dhcp-options-id`, `--vpc-id`, `--dry-run` | DHCP options and VPC must exist | NATS `ec2.AssociateDhcpOptions` → daemon links DHCP options to VPC → update VPC metadata → return success | 1. Associate with VPC<br>2. Missing DHCP options ID (error)<br>3. Missing VPC ID (error) | **NOT STARTED** |
+
+## EC2 - Capacity Reservations
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-capacity-reservation` | — | `--instance-type`, `--instance-platform`, `--availability-zone`, `--instance-count`, `--end-date`, `--end-date-type`, `--instance-match-criteria`, `--tag-specifications` | None | NATS `ec2.CreateCapacityReservation` → daemon stores reservation in JetStream KV with cr-ID → return reservation with state=active | 1. Create with required fields<br>2. Missing instance type (error)<br>3. Verify in describe | **NOT STARTED** |
+| `cancel-capacity-reservation` | — | `--capacity-reservation-id`, `--dry-run` | Reservation must exist | NATS `ec2.CancelCapacityReservation` → daemon marks reservation as cancelled in KV → return success | 1. Cancel active reservation<br>2. Cancel non-existent (error) | **NOT STARTED** |
+| `describe-capacity-reservations` | — | `--capacity-reservation-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeCapacityReservations` → daemon lists reservations from KV → return list with state and counts | 1. List all reservations<br>2. Filter by ID<br>3. Filter by instance type | **NOT STARTED** |
+| `modify-capacity-reservation` | — | `--capacity-reservation-id`, `--instance-count`, `--end-date`, `--end-date-type` | Reservation must exist | NATS `ec2.ModifyCapacityReservation` → daemon updates reservation in KV → return success | 1. Modify instance count<br>2. Modify end date<br>3. Missing reservation ID (error) | **NOT STARTED** |
+
+## EC2 - Elastic IP
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `allocate-address` | — | `--domain` (vpc), `--tag-specifications`, `--dry-run` | None | NATS `ec2.AllocateAddress` → daemon allocates IP from pool → stores in JetStream KV with eipalloc-ID → return AllocationId, PublicIp, Domain | 1. Allocate EIP<br>2. Verify in describe-addresses<br>3. Allocate without --domain uses default | **NOT STARTED** |
+| `release-address` | — | `--allocation-id`, `--dry-run` | EIP must exist and be disassociated | NATS `ec2.ReleaseAddress` → daemon verifies not associated → release IP back to pool → delete from KV → return success | 1. Release disassociated EIP<br>2. Release associated EIP (error)<br>3. Release non-existent (error) | **NOT STARTED** |
+| `associate-address` | — | `--allocation-id`, `--instance-id` or `--network-interface-id`, `--dry-run` | EIP and target must exist | NATS `ec2.AssociateAddress` → daemon links EIP to instance/ENI → configure OVS DNAT/SNAT rules → return eipassoc-ID | 1. Associate with instance<br>2. Re-associate (moves EIP)<br>3. Associate non-existent EIP (error) | **NOT STARTED** |
+| `disassociate-address` | — | `--association-id`, `--dry-run` | Association must exist | NATS `ec2.DisassociateAddress` → daemon unlinks EIP from instance/ENI → remove OVS DNAT/SNAT rules → return success | 1. Disassociate existing<br>2. Disassociate non-existent (error) | **NOT STARTED** |
+| `describe-addresses` | — | `--allocation-ids`, `--filters`, `--public-ips` | None | NATS `ec2.DescribeAddresses` → daemon lists EIPs from KV → return list with association info | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP | **NOT STARTED** |
+
+## EC2 - NAT Gateway
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-nat-gateway` | — | `--subnet-id`, `--allocation-id`, `--connectivity-type`, `--tag-specifications` | Subnet must exist, EIP must be allocated (for public type) | NATS `ec2.CreateNatGateway` → daemon creates NAT gateway in JetStream KV with nat-ID → configure OVS SNAT rules → state=pending→available → return NatGateway | 1. Create public NAT gateway<br>2. Create private NAT gateway (no allocation-id)<br>3. Invalid subnet (error) | **NOT STARTED** |
+| `delete-nat-gateway` | — | `--nat-gateway-id`, `--dry-run` | NAT gateway must exist | NATS `ec2.DeleteNatGateway` → daemon marks as deleting → remove OVS SNAT rules → delete from KV → return NatGatewayId | 1. Delete existing NAT gateway<br>2. Delete non-existent (error) | **NOT STARTED** |
+| `describe-nat-gateways` | — | `--nat-gateway-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNatGateways` → daemon lists NAT gateways from KV → return list with state, subnet, addresses | 1. List all NAT gateways<br>2. Filter by ID<br>3. Filter by VPC ID | **NOT STARTED** |
+| `assign-private-nat-gateway-address` | — | `--nat-gateway-id`, `--private-ip-addresses` | NAT gateway must exist (private type) | NATS `ec2.AssignPrivateNatGatewayAddress` → daemon assigns private IPs to NAT gateway → return assigned addresses | 1. Assign address<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
+| `associate-nat-gateway-address` | — | `--nat-gateway-id`, `--allocation-ids` | NAT gateway and EIPs must exist | NATS `ec2.AssociateNatGatewayAddress` → daemon associates EIPs with NAT gateway → return association info | 1. Associate EIP<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
+
+## EC2 - VPC Networking (Core)
+
+Requires Open vSwitch (`apt install openvswitch-switch openvswitch-common`). Single AZ for Hive v1.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-vpc` | — | `--cidr-block`, `--tag-specifications`, `--instance-tenancy` | None | Create OVS bridge for VPC → assign vpc-ID → store VPC metadata in NATS KV → configure CIDR range → return VPC with state=available | 1. Create VPC with /16 CIDR<br>2. Overlapping CIDR (error)<br>3. Invalid CIDR format (error)<br>4. Verify in describe-vpcs | **NOT STARTED** |
+| `delete-vpc` | — | `--vpc-id` | VPC must be empty (no subnets, gateways, instances) | Verify no dependent resources → delete OVS bridge → remove metadata → return success | 1. Delete empty VPC<br>2. Delete VPC with subnets (error: DependencyViolation)<br>3. Delete non-existent VPC (error) | **NOT STARTED** |
+| `describe-vpcs` | — | `--vpc-ids`, `--filters`, `--max-results` | None | Query NATS KV for VPC metadata → return VPC list with CIDR, state, tags | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Filter by CIDR block | **NOT STARTED** |
+| `modify-vpc-attribute` | — | `--vpc-id`, `--enable-dns-support`, `--enable-dns-hostnames` | VPC must exist | NATS `ec2.ModifyVpcAttribute` → daemon updates VPC attribute in KV → return success | 1. Enable DNS support<br>2. Enable DNS hostnames<br>3. Missing VPC ID (error) | **NOT STARTED** |
+| `associate-vpc-cidr-block` | — | `--vpc-id`, `--cidr-block` | VPC must exist | NATS `ec2.AssociateVpcCidrBlock` → daemon adds secondary CIDR to VPC metadata → return association | 1. Add secondary CIDR<br>2. Overlapping CIDR (error)<br>3. Max CIDR blocks exceeded (error) | **NOT STARTED** |
+| `disassociate-vpc-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateVpcCidrBlock` → daemon removes secondary CIDR from VPC → return success | 1. Remove secondary CIDR<br>2. Remove primary CIDR (error) | **NOT STARTED** |
+| `create-subnet` | — | `--vpc-id`, `--cidr-block`, `--availability-zone`, `--tag-specifications` | VPC must exist | Validate CIDR within VPC range → create OVS port group with VLAN tag → assign subnet-ID → store metadata | 1. Create subnet within VPC CIDR<br>2. Subnet CIDR outside VPC range (error)<br>3. Overlapping subnet CIDRs (error) | **NOT STARTED** |
+| `delete-subnet` | — | `--subnet-id` | Subnet must be empty (no instances) | Verify no instances in subnet → remove OVS port group → delete metadata | 1. Delete empty subnet<br>2. Delete subnet with instances (error) | **NOT STARTED** |
+| `describe-subnets` | — | `--subnet-ids`, `--filters`, `--max-results` | None | Query NATS KV for subnet metadata → return subnet list | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by AZ | **NOT STARTED** |
+| `modify-subnet-attribute` | — | `--subnet-id`, `--map-public-ip-on-launch`, `--assign-ipv6-address-on-creation` | Subnet must exist | NATS `ec2.ModifySubnetAttribute` → daemon updates subnet attribute in KV → return success | 1. Enable auto-assign public IP<br>2. Missing subnet ID (error) | **NOT STARTED** |
+| `associate-subnet-cidr-block` | — | `--subnet-id`, `--ipv6-cidr-block` | Subnet must exist | NATS `ec2.AssociateSubnetCidrBlock` → daemon adds IPv6 CIDR to subnet → return association | 1. Add IPv6 CIDR<br>2. Missing subnet ID (error) | **NOT STARTED** |
+| `disassociate-subnet-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateSubnetCidrBlock` → daemon removes IPv6 CIDR from subnet → return success | 1. Remove IPv6 CIDR<br>2. Invalid association (error) | **NOT STARTED** |
+| `create-security-group` | — | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | VPC must exist | Create security group metadata → create default OVS flow rules (deny all inbound, allow all outbound) → assign sg-ID | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error)<br>3. Verify default rules | **NOT STARTED** |
+| `delete-security-group` | — | `--group-id` | SG must not be in use by instances | Verify no instances reference SG → remove OVS flow rules → delete metadata | 1. Delete unused SG<br>2. Delete SG in use (error) | **NOT STARTED** |
+| `describe-security-groups` | — | `--group-ids`, `--group-names`, `--filters`, `--max-results` | None | Query NATS KV for SG metadata → return SG list with rules | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name | **NOT STARTED** |
+| `authorize-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr`, `--source-group` | SG must exist | Add inbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Duplicate rule (idempotent) | **NOT STARTED** |
+| `authorize-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist | Add outbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR | **NOT STARTED** |
+| `revoke-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove inbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
+| `revoke-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove outbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
+| `create-internet-gateway` | — | `--tag-specifications` | None | Create IGW metadata → assign igw-ID → configure OVS NAT bridge for external access | 1. Create IGW<br>2. Verify in describe-internet-gateways | **NOT STARTED** |
+| `attach-internet-gateway` | — | `--internet-gateway-id`, `--vpc-id` | IGW and VPC must exist | Link IGW to VPC → configure OVS flows for internet routing → update metadata | 1. Attach IGW to VPC<br>2. Attach already-attached IGW (error)<br>3. Attach to non-existent VPC (error) | **NOT STARTED** |
+| `detach-internet-gateway` | — | `--internet-gateway-id`, `--vpc-id` | IGW must be attached to VPC | Unlink IGW from VPC → remove OVS internet routing flows | 1. Detach attached IGW<br>2. Detach unattached IGW (error) | **NOT STARTED** |
+| `delete-internet-gateway` | — | `--internet-gateway-id` | IGW must be detached | Verify IGW detached → remove OVS NAT bridge → delete metadata | 1. Delete detached IGW<br>2. Delete attached IGW (error) | **NOT STARTED** |
+| `describe-internet-gateways` | — | `--internet-gateway-ids`, `--filters`, `--max-results` | None | Query NATS KV for IGW metadata → return list with attachment info | 1. List all IGWs<br>2. Filter by attachment VPC | **NOT STARTED** |
+
+## EC2 - Route Tables
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-route-table` | — | `--vpc-id`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateRouteTable` → daemon creates route table in KV with rtb-ID → add default local route for VPC CIDR → return route table | 1. Create route table<br>2. Verify default local route<br>3. Missing VPC ID (error) | **NOT STARTED** |
+| `delete-route-table` | — | `--route-table-id`, `--dry-run` | Route table must exist and not be main route table | NATS `ec2.DeleteRouteTable` → daemon verifies not main table → delete from KV → return success | 1. Delete non-main route table<br>2. Delete main route table (error)<br>3. Delete non-existent (error) | **NOT STARTED** |
+| `describe-route-tables` | — | `--route-table-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeRouteTables` → daemon lists route tables from KV → return tables with routes | 1. List all route tables<br>2. Filter by VPC | **NOT STARTED** |
+| `create-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` or `--egress-only-internet-gateway-id` | Route table and target must exist | NATS `ec2.CreateRoute` → daemon adds route entry → configure OVS flow for destination CIDR → update metadata | 1. Add internet route via IGW<br>2. Add route to NAT gateway<br>3. Add route to EIGW<br>4. Conflicting route (error) | **NOT STARTED** |
+| `delete-route` | — | `--route-table-id`, `--destination-cidr-block` | Route must exist | NATS `ec2.DeleteRoute` → daemon removes route entry → remove OVS flow → update metadata | 1. Delete existing route<br>2. Delete non-existent route (error)<br>3. Delete local route (error) | **NOT STARTED** |
+| `replace-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` | Route must exist | NATS `ec2.ReplaceRoute` → daemon replaces route target → update OVS flow → update metadata | 1. Replace IGW route with NAT<br>2. Replace non-existent route (error) | **NOT STARTED** |
+| `associate-route-table` | — | `--route-table-id`, `--subnet-id` or `--gateway-id` | Route table and target must exist | NATS `ec2.AssociateRouteTable` → daemon links route table to subnet/gateway → return rtbassoc-ID | 1. Associate with subnet<br>2. Missing route table ID (error) | **NOT STARTED** |
+| `disassociate-route-table` | — | `--association-id` | Association must exist, not main table association | NATS `ec2.DisassociateRouteTable` → daemon unlinks route table → return success | 1. Disassociate existing<br>2. Disassociate main table (error) | **NOT STARTED** |
+
+## EC2 - VPC Peering
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-vpc-peering-connection` | — | `--vpc-id`, `--peer-vpc-id`, `--peer-owner-id`, `--peer-region`, `--tag-specifications` | Both VPCs must exist | NATS `ec2.CreateVpcPeeringConnection` → daemon creates peering in KV with pcx-ID → state=pending-acceptance → return peering connection | 1. Create peering between two VPCs<br>2. Missing VPC ID (error)<br>3. Overlapping CIDRs (error) | **NOT STARTED** |
+| `accept-vpc-peering-connection` | — | `--vpc-peering-connection-id`, `--dry-run` | Peering must exist in pending-acceptance state | NATS `ec2.AcceptVpcPeeringConnection` → daemon updates state to active → configure OVS cross-VPC flows → return peering | 1. Accept pending peering<br>2. Accept already active (error) | **NOT STARTED** |
+| `reject-vpc-peering-connection` | — | `--vpc-peering-connection-id`, `--dry-run` | Peering must exist in pending-acceptance state | NATS `ec2.RejectVpcPeeringConnection` → daemon updates state to rejected → return peering | 1. Reject pending peering<br>2. Reject already active (error) | **NOT STARTED** |
+| `delete-vpc-peering-connection` | — | `--vpc-peering-connection-id`, `--dry-run` | Peering must exist | NATS `ec2.DeleteVpcPeeringConnection` → daemon removes OVS cross-VPC flows → delete from KV → return success | 1. Delete active peering<br>2. Delete non-existent (error) | **NOT STARTED** |
+| `describe-vpc-peering-connections` | — | `--vpc-peering-connection-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeVpcPeeringConnections` → daemon lists peerings from KV → return list with state | 1. List all peerings<br>2. Filter by ID<br>3. Filter by VPC | **NOT STARTED** |
+| `modify-vpc-peering-connection-options` | — | `--vpc-peering-connection-id`, `--requester-peering-connection-options`, `--accepter-peering-connection-options` | Peering must be active | NATS `ec2.ModifyVpcPeeringConnectionOptions` → daemon updates peering options in KV → return modified options | 1. Enable DNS resolution<br>2. Missing peering ID (error) | **NOT STARTED** |
+
+## EC2 - VPC Endpoints
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-vpc-endpoint` | — | `--vpc-id`, `--service-name`, `--vpc-endpoint-type` (Gateway/Interface), `--route-table-ids`, `--subnet-ids`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateVpcEndpoint` → daemon creates endpoint in KV with vpce-ID → return endpoint | 1. Create Gateway endpoint for S3<br>2. Create Interface endpoint<br>3. Missing VPC ID (error) | **NOT STARTED** |
+| `delete-vpc-endpoints` | — | `--vpc-endpoint-ids`, `--dry-run` | Endpoints must exist | NATS `ec2.DeleteVpcEndpoints` → daemon deletes endpoints from KV → return success/failure per endpoint | 1. Delete existing endpoint<br>2. Delete non-existent (error) | **NOT STARTED** |
+| `describe-vpc-endpoints` | — | `--vpc-endpoint-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeVpcEndpoints` → daemon lists endpoints from KV → return list with state and type | 1. List all endpoints<br>2. Filter by VPC<br>3. Filter by service name | **NOT STARTED** |
+| `describe-vpc-endpoint-services` | — | `--service-names`, `--filters`, `--max-results` | None | NATS `ec2.DescribeVpcEndpointServices` → daemon returns available services (S3, DynamoDB equivalents) → return service list | 1. List available services<br>2. Filter by service name | **NOT STARTED** |
+| `modify-vpc-endpoint` | — | `--vpc-endpoint-id`, `--add-route-table-ids`, `--remove-route-table-ids`, `--add-subnet-ids`, `--remove-subnet-ids` | Endpoint must exist | NATS `ec2.ModifyVpcEndpoint` → daemon updates endpoint route tables/subnets in KV → return success | 1. Add route table<br>2. Remove subnet<br>3. Missing endpoint ID (error) | **NOT STARTED** |
+
+## EC2 - VPN Gateway & Customer Gateway
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-customer-gateway` | — | `--type` (ipsec.1), `--bgp-asn`, `--ip-address`, `--tag-specifications` | None | NATS `ec2.CreateCustomerGateway` → daemon stores in JetStream KV with cgw-ID → return customer gateway | 1. Create with BGP ASN and IP<br>2. Missing required fields (error) | **NOT STARTED** |
+| `delete-customer-gateway` | — | `--customer-gateway-id`, `--dry-run` | Must exist, no VPN connections | NATS `ec2.DeleteCustomerGateway` → daemon verifies no active connections → delete from KV → return success | 1. Delete existing<br>2. Delete with active VPN (error) | **NOT STARTED** |
+| `describe-customer-gateways` | — | `--customer-gateway-ids`, `--filters` | None | NATS `ec2.DescribeCustomerGateways` → daemon lists from KV → return list with BGP/IP info | 1. List all<br>2. Filter by ID | **NOT STARTED** |
+| `create-vpn-gateway` | — | `--type` (ipsec.1), `--amazon-side-asn`, `--tag-specifications` | None | NATS `ec2.CreateVpnGateway` → daemon stores in JetStream KV with vgw-ID → return VPN gateway | 1. Create VPN gateway<br>2. Verify in describe | **NOT STARTED** |
+| `delete-vpn-gateway` | — | `--vpn-gateway-id`, `--dry-run` | Must be detached | NATS `ec2.DeleteVpnGateway` → daemon verifies detached → delete from KV → return success | 1. Delete detached gateway<br>2. Delete attached (error) | **NOT STARTED** |
+| `attach-vpn-gateway` | — | `--vpn-gateway-id`, `--vpc-id` | Both must exist | NATS `ec2.AttachVpnGateway` → daemon links VPN gateway to VPC → return attachment | 1. Attach to VPC<br>2. Already attached (error) | **NOT STARTED** |
+| `detach-vpn-gateway` | — | `--vpn-gateway-id`, `--vpc-id` | Must be attached | NATS `ec2.DetachVpnGateway` → daemon unlinks VPN gateway from VPC → return success | 1. Detach from VPC<br>2. Not attached (error) | **NOT STARTED** |
+| `describe-vpn-gateways` | — | `--vpn-gateway-ids`, `--filters` | None | NATS `ec2.DescribeVpnGateways` → daemon lists from KV → return list with attachment info | 1. List all<br>2. Filter by VPC attachment | **NOT STARTED** |
+
+## EC2 - VPN Connections
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-vpn-connection` | — | `--type` (ipsec.1), `--customer-gateway-id`, `--vpn-gateway-id`, `--options`, `--tag-specifications` | Customer gateway and VPN gateway must exist | NATS `ec2.CreateVpnConnection` → daemon stores connection in JetStream KV with vpn-ID → return connection with tunnel info | 1. Create IPsec connection<br>2. Missing customer gateway (error)<br>3. Missing VPN gateway (error) | **NOT STARTED** |
+| `delete-vpn-connection` | — | `--vpn-connection-id`, `--dry-run` | Connection must exist | NATS `ec2.DeleteVpnConnection` → daemon deletes from KV → return success | 1. Delete existing connection<br>2. Delete non-existent (error) | **NOT STARTED** |
+| `describe-vpn-connections` | — | `--vpn-connection-ids`, `--filters` | None | NATS `ec2.DescribeVpnConnections` → daemon lists from KV → return list with state and tunnel info | 1. List all connections<br>2. Filter by ID<br>3. Filter by VPN gateway | **NOT STARTED** |
+| `modify-vpn-connection` | — | `--vpn-connection-id`, `--vpn-gateway-id`, `--customer-gateway-id` | Connection must exist | NATS `ec2.ModifyVpnConnection` → daemon updates connection target in KV → return modified connection | 1. Change VPN gateway<br>2. Missing connection ID (error) | **NOT STARTED** |
+| `modify-vpn-connection-options` | — | `--vpn-connection-id`, `--local-ipv4-network-cidr`, `--remote-ipv4-network-cidr` | Connection must exist | NATS `ec2.ModifyVpnConnectionOptions` → daemon updates tunnel options in KV → return modified connection | 1. Modify CIDR ranges<br>2. Missing connection ID (error) | **NOT STARTED** |
+
+## EC2 - Network ACLs
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-network-acl` | — | `--vpc-id`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateNetworkAcl` → daemon creates NACL in KV with acl-ID → add default deny-all rules → return NACL | 1. Create NACL in VPC<br>2. Missing VPC ID (error)<br>3. Verify default deny rules | **NOT STARTED** |
+| `delete-network-acl` | — | `--network-acl-id`, `--dry-run` | Must not be default NACL | NATS `ec2.DeleteNetworkAcl` → daemon verifies not default → delete from KV → return success | 1. Delete non-default NACL<br>2. Delete default NACL (error) | **NOT STARTED** |
+| `describe-network-acls` | — | `--network-acl-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNetworkAcls` → daemon lists NACLs from KV → return list with entries | 1. List all NACLs<br>2. Filter by VPC | **NOT STARTED** |
+| `create-network-acl-entry` | — | `--network-acl-id`, `--rule-number`, `--protocol`, `--rule-action` (allow/deny), `--cidr-block`, `--ingress`/`--egress`, `--port-range` | NACL must exist | NATS `ec2.CreateNetworkAclEntry` → daemon adds rule entry to NACL in KV → configure OVS flow → return success | 1. Add ingress allow rule<br>2. Add egress deny rule<br>3. ICMP type/code parsing<br>4. Missing NACL ID (error) | **NOT STARTED** |
+| `delete-network-acl-entry` | — | `--network-acl-id`, `--rule-number`, `--ingress`/`--egress` | Entry must exist | NATS `ec2.DeleteNetworkAclEntry` → daemon removes rule from NACL → remove OVS flow → return success | 1. Delete existing entry<br>2. Delete non-existent entry (error) | **NOT STARTED** |
+| `replace-network-acl-association` | — | `--association-id`, `--network-acl-id` | Both must exist | NATS `ec2.ReplaceNetworkAclAssociation` → daemon swaps NACL for subnet → return new association ID | 1. Replace association<br>2. Missing association ID (error) | **NOT STARTED** |
+| `replace-network-acl-entry` | — | `--network-acl-id`, `--rule-number`, `--protocol`, `--rule-action`, `--cidr-block`, `--ingress`/`--egress`, `--port-range` | Entry must exist | NATS `ec2.ReplaceNetworkAclEntry` → daemon replaces rule in KV → update OVS flow → return success | 1. Replace existing entry<br>2. Replace non-existent entry (error) | **NOT STARTED** |
+
+## EC2 - Prefix Lists
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-managed-prefix-list` | — | `--prefix-list-name`, `--address-family` (IPv4/IPv6), `--max-entries`, `--entries`, `--tag-specifications` | None | NATS `ec2.CreateManagedPrefixList` → daemon stores prefix list in JetStream KV with pl-ID → return prefix list with version | 1. Create with IPv4 entries<br>2. Name, address family, max entries required<br>3. Verify in describe | **NOT STARTED** |
+| `delete-managed-prefix-list` | — | `--prefix-list-id`, `--dry-run` | Must exist, not referenced by security groups/route tables | NATS `ec2.DeleteManagedPrefixList` → daemon verifies not referenced → delete from KV → return success | 1. Delete unreferenced list<br>2. Delete referenced list (error) | **NOT STARTED** |
+| `describe-managed-prefix-lists` | — | `--prefix-list-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeManagedPrefixLists` → daemon lists from KV → return list with state and version | 1. List all prefix lists<br>2. Filter by ID | **NOT STARTED** |
+| `modify-managed-prefix-list` | — | `--prefix-list-id`, `--current-version`, `--add-entries`, `--remove-entries`, `--prefix-list-name` | Must exist, current version must match | NATS `ec2.ModifyManagedPrefixList` → daemon validates version → apply changes → increment version → return updated list | 1. Add entries<br>2. Remove entries<br>3. Version mismatch (error) | **NOT STARTED** |
+| `get-managed-prefix-list-entries` | — | `--prefix-list-id`, `--target-version`, `--max-results` | Prefix list must exist | NATS `ec2.GetManagedPrefixListEntries` → daemon reads entries from KV → return entry list | 1. List entries<br>2. List entries at specific version | **NOT STARTED** |
+| `get-managed-prefix-list-associations` | — | `--prefix-list-id`, `--max-results` | Prefix list must exist | NATS `ec2.GetManagedPrefixListAssociations` → daemon finds references in SGs/route tables → return association list | 1. List associations<br>2. No associations returns empty | **NOT STARTED** |
+
+## EC2 - Network Interfaces
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-network-interface` | — | `--subnet-id`, `--description`, `--private-ip-address`, `--groups`, `--tag-specifications` | Subnet must exist | NATS `ec2.CreateNetworkInterface` → daemon creates ENI in KV with eni-ID → assign private IP from subnet pool → return ENI | 1. Create in subnet<br>2. Create with specific private IP<br>3. Missing subnet ID (error) | **NOT STARTED** |
+| `delete-network-interface` | — | `--network-interface-id`, `--dry-run` | Must not be attached | NATS `ec2.DeleteNetworkInterface` → daemon verifies detached → release IP → delete from KV → return success | 1. Delete detached ENI<br>2. Delete attached ENI (error) | **NOT STARTED** |
+| `describe-network-interfaces` | — | `--network-interface-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNetworkInterfaces` → daemon lists ENIs from KV → return list with IP/attachment info | 1. List all ENIs<br>2. Filter by subnet<br>3. Filter by attachment | **NOT STARTED** |
+| `attach-network-interface` | — | `--network-interface-id`, `--instance-id`, `--device-index` | ENI and instance must exist | NATS `ec2.AttachNetworkInterface` → daemon attaches OVS port to instance QEMU process → configure MAC/IP → return eni-attach-ID | 1. Attach ENI to instance<br>2. Already attached (error)<br>3. Missing device index (error) | **NOT STARTED** |
+| `detach-network-interface` | — | `--attachment-id`, `--force` | ENI must be attached | NATS `ec2.DetachNetworkInterface` → daemon detaches OVS port from instance → return success | 1. Detach ENI<br>2. Force detach<br>3. Not attached (error) | **NOT STARTED** |
+| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--source-dest-check`, `--groups` | ENI must exist | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon updates ENI attributes in KV → return success | 1. Modify description<br>2. Toggle source/dest check<br>3. Update security groups | **NOT STARTED** |
+| `assign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` or `--secondary-private-ip-address-count` | ENI must exist | NATS `ec2.AssignPrivateIpAddresses` → daemon allocates IPs from subnet pool → assign to ENI → return assigned IPs | 1. Assign specific IPs<br>2. Assign by count<br>3. Cannot specify both (error) | **NOT STARTED** |
+| `unassign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` | IPs must be assigned to ENI | NATS `ec2.UnassignPrivateIpAddresses` → daemon releases IPs → return success | 1. Unassign secondary IP<br>2. Unassign primary IP (error) | **NOT STARTED** |
+
+## EC2 - Launch Templates
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-launch-template` | — | `--launch-template-name`, `--launch-template-data` (ImageId, InstanceType, etc.), `--tag-specifications` | None | NATS `ec2.CreateLaunchTemplate` → daemon stores template in JetStream KV with lt-ID → version 1 → return template metadata | 1. Create template with full config<br>2. Duplicate name (error)<br>3. Verify in describe output | **NOT STARTED** |
+| `create-launch-template-version` | — | `--launch-template-id` or `--launch-template-name`, `--launch-template-data`, `--source-version` | Template must exist | NATS `ec2.CreateLaunchTemplateVersion` → daemon creates new version in KV → return version details | 1. Create version from scratch<br>2. Create version from source version<br>3. Template not found (error) | **NOT STARTED** |
+| `delete-launch-template` | — | `--launch-template-id` or `--launch-template-name`, `--dry-run` | Template must exist | NATS `ec2.DeleteLaunchTemplate` → daemon deletes template and all versions from KV → return deleted template info | 1. Delete by ID<br>2. Delete by name<br>3. Non-existent template (error) | **NOT STARTED** |
+| `describe-launch-templates` | — | `--launch-template-ids`, `--launch-template-names`, `--filters` | None | NATS `ec2.DescribeLaunchTemplates` → daemon lists templates from KV → return list | 1. List all templates<br>2. Filter by name<br>3. Filter by ID | **NOT STARTED** |
+| `describe-launch-template-versions` | — | `--launch-template-id` or `--launch-template-name`, `--versions`, `--min-version`, `--max-version` | Template must exist | NATS `ec2.DescribeLaunchTemplateVersions` → daemon lists versions from KV → return version list with data | 1. List all versions<br>2. List specific version<br>3. Template not found (error) | **NOT STARTED** |
+
+## EC2 - Misc Operations
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `delete-network-interface-permission` | — | `--network-interface-permission-id`, `--force` | Permission must exist | NATS `ec2.DeleteNetworkInterfacePermission` → daemon deletes permission from KV → return success | 1. Delete existing permission<br>2. Force delete | **NOT STARTED** |
+| `enable-address-transfer` | — | `--allocation-id`, `--transfer-account-id` | EIP must exist | NATS `ec2.EnableAddressTransfer` → daemon stores transfer in JetStream KV with 7-day expiration → return AllocationId and TransferAccountId | 1. Enable transfer<br>2. Verify transfer stored with expiration | **NOT STARTED** |
+| `disable-address-transfer` | — | `--allocation-id` | Transfer must exist | NATS `ec2.DisableAddressTransfer` → daemon updates transfer status to disabled in KV → return status | 1. Disable active transfer<br>2. Disable non-existent transfer (error) | **NOT STARTED** |
+
+## EBS Direct API
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `start-snapshot` | — | `--volume-size`, `--parent-snapshot-id`, `--description`, `--encrypted` | None | Initialize snapshot in viperblock → return snap-ID for block-level writes | 1. Start new snapshot<br>2. Start incremental from parent | **NOT STARTED** |
+| `put-snapshot-block` | — | `--snapshot-id`, `--block-index`, `--block-data`, `--checksum` | start-snapshot | Write block data to snapshot at specified index in viperblock → verify checksum | 1. Write single block<br>2. Write with bad checksum (error) | **NOT STARTED** |
+| `get-snapshot-block` | — | `--snapshot-id`, `--block-index` | Snapshot must exist | Read block data from snapshot at specified index → return data with checksum | 1. Read existing block<br>2. Read sparse block (zeros) | **NOT STARTED** |
+| `complete-snapshot` | — | `--snapshot-id`, `--changed-blocks-count` | start-snapshot, put-snapshot-block | Finalize snapshot → mark as completed → sync to Predastore | 1. Complete valid snapshot<br>2. Complete with wrong block count (error) | **NOT STARTED** |
+| `list-snapshot-blocks` | — | `--snapshot-id`, `--max-results`, `--next-token` | Snapshot must exist | List all non-empty blocks in snapshot with tokens and sizes | 1. List blocks of populated snapshot<br>2. List blocks of empty snapshot | **NOT STARTED** |
+| `list-changed-blocks` | — | `--second-snapshot-id`, `--first-snapshot-id`, `--max-results` | Snapshots must exist | Compare two snapshots → return list of differing block indexes | 1. Compare parent and child snapshots<br>2. Compare unrelated snapshots | **NOT STARTED** |
+
+## S3 (via Predastore)
+
+Currently S3 operations are handled directly by Predastore. Consider moving control/data plane to Hive format for AWS gateway integration.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `s3 mb` (CreateBucket) | — | `s3://bucket-name` | None | NATS `s3.createbucket` → Predastore creates bucket → return success | 1. Create valid bucket<br>2. Duplicate bucket name (error)<br>3. Invalid bucket name (error) | **NOT STARTED** (Predastore handles directly) |
+| `s3 rb` (DeleteBucket) | — | `s3://bucket-name`, `--force` | Bucket must exist | NATS `s3.deletebucket` → verify bucket empty (or force) → Predastore deletes bucket | 1. Delete empty bucket<br>2. Delete non-empty bucket (error)<br>3. Force delete non-empty bucket | **NOT STARTED** |
+| `s3 ls` (ListBuckets/Objects) | — | `s3://bucket/prefix`, `--recursive` | None | NATS `s3.listobjects` → Predastore lists objects with prefix → return list | 1. List all buckets<br>2. List objects in bucket<br>3. List with prefix filter | **NOT STARTED** |
+| `s3 cp` (PutObject/GetObject) | — | `source dest`, `--recursive`, `--acl` | Bucket must exist | NATS `s3.putobject`/`s3.getobject` → Predastore stores/retrieves with Reed-Solomon encoding | 1. Upload single file<br>2. Download single file<br>3. Recursive upload/download<br>4. Large file (multipart) | **NOT STARTED** |
+| `s3 rm` (DeleteObject) | — | `s3://bucket/key`, `--recursive` | Object must exist | NATS `s3.deleteobject` → Predastore deletes object shards | 1. Delete single object<br>2. Recursive delete<br>3. Delete non-existent (idempotent) | **NOT STARTED** |
+| `s3 sync` | — | `source dest`, `--delete`, `--exclude`, `--include` | Bucket must exist | Compare source and dest → upload changed/new files → optionally delete removed files | 1. Sync local dir to S3<br>2. Sync with --delete<br>3. Sync with exclude filter | **NOT STARTED** |
+
+## IAM
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-user` | — | `--user-name`, `--path`, `--tags` | None | Store user in JetStream KV → generate AIDA-prefixed UserId → return User with ARN | 1. Create user<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing username (error) | **NOT STARTED** |
+| `get-user` | — | `--user-name` | User must exist | Read user from JetStream KV → return User details | 1. Get existing user<br>2. Get non-existent user (NoSuchEntity) | **NOT STARTED** |
+| `list-users` | — | `--path-prefix`, `--max-items`, `--marker` | None | List users from JetStream KV → return user list | 1. List all users<br>2. List with path prefix<br>3. Empty list | **NOT STARTED** |
+| `delete-user` | — | `--user-name` | User must exist, no access keys/policies | Remove user from JetStream KV → return success | 1. Delete user with no keys<br>2. Delete user with keys (error)<br>3. Delete non-existent (NoSuchEntity) | **NOT STARTED** |
+| `create-access-key` | — | `--user-name` | User must exist | Generate access key ID (AKIA prefix) and secret → store in JetStream KV → return credentials | 1. Create key for user<br>2. Max keys exceeded (error) | **NOT STARTED** (gateway stub exists) |
+| `delete-access-key` | — | `--access-key-id`, `--user-name` | Key must exist | Remove access key from JetStream KV → return success | 1. Delete existing key<br>2. Delete non-existent key (error) | **NOT STARTED** |
+| `list-access-keys` | — | `--user-name`, `--max-items` | None | Query JetStream KV for user's access keys → return list | 1. List keys for user<br>2. No keys returns empty | **NOT STARTED** |
+| `create-policy` | — | `--policy-name`, `--policy-document`, `--path`, `--description`, `--tags` | None | Validate JSON policy document → store in JetStream KV → generate policy ARN → return Policy | 1. Create with valid document<br>2. Invalid JSON (error)<br>3. Duplicate name (EntityAlreadyExists) | **NOT STARTED** |
+| `delete-policy` | — | `--policy-arn` | Policy must exist, not attached to any user | Remove policy from JetStream KV → return success | 1. Delete unattached policy<br>2. Delete attached policy (error) | **NOT STARTED** |
+| `attach-user-policy` | — | `--user-name`, `--policy-arn` | User and policy must exist | Link policy to user in JetStream KV → return success | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent user (NoSuchEntity) | **NOT STARTED** |
+| `detach-user-policy` | — | `--user-name`, `--policy-arn` | User and policy must exist, policy attached | Unlink policy from user in JetStream KV → return success | 1. Detach attached policy<br>2. Not attached (error) | **NOT STARTED** |
+| `list-attached-user-policies` | — | `--user-name`, `--path-prefix`, `--max-items` | User must exist | List policies attached to user from JetStream KV → return policy list | 1. List attached policies<br>2. No policies returns empty | **NOT STARTED** |

docs/DESIGN.md

@@ -0,0 +1,373 @@
+# Hive Architecture
+
+Hive is an AWS-compatible infrastructure platform for bare-metal, edge, and on-premise deployments. It provides EC2-compatible VM orchestration backed by QEMU/KVM, with storage provided by companion projects Viperblock (EBS) and Predastore (S3).
+
+## High-Level Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                              AWS SDK / CLI                                  │
+│                    (aws ec2 run-instances --endpoint-url ...)               │
+└─────────────────────────────────────────────────────────────────────────────┘
+                                      │
+                                      │ HTTPS (SigV4 Auth)
+                                      ▼
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                           AWS Gateway (Port 9999)                           │
+│                                                                             │
+│   • TLS termination                    • AWS SigV4 authentication           │
+│   • EC2 Query Protocol parsing         • Routes to service handlers         │
+│   • XML response generation            • NATS client connection             │
+│                                                                             │
+│   File: hive/services/awsgw/awsgw.go                                        │
+└─────────────────────────────────────────────────────────────────────────────┘
+                                      │
+                                      │ JSON over NATS
+                                      ▼
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                            NATS Message Broker                              │
+│                                                                             │
+│   Topics:                              Queue Groups:                        │
+│   • ec2.RunInstances                   • hive-workers (load balanced)       │
+│   • ec2.DescribeInstances              • No queue (fan-out to all nodes)    │
+│   • ec2.cmd.{instanceID}              (per-instance start/stop/terminate)   │
+│   • ebs.mount / ebs.unmount                                                 │
+└─────────────────────────────────────────────────────────────────────────────┘
+                                      │
+                    ┌─────────────────┼─────────────────┐
+                    ▼                 ▼                 ▼
+┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐
+│   Hive Daemon        │ │   Hive Daemon        │ │   Hive Daemon        │
+│   (Node 1)           │ │   (Node 2)           │ │   (Node N)           │
+│                      │ │                      │ │                      │
+│ • NATS subscriber    │ │ • NATS subscriber    │ │ • NATS subscriber    │
+│ • Resource manager   │ │ • Resource manager   │ │ • Resource manager   │
+│ • QEMU/KVM launcher  │ │ • QEMU/KVM launcher  │ │ • QEMU/KVM launcher  │
+│ • QMP monitoring     │ │ • QMP monitoring     │ │ • QMP monitoring     │
+│                      │ │                      │ │                      │
+│ File: daemon.go      │ │ File: daemon.go      │ │ File: daemon.go      │
+└──────────────────────┘ └──────────────────────┘ └──────────────────────┘
+          │                       │                       │
+          ▼                       ▼                       ▼
+┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐
+│   QEMU/KVM VMs       │ │   QEMU/KVM VMs       │ │   QEMU/KVM VMs       │
+└──────────────────────┘ └──────────────────────┘ └──────────────────────┘
+```
+
+## Request Flow
+
+### 1. AWS SDK Request
+
+Users interact with Hive using standard AWS SDKs or the AWS CLI by pointing to a custom endpoint:
+
+```bash
+aws --endpoint-url https://localhost:9999 --no-verify-ssl ec2 run-instances \
+    --image-id ami-debian12 \
+    --instance-type t3.micro \
+    --key-name my-keypair
+```
+
+The AWS SDK formats this as an HTTP POST with:
+- AWS SigV4 authentication headers
+- EC2 Query Protocol body (Action=RunInstances&ImageId=ami-debian12...)
+
+### 2. AWS Gateway
+
+The gateway (`hive/services/awsgw/awsgw.go:57-107`) is the entry point:
+
+```go
+// awsgw.go:74 - Connect to NATS
+natsConn, err := nats.Connect(nodeConfig.NATS.Host, opts...)
+
+// awsgw.go:89-94 - Create gateway with NATS connection
+gw := gateway.GatewayConfig{
+    NATSConn: natsConn,
+    Config:   nodeConfig.AWSGW.Config,
+}
+
+// awsgw.go:103 - Start TLS listener
+log.Fatal(app.ListenTLS(nodeConfig.AWSGW.Host, nodeConfig.AWSGW.TLSCert, nodeConfig.AWSGW.TLSKey))
+```
+
+Request routing (`hive/gateway/gateway.go:125-156`):
+
+1. **Authentication**: SigV4 middleware validates AWS credentials (`gateway.go:104`)
+2. **Service Detection**: Extracts service name (ec2, iam, account) from Authorization header (`gateway.go:158-186`)
+3. **Action Dispatch**: Routes to service-specific handler (`gateway.go:136-145`)
+
+```go
+// gateway.go:136-145
+switch svc {
+case "ec2":
+    err = gw.EC2_Request(ctx)
+case "account":
+    err = gw.Account_Request(ctx)
+case "iam":
+    err = gw.IAM_Request(ctx)
+}
+```
+
+### 3. EC2 Handler
+
+The EC2 handler (`hive/gateway/ec2.go`) parses the Action parameter and delegates to specific handlers:
+
+```go
+// ec2.go - Action routing
+switch action {
+case "RunInstances":
+    output, err := gateway_ec2_instance.RunInstances(input, gw.NATSConn)
+case "DescribeInstances":
+    output, err := gateway_ec2_instance.DescribeInstances(input, gw.NATSConn, expectedNodes)
+// ... more actions
+}
+```
+
+### 4. NATS Messaging
+
+The gateway communicates with daemons via NATS request/response (`hive/handlers/ec2/instance/service_nats.go:24-57`):
+
+```go
+// service_nats.go:24-57
+func (s *NATSInstanceService) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) {
+    // Marshal input to JSON
+    jsonData, err := json.Marshal(input)
+
+    // Send request to daemon via NATS with 5 minute timeout
+    msg, err := s.natsConn.Request("ec2.RunInstances", jsonData, 5*time.Minute)
+
+    // Unmarshal response
+    var reservation ec2.Reservation
+    err = json.Unmarshal(msg.Data, &reservation)
+
+    return &reservation, nil
+}
+```
+
+### 5. Daemon Processing
+
+Daemons (`hive/daemon/daemon.go`) subscribe to NATS topics and handle requests:
+
+```go
+// daemon.go:319 - Subscribe with queue group for load balancing
+d.natsSubscriptions["ec2.RunInstances"], err = d.natsConn.QueueSubscribe(
+    "ec2.RunInstances",
+    "hive-workers",      // Queue group - only one daemon handles each request
+    d.handleEC2RunInstances,
+)
+
+// daemon.go:373 - Subscribe without queue group for fan-out
+d.natsSubscriptions["ec2.DescribeInstances"], err = d.natsConn.Subscribe(
+    "ec2.DescribeInstances",
+    d.handleEC2DescribeInstances,  // All daemons respond
+)
+```
+
+**Queue Groups**: Topics subscribed with a queue group (`hive-workers`) are load-balanced - only one daemon handles each request. Topics without a queue group fan out to all daemons.
+
+### 6. VM Launch
+
+When a daemon handles `RunInstances`:
+
+1. **Resource Check**: Validates CPU/memory availability (`daemon.go:66-74`)
+2. **Volume Generation**: Creates boot, cloud-init, and EFI volumes via Viperblock
+3. **Volume Mount**: Sends `ebs.mount` request to Viperblock, receives NBD URI
+4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
+5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
+
+```go
+// daemon.go - Volume mounting via NATS
+msg, err := d.natsConn.Request("ebs.mount", ebsMountRequest, 10*time.Second)
+
+// daemon.go - QEMU launch with NBD storage
+cmd := exec.Command("qemu-system-x86_64",
+    "-drive", fmt.Sprintf("file=nbd:%s,format=raw", nbdURI),
+    // ... additional QEMU args
+)
+```
+
+## Key Components
+
+### Daemon Structure
+
+```go
+// daemon.go:76-102
+type Daemon struct {
+    node            string                  // Node identifier
+    clusterConfig   *config.ClusterConfig   // Cluster-wide configuration
+    config          *config.Config          // Node-specific configuration
+    natsConn        *nats.Conn              // NATS connection
+    resourceMgr     *ResourceManager        // CPU/Memory tracking
+    instanceService *InstanceServiceImpl    // EC2 instance operations
+    keyService      *KeyServiceImpl         // SSH key operations
+    imageService    *ImageServiceImpl       // AMI operations
+    Instances       vm.Instances            // Local VMs
+    clusterApp      *fiber.App              // HTTP cluster manager
+}
+```
+
+### Resource Manager
+
+Tracks available and allocated CPU/memory to prevent overcommit:
+
+```go
+// daemon.go:66-74
+type ResourceManager struct {
+    mu            sync.RWMutex
+    availableVCPU int
+    availableMem  float64
+    allocatedVCPU int
+    allocatedMem  float64
+    instanceTypes map[string]InstanceType  // t3.micro, t3.small, etc.
+}
+```
+
+### NATS Topics
+
+| Topic | Queue Group | Purpose |
+|-------|-------------|---------|
+| `ec2.RunInstances` | `hive-workers` | Launch new instances |
+| `ec2.DescribeInstances` | None (fan-out) | Query all nodes for instances |
+| `ec2.cmd.{instanceID}` | None (owner only) | Per-instance commands (start/stop/terminate, attach/detach volume) |
+| `ec2.CreateKeyPair` | `hive-workers` | Generate SSH keypair |
+| `ec2.DescribeKeyPairs` | `hive-workers` | List SSH keypairs |
+| `ec2.DescribeImages` | `hive-workers` | List AMIs |
+| `ebs.mount` | - | Mount volume via Viperblock |
+| `ebs.unmount` | - | Unmount volume |
+| `hive.admin.{node}.health` | None | Node health checks |
+
+### Multi-Node Aggregation
+
+For operations that need data from all nodes (like `DescribeInstances`), the gateway uses inbox-based fan-out (`hive/gateway/ec2/instance/DescribeInstances.go`):
+
+```go
+// DescribeInstances.go:16-111
+func DescribeInstances(...) {
+    // Create unique inbox for collecting responses
+    inbox := nats.NewInbox()
+    sub, err := natsConn.SubscribeSync(inbox)
+
+    // Publish to all nodes (no queue group = all daemons respond)
+    err = natsConn.PublishRequest("ec2.DescribeInstances", inbox, jsonData)
+
+    // Collect responses from all nodes with 3-second timeout
+    for {
+        msg, err := sub.NextMsg(3 * time.Second)
+        allReservations = append(allReservations, nodeOutput.Reservations...)
+    }
+
+    return allReservations
+}
+```
+
+## Storage Integration
+
+### Viperblock (EBS)
+
+Block storage volumes are managed via NATS messages to Viperblock:
+
+```go
+// Volume mount request
+type EBSRequest struct {
+    Name      string  // Volume name (vol-xxxxx)
+    VolType   string  // gp2, io1, etc.
+    Boot      bool    // Boot volume flag
+    EFI       bool    // EFI boot volume
+    CloudInit bool    // Cloud-init volume
+    NBDURI    string  // NBD URI returned from mount
+}
+```
+
+Viperblock responds with an NBD URI that QEMU uses to access the block device.
+
+### Predastore (S3)
+
+Object storage used for:
+- AMI image storage
+- SSH public key storage (`/bucket/ec2/{key-name}.pub`)
+- Volume metadata and configuration
+
+## Configuration
+
+Cluster configuration (`hive/config/config.go`):
+
+```go
+type ClusterConfig struct {
+    Epoch   uint64            // Incremented on config changes
+    Node    string            // This node's identifier
+    Version string            // Hive version
+    Nodes   map[string]Config // Configuration for all nodes
+}
+
+type Config struct {
+    Node, Host, Region, AZ string
+    DataDir    string
+    Daemon     DaemonConfig     // Cluster manager port
+    NATS       NATSConfig       // NATS broker address
+    Predastore PredastoreConfig // S3 endpoint
+    AWSGW      AWSGWConfig      // Gateway port, TLS certs
+    AccessKey, SecretKey string
+}
+```
+
+## File Structure
+
+```
+hive/
+├── hive/
+│   ├── services/
+│   │   └── awsgw/
+│   │       └── awsgw.go        # AWS Gateway service entry point
+│   ├── gateway/
+│   │   ├── gateway.go          # Request routing, auth middleware
+│   │   ├── ec2.go              # EC2 action dispatcher
+│   │   └── ec2/
+│   │       └── instance/
+│   │           ├── RunInstances.go
+│   │           └── DescribeInstances.go
+│   ├── handlers/
+│   │   └── ec2/
+│   │       ├── instance/
+│   │       │   ├── service_nats.go  # NATS request/response
+│   │       │   └── service_impl.go  # Business logic
+│   │       ├── key/
+│   │       └── image/
+│   ├── daemon/
+│   │   └── daemon.go           # Daemon entry point, NATS subscriptions
+│   ├── config/
+│   │   └── config.go           # Configuration structures
+│   ├── vm/
+│   │   └── instance.go         # VM instance representation
+│   └── qmp/
+│       └── qmp.go              # QEMU Machine Protocol client
+└── cmd/
+    └── hive/
+        └── main.go             # CLI entry point
+```
+
+## Development
+
+### Running the Stack
+
+1. Start NATS broker
+2. Start Predastore (S3)
+3. Start Viperblock (EBS)
+4. Start Hive Gateway (`hive awsgw`)
+5. Start Hive Daemon(s) (`hive daemon`)
+
+### Testing with AWS CLI
+
+```bash
+# Set endpoint
+export AWS_ENDPOINT_URL=https://localhost:9999
+
+# Run instance
+aws --no-verify-ssl ec2 run-instances \
+    --image-id ami-debian12 \
+    --instance-type t3.micro
+
+# List instances (queries all nodes)
+aws --no-verify-ssl ec2 describe-instances
+
+# Terminate instance
+aws --no-verify-ssl ec2 terminate-instances --instance-ids i-xxxxx
+```

go.mod

@@ -1,22 +1,25 @@
 module github.com/mulgadc/hive
 
-go 1.25
+go 1.26.0
 
 require (
+	github.com/NYTimes/gziphandler v1.1.1
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/davecgh/go-spew v1.1.1
 	github.com/gofiber/fiber/v2 v2.52.9
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
-	github.com/mulgadc/predastore v0.1.5
-	github.com/mulgadc/viperblock v0.1.1
+	github.com/klauspost/cpuid/v2 v2.3.0
+	github.com/mulgadc/predastore v1.0.1
+	github.com/mulgadc/viperblock v1.0.15
 	github.com/nats-io/nats-server/v2 v2.11.6
 	github.com/nats-io/nats.go v1.43.0
+	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/pterm/pterm v0.12.82
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1
-	github.com/stretchr/testify v1.10.0
+	github.com/stretchr/testify v1.11.1
 	go.uber.org/automaxprocs v1.6.0
+	golang.org/x/net v0.47.0
 	gopkg.in/ini.v1 v1.67.0
 )
 
@@ -24,43 +27,108 @@ require (
 	atomicgo.dev/cursor v0.2.0 // indirect
 	atomicgo.dev/keyboard v0.2.9 // indirect
 	atomicgo.dev/schedule v0.1.0 // indirect
+	cloud.google.com/go v0.121.2 // indirect
+	cloud.google.com/go/auth v0.16.5 // indirect
+	cloud.google.com/go/compute/metadata v0.8.0 // indirect
+	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/anthropics/anthropic-sdk-go v1.19.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/buraksezer/consistent v0.10.0 // indirect
+	github.com/ccojocar/zxcvbn-go v1.0.4 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/console v1.0.5 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgraph-io/badger/v4 v4.8.0 // indirect
+	github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/go-chi/chi/v5 v5.2.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/google/flatbuffers v25.2.10+incompatible // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/go-tpm v0.9.5 // indirect
-	github.com/gookit/color v1.5.4 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/gookit/color v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/hashicorp/go-hclog v1.6.2 // indirect
+	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
+	github.com/hashicorp/go-metrics v0.5.4 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/hashicorp/raft v1.7.3 // indirect
+	github.com/hashicorp/raft-boltdb/v2 v2.3.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/reedsolomon v1.12.6 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/highwayhash v1.0.3 // indirect
 	github.com/nats-io/jwt/v2 v2.7.4 // indirect
 	github.com/nats-io/nkeys v0.4.11 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/openai/openai-go/v3 v3.8.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/quic-go/quic-go v0.57.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/securego/gosec/v2 v2.22.11 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.12.0 // indirect
 	github.com/spf13/cast v1.7.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.57.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.etcd.io/bbolt v1.3.5 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.39.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/text v0.26.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
+	golang.org/x/mod v0.30.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 // indirect
+	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
+	golang.org/x/tools v0.39.0 // indirect
+	golang.org/x/vuln v1.1.4 // indirect
+	google.golang.org/genai v1.37.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
+	google.golang.org/grpc v1.75.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	honnef.co/go/tools v0.6.1 // indirect
+)
+
+tool (
+	github.com/securego/gosec/v2/cmd/gosec
+	golang.org/x/vuln/cmd/govulncheck
+	honnef.co/go/tools/cmd/staticcheck
 )

go.sum

@@ -6,6 +6,16 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
 atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
 atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
 atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.121.2 h1:v2qQpN6Dx9x2NmwrqlesOt3Ys4ol5/lFZ6Mg1B7OJCg=
+cloud.google.com/go v0.121.2/go.mod h1:nRFlrHq39MNVWu+zESP2PosMWA0ryJw8KUBZ2iZpxbw=
+cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI=
+cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ=
+cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA=
+cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw=
+github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
+github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
 github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
 github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII=
@@ -15,13 +25,82 @@ github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/
 github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE=
 github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4=
 github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY=
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/anthropics/anthropic-sdk-go v1.19.0 h1:mO6E+ffSzLRvR/YUH9KJC0uGw0uV8GjISIuzem//3KE=
+github.com/anthropics/anthropic-sdk-go v1.19.0/go.mod h1:WTz31rIUHUHqai2UslPpw5CwXrQP3geYBioRV4WOLvE=
 github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op h1:+OSa/t11TFhqfrX0EOSqQBDJ0YlpmK0rDSiB19dg9M0=
 github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
+github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
+github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
+github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
+github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
+github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk=
+github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.5/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCdYW9ltGQYrFWg8XdkGDd8=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso=
+github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
+github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU=
+github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
+github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc=
+github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
+github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc=
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
@@ -29,32 +108,126 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/badger/v4 v4.8.0 h1:JYph1ChBijCw8SLeybvPINizbDKWZ5n/GYbz2yhN/bs=
+github.com/dgraph-io/badger/v4 v4.8.0/go.mod h1:U6on6e8k/RTbUWxqKR0MvugJuVmkxSNc79ap4917h4w=
+github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=
+github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
+github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
+github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
+github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
 github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
-github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
+github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw=
 github.com/gofiber/fiber/v2 v2.52.9/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
+github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
+github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU=
 github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY=
+github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
+github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
+github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
+github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
+github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
+github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
+github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
+github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
 github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo=
-github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
-github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
+github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
+github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I=
+github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY=
+github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI=
+github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
+github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0=
+github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4=
+github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
+github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hashicorp/raft v1.7.3 h1:DxpEqZJysHN0wK+fviai5mFcSYsCkNpFUl1xpAW8Rbo=
+github.com/hashicorp/raft v1.7.3/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ=
+github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ=
+github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
+github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
+github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
 github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -62,8 +235,13 @@ github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYW
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
-github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
-github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/klauspost/reedsolomon v1.12.6 h1:8pqE9aECQG/ZFitiUD1xK/E83zwosBAZtE3UbuZM8TQ=
+github.com/klauspost/reedsolomon v1.12.6/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -73,20 +251,33 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
 github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4=
+github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
+github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
 github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q=
 github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
-github.com/mulgadc/predastore v0.1.5 h1:7SySkQQ5MQ1NTcKcB14pCUIJipq0YVgCJZeKM6ytrLM=
-github.com/mulgadc/predastore v0.1.5/go.mod h1:BhMymRV3zMTu2S+5pJnNDDm5+AGakjhSc/PKX0q8Xfg=
-github.com/mulgadc/viperblock v0.1.1 h1:R1DVX1S+/xPPMQ4LIFkwIX7Es+RS3Wgdd8bsWLTEvgY=
-github.com/mulgadc/viperblock v0.1.1/go.mod h1:zCjdQ76fi2Qs2a5ign9A/Uqgbz77xts5WV0OR1be48I=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mulgadc/predastore v1.0.1 h1:Qohnnn3a2VxLI41GABHPxcV6cTAv009+mfhzFZ91RTc=
+github.com/mulgadc/predastore v1.0.1/go.mod h1:/ktr/gfFMECGCKFiz9Ng0DgPii/BhaGNjDxaIUgBEd0=
+github.com/mulgadc/viperblock v1.0.15 h1:QZ+oV4JE2uPxaK+XtcIr1RelAZHwTW00Ix7/ZNGsXpI=
+github.com/mulgadc/viperblock v1.0.15/go.mod h1:dV1eRKaIjH8YfuCk2Bd4hxKd3Rx5mWfO6gelJ9SbbeY=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI=
 github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
 github.com/nats-io/nats-server/v2 v2.11.6 h1:4VXRjbTUFKEB+7UoaKL3F5Y83xC7MxPoIONOnGgpkHw=
@@ -97,12 +288,40 @@ github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
 github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
+github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
+github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
+github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
+github.com/openai/openai-go/v3 v3.8.1 h1:b+YWsmwqXnbpSHWQEntZAkKciBZ5CJXwL68j+l59UDg=
+github.com/openai/openai-go/v3 v3.8.1/go.mod h1:UOpNxkqC9OdNXNUfpNByKOtB4jAL0EssQXq5p8gO0Xs=
+github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
+github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
 github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
 github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
 github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE=
@@ -112,18 +331,29 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b
 github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
 github.com/pterm/pterm v0.12.82 h1:+D9wYhCaeaK0FIQoZtqbNQuNpe2lB2tajKKsTd5paVQ=
 github.com/pterm/pterm v0.12.82/go.mod h1:TyuyrPjnxfwP+ccJdBTeWHtd/e0ybQHkOS/TakajZCw=
+github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10=
+github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
 github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
+github.com/securego/gosec/v2 v2.22.11 h1:tW+weM/hCM/GX3iaCV91d5I6hqaRT2TPsFM1+USPXwg=
+github.com/securego/gosec/v2 v2.22.11/go.mod h1:KE4MW/eH0GLWztkbt4/7XpyH0zJBBnu7sYB4l6Wn7Mw=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
 github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
 github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
@@ -135,14 +365,30 @@ github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
 github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=
@@ -155,34 +401,91 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
+go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
+go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
+go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
+go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
+go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
+go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
+go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
+go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
 go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
 go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
 go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
-golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
-golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
-golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
+golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
+golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -190,39 +493,77 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo=
+golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
 golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
+golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
+golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
+golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
+golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8=
+golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I=
+golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
+gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genai v1.37.0 h1:dgp71k1wQ+/+APdZrN3LFgAGnVnr5IdTF1Oj0Dg+BQc=
+google.golang.org/genai v1.37.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo=
+google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
+google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI=
+honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4=

hive/admin/admin.go

@@ -0,0 +1,651 @@
+package admin
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	toml "github.com/pelletier/go-toml/v2"
+	"gopkg.in/ini.v1"
+)
+
+// RemoteNode holds basic info about a remote cluster node for config generation.
+type RemoteNode struct {
+	Name     string
+	Host     string
+	Region   string
+	AZ       string
+	Services []string
+}
+
+type ConfigSettings struct {
+	AccessKey string
+	SecretKey string
+	AccountID string
+	Region    string
+	NatsToken string
+	DataDir   string
+
+	// Add more fields as needed
+	Node   string
+	Az     string
+	Port   string
+	BindIP string
+
+	// Cluster settings
+	ClusterBindIP string
+	ClusterRoutes []string
+	ClusterName   string
+
+	// Predastore multi-node
+	PredastoreNodeID int
+
+	// Node capabilities
+	Services []string
+
+	// Other nodes in the cluster (for config source of truth)
+	RemoteNodes []RemoteNode
+}
+
+// PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
+type PredastoreNodeConfig struct {
+	ID   int
+	Host string
+}
+
+type ConfigFile struct {
+	Name     string
+	Path     string
+	Template string
+}
+
+func GenerateConfigFiles(configs []ConfigFile, configSettings ConfigSettings) error {
+
+	for _, cfg := range configs {
+		if err := GenerateConfigFile(cfg.Path, cfg.Template, configSettings); err != nil {
+			return fmt.Errorf("error creating %s: %v", cfg.Name, err)
+		}
+		fmt.Printf("✅ Created: %s\n", cfg.Name)
+	}
+
+	return nil
+}
+
+// generateConfigFile creates a configuration file from a template
+func GenerateConfigFile(configPath string, configTemplate string, configSettings ConfigSettings) error {
+	// Parse the embedded template
+	tmpl, err := template.New("config").Parse(configTemplate)
+	if err != nil {
+		return fmt.Errorf("failed to parse template: %w", err)
+	}
+
+	// Create file with secure permissions
+	f, err := os.OpenFile(configPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to create config file: %w", err)
+	}
+	defer f.Close()
+
+	// Execute template
+	if err := tmpl.Execute(f, configSettings); err != nil {
+		return fmt.Errorf("failed to execute template: %w", err)
+	}
+
+	return nil
+}
+
+func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (caCertPath string) {
+	// Certificate paths
+	caCertPath = filepath.Join(configDir, "ca.pem")
+	caKeyPath := filepath.Join(configDir, "ca.key")
+	serverCertPath := filepath.Join(configDir, "server.pem")
+	serverKeyPath := filepath.Join(configDir, "server.key")
+
+	// Check if we need to generate certificates
+	needsGeneration := force ||
+		!FileExists(caCertPath) || !FileExists(caKeyPath) ||
+		!FileExists(serverCertPath) || !FileExists(serverKeyPath)
+
+	if needsGeneration {
+		fmt.Println("\n🔐 Generating Certificate Authority and SSL certificates...")
+
+		// Step 1: Generate CA certificate
+		if err := GenerateCACert(caCertPath, caKeyPath); err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating CA certificate: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ CA certificate generated:\n")
+		fmt.Printf("   CA Certificate: %s\n", caCertPath)
+		fmt.Printf("   CA Key: %s\n", caKeyPath)
+
+		// Step 2: Generate server certificate signed by CA (with bind IP in SANs)
+		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP); err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Server certificate generated (signed by CA):\n")
+		fmt.Printf("   Certificate: %s\n", serverCertPath)
+		fmt.Printf("   Key: %s\n", serverKeyPath)
+
+		// Print instructions for adding CA to system trust store
+		fmt.Println("\n📋 To trust the Hive CA system-wide (recommended):")
+		fmt.Printf("   sudo cp %s /usr/local/share/ca-certificates/hive-ca.crt\n", caCertPath)
+		fmt.Println("   sudo update-ca-certificates")
+		fmt.Println("\n   This allows AWS CLI and other tools to trust Hive services automatically.")
+	} else {
+		fmt.Println("\n✅ CA and SSL certificates already exist")
+	}
+
+	return caCertPath
+}
+
+// GenerateServerCertOnly generates a server certificate signed by an existing CA.
+// Used by joining nodes that receive the CA from the leader.
+func GenerateServerCertOnly(configDir string, bindIP string) error {
+	caCertPath := filepath.Join(configDir, "ca.pem")
+	caKeyPath := filepath.Join(configDir, "ca.key")
+	serverCertPath := filepath.Join(configDir, "server.pem")
+	serverKeyPath := filepath.Join(configDir, "server.key")
+
+	// Verify CA files exist
+	if !FileExists(caCertPath) || !FileExists(caKeyPath) {
+		return fmt.Errorf("CA files not found in %s", configDir)
+	}
+
+	// Generate server cert signed by CA with this node's bind IP
+	return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP)
+}
+
+func CreateServiceDirectories(hiveRoot string) {
+
+	// Create additional directories
+	dirs := []string{
+		filepath.Join(hiveRoot, "images"),
+		filepath.Join(hiveRoot, "amis"),
+		filepath.Join(hiveRoot, "volumes"),
+		filepath.Join(hiveRoot, "state"),
+		filepath.Join(hiveRoot, "logs"),
+		filepath.Join(hiveRoot, "nats"),
+		filepath.Join(hiveRoot, "predastore"),
+		filepath.Join(hiveRoot, "viperblock"),
+		filepath.Join(hiveRoot, "hive"),
+	}
+
+	fmt.Println("\n📁 Creating directory structure...")
+	for _, dir := range dirs {
+
+		// Check if directory exists
+		if _, err := os.Stat(dir); os.IsNotExist(err) {
+			if err := os.MkdirAll(dir, 0750); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Could not create %s: %v\n", dir, err)
+			}
+		}
+	}
+	fmt.Printf("✅ Directory structure created in %s\n", hiveRoot)
+
+}
+
+// Helper functions
+
+func FileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+// updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs
+func UpdateAWSINIFile(path, section string, values map[string]string) error {
+	var cfg *ini.File
+	var err error
+
+	// Load existing file or create new one
+	if FileExists(path) {
+		cfg, err = ini.Load(path)
+		if err != nil {
+			return fmt.Errorf("failed to load INI file: %w", err)
+		}
+	} else {
+		cfg = ini.Empty()
+	}
+
+	// Get or create section
+	sec, err := cfg.NewSection(section)
+	if err != nil {
+		// Section already exists, get it
+		sec, err = cfg.GetSection(section)
+		if err != nil {
+			return fmt.Errorf("failed to get section: %w", err)
+		}
+	}
+
+	// Set key-value pairs
+	for key, value := range values {
+		sec.Key(key).SetValue(value)
+	}
+
+	// Save with proper permissions
+	return cfg.SaveTo(path)
+}
+
+// generateAWSAccessKey generates an AWS-style access key
+// Format: AKIA + 16 random uppercase alphanumeric characters
+func GenerateAWSAccessKey() string {
+	const prefix = "AKIA"
+	const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	const length = 16
+
+	result := make([]byte, length)
+	for i := range result {
+		num, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charset))))
+		result[i] = charset[num.Int64()]
+	}
+
+	return prefix + string(result)
+}
+
+// generateAWSSecretKey generates an AWS-style secret key
+// 40 character base64-encoded string
+func GenerateAWSSecretKey() string {
+	bytes := make([]byte, 30) // 30 bytes = 40 chars in base64
+	if _, err := rand.Read(bytes); err != nil {
+		panic(err)
+	}
+	return base64.StdEncoding.EncodeToString(bytes)
+}
+
+// generateAccountID generates a 12-digit AWS account ID
+func GenerateAccountID() string {
+	// Generate random 12-digit number
+	num, _ := rand.Int(rand.Reader, big.NewInt(900000000000))
+	accountID := num.Int64() + 100000000000 // Ensure it's 12 digits
+	return fmt.Sprintf("%012d", accountID)
+}
+
+// generateNATSToken generates a secure random token for NATS
+func GenerateNATSToken() string {
+	bytes := make([]byte, 32)
+	if _, err := rand.Read(bytes); err != nil {
+		panic(err)
+	}
+	return "nats_" + base64.URLEncoding.EncodeToString(bytes)[:32]
+}
+
+// GenerateCACert generates a Certificate Authority certificate and key
+func GenerateCACert(caCertPath, caKeyPath string) error {
+	// Generate CA private key
+	caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return fmt.Errorf("failed to generate CA private key: %w", err)
+	}
+
+	// Create CA certificate template
+	notBefore := time.Now()
+	notAfter := notBefore.Add(3650 * 24 * time.Hour) // 10 years
+
+	serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %w", err)
+	}
+
+	caTemplate := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "Hive Local CA",
+			Organization: []string{"Hive Platform"},
+		},
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		MaxPathLen:            1,
+	}
+
+	// Self-sign the CA certificate
+	caDerBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caPrivateKey.PublicKey, caPrivateKey)
+	if err != nil {
+		return fmt.Errorf("failed to create CA certificate: %w", err)
+	}
+
+	// Write CA certificate to file
+	caCertOut, err := os.Create(caCertPath)
+	if err != nil {
+		return fmt.Errorf("failed to create CA cert file: %w", err)
+	}
+	defer caCertOut.Close()
+
+	if err := pem.Encode(caCertOut, &pem.Block{Type: "CERTIFICATE", Bytes: caDerBytes}); err != nil {
+		return fmt.Errorf("failed to write CA cert: %w", err)
+	}
+
+	// Write CA private key to file
+	caKeyOut, err := os.OpenFile(caKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to create CA key file: %w", err)
+	}
+	defer caKeyOut.Close()
+
+	caPrivBytes, err := x509.MarshalPKCS8PrivateKey(caPrivateKey)
+	if err != nil {
+		return fmt.Errorf("failed to marshal CA private key: %w", err)
+	}
+
+	if err := pem.Encode(caKeyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: caPrivBytes}); err != nil {
+		return fmt.Errorf("failed to write CA key: %w", err)
+	}
+
+	return nil
+}
+
+// GenerateSignedCert generates a server certificate signed by the CA.
+// extraIPs are additional IP addresses to include in the certificate's SANs.
+func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error {
+	// Load CA certificate
+	caCertPEM, err := os.ReadFile(caCertPath)
+	if err != nil {
+		return fmt.Errorf("failed to read CA cert: %w", err)
+	}
+
+	caCertBlock, _ := pem.Decode(caCertPEM)
+	if caCertBlock == nil {
+		return fmt.Errorf("failed to decode CA cert PEM")
+	}
+
+	caCert, err := x509.ParseCertificate(caCertBlock.Bytes)
+	if err != nil {
+		return fmt.Errorf("failed to parse CA cert: %w", err)
+	}
+
+	// Load CA private key
+	caKeyPEM, err := os.ReadFile(caKeyPath)
+	if err != nil {
+		return fmt.Errorf("failed to read CA key: %w", err)
+	}
+
+	caKeyBlock, _ := pem.Decode(caKeyPEM)
+	if caKeyBlock == nil {
+		return fmt.Errorf("failed to decode CA key PEM")
+	}
+
+	caPrivateKey, err := x509.ParsePKCS8PrivateKey(caKeyBlock.Bytes)
+	if err != nil {
+		return fmt.Errorf("failed to parse CA private key: %w", err)
+	}
+
+	caRSAKey, ok := caPrivateKey.(*rsa.PrivateKey)
+	if !ok {
+		return fmt.Errorf("CA key is not RSA")
+	}
+
+	// Generate server private key
+	serverPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return fmt.Errorf("failed to generate server private key: %w", err)
+	}
+
+	// Create server certificate template
+	notBefore := time.Now()
+	notAfter := notBefore.Add(365 * 24 * time.Hour) // 1 year for server certs
+
+	serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %w", err)
+	}
+
+	// Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node)
+	ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
+	for _, ip := range extraIPs {
+		if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" {
+			if parsed := net.ParseIP(ip); parsed != nil {
+				ipAddresses = append(ipAddresses, parsed)
+			}
+		}
+	}
+
+	serverTemplate := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "localhost",
+			Organization: []string{"Hive Platform"},
+		},
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              []string{"localhost"},
+		IPAddresses:           ipAddresses,
+	}
+
+	// Sign the server certificate with the CA
+	serverDerBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, caCert, &serverPrivateKey.PublicKey, caRSAKey)
+	if err != nil {
+		return fmt.Errorf("failed to create server certificate: %w", err)
+	}
+
+	// Write server certificate to file
+	certOut, err := os.Create(certPath)
+	if err != nil {
+		return fmt.Errorf("failed to create cert file: %w", err)
+	}
+	defer certOut.Close()
+
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: serverDerBytes}); err != nil {
+		return fmt.Errorf("failed to write cert: %w", err)
+	}
+
+	// Write server private key to file
+	keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to create key file: %w", err)
+	}
+	defer keyOut.Close()
+
+	privBytes, err := x509.MarshalPKCS8PrivateKey(serverPrivateKey)
+	if err != nil {
+		return fmt.Errorf("failed to marshal private key: %w", err)
+	}
+
+	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
+		return fmt.Errorf("failed to write key: %w", err)
+	}
+
+	return nil
+}
+
+// generateSelfSignedCert generates a self-signed SSL certificate (legacy, kept for compatibility)
+func GenerateSelfSignedCert(certPath, keyPath string) error {
+	// Generate private key
+	privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return fmt.Errorf("failed to generate private key: %w", err)
+	}
+
+	// Create certificate template
+	notBefore := time.Now()
+	notAfter := notBefore.Add(3650 * 24 * time.Hour) // 10 years
+
+	serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %w", err)
+	}
+
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "localhost",
+			Organization: []string{"Hive Platform"},
+		},
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              []string{"localhost"},
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
+	}
+
+	// Create certificate
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		return fmt.Errorf("failed to create certificate: %w", err)
+	}
+
+	// Write certificate to file
+	certOut, err := os.Create(certPath)
+	if err != nil {
+		return fmt.Errorf("failed to create cert file: %w", err)
+	}
+	defer certOut.Close()
+
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return fmt.Errorf("failed to write cert: %w", err)
+	}
+
+	// Write private key to file
+	keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to create key file: %w", err)
+	}
+	defer keyOut.Close()
+
+	privBytes, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		return fmt.Errorf("failed to marshal private key: %w", err)
+	}
+
+	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
+		return fmt.Errorf("failed to write key: %w", err)
+	}
+
+	return nil
+}
+
+// SetupAWSCredentials updates ~/.aws/credentials and ~/.aws/config.
+// bindIP is the IP the AWS gateway listens on. If empty or "0.0.0.0", defaults to "localhost".
+func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string) error {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return err
+	}
+
+	awsDir := filepath.Join(homeDir, ".aws")
+	if err := os.MkdirAll(awsDir, 0700); err != nil {
+		return err
+	}
+
+	credPath := filepath.Join(awsDir, "credentials")
+	configPath := filepath.Join(awsDir, "config")
+
+	// Determine profile name
+	//profileName := "default"
+
+	// Use hive as the default profile
+	profileName := "hive"
+
+	if FileExists(credPath) {
+		// Check if default profile already exists
+		cfg, err := ini.Load(credPath)
+		if err == nil && cfg.HasSection("default") {
+			profileName = "hive"
+		}
+	}
+
+	// Update credentials file
+	if err := UpdateAWSINIFile(credPath, profileName, map[string]string{
+		"aws_access_key_id":     accessKey,
+		"aws_secret_access_key": secretKey,
+	}); err != nil {
+		return err
+	}
+
+	// Update config file
+	configSection := profileName
+	if profileName != "default" {
+		configSection = "profile " + profileName
+	}
+
+	endpointHost := bindIP
+	if endpointHost == "" || endpointHost == "0.0.0.0" {
+		endpointHost = "localhost"
+	}
+
+	if err := UpdateAWSINIFile(configPath, configSection, map[string]string{
+		"region":       region,
+		"endpoint_url": fmt.Sprintf("https://%s:9999", endpointHost),
+		"ca_bundle":    certPath,
+		"output":       "json",
+	}); err != nil {
+		return err
+	}
+
+	fmt.Printf("   Profile: %s\n", profileName)
+	if profileName != "default" {
+		fmt.Printf("   Use: export AWS_PROFILE=%s\n", profileName)
+	}
+
+	return nil
+}
+
+// GenerateMultiNodePredastoreConfig produces a complete predastore.toml for a
+// multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
+// and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
+func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region string) (string, error) {
+	if len(nodes) < 3 {
+		return "", fmt.Errorf("multi-node predastore requires at least 3 nodes, got %d", len(nodes))
+	}
+
+	data := struct {
+		Nodes     []PredastoreNodeConfig
+		AccessKey string
+		SecretKey string
+		Region    string
+	}{nodes, accessKey, secretKey, region}
+
+	tmpl, err := template.New("predastore-multinode").Parse(templateStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse predastore template: %w", err)
+	}
+
+	var b strings.Builder
+	if err := tmpl.Execute(&b, data); err != nil {
+		return "", fmt.Errorf("failed to execute predastore template: %w", err)
+	}
+
+	return b.String(), nil
+}
+
+// FindNodeIDByIP returns the node ID for the given IP in the node list,
+// or 0 if the IP is not found.
+func FindNodeIDByIP(nodes []PredastoreNodeConfig, ip string) int {
+	for _, n := range nodes {
+		if n.Host == ip {
+			return n.ID
+		}
+	}
+	return 0
+}
+
+// ParsePredastoreNodeIDFromConfig parses a predastore.toml string and returns
+// the node ID whose host matches the given IP, or 0 if not found.
+func ParsePredastoreNodeIDFromConfig(tomlContent string, ip string) int {
+	var cfg struct {
+		DB []PredastoreNodeConfig `toml:"db"`
+	}
+	if err := toml.Unmarshal([]byte(tomlContent), &cfg); err != nil {
+		slog.Warn("Failed to parse predastore.toml content", "error", err)
+		return 0
+	}
+	return FindNodeIDByIP(cfg.DB, ip)
+}

hive/admin/admin_test.go

@@ -0,0 +1,621 @@
+package admin
+
+import (
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/pem"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// --- Key / Token generation ---
+
+func TestGenerateAWSAccessKey_Format(t *testing.T) {
+	key := GenerateAWSAccessKey()
+	assert.Len(t, key, 20)
+	assert.True(t, strings.HasPrefix(key, "AKIA"))
+	for _, c := range key[4:] {
+		assert.True(t, (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'),
+			"unexpected character %c in access key suffix", c)
+	}
+}
+
+func TestGenerateAWSAccessKey_Uniqueness(t *testing.T) {
+	assert.NotEqual(t, GenerateAWSAccessKey(), GenerateAWSAccessKey())
+}
+
+func TestGenerateAWSSecretKey_Format(t *testing.T) {
+	key := GenerateAWSSecretKey()
+	assert.Len(t, key, 40)
+	_, err := base64.StdEncoding.DecodeString(key)
+	assert.NoError(t, err, "secret key should be valid base64")
+}
+
+func TestGenerateAWSSecretKey_Uniqueness(t *testing.T) {
+	assert.NotEqual(t, GenerateAWSSecretKey(), GenerateAWSSecretKey())
+}
+
+func TestGenerateAccountID_Format(t *testing.T) {
+	id := GenerateAccountID()
+	assert.Len(t, id, 12)
+	for _, c := range id {
+		assert.True(t, c >= '0' && c <= '9', "expected digit, got %c", c)
+	}
+	// First digit should not be 0
+	assert.NotEqual(t, byte('0'), id[0], "12-digit account ID should not start with 0")
+}
+
+func TestGenerateAccountID_Uniqueness(t *testing.T) {
+	assert.NotEqual(t, GenerateAccountID(), GenerateAccountID())
+}
+
+func TestGenerateNATSToken_Format(t *testing.T) {
+	token := GenerateNATSToken()
+	assert.True(t, strings.HasPrefix(token, "nats_"))
+	assert.Len(t, token, 37) // 5 prefix + 32 random
+	// URL-safe base64: no '+' or '/'
+	assert.NotContains(t, token, "+")
+	assert.NotContains(t, token, "/")
+}
+
+func TestGenerateNATSToken_Uniqueness(t *testing.T) {
+	assert.NotEqual(t, GenerateNATSToken(), GenerateNATSToken())
+}
+
+// --- Config file generation ---
+
+func TestGenerateConfigFile_Success(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.conf")
+
+	tmpl := "region = {{.Region}}\nnode = {{.Node}}"
+	settings := ConfigSettings{Region: "us-east-1", Node: "node1"}
+
+	err := GenerateConfigFile(path, tmpl, settings)
+	require.NoError(t, err)
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.Contains(t, string(data), "region = us-east-1")
+	assert.Contains(t, string(data), "node = node1")
+
+	info, _ := os.Stat(path)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+}
+
+func TestGenerateConfigFile_InvalidTemplate(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "bad.conf")
+	err := GenerateConfigFile(path, "{{.Unclosed", ConfigSettings{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to parse template")
+}
+
+func TestGenerateConfigFile_InvalidPath(t *testing.T) {
+	err := GenerateConfigFile("/nonexistent/dir/file.conf", "ok", ConfigSettings{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to create config file")
+}
+
+func TestGenerateConfigFile_OverwritesExisting(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "overwrite.conf")
+
+	require.NoError(t, GenerateConfigFile(path, "old={{.Region}}", ConfigSettings{Region: "old"}))
+	require.NoError(t, GenerateConfigFile(path, "new={{.Region}}", ConfigSettings{Region: "new"}))
+
+	data, _ := os.ReadFile(path)
+	assert.Contains(t, string(data), "new=new")
+	assert.NotContains(t, string(data), "old")
+}
+
+func TestGenerateConfigFiles_AllSucceed(t *testing.T) {
+	dir := t.TempDir()
+	configs := []ConfigFile{
+		{Name: "a", Path: filepath.Join(dir, "a.conf"), Template: "a={{.Region}}"},
+		{Name: "b", Path: filepath.Join(dir, "b.conf"), Template: "b={{.Node}}"},
+	}
+	err := GenerateConfigFiles(configs, ConfigSettings{Region: "us-west-2", Node: "n1"})
+	require.NoError(t, err)
+
+	for _, cfg := range configs {
+		assert.True(t, FileExists(cfg.Path))
+	}
+}
+
+func TestGenerateConfigFiles_StopsOnFirstError(t *testing.T) {
+	dir := t.TempDir()
+	configs := []ConfigFile{
+		{Name: "ok", Path: filepath.Join(dir, "ok.conf"), Template: "ok"},
+		{Name: "bad", Path: "/nonexistent/dir/bad.conf", Template: "bad"},
+		{Name: "never", Path: filepath.Join(dir, "never.conf"), Template: "never"},
+	}
+	err := GenerateConfigFiles(configs, ConfigSettings{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "bad")
+	assert.False(t, FileExists(filepath.Join(dir, "never.conf")))
+}
+
+// --- FileExists ---
+
+func TestFileExists(t *testing.T) {
+	dir := t.TempDir()
+	existing := filepath.Join(dir, "exists.txt")
+	require.NoError(t, os.WriteFile(existing, []byte("hi"), 0644))
+
+	assert.True(t, FileExists(existing))
+	assert.False(t, FileExists(filepath.Join(dir, "nope.txt")))
+	assert.True(t, FileExists(dir)) // directory also returns true
+}
+
+// --- UpdateAWSINIFile ---
+
+func TestUpdateAWSINIFile_CreateNew(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "credentials")
+
+	err := UpdateAWSINIFile(path, "hive", map[string]string{
+		"aws_access_key_id":     "AKIATEST",
+		"aws_secret_access_key": "secrettest",
+	})
+	require.NoError(t, err)
+
+	data, _ := os.ReadFile(path)
+	content := string(data)
+	assert.Contains(t, content, "[hive]")
+	assert.Contains(t, content, "AKIATEST")
+	assert.Contains(t, content, "secrettest")
+}
+
+func TestUpdateAWSINIFile_UpdateExisting(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "credentials")
+
+	// Create with initial value
+	require.NoError(t, UpdateAWSINIFile(path, "hive", map[string]string{"key": "old"}))
+	// Update
+	require.NoError(t, UpdateAWSINIFile(path, "hive", map[string]string{"key": "new"}))
+
+	data, _ := os.ReadFile(path)
+	content := string(data)
+	assert.Contains(t, content, "new")
+}
+
+func TestUpdateAWSINIFile_AddNewSection(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "credentials")
+
+	require.NoError(t, UpdateAWSINIFile(path, "default", map[string]string{"key": "default-val"}))
+	require.NoError(t, UpdateAWSINIFile(path, "hive", map[string]string{"key": "hive-val"}))
+
+	data, _ := os.ReadFile(path)
+	content := string(data)
+	assert.Contains(t, content, "[default]")
+	assert.Contains(t, content, "[hive]")
+	assert.Contains(t, content, "default-val")
+	assert.Contains(t, content, "hive-val")
+}
+
+// --- SetupAWSCredentials ---
+
+func TestSetupAWSCredentials_CreatesFiles(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("HOME", dir)
+
+	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/path/to/ca.pem", "")
+	require.NoError(t, err)
+
+	credData, _ := os.ReadFile(filepath.Join(dir, ".aws", "credentials"))
+	configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
+
+	assert.Contains(t, string(credData), "AKIATEST123")
+	assert.Contains(t, string(credData), "secret123")
+	assert.Contains(t, string(configData), "us-east-1")
+	assert.Contains(t, string(configData), "https://localhost:9999")
+	assert.Contains(t, string(configData), "/path/to/ca.pem")
+}
+
+func TestSetupAWSCredentials_PreservesExistingProfiles(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("HOME", dir)
+
+	awsDir := filepath.Join(dir, ".aws")
+	require.NoError(t, os.MkdirAll(awsDir, 0700))
+	require.NoError(t, UpdateAWSINIFile(filepath.Join(awsDir, "credentials"), "default", map[string]string{
+		"aws_access_key_id": "EXISTING_KEY",
+	}))
+
+	err := SetupAWSCredentials("NEWAKEY", "NEWSECRET", "us-west-2", "/ca.pem", "")
+	require.NoError(t, err)
+
+	data, _ := os.ReadFile(filepath.Join(awsDir, "credentials"))
+	content := string(data)
+	assert.Contains(t, content, "EXISTING_KEY")
+	assert.Contains(t, content, "NEWAKEY")
+}
+
+func TestSetupAWSCredentials_UsesBindIP(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("HOME", dir)
+
+	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "10.11.12.1")
+	require.NoError(t, err)
+
+	configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
+	assert.Contains(t, string(configData), "https://10.11.12.1:9999")
+	assert.NotContains(t, string(configData), "localhost")
+}
+
+func TestSetupAWSCredentials_FallsBackToLocalhostForWildcard(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("HOME", dir)
+
+	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "0.0.0.0")
+	require.NoError(t, err)
+
+	configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
+	assert.Contains(t, string(configData), "https://localhost:9999")
+}
+
+// --- Certificate generation ---
+
+func TestGenerateCACert_CreatesValidCA(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "ca.pem")
+	keyPath := filepath.Join(dir, "ca.key")
+
+	err := GenerateCACert(certPath, keyPath)
+	require.NoError(t, err)
+
+	// Parse certificate
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	require.NotNil(t, block)
+	assert.Equal(t, "CERTIFICATE", block.Type)
+
+	cert, err := x509.ParseCertificate(block.Bytes)
+	require.NoError(t, err)
+	assert.True(t, cert.IsCA)
+	assert.Equal(t, "Hive Local CA", cert.Subject.CommonName)
+	assert.NotZero(t, cert.KeyUsage&x509.KeyUsageCertSign)
+
+	// Parse key
+	keyPEM, _ := os.ReadFile(keyPath)
+	keyBlock, _ := pem.Decode(keyPEM)
+	require.NotNil(t, keyBlock)
+	assert.Equal(t, "PRIVATE KEY", keyBlock.Type)
+
+	// Verify key file permissions
+	info, _ := os.Stat(keyPath)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+}
+
+// TestGenerateSignedCert uses subtests sharing a single CA to avoid repeated 4096-bit key generation.
+func TestGenerateSignedCert(t *testing.T) {
+	t.Parallel()
+	// Generate CA once for all subtests (~0.7s instead of ~0.7s x 3)
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	t.Run("CreatesValidCert", func(t *testing.T) {
+		t.Parallel()
+		dir := t.TempDir()
+		certPath := filepath.Join(dir, "server.pem")
+		keyPath := filepath.Join(dir, "server.key")
+
+		require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath))
+
+		certPEM, _ := os.ReadFile(certPath)
+		block, _ := pem.Decode(certPEM)
+		require.NotNil(t, block)
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		require.NoError(t, err)
+		assert.False(t, cert.IsCA)
+		assert.Equal(t, "localhost", cert.Subject.CommonName)
+		assert.Contains(t, cert.DNSNames, "localhost")
+
+		hasLoopback := false
+		for _, ip := range cert.IPAddresses {
+			if ip.Equal(net.ParseIP("127.0.0.1")) {
+				hasLoopback = true
+			}
+		}
+		assert.True(t, hasLoopback)
+
+		// Verify against CA
+		caCertPEM, _ := os.ReadFile(caCertPath)
+		caBlock, _ := pem.Decode(caCertPEM)
+		caCert, _ := x509.ParseCertificate(caBlock.Bytes)
+		pool := x509.NewCertPool()
+		pool.AddCert(caCert)
+		_, err = cert.Verify(x509.VerifyOptions{Roots: pool})
+		assert.NoError(t, err)
+
+		info, _ := os.Stat(keyPath)
+		assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+	})
+
+	t.Run("ExtraIPs", func(t *testing.T) {
+		t.Parallel()
+		dir := t.TempDir()
+		certPath := filepath.Join(dir, "server.pem")
+		keyPath := filepath.Join(dir, "server.key")
+
+		require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath, "192.168.1.100"))
+
+		certPEM, _ := os.ReadFile(certPath)
+		block, _ := pem.Decode(certPEM)
+		cert, _ := x509.ParseCertificate(block.Bytes)
+
+		hasExtraIP := false
+		for _, ip := range cert.IPAddresses {
+			if ip.Equal(net.ParseIP("192.168.1.100")) {
+				hasExtraIP = true
+			}
+		}
+		assert.True(t, hasExtraIP, "cert should contain extra IP 192.168.1.100")
+	})
+
+	t.Run("SkipsDuplicateAndSpecialIPs", func(t *testing.T) {
+		t.Parallel()
+		dir := t.TempDir()
+		certPath := filepath.Join(dir, "server.pem")
+		keyPath := filepath.Join(dir, "server.key")
+
+		require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath, "127.0.0.1", "::1", "0.0.0.0", ""))
+
+		certPEM, _ := os.ReadFile(certPath)
+		block, _ := pem.Decode(certPEM)
+		cert, _ := x509.ParseCertificate(block.Bytes)
+
+		assert.Len(t, cert.IPAddresses, 2)
+	})
+
+	t.Run("InvalidCACert", func(t *testing.T) {
+		t.Parallel()
+		dir := t.TempDir()
+		badCACert := filepath.Join(dir, "bad-ca.pem")
+		require.NoError(t, os.WriteFile(badCACert, []byte("not-a-cert"), 0600))
+
+		err := GenerateSignedCert(filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"), badCACert, caKeyPath)
+		assert.Error(t, err)
+	})
+}
+
+func TestGenerateSelfSignedCert_CreatesValidCert(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "self.pem")
+	keyPath := filepath.Join(dir, "self.key")
+
+	require.NoError(t, GenerateSelfSignedCert(certPath, keyPath))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	require.NotNil(t, block)
+
+	cert, err := x509.ParseCertificate(block.Bytes)
+	require.NoError(t, err)
+	assert.Contains(t, cert.DNSNames, "localhost")
+
+	info, _ := os.Stat(keyPath)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+}
+
+// --- Certificate orchestrator ---
+
+// TestGenerateCertificatesIfNeeded uses subtests to share the initial generation.
+func TestGenerateCertificatesIfNeeded(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	// First call generates all certs
+	caCertPath := GenerateCertificatesIfNeeded(dir, false, "")
+	assert.Equal(t, filepath.Join(dir, "ca.pem"), caCertPath)
+	assert.True(t, FileExists(filepath.Join(dir, "ca.pem")))
+	assert.True(t, FileExists(filepath.Join(dir, "ca.key")))
+	assert.True(t, FileExists(filepath.Join(dir, "server.pem")))
+	assert.True(t, FileExists(filepath.Join(dir, "server.key")))
+
+	t.Run("SkipsWhenAllExist", func(t *testing.T) {
+		caInfo, _ := os.Stat(filepath.Join(dir, "ca.pem"))
+		origModTime := caInfo.ModTime()
+
+		GenerateCertificatesIfNeeded(dir, false, "")
+		caInfo2, _ := os.Stat(filepath.Join(dir, "ca.pem"))
+		assert.Equal(t, origModTime, caInfo2.ModTime())
+	})
+
+	t.Run("ForceRegenerates", func(t *testing.T) {
+		origCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))
+
+		GenerateCertificatesIfNeeded(dir, true, "")
+		newCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))
+		assert.NotEqual(t, origCA, newCA)
+	})
+}
+
+func TestGenerateServerCertOnly(t *testing.T) {
+	t.Parallel()
+	// Generate CA once, reuse for subtests
+	caDir := t.TempDir()
+	require.NoError(t, GenerateCACert(filepath.Join(caDir, "ca.pem"), filepath.Join(caDir, "ca.key")))
+
+	t.Run("Success", func(t *testing.T) {
+		dir := t.TempDir()
+		// Copy CA files into test dir
+		caCert, _ := os.ReadFile(filepath.Join(caDir, "ca.pem"))
+		caKey, _ := os.ReadFile(filepath.Join(caDir, "ca.key"))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), caCert, 0600))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.key"), caKey, 0600))
+
+		err := GenerateServerCertOnly(dir, "10.0.0.5")
+		require.NoError(t, err)
+		assert.True(t, FileExists(filepath.Join(dir, "server.pem")))
+		assert.True(t, FileExists(filepath.Join(dir, "server.key")))
+
+		certPEM, _ := os.ReadFile(filepath.Join(dir, "server.pem"))
+		block, _ := pem.Decode(certPEM)
+		cert, _ := x509.ParseCertificate(block.Bytes)
+
+		hasBindIP := false
+		for _, ip := range cert.IPAddresses {
+			if ip.Equal(net.ParseIP("10.0.0.5")) {
+				hasBindIP = true
+			}
+		}
+		assert.True(t, hasBindIP)
+	})
+
+	t.Run("MissingCA", func(t *testing.T) {
+		dir := t.TempDir()
+		err := GenerateServerCertOnly(dir, "10.0.0.5")
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "CA files not found")
+	})
+}
+
+// --- Directory creation ---
+
+func TestCreateServiceDirectories_CreatesAll(t *testing.T) {
+	dir := t.TempDir()
+	CreateServiceDirectories(dir)
+
+	expected := []string{"images", "amis", "volumes", "state", "logs", "nats", "predastore", "viperblock", "hive"}
+	for _, name := range expected {
+		path := filepath.Join(dir, name)
+		info, err := os.Stat(path)
+		assert.NoError(t, err, "directory %s should exist", name)
+		if err == nil {
+			assert.True(t, info.IsDir())
+		}
+	}
+}
+
+func TestCreateServiceDirectories_Idempotent(t *testing.T) {
+	dir := t.TempDir()
+	CreateServiceDirectories(dir)
+	// Should not error on second call
+	CreateServiceDirectories(dir)
+
+	info, err := os.Stat(filepath.Join(dir, "images"))
+	assert.NoError(t, err)
+	assert.True(t, info.IsDir())
+}
+
+// --- Predastore multi-node config ---
+
+func TestGenerateMultiNodePredastoreConfig_Success(t *testing.T) {
+	tmpl := `{{range .Nodes}}[[db]]
+id = {{.ID}}
+host = "{{.Host}}"
+{{end}}`
+	nodes := []PredastoreNodeConfig{
+		{ID: 1, Host: "10.0.0.1"},
+		{ID: 2, Host: "10.0.0.2"},
+		{ID: 3, Host: "10.0.0.3"},
+	}
+
+	result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1")
+	require.NoError(t, err)
+	assert.Contains(t, result, `host = "10.0.0.1"`)
+	assert.Contains(t, result, `host = "10.0.0.3"`)
+}
+
+func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
+	tmpl := "{{range .Nodes}}{{.ID}}{{end}}"
+
+	_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
+		{ID: 1, Host: "10.0.0.1"},
+		{ID: 2, Host: "10.0.0.2"},
+	}, "AK", "SK", "us-east-1")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "at least 3 nodes")
+}
+
+func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
+	_, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []PredastoreNodeConfig{
+		{ID: 1, Host: "a"}, {ID: 2, Host: "b"}, {ID: 3, Host: "c"},
+	}, "AK", "SK", "us-east-1")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to parse")
+}
+
+// --- FindNodeIDByIP ---
+
+func TestFindNodeIDByIP(t *testing.T) {
+	nodes := []PredastoreNodeConfig{
+		{ID: 1, Host: "10.0.0.1"},
+		{ID: 2, Host: "10.0.0.2"},
+		{ID: 3, Host: "10.0.0.3"},
+	}
+
+	assert.Equal(t, 2, FindNodeIDByIP(nodes, "10.0.0.2"))
+	assert.Equal(t, 0, FindNodeIDByIP(nodes, "10.0.0.99"))
+	assert.Equal(t, 0, FindNodeIDByIP(nil, "10.0.0.1"))
+}
+
+// --- ParsePredastoreNodeIDFromConfig ---
+
+func TestParsePredastoreNodeIDFromConfig(t *testing.T) {
+	tomlContent := `
+[[db]]
+id = 1
+host = "10.0.0.1"
+
+[[db]]
+id = 2
+host = "10.0.0.2"
+
+[[db]]
+id = 3
+host = "10.0.0.3"
+`
+	assert.Equal(t, 2, ParsePredastoreNodeIDFromConfig(tomlContent, "10.0.0.2"))
+	assert.Equal(t, 0, ParsePredastoreNodeIDFromConfig(tomlContent, "10.0.0.99"))
+	assert.Equal(t, 0, ParsePredastoreNodeIDFromConfig("invalid toml {{{", "10.0.0.1"))
+	assert.Equal(t, 0, ParsePredastoreNodeIDFromConfig("", "10.0.0.1"))
+}
+
+// --- Integration: Full config generation flow ---
+
+func TestGenerateConfigFile_HiveTomlTemplate(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	tmpl := `version = "1.0"
+epoch = 1
+node = "{{.Node}}"
+
+[nodes.{{.Node}}]
+region = "{{.Region}}"
+accesskey = "{{.AccessKey}}"
+secretkey = "{{.SecretKey}}"
+`
+
+	settings := ConfigSettings{
+		Node:      "node1",
+		Region:    "us-east-1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+	}
+
+	require.NoError(t, GenerateConfigFile(path, tmpl, settings))
+
+	data, _ := os.ReadFile(path)
+	content := string(data)
+	assert.Contains(t, content, `node = "node1"`)
+	assert.Contains(t, content, `region = "us-east-1"`)
+	assert.Contains(t, content, fmt.Sprintf(`accesskey = "%s"`, settings.AccessKey))
+}

hive/awsec2query/parser.go

@@ -29,7 +29,7 @@ args["TagSpecification.2.Tag.1.Value"] = "Production"
 
 */
 
-func QueryParamsToStruct(params map[string]string, out interface{}) error {
+func QueryParamsToStruct(params map[string]string, out any) error {
 	v := reflect.ValueOf(out)
 
 	// Must be a pointer to a struct
@@ -62,10 +62,17 @@ func setStructFields(v reflect.Value, params map[string]string, prefix string) e
 		// Check for locationName tag (AWS SDK uses this for query parameter names)
 		locationName := fieldType.Tag.Get("locationName")
 
-		// Try both the field name and locationName (if different)
+		// Try the field name, locationName, and title-cased locationName.
+		// AWS query params use title case (e.g. "ResourceId") but some SDK
+		// structs use camelCase locationName (e.g. "resourceId" in DeleteTagsInput
+		// vs "ResourceId" in CreateTagsInput).
 		queryKeys := []string{prefix + fieldName}
 		if locationName != "" && locationName != fieldName {
 			queryKeys = append(queryKeys, prefix+locationName)
+			titled := strings.ToUpper(locationName[:1]) + locationName[1:]
+			if titled != fieldName && titled != locationName {
+				queryKeys = append(queryKeys, prefix+titled)
+			}
 		}
 
 		// Check if this is a simple field (string, int, bool, etc.)

hive/awssmithy/awssmithy.go

@@ -1,16 +0,0 @@
-package awssmithy
-
-type FilterList struct {
-	Name   string
-	Values []string
-}
-
-type DescribeInstancesRequest struct {
-	InstanceIds []string     `xml:"InstanceId"`
-	DryRun      bool         `xml:"DryRun"`
-	Filters     []FilterList `xml:"Filter"`
-
-	NextToken string `xml:"nextToken"`
-
-	MaxResults int `xml:"maxResults"`
-}

hive/config/config.go

@@ -11,21 +11,81 @@ import (
 	"github.com/spf13/viper"
 )
 
+type ClusterConfig struct {
+	Epoch   uint64            `mapstructure:"epoch"`   // bump when leader commits changes
+	Node    string            `mapstructure:"node"`    // my node name
+	Version string            `mapstructure:"version"` // hive version
+	Nodes   map[string]Config `mapstructure:"nodes"`   // full config for every node
+}
+
 // Config holds all configuration for the application
 type Config struct {
+	// Node config
+	Node     string   `mapstructure:"node"`
+	Host     string   `mapstructure:"host"` // Unique hostname or IP of this node
+	Region   string   `mapstructure:"region"`
+	AZ       string   `mapstructure:"az"`
+	DataDir  string   `mapstructure:"data_dir"`
+	Services []string `mapstructure:"services"` // Which services this node runs locally
+
 	Daemon     DaemonConfig     `mapstructure:"daemon"`
 	NATS       NATSConfig       `mapstructure:"nats"`
 	Predastore PredastoreConfig `mapstructure:"predastore"`
+	Viperblock ViperblockConfig `mapstructure:"viperblock"`
 	AWSGW      AWSGWConfig      `mapstructure:"awsgw"`
 
 	// Authentication
-	AccessKey string `mapstructure:"access_key"`
-	SecretKey string `mapstructure:"secret_key"`
-	Host      string `mapstructure:"host"`
+	// TODO: Move to more appropriate setting above
+	AccessKey string `mapstructure:"accesskey"`
+	SecretKey string `mapstructure:"secretkey"`
 	BaseDir   string `mapstructure:"base_dir"`
 	WalDir    string `mapstructure:"wal_dir"`
 }
 
+// NodeBaseDir returns the BaseDir for the current node, or "" if the config
+// is nil, the node name is unset, or the node is not found in the Nodes map.
+func (cc *ClusterConfig) NodeBaseDir() string {
+	if cc == nil || cc.Node == "" {
+		slog.Warn("NodeBaseDir: no config or node name set, using global PID path")
+		return ""
+	}
+	node, ok := cc.Nodes[cc.Node]
+	if !ok {
+		slog.Error("NodeBaseDir: node not found in config", "node", cc.Node)
+		return ""
+	}
+	if node.BaseDir == "" {
+		slog.Warn("NodeBaseDir: BaseDir is empty for node, using global PID path", "node", cc.Node)
+	}
+	return node.BaseDir
+}
+
+// AllServices is the default service list when Services is empty (backward compat).
+var AllServices = []string{"nats", "predastore", "viperblock", "daemon", "awsgw", "ui"}
+
+// HasService reports whether the node runs the named service.
+// An empty Services list means all services (backward compat).
+func (c Config) HasService(name string) bool {
+	services := c.Services
+	if len(services) == 0 {
+		services = AllServices
+	}
+	for _, s := range services {
+		if s == name {
+			return true
+		}
+	}
+	return false
+}
+
+// GetServices returns the configured service list, defaulting to AllServices.
+func (c Config) GetServices() []string {
+	if len(c.Services) == 0 {
+		return AllServices
+	}
+	return c.Services
+}
+
 type AWSGWConfig struct {
 	Host    string `mapstructure:"host"`
 	TLSKey  string `mapstructure:"tlskey"`
@@ -33,16 +93,21 @@ type AWSGWConfig struct {
 	Config  string `mapstructure:"config"`
 
 	Debug         bool `mapstructure:"debug"`
-	ExpectedNodes int  `mapstructure:"expected_nodes"`
+	ExpectedNodes int  `mapstructure:"expected_nodes"` // TODO: Replace with root cluster config
+}
+
+type ViperblockConfig struct {
+	ShardWAL *bool `mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
 }
 
 type PredastoreConfig struct {
 	Host      string `mapstructure:"host"`
 	Bucket    string `mapstructure:"bucket"`
 	Region    string `mapstructure:"region"`
-	AccessKey string `mapstructure:"access_key"`
-	SecretKey string `mapstructure:"secret_key"`
+	AccessKey string `mapstructure:"accesskey"`
+	SecretKey string `mapstructure:"secretkey"`
 	BaseDir   string `mapstructure:"base_dir"`
+	NodeID    int    `mapstructure:"node_id"`
 }
 
 // DaemonConfig holds the daemon configuration
@@ -75,14 +140,26 @@ type EBSRequests struct {
 }
 
 type EBSRequest struct {
-	Name      string
-	VolType   string
-	Boot      bool
-	EFI       bool
-	CloudInit bool
-	NBDURI    string
+	Name                string
+	VolType             string
+	Boot                bool
+	EFI                 bool
+	CloudInit           bool
+	DeleteOnTermination bool
+	NBDURI              string // NBD URI - socket path (nbd:unix:/path.sock) or TCP (nbd://host:port)
+	DeviceName          string // Device name (e.g. /dev/sdf) for hot-plugged volumes
 }
 
+// NBDTransport defines the transport type for NBD connections
+type NBDTransport string
+
+const (
+	// NBDTransportSocket uses Unix domain sockets (faster, local only)
+	NBDTransportSocket NBDTransport = "socket"
+	// NBDTransportTCP uses TCP connections (required for remote/DPU scenarios)
+	NBDTransportTCP NBDTransport = "tcp"
+)
+
 type EBSMountResponse struct {
 	URI     string
 	Mounted bool
@@ -95,6 +172,16 @@ type EBSUnMountResponse struct {
 	Error   string
 }
 
+type EBSSyncRequest struct {
+	Volume string
+}
+
+type EBSSyncResponse struct {
+	Volume string
+	Synced bool
+	Error  string
+}
+
 type EBSDeleteRequest struct {
 	Volume string
 }
@@ -105,26 +192,18 @@ type EBSDeleteResponse struct {
 	Error   string
 }
 
-// EC2, TODO: Move to vm.go or more applicable place
-type EC2Response struct {
-	InstanceID string
-	Hostname   string
+type EBSSnapshotRequest struct {
+	Volume     string
+	SnapshotID string
+}
+
+type EBSSnapshotResponse struct {
+	SnapshotID string
 	Success    bool
-	Status     string
-	Error      string
-}
-
-type EC2DescribeRequest struct {
-	InstanceID string
-}
-
-type EC2DescribeResponse struct {
-	InstanceID string
-	Hostname   string
-	Status     string
 	Error      string
 }
 
+// EC2, TODO: Move to vm.go or more applicable place
 type EC2StartInstancesRequest struct {
 	InstanceID string
 }
@@ -144,47 +223,99 @@ func (ec2StartInstanceResponse EC2StartInstancesResponse) Respond(msg *nats.Msg)
 		return
 	}
 
-	msg.Respond(response)
-
-}
-func (ec2DescribeResponse EC2DescribeResponse) Respond(msg *nats.Msg) {
-
-	response, err := json.Marshal(ec2DescribeResponse)
-	if err != nil {
-		slog.Error("Failed to marshal response", "err", err)
-		return
+	if err := msg.Respond(response); err != nil {
+		slog.Error("Failed to respond to NATS request", "err", err)
 	}
 
-	msg.Respond(response)
-
 }
 
-func (ec2Response EC2Response) Respond(msg *nats.Msg) {
+// Cluster manager API types
+type NodeJoinRequest struct {
+	Node       string `json:"node"`
+	Region     string `json:"region"`
+	AZ         string `json:"az"`
+	DataDir    string `json:"data_dir"`
+	DaemonHost string `json:"daemon_host"` // Host:port where this node's daemon is accessible
+}
 
-	response, err := json.Marshal(ec2Response)
-	if err != nil {
-		slog.Error("Failed to marshal response", "err", err)
-		return
-	}
+// SharedClusterData contains only the shared cluster information (no node-specific top-level fields)
+type SharedClusterData struct {
+	Epoch   uint64            `json:"epoch" toml:"epoch"`
+	Version string            `json:"version" toml:"version"`
+	Nodes   map[string]Config `json:"nodes" toml:"nodes"`
+}
 
-	msg.Respond(response)
+type NodeJoinResponse struct {
+	Success     bool               `json:"success"`
+	Message     string             `json:"message"`
+	SharedData  *SharedClusterData `json:"shared_data,omitempty"`
+	ConfigHash  string             `json:"config_hash,omitempty"`
+	JoiningNode string             `json:"joining_node,omitempty"` // The node name that is joining
+	// CA distribution for per-node certificate generation
+	CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate
+	CAKey  string `json:"ca_key,omitempty"`  // PEM-encoded CA private key
+	// Predastore config distribution for multi-node clusters
+	PredastoreConfig string `json:"predastore_config,omitempty"` // Full predastore.toml content
+}
 
+type NodeHealthResponse struct {
+	Node          string            `json:"node"`
+	Status        string            `json:"status"`
+	ConfigHash    string            `json:"config_hash"`
+	Epoch         uint64            `json:"epoch"`
+	Uptime        int64             `json:"uptime"`
+	Services      []string          `json:"services"`
+	ServiceHealth map[string]string `json:"service_health,omitempty"`
+}
+
+// NodeStatusResponse is returned by the hive.node.status NATS topic (fan-out).
+type NodeStatusResponse struct {
+	Node          string            `json:"node"`
+	Status        string            `json:"status"`
+	Host          string            `json:"host"`
+	Region        string            `json:"region"`
+	AZ            string            `json:"az"`
+	Uptime        int64             `json:"uptime"`
+	Services      []string          `json:"services"`
+	TotalVCPU     int               `json:"total_vcpu"`
+	TotalMemGB    float64           `json:"total_mem_gb"`
+	AllocVCPU     int               `json:"alloc_vcpu"`
+	AllocMemGB    float64           `json:"alloc_mem_gb"`
+	VMCount       int               `json:"vm_count"`
+	InstanceTypes []InstanceTypeCap `json:"instance_types"`
+}
+
+// InstanceTypeCap describes available capacity for one instance type on a node.
+type InstanceTypeCap struct {
+	Name      string  `json:"name"`
+	VCPU      int     `json:"vcpu"`
+	MemoryGB  float64 `json:"memory_gb"`
+	Available int     `json:"available"`
+}
+
+// VMInfo describes a single VM for the cluster stats CLI.
+type VMInfo struct {
+	InstanceID   string  `json:"instance_id"`
+	Status       string  `json:"status"`
+	InstanceType string  `json:"instance_type"`
+	VCPU         int     `json:"vcpu"`
+	MemoryGB     float64 `json:"memory_gb"`
+	LaunchTime   int64   `json:"launch_time"`
+}
+
+// NodeVMsResponse is returned by the hive.node.vms NATS topic (fan-out).
+type NodeVMsResponse struct {
+	Node string   `json:"node"`
+	Host string   `json:"host"`
+	VMs  []VMInfo `json:"vms"`
 }
 
 // LoadConfig loads the configuration from file and environment variables
-func LoadConfig(configPath string) (*Config, error) {
+func LoadConfig(configPath string) (*ClusterConfig, error) {
 	// Set environment variable prefix
 	viper.SetEnvPrefix("HIVE")
 	viper.AutomaticEnv()
 
-	// Set default values
-	//viper.SetDefault("host", "https://localhost:8443/")
-	//viper.SetDefault("base_dir", "/tmp/vb/")
-
-	//viper.SetDefault("daemon.host", "0.0.0.0:4432")
-	//viper.SetDefault("nats.host", "0.0.0.0:4222")
-	//viper.SetDefault("nats.sub.subject", "ec2.>")
-
 	// Try to load config file if it exists
 	if configPath != "" {
 		// Check if file exists
@@ -195,33 +326,18 @@ func LoadConfig(configPath string) (*Config, error) {
 			if err := viper.ReadInConfig(); err != nil {
 				return nil, fmt.Errorf("error reading config file: %w", err)
 			}
-			fmt.Fprintf(os.Stderr, "Using config file: %s\n", viper.ConfigFileUsed())
+			//fmt.Fprintf(os.Stderr, "Using config file: %s\n", viper.ConfigFileUsed())
+
 		} else {
 			fmt.Fprintf(os.Stderr, "Config file not found: %s, using environment variables and defaults\n", configPath)
 		}
 	}
 
 	// Create config struct
-	var config Config
+	var config ClusterConfig
 	if err := viper.Unmarshal(&config); err != nil {
 		return nil, fmt.Errorf("error unmarshaling config: %w", err)
 	}
 
-	// Validate required fields (can be overwritten by env vars)
-	/*
-		if config.NATS.Host == "" {
-			return nil, fmt.Errorf("NATS host is required")
-		}
-
-		if config.AccessKey == "" {
-			return nil, fmt.Errorf("access key is required")
-		}
-		if config.SecretKey == "" {
-			return nil, fmt.Errorf("secret key is required")
-		}
-	*/
-
-	//	fmt.Println("Config: ", config)
-
 	return &config, nil
 }

hive/config/config_test.go

@@ -0,0 +1,382 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func resetViper(t *testing.T) {
+	t.Cleanup(func() { viper.Reset() })
+}
+
+func TestLoadConfig_ValidTOMLFile(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+version = "1.0"
+epoch = 1
+node = "node1"
+
+[nodes.node1]
+node = "node1"
+region = "us-east-1"
+az = "us-east-1a"
+accesskey = "AKIATEST"
+secretkey = "SECRET"
+
+[nodes.node1.daemon]
+host = "127.0.0.1:8080"
+
+[nodes.node1.nats]
+host = "127.0.0.1:4222"
+
+[nodes.node1.nats.acl]
+token = "nats_testtoken"
+
+[nodes.node1.predastore]
+host = "127.0.0.1:8443"
+bucket = "predastore"
+region = "us-east-1"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	assert.Equal(t, uint64(1), cfg.Epoch)
+	assert.Equal(t, "node1", cfg.Node)
+	assert.Equal(t, "1.0", cfg.Version)
+
+	node, ok := cfg.Nodes["node1"]
+	require.True(t, ok, "node1 should exist in Nodes map")
+	assert.Equal(t, "us-east-1", node.Region)
+	assert.Equal(t, "us-east-1a", node.AZ)
+	assert.Equal(t, "AKIATEST", node.AccessKey)
+	assert.Equal(t, "127.0.0.1:8080", node.Daemon.Host)
+	assert.Equal(t, "127.0.0.1:4222", node.NATS.Host)
+	assert.Equal(t, "nats_testtoken", node.NATS.ACL.Token)
+	assert.Equal(t, "127.0.0.1:8443", node.Predastore.Host)
+	assert.Equal(t, "predastore", node.Predastore.Bucket)
+}
+
+func TestLoadConfig_MultipleNodes(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+epoch = 2
+node = "leader"
+version = "2.0"
+
+[nodes.leader]
+region = "us-east-1"
+
+[nodes.follower]
+region = "us-west-2"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+	assert.Len(t, cfg.Nodes, 2)
+	assert.Equal(t, "us-east-1", cfg.Nodes["leader"].Region)
+	assert.Equal(t, "us-west-2", cfg.Nodes["follower"].Region)
+}
+
+func TestLoadConfig_EmptyConfigPath(t *testing.T) {
+	resetViper(t)
+	cfg, err := LoadConfig("")
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+	// All zero values
+	assert.Equal(t, uint64(0), cfg.Epoch)
+	assert.Empty(t, cfg.Node)
+}
+
+func TestLoadConfig_NonexistentFile(t *testing.T) {
+	resetViper(t)
+	cfg, err := LoadConfig("/tmp/nonexistent-hive-config-test-12345.toml")
+	// Not an error - falls through to defaults
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+}
+
+func TestLoadConfig_MalformedTOML(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "bad.toml")
+	require.NoError(t, os.WriteFile(path, []byte("this is not valid toml {{{"), 0600))
+
+	cfg, err := LoadConfig(path)
+	assert.Error(t, err)
+	assert.Nil(t, cfg)
+	assert.Contains(t, err.Error(), "error reading config file")
+}
+
+func TestLoadConfig_PartialConfig(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "partial.toml")
+	require.NoError(t, os.WriteFile(path, []byte(`node = "partial-node"
+epoch = 5
+`), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+	assert.Equal(t, "partial-node", cfg.Node)
+	assert.Equal(t, uint64(5), cfg.Epoch)
+	assert.Empty(t, cfg.Version)
+	assert.Nil(t, cfg.Nodes)
+}
+
+func TestLoadConfig_EnvVarOverrideWithFile(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	// Viper's AutomaticEnv only works for keys Viper already knows about
+	// (from a config file or explicit BindEnv). Provide a minimal config
+	// so Viper registers the "epoch" key, then override via env.
+	require.NoError(t, os.WriteFile(path, []byte(`epoch = 1
+node = "file-node"
+`), 0600))
+
+	t.Setenv("HIVE_NODE", "env-node")
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+	// Env vars override file values for keys Viper knows about
+	assert.Equal(t, "env-node", cfg.Node)
+}
+
+func TestLoadConfig_NestedStructParsing(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "nested.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1]
+region = "ap-southeast-2"
+
+[nodes.n1.daemon]
+host = "0.0.0.0:8080"
+tlskey = "server.key"
+tlscert = "server.pem"
+
+[nodes.n1.nats]
+host = "0.0.0.0:4222"
+
+[nodes.n1.nats.acl]
+token = "secret-token"
+
+[nodes.n1.nats.sub]
+subject = "test-subject"
+
+[nodes.n1.predastore]
+host = "0.0.0.0:8443"
+bucket = "mybucket"
+region = "ap-southeast-2"
+accesskey = "AK"
+secretkey = "SK"
+base_dir = "/data"
+node_id = 1
+
+[nodes.n1.awsgw]
+host = "0.0.0.0:9999"
+debug = true
+expected_nodes = 3
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	n := cfg.Nodes["n1"]
+	assert.Equal(t, "0.0.0.0:8080", n.Daemon.Host)
+	assert.Equal(t, "server.key", n.Daemon.TLSKey)
+	assert.Equal(t, "server.pem", n.Daemon.TLSCert)
+	assert.Equal(t, "0.0.0.0:4222", n.NATS.Host)
+	assert.Equal(t, "secret-token", n.NATS.ACL.Token)
+	assert.Equal(t, "test-subject", n.NATS.Sub.Subject)
+	assert.Equal(t, "0.0.0.0:8443", n.Predastore.Host)
+	assert.Equal(t, "mybucket", n.Predastore.Bucket)
+	assert.Equal(t, "AK", n.Predastore.AccessKey)
+	assert.Equal(t, "/data", n.Predastore.BaseDir)
+	assert.Equal(t, 1, n.Predastore.NodeID)
+	assert.Equal(t, "0.0.0.0:9999", n.AWSGW.Host)
+	assert.True(t, n.AWSGW.Debug)
+	assert.Equal(t, 3, n.AWSGW.ExpectedNodes)
+}
+
+// Tests for HasService / GetServices
+
+func TestHasService_ExplicitList(t *testing.T) {
+	c := Config{Services: []string{"nats", "daemon"}}
+
+	assert.True(t, c.HasService("nats"))
+	assert.True(t, c.HasService("daemon"))
+	assert.False(t, c.HasService("predastore"))
+	assert.False(t, c.HasService("viperblock"))
+	assert.False(t, c.HasService("ui"))
+}
+
+func TestHasService_EmptyListBackwardCompat(t *testing.T) {
+	c := Config{} // no Services set
+
+	// Empty list means all services
+	for _, svc := range AllServices {
+		assert.True(t, c.HasService(svc), "expected %s to be available with empty list", svc)
+	}
+}
+
+func TestHasService_UnknownService(t *testing.T) {
+	c := Config{Services: []string{"nats"}}
+	assert.False(t, c.HasService("unknown"))
+}
+
+func TestGetServices_DefaultsToAll(t *testing.T) {
+	c := Config{}
+	services := c.GetServices()
+	assert.Equal(t, AllServices, services)
+}
+
+func TestGetServices_ExplicitList(t *testing.T) {
+	c := Config{Services: []string{"nats", "predastore"}}
+	services := c.GetServices()
+	assert.Equal(t, []string{"nats", "predastore"}, services)
+}
+
+// Tests for NodeBaseDir
+
+func TestNodeBaseDir_HappyPath(t *testing.T) {
+	cc := &ClusterConfig{
+		Node: "node1",
+		Nodes: map[string]Config{
+			"node1": {BaseDir: "/data/node1"},
+		},
+	}
+	assert.Equal(t, "/data/node1", cc.NodeBaseDir())
+}
+
+func TestNodeBaseDir_NilConfig(t *testing.T) {
+	var cc *ClusterConfig
+	assert.Equal(t, "", cc.NodeBaseDir())
+}
+
+func TestNodeBaseDir_EmptyNode(t *testing.T) {
+	cc := &ClusterConfig{
+		Node: "",
+		Nodes: map[string]Config{
+			"node1": {BaseDir: "/data/node1"},
+		},
+	}
+	assert.Equal(t, "", cc.NodeBaseDir())
+}
+
+func TestNodeBaseDir_NodeNotInMap(t *testing.T) {
+	cc := &ClusterConfig{
+		Node: "missing",
+		Nodes: map[string]Config{
+			"node1": {BaseDir: "/data/node1"},
+		},
+	}
+	assert.Equal(t, "", cc.NodeBaseDir())
+}
+
+func TestNodeBaseDir_EmptyBaseDir(t *testing.T) {
+	cc := &ClusterConfig{
+		Node: "node1",
+		Nodes: map[string]Config{
+			"node1": {BaseDir: ""},
+		},
+	}
+	assert.Equal(t, "", cc.NodeBaseDir())
+}
+
+// Tests for type constants
+
+func TestNBDTransportConstants(t *testing.T) {
+	assert.Equal(t, NBDTransport("socket"), NBDTransportSocket)
+	assert.Equal(t, NBDTransport("tcp"), NBDTransportTCP)
+}
+
+// Tests for ViperblockConfig
+
+func TestLoadConfig_ViperblockShardWAL_Explicit(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1]
+region = "us-east-1"
+
+[nodes.n1.viperblock]
+shardwal = false
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	n := cfg.Nodes["n1"]
+	require.NotNil(t, n.Viperblock.ShardWAL, "ShardWAL should be set when explicitly configured")
+	assert.False(t, *n.Viperblock.ShardWAL)
+}
+
+func TestLoadConfig_ViperblockShardWAL_DefaultNil(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1]
+region = "us-east-1"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	n := cfg.Nodes["n1"]
+	assert.Nil(t, n.Viperblock.ShardWAL, "ShardWAL should be nil when not configured (defaults to false in service)")
+}
+
+func TestLoadConfig_ViperblockShardWAL_True(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "hive.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1]
+region = "us-east-1"
+
+[nodes.n1.viperblock]
+shardwal = true
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	n := cfg.Nodes["n1"]
+	require.NotNil(t, n.Viperblock.ShardWAL)
+	assert.True(t, *n.Viperblock.ShardWAL)
+}

hive/daemon/TESTING.md

@@ -418,3 +418,20 @@ When adding new tests:
    - Add comments explaining Linux-only requirements
    - Use build tags where appropriate
    - Provide alternative test paths for macOS
+
+## Local Docker E2E Tests
+
+For more advanced testing (e.g., if GitHub Actions is failing), run the Docker E2E tests locally before committing. These mirror the CI pipeline exactly.
+
+```bash
+make test-docker-single       # Single-node E2E only
+make test-docker-multi        # Multi-node E2E only
+make test-docker              # Both suites
+```
+
+Requirements:
+- Docker with `--privileged` support
+- `/dev/kvm` available (KVM-capable host)
+- Sibling repos (`viperblock/`, `predastore/`) cloned alongside `hive/`
+
+The Docker build copies local source (including `go.work` if present), so uncommitted changes in sibling repos are picked up automatically.

hive/daemon/health.go

@@ -0,0 +1,274 @@
+package daemon
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"syscall"
+	"time"
+
+	"github.com/mulgadc/hive/hive/vm"
+)
+
+const (
+	maxRestartsInWindow = 3
+	restartWindow       = 10 * time.Minute
+	restartBackoffBase  = 5 * time.Second
+	restartBackoffMax   = 2 * time.Minute
+)
+
+// classifyCrashReason extracts a human-readable crash reason from the error
+// returned by cmd.Wait(). Uses exec.ExitError + syscall.WaitStatus to
+// distinguish OOM kills (SIGKILL), segfaults (SIGSEGV), etc.
+func classifyCrashReason(waitErr error) string {
+	if waitErr == nil {
+		return "clean-exit"
+	}
+
+	var exitErr *exec.ExitError
+	if !errors.As(waitErr, &exitErr) {
+		return "unknown"
+	}
+
+	status, ok := exitErr.Sys().(syscall.WaitStatus)
+	if !ok {
+		return "unknown"
+	}
+
+	if status.Signaled() {
+		switch status.Signal() {
+		case syscall.SIGKILL:
+			return "oom-killed"
+		case syscall.SIGSEGV:
+			return "segfault"
+		case syscall.SIGABRT:
+			return "abort"
+		default:
+			return fmt.Sprintf("signal-%d", status.Signal())
+		}
+	}
+
+	if status.Exited() {
+		return fmt.Sprintf("exit-%d", status.ExitStatus())
+	}
+
+	return "unknown"
+}
+
+// handleInstanceCrash is called by the QEMU launch goroutine after cmd.Wait()
+// returns during runtime (after startup was confirmed successful). It detects
+// the crash reason, transitions the instance to error state, cleans up resources,
+// and schedules a restart if policy allows.
+func (d *Daemon) handleInstanceCrash(instance *vm.VM, waitErr error) {
+	// Guard: if instance is not running, this was an expected exit
+	// (stopInstance/terminateInstance set status before QEMU exits)
+	d.Instances.Mu.Lock()
+	status := instance.Status
+	d.Instances.Mu.Unlock()
+
+	if status != vm.StateRunning {
+		slog.Debug("QEMU exited but instance not in running state, skipping crash handler",
+			"instance", instance.ID, "status", status)
+		return
+	}
+
+	// Guard: coordinated shutdown in progress
+	if d.shuttingDown.Load() {
+		slog.Debug("QEMU exited during coordinated shutdown, skipping crash handler",
+			"instance", instance.ID)
+		return
+	}
+
+	reason := classifyCrashReason(waitErr)
+	slog.Error("VM process crashed", "instance", instance.ID, "reason", reason, "err", waitErr)
+
+	// Transition to error state
+	if err := d.TransitionState(instance, vm.StateError); err != nil {
+		slog.Error("Failed to transition crashed instance to error state",
+			"instance", instance.ID, "err", err)
+	}
+
+	// Update health tracking
+	now := time.Now()
+	d.Instances.Mu.Lock()
+	instance.Health.CrashCount++
+	instance.Health.LastCrashTime = now
+	instance.Health.LastCrashReason = reason
+	if instance.Health.FirstCrashTime.IsZero() {
+		instance.Health.FirstCrashTime = now
+	}
+	instance.Running = false
+	instance.PID = 0
+	d.Instances.Mu.Unlock()
+
+	// Deallocate resources to fix phantom reservation
+	instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
+	if ok && instanceType != nil {
+		slog.Info("Deallocating resources for crashed instance",
+			"instance", instance.ID, "type", instance.InstanceType)
+		d.resourceMgr.deallocate(instanceType)
+	}
+
+	// Clean up stale QMP socket so QEMU can rebind on restart
+	if instance.Config.QMPSocket != "" {
+		_ = os.Remove(instance.Config.QMPSocket)
+	}
+
+	// Unmount EBS volumes (same pattern as stopInstance)
+	d.unmountInstanceVolumes(instance)
+
+	// Persist state
+	if err := d.WriteState(); err != nil {
+		slog.Error("Failed to persist state after crash handling",
+			"instance", instance.ID, "err", err)
+	}
+
+	d.maybeRestartInstance(instance)
+}
+
+// unmountInstanceVolumes sends NATS unmount requests for all volumes attached
+// to the instance and updates their state to "available".
+func (d *Daemon) unmountInstanceVolumes(instance *vm.VM) {
+	instance.EBSRequests.Mu.Lock()
+	defer instance.EBSRequests.Mu.Unlock()
+
+	for _, ebsRequest := range instance.EBSRequests.Requests {
+		ebsUnMountRequest, err := json.Marshal(ebsRequest)
+		if err != nil {
+			slog.Error("Failed to marshal volume payload for crash cleanup",
+				"err", err)
+			continue
+		}
+
+		msg, err := d.natsConn.Request(d.ebsTopic("unmount"), ebsUnMountRequest, 30*time.Second)
+		if err != nil {
+			slog.Error("Failed to unmount volume after crash",
+				"name", ebsRequest.Name, "instance", instance.ID, "err", err)
+		} else {
+			slog.Info("Unmounted volume after crash",
+				"instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data))
+		}
+
+		// Update user-visible volume state to "available"
+		if !ebsRequest.EFI && !ebsRequest.CloudInit {
+			if err := d.volumeService.UpdateVolumeState(ebsRequest.Name, "available", "", ""); err != nil {
+				slog.Error("Failed to update volume state to available after crash",
+					"volumeId", ebsRequest.Name, "err", err)
+			}
+		}
+	}
+}
+
+// maybeRestartInstance checks restart policy and schedules a restart if allowed.
+func (d *Daemon) maybeRestartInstance(instance *vm.VM) {
+	if d.shuttingDown.Load() {
+		slog.Info("Skipping restart during shutdown", "instance", instance.ID)
+		return
+	}
+
+	now := time.Now()
+
+	d.Instances.Mu.Lock()
+	health := &instance.Health
+
+	// If crashes are outside the restart window, reset counters
+	if !health.FirstCrashTime.IsZero() && now.Sub(health.FirstCrashTime) > restartWindow {
+		slog.Info("Crash window expired, resetting counters", "instance", instance.ID)
+		health.CrashCount = 1
+		health.FirstCrashTime = now
+		health.RestartCount = 0
+	}
+
+	// Check if we've exceeded the max restarts in the window
+	if health.CrashCount > maxRestartsInWindow {
+		d.Instances.Mu.Unlock()
+		slog.Error("Instance exceeded max restarts in window, leaving in error state",
+			"instance", instance.ID,
+			"crashes", health.CrashCount,
+			"window", restartWindow,
+			"max", maxRestartsInWindow)
+		return
+	}
+
+	restartCount := health.RestartCount
+	d.Instances.Mu.Unlock()
+
+	// Check resource availability
+	instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
+	if !ok || instanceType == nil {
+		slog.Error("Unknown instance type, cannot restart",
+			"instance", instance.ID, "type", instance.InstanceType)
+		return
+	}
+
+	if d.resourceMgr.canAllocate(instanceType, 1) < 1 {
+		slog.Error("Insufficient resources to restart instance",
+			"instance", instance.ID, "type", instance.InstanceType)
+		return
+	}
+
+	// Calculate exponential backoff
+	delay := restartBackoffBase
+	for i := 0; i < restartCount; i++ {
+		delay *= 2
+		if delay > restartBackoffMax {
+			delay = restartBackoffMax
+			break
+		}
+	}
+
+	slog.Info("Scheduling instance restart",
+		"instance", instance.ID,
+		"delay", delay,
+		"restartCount", restartCount+1)
+
+	time.AfterFunc(delay, func() {
+		d.restartCrashedInstance(instance)
+	})
+}
+
+// restartCrashedInstance re-verifies the instance is still in error state
+// and relaunches it via LaunchInstance.
+func (d *Daemon) restartCrashedInstance(instance *vm.VM) {
+	d.Instances.Mu.Lock()
+	if instance.Status != vm.StateError {
+		d.Instances.Mu.Unlock()
+		slog.Info("Instance no longer in error state, skipping restart",
+			"instance", instance.ID, "status", instance.Status)
+		return
+	}
+
+	if d.shuttingDown.Load() {
+		d.Instances.Mu.Unlock()
+		slog.Info("Daemon shutting down, skipping restart", "instance", instance.ID)
+		return
+	}
+
+	instance.Health.RestartCount++
+	d.Instances.Mu.Unlock()
+
+	slog.Info("Restarting crashed instance",
+		"instance", instance.ID,
+		"restartCount", instance.Health.RestartCount)
+
+	// Transition Error → Pending (valid now that we added it to the transitions map)
+	if err := d.TransitionState(instance, vm.StatePending); err != nil {
+		slog.Error("Failed to transition instance to pending for restart",
+			"instance", instance.ID, "err", err)
+		return
+	}
+
+	// LaunchInstance handles the full relaunch flow:
+	// check PID → MountVolumes → StartInstance → CreateQMPClient → NATS subscribe → Running
+	if err := d.LaunchInstance(instance); err != nil {
+		slog.Error("Failed to restart crashed instance",
+			"instance", instance.ID, "err", err)
+		if err := d.TransitionState(instance, vm.StateError); err != nil {
+			slog.Error("Failed to transition instance back to error after restart failure",
+				"instance", instance.ID, "err", err)
+		}
+	}
+}

hive/daemon/health_test.go

@@ -0,0 +1,202 @@
+package daemon
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/vm"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClassifyCrashReason_CleanExit(t *testing.T) {
+	assert.Equal(t, "clean-exit", classifyCrashReason(nil))
+}
+
+func TestClassifyCrashReason_OOMKill(t *testing.T) {
+	// Start a process and kill it with SIGKILL to get a real ExitError
+	cmd := exec.Command("sleep", "60")
+	require.NoError(t, cmd.Start())
+
+	require.NoError(t, cmd.Process.Kill()) // sends SIGKILL
+	waitErr := cmd.Wait()
+	require.Error(t, waitErr)
+
+	assert.Equal(t, "oom-killed", classifyCrashReason(waitErr))
+}
+
+func TestClassifyCrashReason_ExitCode(t *testing.T) {
+	// Run a command that exits with a non-zero code
+	cmd := exec.Command("sh", "-c", "exit 42")
+	waitErr := cmd.Run()
+	require.Error(t, waitErr)
+
+	assert.Equal(t, "exit-42", classifyCrashReason(waitErr))
+}
+
+func TestClassifyCrashReason_Unknown(t *testing.T) {
+	// A non-ExitError error
+	err := fmt.Errorf("some random error")
+	assert.Equal(t, "unknown", classifyCrashReason(err))
+}
+
+func TestHandleInstanceCrash_SkipsNonRunning(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	d := &Daemon{
+		natsConn:    nc,
+		resourceMgr: NewResourceManager(),
+		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+	}
+
+	instance := &vm.VM{
+		ID:     "i-test-stopped",
+		Status: vm.StateStopped,
+	}
+	d.Instances.VMS[instance.ID] = instance
+
+	// Should return without action (no panic, no state change)
+	d.handleInstanceCrash(instance, fmt.Errorf("test error"))
+	assert.Equal(t, vm.StateStopped, instance.Status)
+}
+
+func TestHandleInstanceCrash_SkipsShuttingDown(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	d := &Daemon{
+		natsConn:    nc,
+		resourceMgr: NewResourceManager(),
+		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+	}
+	d.shuttingDown.Store(true)
+
+	instance := &vm.VM{
+		ID:     "i-test-shutdown",
+		Status: vm.StateRunning,
+	}
+	d.Instances.VMS[instance.ID] = instance
+
+	d.handleInstanceCrash(instance, fmt.Errorf("test error"))
+
+	// Status unchanged because shuttingDown is true
+	assert.Equal(t, vm.StateRunning, instance.Status)
+}
+
+func TestMaybeRestart_ExceedsMaxInWindow(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	d := &Daemon{
+		node:     "test-node",
+		natsConn: nc,
+		config: &config.Config{
+			Services: []string{"daemon"},
+		},
+		resourceMgr: NewResourceManager(),
+		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+	}
+
+	// Find an allocatable instance type
+	var allocType string
+	for typeName := range d.resourceMgr.instanceTypes {
+		allocType = typeName
+		break
+	}
+	require.NotEmpty(t, allocType)
+
+	instance := &vm.VM{
+		ID:           "i-test-maxcrash",
+		Status:       vm.StateError,
+		InstanceType: allocType,
+		Health: vm.InstanceHealthState{
+			CrashCount:     maxRestartsInWindow + 1, // exceeds limit
+			FirstCrashTime: time.Now(),
+			RestartCount:   maxRestartsInWindow,
+		},
+	}
+	d.Instances.VMS[instance.ID] = instance
+
+	// Should not schedule a restart (no panic, instance stays in error)
+	d.maybeRestartInstance(instance)
+
+	// Instance should remain in error state
+	assert.Equal(t, vm.StateError, instance.Status)
+}
+
+func TestMaybeRestart_ResetsAfterWindow(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	d := &Daemon{
+		node:     "test-node",
+		natsConn: nc,
+		config: &config.Config{
+			Services: []string{"daemon"},
+		},
+		resourceMgr: NewResourceManager(),
+		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+	}
+
+	// Find an allocatable instance type
+	var allocType string
+	for typeName := range d.resourceMgr.instanceTypes {
+		allocType = typeName
+		break
+	}
+	require.NotEmpty(t, allocType)
+
+	instance := &vm.VM{
+		ID:           "i-test-windowreset",
+		Status:       vm.StateError,
+		InstanceType: allocType,
+		Health: vm.InstanceHealthState{
+			CrashCount:     5,
+			FirstCrashTime: time.Now().Add(-restartWindow - time.Minute), // outside window
+			RestartCount:   5,
+		},
+	}
+	d.Instances.VMS[instance.ID] = instance
+
+	// maybeRestartInstance should reset counters and schedule a restart
+	// (restart will fail because there's no real QEMU, but counters should reset)
+	d.maybeRestartInstance(instance)
+
+	// Counters should be reset
+	assert.Equal(t, 1, instance.Health.CrashCount)
+	assert.Equal(t, 0, instance.Health.RestartCount)
+}
+
+func TestRestartBackoff_Exponential(t *testing.T) {
+	// Verify the backoff calculation logic
+	expected := []time.Duration{
+		5 * time.Second,   // restart 0
+		10 * time.Second,  // restart 1
+		20 * time.Second,  // restart 2
+		40 * time.Second,  // restart 3
+		80 * time.Second,  // restart 4
+		120 * time.Second, // restart 5 (capped at 2min)
+		120 * time.Second, // restart 6 (capped at 2min)
+	}
+
+	for i, want := range expected {
+		delay := restartBackoffBase
+		for j := 0; j < i; j++ {
+			delay *= 2
+			if delay > restartBackoffMax {
+				delay = restartBackoffMax
+				break
+			}
+		}
+		assert.Equal(t, want, delay, "Backoff mismatch at restart count %d", i)
+	}
+}

hive/daemon/heartbeat.go

@@ -0,0 +1,69 @@
+package daemon
+
+import (
+	"log/slog"
+	"time"
+)
+
+const heartbeatInterval = 10 * time.Second
+
+// startHeartbeat launches a goroutine that publishes this daemon's heartbeat
+// to the cluster-state KV store every heartbeatInterval. It fires immediately
+// on start, then repeats on a ticker. The goroutine exits when d.ctx is cancelled.
+func (d *Daemon) startHeartbeat() {
+	if d.jsManager == nil {
+		slog.Warn("JetStream not initialized, skipping heartbeat")
+		return
+	}
+
+	go func() {
+		// Fire immediately on startup
+		d.publishHeartbeat()
+
+		ticker := time.NewTicker(heartbeatInterval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-d.ctx.Done():
+				slog.Info("Heartbeat goroutine stopping")
+				return
+			case <-ticker.C:
+				d.publishHeartbeat()
+			}
+		}
+	}()
+
+	slog.Info("Heartbeat started", "interval", heartbeatInterval)
+}
+
+// publishHeartbeat builds and writes a heartbeat entry to KV.
+func (d *Daemon) publishHeartbeat() {
+	h := d.buildHeartbeat()
+	if err := d.jsManager.WriteHeartbeat(h); err != nil {
+		slog.Warn("Failed to publish heartbeat", "error", err)
+	} else {
+		slog.Debug("Heartbeat published", "node", h.Node, "vms", h.VMCount)
+	}
+}
+
+// buildHeartbeat constructs a Heartbeat from the daemon's current state.
+func (d *Daemon) buildHeartbeat() *Heartbeat {
+	totalVCPU, totalMem, allocVCPU, allocMem, _ := d.resourceMgr.GetResourceStats()
+
+	d.Instances.Mu.Lock()
+	vmCount := len(d.Instances.VMS)
+	d.Instances.Mu.Unlock()
+
+	return &Heartbeat{
+		Node:          d.node,
+		Epoch:         d.clusterConfig.Epoch,
+		Timestamp:     time.Now().UTC().Format(time.RFC3339),
+		Services:      d.config.GetServices(),
+		VMCount:       vmCount,
+		AllocatedVCPU: allocVCPU,
+		AvailableVCPU: totalVCPU - allocVCPU,
+		AllocatedMem:  allocMem,
+		AvailableMem:  totalMem - allocMem,
+	}
+}

hive/daemon/heartbeat_test.go

@@ -0,0 +1,126 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/vm"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestBuildHeartbeat verifies that buildHeartbeat populates the struct from daemon state.
+func TestBuildHeartbeat(t *testing.T) {
+	d := &Daemon{
+		node: "test-node",
+		clusterConfig: &config.ClusterConfig{
+			Epoch: 5,
+		},
+		config: &config.Config{
+			Services: []string{"daemon", "nats", "viperblock"},
+		},
+		resourceMgr: NewResourceManager(),
+		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+	}
+
+	h := d.buildHeartbeat()
+
+	assert.Equal(t, "test-node", h.Node)
+	assert.Equal(t, uint64(5), h.Epoch)
+	assert.NotEmpty(t, h.Timestamp)
+	assert.Equal(t, []string{"daemon", "nats", "viperblock"}, h.Services)
+	assert.Equal(t, 0, h.VMCount)
+	assert.Equal(t, 0, h.AllocatedVCPU)
+	assert.Greater(t, h.AvailableVCPU, 0)
+	assert.Greater(t, h.AvailableMem, 0.0)
+}
+
+// TestHeartbeatReflectsAllocation verifies that allocating resources changes the heartbeat values.
+func TestHeartbeatReflectsAllocation(t *testing.T) {
+	d := &Daemon{
+		node: "alloc-node",
+		clusterConfig: &config.ClusterConfig{
+			Epoch: 1,
+		},
+		config: &config.Config{
+			Services: []string{"daemon"},
+		},
+		resourceMgr: NewResourceManager(),
+		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
+	}
+
+	// Take a heartbeat before allocation
+	before := d.buildHeartbeat()
+
+	// Find an instance type we can allocate
+	var allocType string
+	for typeName := range d.resourceMgr.instanceTypes {
+		if d.resourceMgr.canAllocate(d.resourceMgr.instanceTypes[typeName], 1) >= 1 {
+			allocType = typeName
+			break
+		}
+	}
+	require.NotEmpty(t, allocType, "Should have at least one allocatable instance type")
+
+	// Allocate resources
+	err := d.resourceMgr.allocate(d.resourceMgr.instanceTypes[allocType])
+	require.NoError(t, err)
+
+	// Add a VM to the instance map
+	d.Instances.Mu.Lock()
+	d.Instances.VMS["i-test-001"] = &vm.VM{
+		ID:           "i-test-001",
+		Status:       vm.StateRunning,
+		InstanceType: allocType,
+	}
+	d.Instances.Mu.Unlock()
+
+	// Take a heartbeat after allocation
+	after := d.buildHeartbeat()
+
+	assert.Equal(t, 1, after.VMCount, "Should reflect 1 VM")
+	assert.Greater(t, after.AllocatedVCPU, before.AllocatedVCPU, "AllocatedVCPU should increase")
+	assert.Less(t, after.AvailableVCPU, before.AvailableVCPU, "AvailableVCPU should decrease")
+}
+
+// TestHeartbeatKVRoundTrip verifies that heartbeat can be written to and read from KV.
+func TestHeartbeatKVRoundTrip(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	err = jsm.InitClusterStateBucket()
+	require.NoError(t, err)
+
+	h := &Heartbeat{
+		Node:          "kv-test-node",
+		Epoch:         3,
+		Timestamp:     "2025-01-01T00:00:00Z",
+		Services:      []string{"daemon", "nats"},
+		VMCount:       2,
+		AllocatedVCPU: 4,
+		AvailableVCPU: 12,
+		AllocatedMem:  8.0,
+		AvailableMem:  24.0,
+	}
+
+	err = jsm.WriteHeartbeat(h)
+	require.NoError(t, err)
+
+	loaded, err := jsm.ReadHeartbeat("kv-test-node")
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+
+	assert.Equal(t, h.Node, loaded.Node)
+	assert.Equal(t, h.Epoch, loaded.Epoch)
+	assert.Equal(t, h.Timestamp, loaded.Timestamp)
+	assert.Equal(t, h.Services, loaded.Services)
+	assert.Equal(t, h.VMCount, loaded.VMCount)
+	assert.Equal(t, h.AllocatedVCPU, loaded.AllocatedVCPU)
+	assert.Equal(t, h.AvailableVCPU, loaded.AvailableVCPU)
+	assert.Equal(t, h.AllocatedMem, loaded.AllocatedMem)
+	assert.Equal(t, h.AvailableMem, loaded.AvailableMem)
+}

hive/daemon/jetstream.go

@@ -0,0 +1,482 @@
+package daemon
+
+import (
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/hive/hive/vm"
+	"github.com/nats-io/nats.go"
+)
+
+const (
+	// InstanceStateBucket is the name of the KV bucket for storing instance state
+	InstanceStateBucket = "hive-instance-state"
+	// ClusterStateBucket is the name of the KV bucket for cluster state (heartbeats, shutdown markers, service maps)
+	ClusterStateBucket = "hive-cluster-state"
+	// InstanceStatePrefix is the key prefix for per-node instance state entries
+	InstanceStatePrefix = "node."
+	// StoppedInstancePrefix is the key prefix for stopped instances in shared KV
+	StoppedInstancePrefix = "instance."
+)
+
+// JetStreamManager manages JetStream KV store operations for instance state
+type JetStreamManager struct {
+	js        nats.JetStreamContext
+	kv        nats.KeyValue // hive-instance-state
+	clusterKV nats.KeyValue // hive-cluster-state
+	replicas  int
+}
+
+// NewJetStreamManager creates a new JetStreamManager from a NATS connection.
+// replicas specifies the number of replicas for the KV bucket (typically matches cluster node count).
+func NewJetStreamManager(nc *nats.Conn, replicas int) (*JetStreamManager, error) {
+	js, err := nc.JetStream()
+	if err != nil {
+		return nil, err
+	}
+
+	return &JetStreamManager{
+		js:       js,
+		replicas: replicas,
+	}, nil
+}
+
+// InitKVBucket initializes the KV bucket, creating it if it doesn't exist
+func (m *JetStreamManager) InitKVBucket() error {
+	// Try to get the existing bucket first
+	kv, err := m.js.KeyValue(InstanceStateBucket)
+	if err != nil {
+		if errors.Is(err, nats.ErrBucketNotFound) {
+			// Bucket doesn't exist, create it
+			slog.Debug("Creating JetStream KV bucket", "bucket", InstanceStateBucket, "replicas", m.replicas)
+			kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{
+				Bucket:      InstanceStateBucket,
+				Description: "Hive instance state storage",
+				History:     1,          // Only keep latest value
+				Replicas:    m.replicas, // Replication across cluster nodes
+			})
+			if err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	} else {
+		slog.Debug("Connected to existing JetStream KV bucket", "bucket", InstanceStateBucket)
+	}
+
+	m.kv = kv
+	return nil
+}
+
+// InitClusterStateBucket initializes the cluster-state KV bucket, creating it if it doesn't exist.
+func (m *JetStreamManager) InitClusterStateBucket() error {
+	kv, err := m.js.KeyValue(ClusterStateBucket)
+	if err != nil {
+		if errors.Is(err, nats.ErrBucketNotFound) {
+			slog.Debug("Creating JetStream KV bucket", "bucket", ClusterStateBucket, "replicas", m.replicas)
+			kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{
+				Bucket:      ClusterStateBucket,
+				Description: "Hive cluster state (heartbeats, shutdown markers, service maps)",
+				History:     1,
+				Replicas:    m.replicas,
+				TTL:         1 * time.Hour,
+			})
+			if err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	} else {
+		slog.Debug("Connected to existing JetStream KV bucket", "bucket", ClusterStateBucket)
+	}
+
+	m.clusterKV = kv
+	return nil
+}
+
+// Heartbeat represents a daemon's periodic health status published to cluster KV.
+type Heartbeat struct {
+	Node          string   `json:"node"`
+	Epoch         uint64   `json:"epoch"`
+	Timestamp     string   `json:"timestamp"`
+	Services      []string `json:"services"`
+	VMCount       int      `json:"vm_count"`
+	AllocatedVCPU int      `json:"allocated_vcpu"`
+	AvailableVCPU int      `json:"available_vcpu"`
+	AllocatedMem  float64  `json:"allocated_mem_gb"`
+	AvailableMem  float64  `json:"available_mem_gb"`
+}
+
+// WriteHeartbeat writes a heartbeat entry for the given node to the cluster-state KV.
+func (m *JetStreamManager) WriteHeartbeat(h *Heartbeat) error {
+	if m.clusterKV == nil {
+		return errors.New("cluster state KV not initialized")
+	}
+	data, err := json.Marshal(h)
+	if err != nil {
+		return err
+	}
+	_, err = m.clusterKV.Put("heartbeat."+h.Node, data)
+	return err
+}
+
+// ReadHeartbeat reads the heartbeat entry for the given node from the cluster-state KV.
+func (m *JetStreamManager) ReadHeartbeat(nodeID string) (*Heartbeat, error) {
+	if m.clusterKV == nil {
+		return nil, errors.New("cluster state KV not initialized")
+	}
+	entry, err := m.clusterKV.Get("heartbeat." + nodeID)
+	if err != nil {
+		return nil, err
+	}
+	var h Heartbeat
+	if err := json.Unmarshal(entry.Value(), &h); err != nil {
+		return nil, err
+	}
+	return &h, nil
+}
+
+// ClusterShutdownState tracks the coordinated cluster shutdown progress in KV.
+type ClusterShutdownState struct {
+	Initiator  string            `json:"initiator"`
+	Phase      string            `json:"phase"`
+	Started    string            `json:"started"`
+	Timeout    string            `json:"timeout"`
+	Force      bool              `json:"force"`
+	NodesTotal int               `json:"nodes_total"`
+	NodesAcked map[string]string `json:"nodes_acked"`
+}
+
+// WriteClusterShutdown writes the cluster shutdown state to KV.
+func (m *JetStreamManager) WriteClusterShutdown(state *ClusterShutdownState) error {
+	if m.clusterKV == nil {
+		return errors.New("cluster state KV not initialized")
+	}
+	data, err := json.Marshal(state)
+	if err != nil {
+		return err
+	}
+	_, err = m.clusterKV.Put("cluster.shutdown", data)
+	return err
+}
+
+// ReadClusterShutdown reads the cluster shutdown state from KV.
+func (m *JetStreamManager) ReadClusterShutdown() (*ClusterShutdownState, error) {
+	if m.clusterKV == nil {
+		return nil, errors.New("cluster state KV not initialized")
+	}
+	entry, err := m.clusterKV.Get("cluster.shutdown")
+	if err != nil {
+		return nil, err
+	}
+	var state ClusterShutdownState
+	if err := json.Unmarshal(entry.Value(), &state); err != nil {
+		return nil, err
+	}
+	return &state, nil
+}
+
+// DeleteClusterShutdown removes the cluster shutdown state from KV.
+func (m *JetStreamManager) DeleteClusterShutdown() error {
+	if m.clusterKV == nil {
+		return errors.New("cluster state KV not initialized")
+	}
+	err := m.clusterKV.Delete("cluster.shutdown")
+	if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
+		return err
+	}
+	return nil
+}
+
+// WriteShutdownMarker writes a shutdown marker for the given node to the cluster-state KV.
+func (m *JetStreamManager) WriteShutdownMarker(nodeID string) error {
+	if m.clusterKV == nil {
+		return errors.New("cluster state KV not initialized")
+	}
+	data, _ := json.Marshal(map[string]any{
+		"node":      nodeID,
+		"timestamp": time.Now().UTC().Format(time.RFC3339),
+	})
+	_, err := m.clusterKV.Put("shutdown."+nodeID, data)
+	return err
+}
+
+// ReadShutdownMarker checks if a clean shutdown marker exists for the given node.
+func (m *JetStreamManager) ReadShutdownMarker(nodeID string) (bool, error) {
+	if m.clusterKV == nil {
+		return false, errors.New("cluster state KV not initialized")
+	}
+	_, err := m.clusterKV.Get("shutdown." + nodeID)
+	if errors.Is(err, nats.ErrKeyNotFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// DeleteShutdownMarker removes the shutdown marker for the given node.
+func (m *JetStreamManager) DeleteShutdownMarker(nodeID string) error {
+	if m.clusterKV == nil {
+		return errors.New("cluster state KV not initialized")
+	}
+	err := m.clusterKV.Delete("shutdown." + nodeID)
+	if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
+		return err
+	}
+	return nil
+}
+
+// WriteServiceManifest writes the service manifest for the given node to the cluster-state KV.
+func (m *JetStreamManager) WriteServiceManifest(nodeID string, services []string, natsHost, predastoreHost string) error {
+	if m.clusterKV == nil {
+		return errors.New("cluster state KV not initialized")
+	}
+	data, _ := json.Marshal(map[string]any{
+		"node":            nodeID,
+		"services":        services,
+		"nats_host":       natsHost,
+		"predastore_host": predastoreHost,
+		"timestamp":       time.Now().UTC().Format(time.RFC3339),
+	})
+	_, err := m.clusterKV.Put("node."+nodeID+".services", data)
+	return err
+}
+
+// WriteState writes the instance state to the KV store for the given node.
+// It acquires instances.Mu internally.
+func (m *JetStreamManager) WriteState(nodeID string, instances *vm.Instances) error {
+	instances.Mu.Lock()
+	defer instances.Mu.Unlock()
+
+	if m.kv == nil {
+		return errors.New("KV bucket not initialized")
+	}
+
+	// Create a struct without the mutex to avoid copying the lock
+	state := struct {
+		VMS map[string]*vm.VM `json:"vms"`
+	}{
+		VMS: instances.VMS,
+	}
+
+	jsonData, err := json.Marshal(state)
+	if err != nil {
+		return err
+	}
+
+	key := InstanceStatePrefix + nodeID
+	_, err = m.kv.Put(key, jsonData)
+	if err != nil {
+		return err
+	}
+
+	slog.Debug("Wrote state to JetStream KV", "key", key, "instances", len(instances.VMS))
+	return nil
+}
+
+// LoadState loads the instance state from the KV store for the given node
+func (m *JetStreamManager) LoadState(nodeID string) (*vm.Instances, error) {
+	if m.kv == nil {
+		return nil, errors.New("KV bucket not initialized")
+	}
+
+	key := InstanceStatePrefix + nodeID
+	entry, err := m.kv.Get(key)
+	if err != nil {
+		if errors.Is(err, nats.ErrKeyNotFound) {
+			// Key doesn't exist, return empty state
+			slog.Debug("No existing state in JetStream KV, returning empty state", "key", key)
+			return &vm.Instances{VMS: make(map[string]*vm.VM)}, nil
+		}
+		return nil, err
+	}
+
+	var instances vm.Instances
+	if err := json.Unmarshal(entry.Value(), &instances); err != nil {
+		return nil, err
+	}
+
+	// Ensure the VMS map is initialized
+	if instances.VMS == nil {
+		instances.VMS = make(map[string]*vm.VM)
+	}
+
+	slog.Debug("Loaded state from JetStream KV", "key", key, "instances", len(instances.VMS))
+	return &instances, nil
+}
+
+// DeleteState removes the instance state from the KV store for the given node
+func (m *JetStreamManager) DeleteState(nodeID string) error {
+	if m.kv == nil {
+		return errors.New("KV bucket not initialized")
+	}
+
+	key := InstanceStatePrefix + nodeID
+	err := m.kv.Delete(key)
+	if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
+		return err
+	}
+
+	slog.Debug("Deleted state from JetStream KV", "key", key)
+	return nil
+}
+
+// UpdateReplicas updates the replica count for the KV bucket's underlying stream.
+// This should be called when new nodes join the cluster.
+// Note: Increasing replicas requires the new replica count of NATS servers to be available.
+func (m *JetStreamManager) UpdateReplicas(newReplicas int) error {
+	if m.js == nil {
+		return errors.New("JetStream context not initialized")
+	}
+
+	// KV buckets are backed by streams with name "KV_<bucket>"
+	streamName := "KV_" + InstanceStateBucket
+
+	// Get current stream info
+	streamInfo, err := m.js.StreamInfo(streamName)
+	if err != nil {
+		return err
+	}
+
+	currentReplicas := streamInfo.Config.Replicas
+	if currentReplicas >= newReplicas {
+		slog.Debug("Stream already has sufficient replicas", "current", currentReplicas, "requested", newReplicas)
+		return nil
+	}
+
+	// Update the stream config with new replica count
+	streamInfo.Config.Replicas = newReplicas
+
+	_, err = m.js.UpdateStream(&streamInfo.Config)
+	if err != nil {
+		return err
+	}
+
+	m.replicas = newReplicas
+	slog.Info("Updated JetStream KV bucket replicas", "bucket", InstanceStateBucket, "oldReplicas", currentReplicas, "newReplicas", newReplicas)
+
+	// Also update the cluster-state bucket if it exists
+	if m.clusterKV != nil {
+		clusterStreamName := "KV_" + ClusterStateBucket
+		clusterInfo, err := m.js.StreamInfo(clusterStreamName)
+		if err == nil && clusterInfo.Config.Replicas < newReplicas {
+			clusterInfo.Config.Replicas = newReplicas
+			if _, err := m.js.UpdateStream(&clusterInfo.Config); err != nil {
+				slog.Warn("Failed to update cluster-state bucket replicas", "error", err)
+			} else {
+				slog.Info("Updated cluster-state bucket replicas", "bucket", ClusterStateBucket, "newReplicas", newReplicas)
+			}
+		}
+	}
+
+	return nil
+}
+
+// WriteStoppedInstance writes a stopped instance to the shared KV store.
+func (m *JetStreamManager) WriteStoppedInstance(instanceID string, instance *vm.VM) error {
+	if m.kv == nil {
+		return errors.New("KV bucket not initialized")
+	}
+
+	jsonData, err := json.Marshal(instance)
+	if err != nil {
+		return err
+	}
+
+	key := StoppedInstancePrefix + instanceID
+	_, err = m.kv.Put(key, jsonData)
+	if err != nil {
+		return err
+	}
+
+	slog.Debug("Wrote stopped instance to JetStream KV", "key", key, "instanceId", instanceID)
+	return nil
+}
+
+// LoadStoppedInstance loads a stopped instance from the shared KV store.
+// Returns nil, nil if the key does not exist.
+func (m *JetStreamManager) LoadStoppedInstance(instanceID string) (*vm.VM, error) {
+	if m.kv == nil {
+		return nil, errors.New("KV bucket not initialized")
+	}
+
+	key := StoppedInstancePrefix + instanceID
+	entry, err := m.kv.Get(key)
+	if err != nil {
+		if errors.Is(err, nats.ErrKeyNotFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	var instance vm.VM
+	if err := json.Unmarshal(entry.Value(), &instance); err != nil {
+		return nil, err
+	}
+
+	return &instance, nil
+}
+
+// DeleteStoppedInstance removes a stopped instance from the shared KV store.
+// It is idempotent — deleting a non-existent key is not an error.
+func (m *JetStreamManager) DeleteStoppedInstance(instanceID string) error {
+	if m.kv == nil {
+		return errors.New("KV bucket not initialized")
+	}
+
+	key := StoppedInstancePrefix + instanceID
+	err := m.kv.Delete(key)
+	if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
+		return err
+	}
+
+	slog.Debug("Deleted stopped instance from JetStream KV", "key", key)
+	return nil
+}
+
+// ListStoppedInstances returns all stopped instances from the shared KV store.
+func (m *JetStreamManager) ListStoppedInstances() ([]*vm.VM, error) {
+	if m.kv == nil {
+		return nil, errors.New("KV bucket not initialized")
+	}
+
+	keys, err := m.kv.Keys()
+	if err != nil {
+		if errors.Is(err, nats.ErrNoKeysFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	var instances []*vm.VM
+	for _, key := range keys {
+		if !strings.HasPrefix(key, StoppedInstancePrefix) {
+			continue
+		}
+
+		entry, err := m.kv.Get(key)
+		if err != nil {
+			if errors.Is(err, nats.ErrKeyNotFound) {
+				continue
+			}
+			return nil, err
+		}
+
+		var instance vm.VM
+		if err := json.Unmarshal(entry.Value(), &instance); err != nil {
+			slog.Error("Failed to unmarshal stopped instance", "key", key, "err", err)
+			continue
+		}
+
+		instances = append(instances, &instance)
+	}
+
+	return instances, nil
+}

hive/daemon/jetstream_test.go

@@ -0,0 +1,573 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/mulgadc/hive/hive/vm"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestJetStreamManager_WriteAndLoadState tests round-trip write and load of instance state
+func TestJetStreamManager_WriteAndLoadState(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	// Connect to NATS
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err, "Failed to connect to NATS")
+	defer nc.Close()
+
+	// Create JetStreamManager
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err, "Failed to create JetStreamManager")
+
+	// Initialize the KV bucket
+	err = jsm.InitKVBucket()
+	require.NoError(t, err, "Failed to init KV bucket")
+
+	// Create test instances
+	testNodeID := "test-node-1"
+	testInstances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-test-001": {
+				ID:           "i-test-001",
+				Status:       vm.StateRunning,
+				InstanceType: "t3.micro",
+			},
+			"i-test-002": {
+				ID:           "i-test-002",
+				Status:       vm.StateStopped,
+				InstanceType: "t3.small",
+			},
+		},
+	}
+
+	// Write state
+	err = jsm.WriteState(testNodeID, testInstances)
+	require.NoError(t, err, "Failed to write state")
+
+	// Load state
+	loadedInstances, err := jsm.LoadState(testNodeID)
+	require.NoError(t, err, "Failed to load state")
+	require.NotNil(t, loadedInstances, "Loaded instances should not be nil")
+
+	// Verify the loaded state matches
+	assert.Len(t, loadedInstances.VMS, 2, "Should have 2 instances")
+	assert.NotNil(t, loadedInstances.VMS["i-test-001"], "Should have i-test-001")
+	assert.NotNil(t, loadedInstances.VMS["i-test-002"], "Should have i-test-002")
+	assert.Equal(t, vm.StateRunning, loadedInstances.VMS["i-test-001"].Status)
+	assert.Equal(t, vm.StateStopped, loadedInstances.VMS["i-test-002"].Status)
+	assert.Equal(t, "t3.micro", loadedInstances.VMS["i-test-001"].InstanceType)
+	assert.Equal(t, "t3.small", loadedInstances.VMS["i-test-002"].InstanceType)
+}
+
+// TestJetStreamManager_LoadState_KeyNotFound tests that LoadState returns empty state when key doesn't exist
+func TestJetStreamManager_LoadState_KeyNotFound(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	// Load state for a non-existent node
+	instances, err := jsm.LoadState("non-existent-node")
+	require.NoError(t, err, "Should not error when key not found")
+	require.NotNil(t, instances, "Should return non-nil instances")
+	assert.Empty(t, instances.VMS, "Should return empty VMS map")
+}
+
+// TestJetStreamManager_BucketCreation tests that InitKVBucket creates the bucket when it doesn't exist
+func TestJetStreamManager_BucketCreation(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	// InitKVBucket should create the bucket
+	err = jsm.InitKVBucket()
+	require.NoError(t, err, "Should create bucket without error")
+
+	// Verify the bucket exists by checking jsm.kv is set
+	assert.NotNil(t, jsm.kv, "KV bucket should be initialized")
+}
+
+// TestJetStreamManager_BucketReconnection tests that InitKVBucket connects to existing bucket
+func TestJetStreamManager_BucketReconnection(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	// First connection - create the bucket
+	nc1, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+
+	jsm1, err := NewJetStreamManager(nc1, 1)
+	require.NoError(t, err)
+
+	err = jsm1.InitKVBucket()
+	require.NoError(t, err, "First InitKVBucket should succeed")
+
+	// Write some test data
+	testInstances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-persist": {
+				ID:     "i-persist",
+				Status: vm.StateRunning,
+			},
+		},
+	}
+	err = jsm1.WriteState("persist-node", testInstances)
+	require.NoError(t, err)
+
+	nc1.Close()
+
+	// Second connection - should connect to existing bucket
+	nc2, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc2.Close()
+
+	jsm2, err := NewJetStreamManager(nc2, 1)
+	require.NoError(t, err)
+
+	err = jsm2.InitKVBucket()
+	require.NoError(t, err, "Second InitKVBucket should succeed (reconnect)")
+
+	// Verify data persisted
+	loadedInstances, err := jsm2.LoadState("persist-node")
+	require.NoError(t, err)
+	assert.NotEmpty(t, loadedInstances.VMS, "Should have persisted instances")
+	assert.NotNil(t, loadedInstances.VMS["i-persist"], "Should have i-persist")
+	assert.Equal(t, vm.StateRunning, loadedInstances.VMS["i-persist"].Status)
+}
+
+// TestJetStreamManager_DeleteState tests deleting state from the KV store
+func TestJetStreamManager_DeleteState(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	// Write state
+	testNodeID := "delete-test-node"
+	testInstances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-delete-me": {
+				ID:     "i-delete-me",
+				Status: vm.StateRunning,
+			},
+		},
+	}
+	err = jsm.WriteState(testNodeID, testInstances)
+	require.NoError(t, err)
+
+	// Verify state exists
+	loadedInstances, err := jsm.LoadState(testNodeID)
+	require.NoError(t, err)
+	assert.NotEmpty(t, loadedInstances.VMS)
+
+	// Delete state
+	err = jsm.DeleteState(testNodeID)
+	require.NoError(t, err, "Should delete state without error")
+
+	// Verify state is gone (should return empty state)
+	loadedInstances, err = jsm.LoadState(testNodeID)
+	require.NoError(t, err)
+	assert.Empty(t, loadedInstances.VMS, "Should return empty state after deletion")
+}
+
+// TestJetStreamManager_DeleteState_NonExistent tests deleting state that doesn't exist
+func TestJetStreamManager_DeleteState_NonExistent(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	// Delete non-existent state should not error
+	err = jsm.DeleteState("non-existent-node")
+	require.NoError(t, err, "Deleting non-existent state should not error")
+}
+
+// TestJetStreamManager_WriteState_UpdateExisting tests that writing state updates existing entry
+func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	testNodeID := "update-test-node"
+
+	// Write initial state
+	initialInstances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-initial": {
+				ID:     "i-initial",
+				Status: vm.StateRunning,
+			},
+		},
+	}
+	err = jsm.WriteState(testNodeID, initialInstances)
+	require.NoError(t, err)
+
+	// Update state with different instances
+	updatedInstances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-initial": {
+				ID:     "i-initial",
+				Status: vm.StateStopped, // Changed status
+			},
+			"i-new": { // Added new instance
+				ID:     "i-new",
+				Status: vm.StateRunning,
+			},
+		},
+	}
+	err = jsm.WriteState(testNodeID, updatedInstances)
+	require.NoError(t, err)
+
+	// Load and verify updated state
+	loadedInstances, err := jsm.LoadState(testNodeID)
+	require.NoError(t, err)
+	assert.Len(t, loadedInstances.VMS, 2, "Should have 2 instances")
+	assert.Equal(t, vm.StateStopped, loadedInstances.VMS["i-initial"].Status, "Status should be updated")
+	assert.NotNil(t, loadedInstances.VMS["i-new"], "Should have new instance")
+}
+
+// TestJetStreamManager_MultipleNodes tests storing state for multiple nodes
+func TestJetStreamManager_MultipleNodes(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	// Write state for node-1
+	node1Instances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-node1-001": {ID: "i-node1-001", Status: vm.StateRunning},
+		},
+	}
+	err = jsm.WriteState("node-1", node1Instances)
+	require.NoError(t, err)
+
+	// Write state for node-2
+	node2Instances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-node2-001": {ID: "i-node2-001", Status: vm.StateStopped},
+			"i-node2-002": {ID: "i-node2-002", Status: vm.StateRunning},
+		},
+	}
+	err = jsm.WriteState("node-2", node2Instances)
+	require.NoError(t, err)
+
+	// Load and verify node-1 state
+	loadedNode1, err := jsm.LoadState("node-1")
+	require.NoError(t, err)
+	assert.Len(t, loadedNode1.VMS, 1)
+	assert.NotNil(t, loadedNode1.VMS["i-node1-001"])
+
+	// Load and verify node-2 state
+	loadedNode2, err := jsm.LoadState("node-2")
+	require.NoError(t, err)
+	assert.Len(t, loadedNode2.VMS, 2)
+	assert.NotNil(t, loadedNode2.VMS["i-node2-001"])
+	assert.NotNil(t, loadedNode2.VMS["i-node2-002"])
+
+	// Verify node isolation - node-1 doesn't have node-2's instances
+	_, exists := loadedNode1.VMS["i-node2-001"]
+	assert.False(t, exists, "Node-1 should not have node-2's instances")
+}
+
+// TestJetStreamManager_KVNotInitialized tests error handling when KV is not initialized
+func TestJetStreamManager_KVNotInitialized(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	// Create JetStreamManager but don't call InitKVBucket
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	testInstances := &vm.Instances{VMS: make(map[string]*vm.VM)}
+	err = jsm.WriteState("test-node", testInstances)
+	assert.Error(t, err, "WriteState should error when KV not initialized")
+
+	_, err = jsm.LoadState("test-node")
+	assert.Error(t, err, "LoadState should error when KV not initialized")
+
+	err = jsm.DeleteState("test-node")
+	assert.Error(t, err, "DeleteState should error when KV not initialized")
+}
+
+// TestJetStreamManager_UpdateReplicas tests updating replica count for the KV bucket
+func TestJetStreamManager_UpdateReplicas(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	// Create with 1 replica (typical for single node startup)
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	// Verify initial replica count
+	js, _ := nc.JetStream()
+	streamInfo, err := js.StreamInfo("KV_" + InstanceStateBucket)
+	require.NoError(t, err)
+	assert.Equal(t, 1, streamInfo.Config.Replicas, "Should start with 1 replica")
+
+	// Try to update to same replica count (should be a no-op)
+	err = jsm.UpdateReplicas(1)
+	assert.NoError(t, err, "Updating to same replica count should succeed")
+
+	// Note: Increasing replicas beyond 1 requires additional NATS servers in the cluster,
+	// which we don't have in the test environment. In a single-node test server,
+	// attempting to increase replicas will fail with "insufficient resources" error.
+	// This test verifies the basic functionality works.
+}
+
+// TestJetStreamManager_UpdateReplicas_NoInit tests UpdateReplicas when JS not initialized
+func TestJetStreamManager_UpdateReplicas_NoInit(t *testing.T) {
+	// Test with nil JetStream context
+	jsm := &JetStreamManager{
+		js:       nil,
+		replicas: 1,
+	}
+
+	err := jsm.UpdateReplicas(3)
+	assert.Error(t, err, "UpdateReplicas should error when JetStream not initialized")
+}
+
+// --- Stopped instance KV tests ---
+
+// TestJetStreamManager_WriteAndLoadStoppedInstance tests round-trip write and load of a stopped instance
+func TestJetStreamManager_WriteAndLoadStoppedInstance(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	testVM := &vm.VM{
+		ID:           "i-stopped-001",
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		LastNode:     "node-1",
+	}
+
+	// Write stopped instance
+	err = jsm.WriteStoppedInstance(testVM.ID, testVM)
+	require.NoError(t, err)
+
+	// Load stopped instance
+	loaded, err := jsm.LoadStoppedInstance(testVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+	assert.Equal(t, testVM.ID, loaded.ID)
+	assert.Equal(t, vm.StateStopped, loaded.Status)
+	assert.Equal(t, "t3.micro", loaded.InstanceType)
+	assert.Equal(t, "node-1", loaded.LastNode)
+
+	// Cleanup
+	_ = jsm.DeleteStoppedInstance(testVM.ID)
+}
+
+// TestJetStreamManager_LoadStoppedInstance_NotFound tests that LoadStoppedInstance returns nil for missing key
+func TestJetStreamManager_LoadStoppedInstance_NotFound(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	loaded, err := jsm.LoadStoppedInstance("i-nonexistent")
+	require.NoError(t, err)
+	assert.Nil(t, loaded)
+}
+
+// TestJetStreamManager_DeleteStoppedInstance tests deleting a stopped instance (including non-existent)
+func TestJetStreamManager_DeleteStoppedInstance(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	testVM := &vm.VM{
+		ID:     "i-stopped-del",
+		Status: vm.StateStopped,
+	}
+
+	// Write then delete
+	err = jsm.WriteStoppedInstance(testVM.ID, testVM)
+	require.NoError(t, err)
+
+	err = jsm.DeleteStoppedInstance(testVM.ID)
+	require.NoError(t, err)
+
+	// Verify it's gone
+	loaded, err := jsm.LoadStoppedInstance(testVM.ID)
+	require.NoError(t, err)
+	assert.Nil(t, loaded)
+
+	// Delete non-existent should not error
+	err = jsm.DeleteStoppedInstance("i-does-not-exist")
+	require.NoError(t, err)
+}
+
+// TestJetStreamManager_ListStoppedInstances tests listing multiple stopped instances
+func TestJetStreamManager_ListStoppedInstances(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	// Write multiple stopped instances
+	vms := []*vm.VM{
+		{ID: "i-list-001", Status: vm.StateStopped, InstanceType: "t3.micro"},
+		{ID: "i-list-002", Status: vm.StateStopped, InstanceType: "t3.small"},
+		{ID: "i-list-003", Status: vm.StateStopped, InstanceType: "t3.medium"},
+	}
+
+	for _, v := range vms {
+		err = jsm.WriteStoppedInstance(v.ID, v)
+		require.NoError(t, err)
+	}
+
+	// List stopped instances
+	instances, err := jsm.ListStoppedInstances()
+	require.NoError(t, err)
+
+	// Should contain at least the 3 we wrote (may also contain instances from other tests)
+	instanceIDs := make(map[string]bool)
+	for _, inst := range instances {
+		instanceIDs[inst.ID] = true
+	}
+	assert.True(t, instanceIDs["i-list-001"])
+	assert.True(t, instanceIDs["i-list-002"])
+	assert.True(t, instanceIDs["i-list-003"])
+
+	// Cleanup
+	for _, v := range vms {
+		_ = jsm.DeleteStoppedInstance(v.ID)
+	}
+}
+
+// TestJetStreamManager_StoppedInstances_NoInterference tests that stopped instances don't interfere with per-node state
+func TestJetStreamManager_StoppedInstances_NoInterference(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	err = jsm.InitKVBucket()
+	require.NoError(t, err)
+
+	// Write per-node state
+	nodeInstances := &vm.Instances{
+		VMS: map[string]*vm.VM{
+			"i-running-001": {ID: "i-running-001", Status: vm.StateRunning},
+		},
+	}
+	err = jsm.WriteState("interference-test-node", nodeInstances)
+	require.NoError(t, err)
+
+	// Write stopped instance
+	stoppedVM := &vm.VM{ID: "i-stopped-interf", Status: vm.StateStopped}
+	err = jsm.WriteStoppedInstance(stoppedVM.ID, stoppedVM)
+	require.NoError(t, err)
+
+	// Verify per-node state is unaffected
+	loaded, err := jsm.LoadState("interference-test-node")
+	require.NoError(t, err)
+	assert.Len(t, loaded.VMS, 1)
+	assert.NotNil(t, loaded.VMS["i-running-001"])
+
+	// Verify stopped instance loads independently
+	loadedStopped, err := jsm.LoadStoppedInstance(stoppedVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, loadedStopped)
+	assert.Equal(t, "i-stopped-interf", loadedStopped.ID)
+
+	// Cleanup
+	_ = jsm.DeleteStoppedInstance(stoppedVM.ID)
+	_ = jsm.DeleteState("interference-test-node")
+}
+
+// TestJetStreamManager_StoppedInstance_KVNotInitialized tests error handling when KV is not initialized
+func TestJetStreamManager_StoppedInstance_KVNotInitialized(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	// Don't call InitKVBucket
+
+	err = jsm.WriteStoppedInstance("i-test", &vm.VM{})
+	assert.Error(t, err)
+
+	_, err = jsm.LoadStoppedInstance("i-test")
+	assert.Error(t, err)
+
+	err = jsm.DeleteStoppedInstance("i-test")
+	assert.Error(t, err)
+
+	_, err = jsm.ListStoppedInstances()
+	assert.Error(t, err)
+}

hive/daemon/main_test.go

@@ -0,0 +1,73 @@
+package daemon
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+)
+
+// Shared NATS servers for all daemon tests — started once in TestMain.
+var (
+	sharedNATSServer *server.Server
+	sharedNATSURL    string
+	sharedJSServer   *server.Server
+	sharedJSNATSURL  string
+)
+
+func TestMain(m *testing.M) {
+	// Start a plain NATS server (no JetStream)
+	ns, err := server.NewServer(&server.Options{
+		Host:   "127.0.0.1",
+		Port:   -1,
+		NoLog:  true,
+		NoSigs: true,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to create shared NATS server: %v\n", err)
+		os.Exit(1)
+	}
+	go ns.Start()
+	if !ns.ReadyForConnections(5 * time.Second) {
+		fmt.Fprintln(os.Stderr, "Shared NATS server failed to start")
+		os.Exit(1)
+	}
+	sharedNATSServer = ns
+	sharedNATSURL = ns.ClientURL()
+
+	// Start a JetStream-enabled NATS server
+	jsTmpDir, err := os.MkdirTemp("", "nats-js-shared-*")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to create JetStream temp dir: %v\n", err)
+		os.Exit(1)
+	}
+	jsNS, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  jsTmpDir,
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to create shared JetStream NATS server: %v\n", err)
+		os.Exit(1)
+	}
+	go jsNS.Start()
+	if !jsNS.ReadyForConnections(5 * time.Second) {
+		fmt.Fprintln(os.Stderr, "Shared JetStream NATS server failed to start")
+		os.Exit(1)
+	}
+	sharedJSServer = jsNS
+	sharedJSNATSURL = jsNS.ClientURL()
+
+	code := m.Run()
+
+	ns.Shutdown()
+	jsNS.Shutdown()
+	os.RemoveAll(jsTmpDir)
+
+	os.Exit(code)
+}

hive/daemon/shutdown.go

@@ -0,0 +1,276 @@
+package daemon
+
+import (
+	"encoding/json"
+	"log/slog"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats.go"
+)
+
+// ShutdownRequest is sent by the coordinator to each daemon per phase.
+type ShutdownRequest struct {
+	Phase   string `json:"phase"`
+	Force   bool   `json:"force"`
+	Timeout int    `json:"timeout_seconds"`
+}
+
+// ShutdownACK is the response from a daemon after completing a shutdown phase.
+type ShutdownACK struct {
+	Node    string   `json:"node"`
+	Phase   string   `json:"phase"`
+	Stopped []string `json:"stopped,omitempty"`
+	Error   string   `json:"error,omitempty"`
+}
+
+// ShutdownProgress is published by daemons during the DRAIN phase to report VM shutdown progress.
+type ShutdownProgress struct {
+	Node      string `json:"node"`
+	Phase     string `json:"phase"`
+	Total     int    `json:"total"`
+	Remaining int    `json:"remaining"`
+}
+
+// handleShutdownGate stops the API gateway and UI, then sets shuttingDown flag
+// so the daemon rejects new work. Phase: GATE.
+func (d *Daemon) handleShutdownGate(msg *nats.Msg) {
+	var req ShutdownRequest
+	if err := json.Unmarshal(msg.Data, &req); err != nil {
+		slog.Error("handleShutdownGate: failed to unmarshal request", "error", err)
+		return
+	}
+
+	slog.Info("Shutdown GATE phase starting", "node", d.node, "force", req.Force)
+
+	var stopped []string
+
+	// Stop AWSGW
+	if d.config.HasService("awsgw") {
+		if err := utils.StopProcess("awsgw"); err != nil {
+			slog.Warn("Failed to stop awsgw", "error", err)
+		} else {
+			stopped = append(stopped, "awsgw")
+		}
+	}
+
+	// Stop UI
+	if d.config.HasService("ui") {
+		if err := utils.StopProcess("hive-ui"); err != nil {
+			slog.Warn("Failed to stop hive-ui", "error", err)
+		} else {
+			stopped = append(stopped, "hive-ui")
+		}
+	}
+
+	// Set shuttingDown flag so daemon rejects new work
+	d.shuttingDown.Store(true)
+
+	ack := ShutdownACK{
+		Node:    d.node,
+		Phase:   "gate",
+		Stopped: stopped,
+	}
+	d.respondShutdownACK(msg, ack)
+	slog.Info("Shutdown GATE phase complete", "node", d.node, "stopped", stopped)
+}
+
+// handleShutdownDrain gracefully stops all running VMs, writes shutdown marker
+// and persists state. Phase: DRAIN.
+func (d *Daemon) handleShutdownDrain(msg *nats.Msg) {
+	var req ShutdownRequest
+	if err := json.Unmarshal(msg.Data, &req); err != nil {
+		slog.Error("handleShutdownDrain: failed to unmarshal request", "error", err)
+		return
+	}
+
+	slog.Info("Shutdown DRAIN phase starting", "node", d.node)
+
+	// Count total VMs
+	d.Instances.Mu.Lock()
+	total := len(d.Instances.VMS)
+	d.Instances.Mu.Unlock()
+
+	// Publish initial progress
+	d.publishShutdownProgress("drain", total, total)
+
+	// Stop all instances (graceful shutdown, no volume deletion)
+	if total > 0 {
+		d.Instances.Mu.Lock()
+		vms := d.Instances.VMS
+		d.Instances.Mu.Unlock()
+
+		if err := d.stopInstance(vms, false); err != nil {
+			slog.Error("Failed to stop instances during DRAIN", "error", err)
+			ack := ShutdownACK{
+				Node:  d.node,
+				Phase: "drain",
+				Error: err.Error(),
+			}
+			d.respondShutdownACK(msg, ack)
+			return
+		}
+	}
+
+	// Publish final progress
+	d.publishShutdownProgress("drain", total, 0)
+
+	// Write shutdown marker and persist state
+	if d.jsManager != nil {
+		if err := d.jsManager.WriteShutdownMarker(d.node); err != nil {
+			slog.Error("Failed to write shutdown marker during DRAIN", "error", err)
+		}
+	}
+	if err := d.WriteState(); err != nil {
+		slog.Error("Failed to write state during DRAIN", "error", err)
+	}
+
+	ack := ShutdownACK{
+		Node:  d.node,
+		Phase: "drain",
+	}
+	d.respondShutdownACK(msg, ack)
+	slog.Info("Shutdown DRAIN phase complete", "node", d.node, "vms_stopped", total)
+}
+
+// handleShutdownStorage stops viperblock and cleans up orphan nbdkit processes. Phase: STORAGE.
+func (d *Daemon) handleShutdownStorage(msg *nats.Msg) {
+	var req ShutdownRequest
+	if err := json.Unmarshal(msg.Data, &req); err != nil {
+		slog.Error("handleShutdownStorage: failed to unmarshal request", "error", err)
+		return
+	}
+
+	slog.Info("Shutdown STORAGE phase starting", "node", d.node)
+
+	var stopped []string
+
+	if d.config.HasService("viperblock") {
+		if err := utils.StopProcess("viperblock"); err != nil {
+			slog.Warn("Failed to stop viperblock", "error", err)
+		} else {
+			stopped = append(stopped, "viperblock")
+		}
+	}
+
+	// Best-effort cleanup of orphaned nbdkit processes
+	if err := exec.Command("pkill", "-f", "nbdkit").Run(); err != nil {
+		slog.Debug("pkill nbdkit (best-effort)", "result", err)
+	}
+
+	ack := ShutdownACK{
+		Node:    d.node,
+		Phase:   "storage",
+		Stopped: stopped,
+	}
+	d.respondShutdownACK(msg, ack)
+	slog.Info("Shutdown STORAGE phase complete", "node", d.node, "stopped", stopped)
+}
+
+// handleShutdownPersist stops predastore. Phase: PERSIST.
+func (d *Daemon) handleShutdownPersist(msg *nats.Msg) {
+	var req ShutdownRequest
+	if err := json.Unmarshal(msg.Data, &req); err != nil {
+		slog.Error("handleShutdownPersist: failed to unmarshal request", "error", err)
+		return
+	}
+
+	slog.Info("Shutdown PERSIST phase starting", "node", d.node)
+
+	var stopped []string
+
+	if d.config.HasService("predastore") {
+		if err := utils.StopProcess("predastore"); err != nil {
+			slog.Warn("Failed to stop predastore", "error", err)
+		} else {
+			stopped = append(stopped, "predastore")
+		}
+	}
+
+	ack := ShutdownACK{
+		Node:    d.node,
+		Phase:   "persist",
+		Stopped: stopped,
+	}
+	d.respondShutdownACK(msg, ack)
+	slog.Info("Shutdown PERSIST phase complete", "node", d.node, "stopped", stopped)
+}
+
+// handleShutdownInfra stops NATS and exits the daemon. Phase: INFRA.
+// No ACK is sent because NATS is going down — this is fire-and-forget from the coordinator.
+func (d *Daemon) handleShutdownInfra(msg *nats.Msg) {
+	var req ShutdownRequest
+	if err := json.Unmarshal(msg.Data, &req); err != nil {
+		slog.Error("handleShutdownInfra: failed to unmarshal request", "error", err)
+		return
+	}
+
+	slog.Info("Shutdown INFRA phase starting", "node", d.node)
+
+	// Cancel context to stop heartbeat and other goroutines
+	d.cancel()
+
+	// Unsubscribe from all NATS topics
+	for topic, sub := range d.natsSubscriptions {
+		slog.Debug("Unsubscribing from NATS", "topic", topic)
+		if err := sub.Unsubscribe(); err != nil {
+			slog.Warn("Failed to unsubscribe", "topic", topic, "error", err)
+		}
+	}
+
+	// Shutdown cluster manager
+	if d.clusterApp != nil {
+		if err := d.clusterApp.Shutdown(); err != nil {
+			slog.Warn("Failed to shutdown cluster manager", "error", err)
+		}
+	}
+
+	// Close NATS connection
+	d.natsConn.Close()
+
+	// Stop NATS server if this node runs it
+	if d.config.HasService("nats") {
+		if err := utils.StopProcess("nats"); err != nil {
+			slog.Warn("Failed to stop nats", "error", err)
+		} else {
+			slog.Info("NATS server stopped", "node", d.node)
+		}
+	}
+
+	slog.Info("Shutdown INFRA phase complete, exiting", "node", d.node)
+
+	// Give a moment for log flush
+	time.Sleep(500 * time.Millisecond)
+	os.Exit(0)
+}
+
+// respondShutdownACK marshals and sends a ShutdownACK response.
+func (d *Daemon) respondShutdownACK(msg *nats.Msg, ack ShutdownACK) {
+	data, err := json.Marshal(ack)
+	if err != nil {
+		slog.Error("Failed to marshal shutdown ACK", "error", err)
+		return
+	}
+	if err := msg.Respond(data); err != nil {
+		slog.Error("Failed to respond with shutdown ACK", "error", err)
+	}
+}
+
+// publishShutdownProgress publishes a progress update during the DRAIN phase.
+func (d *Daemon) publishShutdownProgress(phase string, total, remaining int) {
+	progress := ShutdownProgress{
+		Node:      d.node,
+		Phase:     phase,
+		Total:     total,
+		Remaining: remaining,
+	}
+	data, err := json.Marshal(progress)
+	if err != nil {
+		return
+	}
+	if err := d.natsConn.Publish("hive.cluster.shutdown.progress", data); err != nil {
+		slog.Debug("Failed to publish shutdown progress", "error", err)
+	}
+}

hive/daemon/shutdown_test.go

@@ -0,0 +1,147 @@
+package daemon
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShutdownACKMarshal verifies JSON round-trip for ShutdownACK.
+func TestShutdownACKMarshal(t *testing.T) {
+	ack := ShutdownACK{
+		Node:    "node1",
+		Phase:   "gate",
+		Stopped: []string{"awsgw", "hive-ui"},
+	}
+
+	data, err := json.Marshal(ack)
+	require.NoError(t, err)
+
+	var decoded ShutdownACK
+	err = json.Unmarshal(data, &decoded)
+	require.NoError(t, err)
+
+	assert.Equal(t, ack.Node, decoded.Node)
+	assert.Equal(t, ack.Phase, decoded.Phase)
+	assert.Equal(t, ack.Stopped, decoded.Stopped)
+	assert.Empty(t, decoded.Error)
+}
+
+// TestShutdownACKWithError verifies JSON round-trip for ShutdownACK with an error.
+func TestShutdownACKWithError(t *testing.T) {
+	ack := ShutdownACK{
+		Node:  "node2",
+		Phase: "drain",
+		Error: "failed to stop VMs",
+	}
+
+	data, err := json.Marshal(ack)
+	require.NoError(t, err)
+
+	var decoded ShutdownACK
+	err = json.Unmarshal(data, &decoded)
+	require.NoError(t, err)
+
+	assert.Equal(t, "failed to stop VMs", decoded.Error)
+	assert.Nil(t, decoded.Stopped)
+}
+
+// TestShutdownRequestMarshal verifies JSON round-trip for ShutdownRequest.
+func TestShutdownRequestMarshal(t *testing.T) {
+	req := ShutdownRequest{
+		Phase:   "drain",
+		Force:   true,
+		Timeout: 120,
+	}
+
+	data, err := json.Marshal(req)
+	require.NoError(t, err)
+
+	var decoded ShutdownRequest
+	err = json.Unmarshal(data, &decoded)
+	require.NoError(t, err)
+
+	assert.Equal(t, "drain", decoded.Phase)
+	assert.True(t, decoded.Force)
+	assert.Equal(t, 120, decoded.Timeout)
+}
+
+// TestShutdownProgressMarshal verifies JSON round-trip for ShutdownProgress.
+func TestShutdownProgressMarshal(t *testing.T) {
+	progress := ShutdownProgress{
+		Node:      "node1",
+		Phase:     "drain",
+		Total:     5,
+		Remaining: 3,
+	}
+
+	data, err := json.Marshal(progress)
+	require.NoError(t, err)
+
+	var decoded ShutdownProgress
+	err = json.Unmarshal(data, &decoded)
+	require.NoError(t, err)
+
+	assert.Equal(t, "node1", decoded.Node)
+	assert.Equal(t, "drain", decoded.Phase)
+	assert.Equal(t, 5, decoded.Total)
+	assert.Equal(t, 3, decoded.Remaining)
+}
+
+// TestClusterShutdownStateKVRoundTrip verifies cluster shutdown state can be stored and retrieved from KV.
+func TestClusterShutdownStateKVRoundTrip(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	err = jsm.InitClusterStateBucket()
+	require.NoError(t, err)
+
+	state := &ClusterShutdownState{
+		Initiator:  "node1",
+		Phase:      "drain",
+		Started:    "2025-01-01T00:00:00Z",
+		Timeout:    "2m0s",
+		Force:      false,
+		NodesTotal: 3,
+		NodesAcked: map[string]string{
+			"node1": "gate",
+			"node2": "gate",
+		},
+	}
+
+	err = jsm.WriteClusterShutdown(state)
+	require.NoError(t, err)
+
+	loaded, err := jsm.ReadClusterShutdown()
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+
+	assert.Equal(t, state.Initiator, loaded.Initiator)
+	assert.Equal(t, state.Phase, loaded.Phase)
+	assert.Equal(t, state.Started, loaded.Started)
+	assert.Equal(t, state.NodesTotal, loaded.NodesTotal)
+	assert.Equal(t, state.Force, loaded.Force)
+	assert.Len(t, loaded.NodesAcked, 2)
+
+	// Cleanup
+	err = jsm.DeleteClusterShutdown()
+	require.NoError(t, err)
+}
+
+// TestShuttingDownFlagSkipsVMStop verifies that the shuttingDown flag is respected.
+func TestShuttingDownFlagSkipsVMStop(t *testing.T) {
+	d := &Daemon{}
+
+	// Default should be false
+	assert.False(t, d.shuttingDown.Load())
+
+	// Set to true (as GATE phase does)
+	d.shuttingDown.Store(true)
+	assert.True(t, d.shuttingDown.Load())
+}

hive/daemon/state.go

@@ -0,0 +1,34 @@
+package daemon
+
+import (
+	"fmt"
+	"log/slog"
+
+	"github.com/mulgadc/hive/hive/vm"
+)
+
+// TransitionState validates and applies a state transition on the given instance.
+// It sets instance.Status, persists via WriteState, and logs.
+// On validation failure, the instance status is unchanged and an error is returned.
+// If WriteState fails, the in-memory status retains the new value (the VM has
+// physically changed state regardless) and an error is returned.
+// The caller must NOT hold d.Instances.Mu; this method acquires it internally.
+func (d *Daemon) TransitionState(instance *vm.VM, target vm.InstanceState) error {
+	d.Instances.Mu.Lock()
+	current := instance.Status
+	if !vm.IsValidTransition(current, target) {
+		d.Instances.Mu.Unlock()
+		return fmt.Errorf("invalid state transition: %s -> %s for instance %s", current, target, instance.ID)
+	}
+	instance.Status = target
+	d.Instances.Mu.Unlock()
+
+	slog.Info("Instance state transition", "instanceId", instance.ID, "from", string(current), "to", string(target))
+
+	if err := d.WriteState(); err != nil {
+		slog.Error("Failed to persist state after transition", "instanceId", instance.ID,
+			"from", string(current), "to", string(target), "err", err)
+		return fmt.Errorf("state transition applied but write failed: %w", err)
+	}
+	return nil
+}

hive/daemon/state_test.go

@@ -0,0 +1,859 @@
+package daemon
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/vm"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// createDaemonWithJetStream creates a daemon backed by an in-process NATS+JetStream server
+// so that TransitionState (which calls WriteState) works end-to-end.
+func createDaemonWithJetStream(t *testing.T) *Daemon {
+	t.Helper()
+
+	jsTmpDir, err := os.MkdirTemp("", "nats-js-state-test-*")
+	require.NoError(t, err)
+	t.Cleanup(func() { os.RemoveAll(jsTmpDir) })
+
+	opts := &server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  jsTmpDir,
+		NoLog:     true,
+		NoSigs:    true,
+	}
+	ns, err := server.NewServer(opts)
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second), "NATS server failed to start")
+	t.Cleanup(func() { ns.Shutdown() })
+
+	natsURL := ns.ClientURL()
+
+	tmpDir, err := os.MkdirTemp("", "hive-state-test-*")
+	require.NoError(t, err)
+	t.Cleanup(func() { os.RemoveAll(tmpDir) })
+
+	clusterCfg := &config.ClusterConfig{
+		Node:  "node-1",
+		Nodes: map[string]config.Config{"node-1": {BaseDir: tmpDir}},
+	}
+	daemon := NewDaemon(clusterCfg)
+	daemon.config = &config.Config{BaseDir: tmpDir}
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	daemon.natsConn = nc
+	daemon.jsManager, err = NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, daemon.jsManager.InitKVBucket())
+
+	return daemon
+}
+
+func TestTransitionState_ValidTransitions(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	tests := []struct {
+		name string
+		from vm.InstanceState
+		to   vm.InstanceState
+	}{
+		{"provisioning->running", vm.StateProvisioning, vm.StateRunning},
+		{"provisioning->error", vm.StateProvisioning, vm.StateError},
+		{"provisioning->shutting-down", vm.StateProvisioning, vm.StateShuttingDown},
+		{"pending->running", vm.StatePending, vm.StateRunning},
+		{"pending->error", vm.StatePending, vm.StateError},
+		{"pending->shutting-down", vm.StatePending, vm.StateShuttingDown},
+		{"running->stopping", vm.StateRunning, vm.StateStopping},
+		{"running->shutting-down", vm.StateRunning, vm.StateShuttingDown},
+		{"running->error", vm.StateRunning, vm.StateError},
+		{"stopping->stopped", vm.StateStopping, vm.StateStopped},
+		{"stopping->shutting-down", vm.StateStopping, vm.StateShuttingDown},
+		{"stopping->error", vm.StateStopping, vm.StateError},
+		{"stopped->running", vm.StateStopped, vm.StateRunning},
+		{"stopped->shutting-down", vm.StateStopped, vm.StateShuttingDown},
+		{"stopped->error", vm.StateStopped, vm.StateError},
+		{"shutting-down->terminated", vm.StateShuttingDown, vm.StateTerminated},
+		{"shutting-down->error", vm.StateShuttingDown, vm.StateError},
+		{"error->running", vm.StateError, vm.StateRunning},
+		{"error->shutting-down", vm.StateError, vm.StateShuttingDown},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			instance := &vm.VM{
+				ID:     "i-test-valid",
+				Status: tt.from,
+			}
+
+			daemon.Instances.Mu.Lock()
+			daemon.Instances.VMS[instance.ID] = instance
+			daemon.Instances.Mu.Unlock()
+
+			err := daemon.TransitionState(instance, tt.to)
+			require.NoError(t, err)
+
+			assert.Equal(t, tt.to, instance.Status)
+		})
+	}
+}
+
+func TestTransitionState_InvalidTransitions(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	tests := []struct {
+		name string
+		from vm.InstanceState
+		to   vm.InstanceState
+	}{
+		{"running->running", vm.StateRunning, vm.StateRunning},
+		{"running->pending", vm.StateRunning, vm.StatePending},
+		{"running->stopped", vm.StateRunning, vm.StateStopped},
+		{"running->terminated", vm.StateRunning, vm.StateTerminated},
+		{"stopped->stopping", vm.StateStopped, vm.StateStopping},
+		{"stopped->terminated", vm.StateStopped, vm.StateTerminated},
+		{"terminated->running", vm.StateTerminated, vm.StateRunning},
+		{"terminated->stopped", vm.StateTerminated, vm.StateStopped},
+		{"stopping->running", vm.StateStopping, vm.StateRunning},
+		{"shutting-down->running", vm.StateShuttingDown, vm.StateRunning},
+		{"shutting-down->stopped", vm.StateShuttingDown, vm.StateStopped},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			instance := &vm.VM{
+				ID:     "i-test-invalid",
+				Status: tt.from,
+			}
+
+			daemon.Instances.Mu.Lock()
+			daemon.Instances.VMS[instance.ID] = instance
+			daemon.Instances.Mu.Unlock()
+
+			err := daemon.TransitionState(instance, tt.to)
+			assert.Error(t, err)
+			assert.Contains(t, err.Error(), "invalid state transition")
+
+			// Status should remain unchanged
+			assert.Equal(t, tt.from, instance.Status)
+		})
+	}
+}
+
+func TestTransitionState_NilEC2Instance(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{
+		ID:       "i-test-nil-ec2",
+		Status:   vm.StateProvisioning,
+		Instance: nil, // no EC2 instance metadata
+	}
+
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instance.ID] = instance
+	daemon.Instances.Mu.Unlock()
+
+	err := daemon.TransitionState(instance, vm.StateRunning)
+	require.NoError(t, err)
+	assert.Equal(t, vm.StateRunning, instance.Status)
+}
+
+// allStates is the canonical list of every InstanceState in the system.
+var allStates = []vm.InstanceState{
+	vm.StateProvisioning,
+	vm.StatePending,
+	vm.StateRunning,
+	vm.StateStopping,
+	vm.StateStopped,
+	vm.StateShuttingDown,
+	vm.StateTerminated,
+	vm.StateError,
+}
+
+// stableStates are states where an instance can rest indefinitely.
+var stableStates = []vm.InstanceState{
+	vm.StateRunning,
+	vm.StateStopped,
+	vm.StateTerminated,
+	vm.StateError,
+}
+
+// transitionalStates are states that must resolve to a stable state.
+var transitionalStates = []vm.InstanceState{
+	vm.StateProvisioning,
+	vm.StatePending,
+	vm.StateStopping,
+	vm.StateShuttingDown,
+}
+
+// TestValidTransitions_MapCoversAllNonTerminalStates verifies that every state
+// except StateTerminated has at least one valid outgoing transition. A missing
+// entry would mean an instance could get permanently stuck.
+func TestValidTransitions_MapCoversAllNonTerminalStates(t *testing.T) {
+	for _, state := range allStates {
+		if state == vm.StateTerminated {
+			continue
+		}
+		targets, ok := vm.ValidTransitions[state]
+		assert.True(t, ok, "state %q missing from ValidTransitions map", state)
+		assert.NotEmpty(t, targets, "state %q has no valid transitions — instances would get stuck", state)
+	}
+}
+
+// TestTerminatedState_HasNoOutgoingTransitions ensures the terminal state is
+// truly terminal — no valid transitions out.
+func TestTerminatedState_HasNoOutgoingTransitions(t *testing.T) {
+	targets, ok := vm.ValidTransitions[vm.StateTerminated]
+	assert.False(t, ok || len(targets) > 0,
+		"StateTerminated should have no valid transitions, got %v", targets)
+}
+
+// TestTransitionalStates_CanReachStableState verifies that every transitional
+// state has at least one direct transition to a stable state. This is the
+// structural guarantee that prevents the "stuck in stopping" bug.
+func TestTransitionalStates_CanReachStableState(t *testing.T) {
+	stableSet := map[vm.InstanceState]bool{}
+	for _, s := range stableStates {
+		stableSet[s] = true
+	}
+
+	for _, state := range transitionalStates {
+		targets := vm.ValidTransitions[state]
+		hasStableTarget := false
+		for _, target := range targets {
+			if stableSet[target] {
+				hasStableTarget = true
+				break
+			}
+		}
+		assert.True(t, hasStableTarget,
+			"transitional state %q has no direct path to a stable state (targets: %v)", state, targets)
+	}
+}
+
+// TestEveryState_CanReachTerminated does a BFS from each non-terminated state
+// to confirm there is always a path to StateTerminated. An unreachable terminal
+// state means instances can never be fully cleaned up.
+func TestEveryState_CanReachTerminated(t *testing.T) {
+	for _, start := range allStates {
+		if start == vm.StateTerminated {
+			continue
+		}
+
+		visited := map[vm.InstanceState]bool{start: true}
+		queue := []vm.InstanceState{start}
+		found := false
+
+		for len(queue) > 0 {
+			current := queue[0]
+			queue = queue[1:]
+			if current == vm.StateTerminated {
+				found = true
+				break
+			}
+			for _, next := range vm.ValidTransitions[current] {
+				if !visited[next] {
+					visited[next] = true
+					queue = append(queue, next)
+				}
+			}
+		}
+
+		assert.True(t, found,
+			"state %q has no path to StateTerminated — instances starting here can never be cleaned up", start)
+	}
+}
+
+// TestStopLifecycle_CompletesFullChain walks the full stop lifecycle:
+// provisioning → running → stopping → stopped, then restart and terminate:
+// stopped → running → shutting-down → terminated.
+func TestStopLifecycle_CompletesFullChain(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{ID: "i-lifecycle-stop", Status: vm.StateProvisioning}
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instance.ID] = instance
+	daemon.Instances.Mu.Unlock()
+
+	chain := []vm.InstanceState{
+		vm.StateRunning,      // launch
+		vm.StateStopping,     // stop requested
+		vm.StateStopped,      // stop completed
+		vm.StateRunning,      // restart
+		vm.StateShuttingDown, // terminate requested
+		vm.StateTerminated,   // terminate completed
+	}
+
+	for _, target := range chain {
+		err := daemon.TransitionState(instance, target)
+		require.NoError(t, err, "transition to %s failed (current: %s)", target, instance.Status)
+	}
+
+	assert.Equal(t, vm.StateTerminated, instance.Status)
+}
+
+// TestStopLifecycle_ErrorRecovery verifies that an instance that hits an error
+// during stop can recover: running → stopping → error → shutting-down → terminated.
+func TestStopLifecycle_ErrorRecovery(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{ID: "i-lifecycle-err", Status: vm.StateRunning}
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instance.ID] = instance
+	daemon.Instances.Mu.Unlock()
+
+	chain := []vm.InstanceState{
+		vm.StateStopping,
+		vm.StateError,        // stop failed
+		vm.StateShuttingDown, // force terminate from error
+		vm.StateTerminated,
+	}
+
+	for _, target := range chain {
+		err := daemon.TransitionState(instance, target)
+		require.NoError(t, err, "transition to %s failed (current: %s)", target, instance.Status)
+	}
+
+	assert.Equal(t, vm.StateTerminated, instance.Status)
+}
+
+// TestShuttingDown_ErrorRecovery verifies: shutting-down → error → shutting-down → terminated.
+func TestShuttingDown_ErrorRecovery(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{ID: "i-shutdown-err", Status: vm.StateShuttingDown}
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instance.ID] = instance
+	daemon.Instances.Mu.Unlock()
+
+	chain := []vm.InstanceState{
+		vm.StateError,        // terminate failed
+		vm.StateShuttingDown, // retry terminate
+		vm.StateTerminated,   // success
+	}
+
+	for _, target := range chain {
+		err := daemon.TransitionState(instance, target)
+		require.NoError(t, err, "transition to %s failed (current: %s)", target, instance.Status)
+	}
+
+	assert.Equal(t, vm.StateTerminated, instance.Status)
+}
+
+// TestTransitionState_ConcurrentTransitions hammers the same instance with
+// concurrent transitions to verify the mutex protects against races and
+// the instance always ends in a valid state.
+func TestTransitionState_ConcurrentTransitions(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{ID: "i-concurrent", Status: vm.StateRunning}
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instance.ID] = instance
+	daemon.Instances.Mu.Unlock()
+
+	// Multiple goroutines race to transition from running.
+	// After one succeeds, the others may chain from the new state
+	// (e.g., running→error, then error→shutting-down), which is correct.
+	targets := []vm.InstanceState{vm.StateStopping, vm.StateShuttingDown, vm.StateError}
+
+	var wg sync.WaitGroup
+	results := make([]error, len(targets))
+
+	for i, target := range targets {
+		wg.Add(1)
+		go func(idx int, tgt vm.InstanceState) {
+			defer wg.Done()
+			results[idx] = daemon.TransitionState(instance, tgt)
+		}(i, target)
+	}
+	wg.Wait()
+
+	// At least one transition must succeed.
+	successes := 0
+	for _, err := range results {
+		if err == nil {
+			successes++
+		}
+	}
+	assert.GreaterOrEqual(t, successes, 1, "at least one concurrent transition should succeed")
+
+	// The final state must be reachable from running via valid transitions.
+	finalState := instance.Status
+	reachable := reachableStates(vm.StateRunning)
+	assert.True(t, reachable[finalState],
+		"instance ended in state %q which is not reachable from running", finalState)
+}
+
+// reachableStates returns all states reachable from start via BFS.
+func reachableStates(start vm.InstanceState) map[vm.InstanceState]bool {
+	visited := map[vm.InstanceState]bool{start: true}
+	queue := []vm.InstanceState{start}
+	for len(queue) > 0 {
+		current := queue[0]
+		queue = queue[1:]
+		for _, next := range vm.ValidTransitions[current] {
+			if !visited[next] {
+				visited[next] = true
+				queue = append(queue, next)
+			}
+		}
+	}
+	return visited
+}
+
+// TestTransitionState_DoubleTransitionSameTarget verifies that transitioning
+// to the same state twice is rejected (no self-loops).
+func TestTransitionState_DoubleTransitionSameTarget(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	for _, state := range allStates {
+		t.Run(string(state), func(t *testing.T) {
+			instance := &vm.VM{ID: fmt.Sprintf("i-double-%s", state), Status: state}
+			daemon.Instances.Mu.Lock()
+			daemon.Instances.VMS[instance.ID] = instance
+			daemon.Instances.Mu.Unlock()
+
+			err := daemon.TransitionState(instance, state)
+			assert.Error(t, err, "self-transition %s -> %s should be rejected", state, state)
+			assert.Equal(t, state, instance.Status, "state should be unchanged after rejected self-transition")
+		})
+	}
+}
+
+// TestTransitionState_InvalidFromTerminated exhaustively checks that
+// terminated cannot transition to any state.
+func TestTransitionState_InvalidFromTerminated(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	for _, target := range allStates {
+		t.Run(fmt.Sprintf("terminated->%s", target), func(t *testing.T) {
+			instance := &vm.VM{ID: fmt.Sprintf("i-term-%s", target), Status: vm.StateTerminated}
+			daemon.Instances.Mu.Lock()
+			daemon.Instances.VMS[instance.ID] = instance
+			daemon.Instances.Mu.Unlock()
+
+			err := daemon.TransitionState(instance, target)
+			assert.Error(t, err, "transition from terminated to %s should be rejected", target)
+			assert.Equal(t, vm.StateTerminated, instance.Status)
+		})
+	}
+}
+
+// TestTransitionState_StoppingMustReachStoppedOrError is the specific
+// regression test for the bug where stopping never reached stopped.
+// It verifies both the happy path and error path from stopping.
+func TestTransitionState_StoppingMustReachStoppedOrError(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	t.Run("happy_path", func(t *testing.T) {
+		instance := &vm.VM{ID: "i-stop-happy", Status: vm.StateStopping}
+		daemon.Instances.Mu.Lock()
+		daemon.Instances.VMS[instance.ID] = instance
+		daemon.Instances.Mu.Unlock()
+
+		err := daemon.TransitionState(instance, vm.StateStopped)
+		require.NoError(t, err)
+		assert.Equal(t, vm.StateStopped, instance.Status)
+	})
+
+	t.Run("error_path", func(t *testing.T) {
+		instance := &vm.VM{ID: "i-stop-error", Status: vm.StateStopping}
+		daemon.Instances.Mu.Lock()
+		daemon.Instances.VMS[instance.ID] = instance
+		daemon.Instances.Mu.Unlock()
+
+		err := daemon.TransitionState(instance, vm.StateError)
+		require.NoError(t, err)
+		assert.Equal(t, vm.StateError, instance.Status)
+	})
+
+	t.Run("force_terminate_path", func(t *testing.T) {
+		instance := &vm.VM{ID: "i-stop-term", Status: vm.StateStopping}
+		daemon.Instances.Mu.Lock()
+		daemon.Instances.VMS[instance.ID] = instance
+		daemon.Instances.Mu.Unlock()
+
+		err := daemon.TransitionState(instance, vm.StateShuttingDown)
+		require.NoError(t, err)
+		assert.Equal(t, vm.StateShuttingDown, instance.Status)
+
+		err = daemon.TransitionState(instance, vm.StateTerminated)
+		require.NoError(t, err)
+		assert.Equal(t, vm.StateTerminated, instance.Status)
+	})
+}
+
+// TestTransitionState_ShuttingDownMustReachTerminatedOrError is the parallel
+// regression test for the terminate flow.
+func TestTransitionState_ShuttingDownMustReachTerminatedOrError(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	t.Run("happy_path", func(t *testing.T) {
+		instance := &vm.VM{ID: "i-shut-happy", Status: vm.StateShuttingDown}
+		daemon.Instances.Mu.Lock()
+		daemon.Instances.VMS[instance.ID] = instance
+		daemon.Instances.Mu.Unlock()
+
+		err := daemon.TransitionState(instance, vm.StateTerminated)
+		require.NoError(t, err)
+		assert.Equal(t, vm.StateTerminated, instance.Status)
+	})
+
+	t.Run("error_path", func(t *testing.T) {
+		instance := &vm.VM{ID: "i-shut-error", Status: vm.StateShuttingDown}
+		daemon.Instances.Mu.Lock()
+		daemon.Instances.VMS[instance.ID] = instance
+		daemon.Instances.Mu.Unlock()
+
+		err := daemon.TransitionState(instance, vm.StateError)
+		require.NoError(t, err)
+		assert.Equal(t, vm.StateError, instance.Status)
+	})
+}
+
+// TestValidTransitions_NoCyclesWithoutStableState ensures there is no cycle
+// of only transitional states. Every cycle in the state graph must pass
+// through at least one stable state.
+func TestValidTransitions_NoCyclesWithoutStableState(t *testing.T) {
+	stableSet := map[vm.InstanceState]bool{}
+	for _, s := range stableStates {
+		stableSet[s] = true
+	}
+
+	// DFS from each transitional state — if we revisit a transitional state
+	// without going through a stable state, that's a problem.
+	for _, start := range transitionalStates {
+		visited := map[vm.InstanceState]bool{}
+		var hasCycle bool
+
+		var dfs func(vm.InstanceState)
+		dfs = func(current vm.InstanceState) {
+			if stableSet[current] {
+				return // reached a stable state, stop this path
+			}
+			if visited[current] {
+				hasCycle = true
+				return
+			}
+			visited[current] = true
+			for _, next := range vm.ValidTransitions[current] {
+				dfs(next)
+			}
+		}
+		dfs(start)
+
+		assert.False(t, hasCycle,
+			"found a cycle of only transitional states starting from %q", start)
+	}
+}
+
+// TestTransitionState_MultipleInstancesIndependent verifies that transitioning
+// one instance does not affect another instance's state.
+func TestTransitionState_MultipleInstancesIndependent(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	inst1 := &vm.VM{ID: "i-multi-1", Status: vm.StateRunning}
+	inst2 := &vm.VM{ID: "i-multi-2", Status: vm.StateRunning}
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[inst1.ID] = inst1
+	daemon.Instances.VMS[inst2.ID] = inst2
+	daemon.Instances.Mu.Unlock()
+
+	err := daemon.TransitionState(inst1, vm.StateStopping)
+	require.NoError(t, err)
+
+	assert.Equal(t, vm.StateStopping, inst1.Status)
+	assert.Equal(t, vm.StateRunning, inst2.Status, "inst2 should be unaffected by inst1's transition")
+}
+
+// --- Recovery / restoreInstances tests ---
+//
+// These tests simulate a daemon restart by writing instance state to JetStream,
+// then calling restoreInstances on a fresh daemon. Since no QEMU process is
+// running, isInstanceProcessRunning returns false for all instances, which
+// exercises the recovery state resolution logic.
+
+// TestRestoreInstances_StoppingFinalizedToStopped verifies that an instance
+// stuck in StateStopping when the daemon died gets finalized to StateStopped
+// and migrated to shared KV.
+func TestRestoreInstances_StoppingFinalizedToStopped(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	// Simulate: daemon was running, instance was in StateStopping, daemon crashed.
+	daemon.Instances.VMS["i-restore-stop"] = &vm.VM{
+		ID:     "i-restore-stop",
+		Status: vm.StateStopping,
+	}
+	require.NoError(t, daemon.WriteState())
+
+	// Clear in-memory state to simulate fresh daemon startup.
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+
+	// restoreInstances loads from JetStream and resolves transitional states.
+	daemon.restoreInstances()
+
+	// Stopped instances are migrated to shared KV and removed from local map
+	_, ok := daemon.Instances.VMS["i-restore-stop"]
+	assert.False(t, ok, "stopping→stopped instance should be migrated to shared KV, not in local map")
+
+	stoppedInst, err := daemon.jsManager.LoadStoppedInstance("i-restore-stop")
+	require.NoError(t, err)
+	require.NotNil(t, stoppedInst, "stopping→stopped instance should exist in shared KV")
+	assert.Equal(t, vm.StateStopped, stoppedInst.Status,
+		"stopping instance should be finalized to stopped on recovery")
+}
+
+// TestRestoreInstances_ShuttingDownFinalizedToTerminated verifies that an
+// instance stuck in StateShuttingDown gets finalized to StateTerminated.
+func TestRestoreInstances_ShuttingDownFinalizedToTerminated(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	daemon.Instances.VMS["i-restore-shut"] = &vm.VM{
+		ID:     "i-restore-shut",
+		Status: vm.StateShuttingDown,
+	}
+	require.NoError(t, daemon.WriteState())
+
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	instance, ok := daemon.Instances.VMS["i-restore-shut"]
+	require.True(t, ok, "instance should be loaded from state")
+	assert.Equal(t, vm.StateTerminated, instance.Status,
+		"shutting-down instance should be finalized to terminated on recovery")
+}
+
+// TestRestoreInstances_TerminatedSkipped verifies that terminated instances
+// are loaded but not relaunched or modified.
+func TestRestoreInstances_TerminatedSkipped(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	daemon.Instances.VMS["i-restore-term"] = &vm.VM{
+		ID:     "i-restore-term",
+		Status: vm.StateTerminated,
+	}
+	require.NoError(t, daemon.WriteState())
+
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	instance, ok := daemon.Instances.VMS["i-restore-term"]
+	require.True(t, ok, "terminated instance should still be loaded")
+	assert.Equal(t, vm.StateTerminated, instance.Status,
+		"terminated instance should remain terminated")
+}
+
+// TestRestoreInstances_UserStoppedMigratedToSharedKV verifies that instances
+// flagged as user-stopped are migrated to shared KV and not relaunched.
+func TestRestoreInstances_UserStoppedMigratedToSharedKV(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	daemon.Instances.VMS["i-restore-userstop"] = &vm.VM{
+		ID:     "i-restore-userstop",
+		Status: vm.StateStopped,
+		Attributes: qmp.Attributes{
+			StopInstance: true,
+		},
+	}
+	require.NoError(t, daemon.WriteState())
+
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	// Stopped instances should be migrated to shared KV
+	_, ok := daemon.Instances.VMS["i-restore-userstop"]
+	assert.False(t, ok, "user-stopped instance should be migrated to shared KV, not in local map")
+
+	stoppedInst, err := daemon.jsManager.LoadStoppedInstance("i-restore-userstop")
+	require.NoError(t, err)
+	require.NotNil(t, stoppedInst, "user-stopped instance should exist in shared KV")
+	assert.Equal(t, vm.StateStopped, stoppedInst.Status,
+		"user-stopped instance should remain stopped in shared KV")
+}
+
+// TestRestoreInstances_RunningResetToPending verifies that a running instance
+// whose QEMU process is dead gets reset to pending for relaunch.
+// LaunchInstance will fail (no QEMU available in tests), but the state
+// should be set to pending before the attempt.
+func TestRestoreInstances_RunningResetToPending(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	daemon.Instances.VMS["i-restore-run"] = &vm.VM{
+		ID:     "i-restore-run",
+		Status: vm.StateRunning,
+	}
+	require.NoError(t, daemon.WriteState())
+
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	instance, ok := daemon.Instances.VMS["i-restore-run"]
+	require.True(t, ok, "running instance should still be loaded")
+	// LaunchInstance fails (no QEMU), so it stays pending or gets marked failed.
+	// Either way it should NOT remain "running" — that would be a lie.
+	assert.NotEqual(t, vm.StateRunning, instance.Status,
+		"instance should not remain running when QEMU process is dead")
+}
+
+// TestRestoreInstances_MixedStates verifies recovery with multiple instances
+// in different states, ensuring each is handled correctly and independently.
+// Stopped instances (including stopping→stopped transitions) are migrated to shared KV.
+func TestRestoreInstances_MixedStates(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	daemon.Instances.VMS["i-mix-stopping"] = &vm.VM{
+		ID: "i-mix-stopping", Status: vm.StateStopping,
+	}
+	daemon.Instances.VMS["i-mix-shutting"] = &vm.VM{
+		ID: "i-mix-shutting", Status: vm.StateShuttingDown,
+	}
+	daemon.Instances.VMS["i-mix-terminated"] = &vm.VM{
+		ID: "i-mix-terminated", Status: vm.StateTerminated,
+	}
+	daemon.Instances.VMS["i-mix-stopped"] = &vm.VM{
+		ID: "i-mix-stopped", Status: vm.StateStopped,
+		Attributes: qmp.Attributes{StopInstance: true},
+	}
+	require.NoError(t, daemon.WriteState())
+
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	// Stopped instances are migrated to shared KV and removed from local map
+	assert.Nil(t, daemon.Instances.VMS["i-mix-stopping"],
+		"stopping→stopped should be migrated to shared KV")
+	stoppedFromStopping, err := daemon.jsManager.LoadStoppedInstance("i-mix-stopping")
+	require.NoError(t, err)
+	require.NotNil(t, stoppedFromStopping, "stopping→stopped should exist in shared KV")
+	assert.Equal(t, vm.StateStopped, stoppedFromStopping.Status)
+
+	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-mix-shutting"].Status,
+		"shutting-down should finalize to terminated")
+	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-mix-terminated"].Status,
+		"terminated should remain terminated")
+
+	assert.Nil(t, daemon.Instances.VMS["i-mix-stopped"],
+		"user-stopped should be migrated to shared KV")
+	stoppedFromUser, err := daemon.jsManager.LoadStoppedInstance("i-mix-stopped")
+	require.NoError(t, err)
+	require.NotNil(t, stoppedFromUser, "user-stopped should exist in shared KV")
+	assert.Equal(t, vm.StateStopped, stoppedFromUser.Status)
+}
+
+// TestRestoreInstances_StatePersistsAfterRecovery verifies that the finalized
+// state is actually persisted to JetStream, so a second restart doesn't
+// re-process the same transitional state.
+func TestRestoreInstances_StatePersistsAfterRecovery(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	daemon.Instances.VMS["i-persist-stop"] = &vm.VM{
+		ID: "i-persist-stop", Status: vm.StateStopping,
+	}
+	daemon.Instances.VMS["i-persist-shut"] = &vm.VM{
+		ID: "i-persist-shut", Status: vm.StateShuttingDown,
+	}
+	require.NoError(t, daemon.WriteState())
+
+	// First restart: finalize transitional states.
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	// stopping→stopped migrates to shared KV
+	assert.Nil(t, daemon.Instances.VMS["i-persist-stop"],
+		"stopping→stopped should be migrated to shared KV")
+	stoppedInst, err := daemon.jsManager.LoadStoppedInstance("i-persist-stop")
+	require.NoError(t, err)
+	require.NotNil(t, stoppedInst)
+	assert.Equal(t, vm.StateStopped, stoppedInst.Status)
+
+	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-persist-shut"].Status)
+
+	// Second restart: stopped instance is in shared KV (not per-node state),
+	// so local map won't have it. Terminated instance persists in per-node state.
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	// Stopped instance should still be in shared KV
+	stoppedInst2, err := daemon.jsManager.LoadStoppedInstance("i-persist-stop")
+	require.NoError(t, err)
+	require.NotNil(t, stoppedInst2, "stopped instance should persist in shared KV through second restart")
+	assert.Equal(t, vm.StateStopped, stoppedInst2.Status)
+
+	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-persist-shut"].Status,
+		"terminated state should persist through second restart")
+}
+
+// TestStatePersistence_RoundTrip verifies that all instance fields survive
+// a write-load cycle through JetStream, simulating a daemon restart.
+func TestStatePersistence_RoundTrip(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	original := &vm.VM{
+		ID:           "i-roundtrip",
+		Status:       vm.StateRunning,
+		InstanceType: "t3.micro",
+		Attributes: qmp.Attributes{
+			StopInstance:      false,
+			TerminateInstance: false,
+		},
+	}
+	daemon.Instances.VMS[original.ID] = original
+	require.NoError(t, daemon.WriteState())
+
+	// Simulate restart: clear and reload.
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	require.NoError(t, daemon.LoadState())
+
+	loaded := daemon.Instances.VMS["i-roundtrip"]
+	require.NotNil(t, loaded)
+	assert.Equal(t, original.ID, loaded.ID)
+	assert.Equal(t, original.Status, loaded.Status)
+	assert.Equal(t, original.InstanceType, loaded.InstanceType)
+	assert.Equal(t, original.Attributes.StopInstance, loaded.Attributes.StopInstance)
+}
+
+// TestRestoreInstances_StoppedInstanceMigratedToSharedKV verifies that after
+// a daemon restart, a stopped instance is migrated to shared KV and can be
+// retrieved by the ec2.start handler.
+func TestRestoreInstances_StoppedInstanceMigratedToSharedKV(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instanceID := "i-restore-start"
+	daemon.Instances.VMS[instanceID] = &vm.VM{
+		ID:           instanceID,
+		InstanceType: "t3.micro",
+		Status:       vm.StateStopped,
+		Attributes:   qmp.Attributes{StopInstance: true},
+	}
+	require.NoError(t, daemon.WriteState())
+
+	// Simulate daemon restart: clear in-memory state and restore from JetStream.
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	daemon.restoreInstances()
+
+	// Instance should not be in local map (migrated to shared KV).
+	_, ok := daemon.Instances.VMS[instanceID]
+	assert.False(t, ok, "stopped instance should be migrated to shared KV, not in local map")
+
+	// Instance should be in shared KV
+	stoppedInst, err := daemon.jsManager.LoadStoppedInstance(instanceID)
+	require.NoError(t, err)
+	require.NotNil(t, stoppedInst, "stopped instance should exist in shared KV")
+	assert.Equal(t, vm.StateStopped, stoppedInst.Status)
+	assert.Equal(t, "node-1", stoppedInst.LastNode,
+		"stopped instance should record last node")
+}

hive/formation/formation.go

@@ -0,0 +1,262 @@
+package formation
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+// NodeInfo describes a node participating in cluster formation.
+type NodeInfo struct {
+	Name      string   `json:"name"`
+	BindIP    string   `json:"bind_ip"`
+	ClusterIP string   `json:"cluster_ip"`
+	Region    string   `json:"region"`
+	AZ        string   `json:"az"`
+	Port      int      `json:"port"`
+	Services  []string `json:"services,omitempty"`
+}
+
+// JoinRequest is the payload POSTed by joining nodes.
+type JoinRequest struct {
+	NodeInfo
+}
+
+// JoinResponse is returned by the formation server on join.
+type JoinResponse struct {
+	Success  bool   `json:"success"`
+	Message  string `json:"message"`
+	Joined   int    `json:"joined"`
+	Expected int    `json:"expected"`
+}
+
+// StatusResponse is returned by the formation server status endpoint.
+type StatusResponse struct {
+	Complete    bool                `json:"complete"`
+	Joined      int                 `json:"joined"`
+	Expected    int                 `json:"expected"`
+	Nodes       map[string]NodeInfo `json:"nodes,omitempty"`
+	Credentials *SharedCredentials  `json:"credentials,omitempty"`
+	CACert      string              `json:"ca_cert,omitempty"`
+	CAKey       string              `json:"ca_key,omitempty"`
+}
+
+// SharedCredentials contains the cluster-wide credentials distributed during formation.
+type SharedCredentials struct {
+	AccessKey   string `json:"access_key"`
+	SecretKey   string `json:"secret_key"`
+	AccountID   string `json:"account_id"`
+	NatsToken   string `json:"nats_token"`
+	ClusterName string `json:"cluster_name"`
+	Region      string `json:"region"`
+}
+
+// FormationServer is a lightweight HTTP server that coordinates cluster formation.
+// Nodes register themselves via POST /formation/join. Once the expected number of
+// nodes have joined, the done channel is closed and full cluster data (credentials,
+// CA, node list) becomes available via GET /formation/status.
+type FormationServer struct {
+	mu          sync.RWMutex
+	expected    int
+	nodes       map[string]NodeInfo
+	credentials *SharedCredentials
+	caCert      string
+	caKey       string
+	done        chan struct{}
+	server      *http.Server
+}
+
+// NewFormationServer creates a new formation server expecting the given number of nodes.
+func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string) *FormationServer {
+	return &FormationServer{
+		expected:    expected,
+		nodes:       make(map[string]NodeInfo),
+		credentials: creds,
+		caCert:      caCert,
+		caKey:       caKey,
+		done:        make(chan struct{}),
+	}
+}
+
+// RegisterNode validates and adds a node. Returns an error for duplicates.
+func (fs *FormationServer) RegisterNode(info NodeInfo) error {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	if info.Name == "" {
+		return fmt.Errorf("node name is required")
+	}
+	if info.BindIP == "" {
+		return fmt.Errorf("bind_ip is required")
+	}
+
+	// Check for duplicate name
+	if _, exists := fs.nodes[info.Name]; exists {
+		return fmt.Errorf("node %q already registered", info.Name)
+	}
+
+	// Check for duplicate bind IP
+	for _, n := range fs.nodes {
+		if n.BindIP == info.BindIP {
+			return fmt.Errorf("bind IP %s already registered by node %q", info.BindIP, n.Name)
+		}
+	}
+
+	fs.nodes[info.Name] = info
+	slog.Info("Node registered", "name", info.Name, "bind_ip", info.BindIP, "joined", len(fs.nodes), "expected", fs.expected)
+
+	if fs.isComplete() {
+		close(fs.done)
+	}
+
+	return nil
+}
+
+// isComplete returns true when we have enough nodes. Must be called with lock held.
+func (fs *FormationServer) isComplete() bool {
+	return len(fs.nodes) >= fs.expected
+}
+
+// IsComplete returns true when the expected number of nodes have registered.
+func (fs *FormationServer) IsComplete() bool {
+	fs.mu.RLock()
+	defer fs.mu.RUnlock()
+	return fs.isComplete()
+}
+
+// WaitForCompletion blocks until all nodes have joined or the timeout fires.
+func (fs *FormationServer) WaitForCompletion(timeout time.Duration) error {
+	select {
+	case <-fs.done:
+		return nil
+	case <-time.After(timeout):
+		fs.mu.RLock()
+		joined := len(fs.nodes)
+		fs.mu.RUnlock()
+		return fmt.Errorf("formation timed out after %s: %d/%d nodes joined", timeout, joined, fs.expected)
+	}
+}
+
+// Nodes returns a copy of the registered nodes.
+func (fs *FormationServer) Nodes() map[string]NodeInfo {
+	fs.mu.RLock()
+	defer fs.mu.RUnlock()
+	out := make(map[string]NodeInfo, len(fs.nodes))
+	for k, v := range fs.nodes {
+		out[k] = v
+	}
+	return out
+}
+
+// Start launches the HTTP server on the given address (e.g. "10.0.0.1:4432").
+func (fs *FormationServer) Start(bindAddr string) error {
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /formation/join", fs.handleJoin)
+	mux.HandleFunc("GET /formation/status", fs.handleStatus)
+	mux.HandleFunc("GET /formation/health", fs.handleHealth)
+
+	fs.server = &http.Server{
+		Addr:              bindAddr,
+		Handler:           mux,
+		ReadTimeout:       10 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		IdleTimeout:       30 * time.Second,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	ln, err := net.Listen("tcp", bindAddr)
+	if err != nil {
+		return fmt.Errorf("formation server listen: %w", err)
+	}
+
+	go func() {
+		if err := fs.server.Serve(ln); err != nil && err != http.ErrServerClosed {
+			slog.Error("Formation server error", "error", err)
+		}
+	}()
+
+	slog.Info("Formation server started", "addr", bindAddr)
+	return nil
+}
+
+// Shutdown gracefully stops the formation server.
+func (fs *FormationServer) Shutdown(ctx context.Context) error {
+	if fs.server == nil {
+		return nil
+	}
+	return fs.server.Shutdown(ctx)
+}
+
+func (fs *FormationServer) handleJoin(w http.ResponseWriter, r *http.Request) {
+	var req JoinRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeJSON(w, http.StatusBadRequest, JoinResponse{
+			Success: false,
+			Message: fmt.Sprintf("invalid request body: %v", err),
+		})
+		return
+	}
+
+	if err := fs.RegisterNode(req.NodeInfo); err != nil {
+		writeJSON(w, http.StatusConflict, JoinResponse{
+			Success: false,
+			Message: err.Error(),
+		})
+		return
+	}
+
+	fs.mu.RLock()
+	joined := len(fs.nodes)
+	fs.mu.RUnlock()
+
+	writeJSON(w, http.StatusOK, JoinResponse{
+		Success:  true,
+		Message:  fmt.Sprintf("node %q registered", req.Name),
+		Joined:   joined,
+		Expected: fs.expected,
+	})
+}
+
+func (fs *FormationServer) handleStatus(w http.ResponseWriter, r *http.Request) {
+	fs.mu.RLock()
+	defer fs.mu.RUnlock()
+
+	resp := StatusResponse{
+		Complete: fs.isComplete(),
+		Joined:   len(fs.nodes),
+		Expected: fs.expected,
+	}
+
+	// Only expose full data when formation is complete
+	if fs.isComplete() {
+		resp.Nodes = make(map[string]NodeInfo, len(fs.nodes))
+		for k, v := range fs.nodes {
+			resp.Nodes[k] = v
+		}
+		resp.Credentials = fs.credentials
+		resp.CACert = fs.caCert
+		resp.CAKey = fs.caKey
+	}
+
+	writeJSON(w, http.StatusOK, resp)
+}
+
+func (fs *FormationServer) handleHealth(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	if _, err := w.Write([]byte("ok")); err != nil {
+		slog.Error("Failed to write health response", "error", err)
+	}
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	if err := json.NewEncoder(w).Encode(v); err != nil {
+		slog.Error("Failed to encode JSON response", "error", err)
+	}
+}

hive/formation/formation_test.go

@@ -0,0 +1,313 @@
+package formation
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testCreds() *SharedCredentials {
+	return &SharedCredentials{
+		AccessKey:   "AKIATEST1234567890AB",
+		SecretKey:   "testSecretKey1234567890",
+		AccountID:   "123456789012",
+		NatsToken:   "nats_testtoken123456",
+		ClusterName: "hive",
+		Region:      "us-west-1",
+	}
+}
+
+func testNode(name, ip string) NodeInfo {
+	return NodeInfo{
+		Name:   name,
+		BindIP: ip,
+		Region: "us-west-1",
+		AZ:     "us-west-1a",
+		Port:   4432,
+	}
+}
+
+func TestNewFormationServer(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem")
+
+	assert.Equal(t, 3, fs.expected)
+	assert.Empty(t, fs.nodes)
+	assert.False(t, fs.IsComplete())
+}
+
+func TestRegisterNode(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "")
+
+	err := fs.RegisterNode(testNode("node1", "10.0.0.1"))
+	require.NoError(t, err)
+
+	nodes := fs.Nodes()
+	assert.Len(t, nodes, 1)
+	assert.Equal(t, "10.0.0.1", nodes["node1"].BindIP)
+}
+
+func TestRegisterDuplicateName(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "")
+
+	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
+	err := fs.RegisterNode(testNode("node1", "10.0.0.2"))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "already registered")
+}
+
+func TestRegisterDuplicateIP(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "")
+
+	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
+	err := fs.RegisterNode(testNode("node2", "10.0.0.1"))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "bind IP")
+}
+
+func TestIsComplete(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(2, testCreds(), "", "")
+
+	assert.False(t, fs.IsComplete())
+	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
+	assert.False(t, fs.IsComplete())
+	require.NoError(t, fs.RegisterNode(testNode("node2", "10.0.0.2")))
+	assert.True(t, fs.IsComplete())
+}
+
+func TestWaitForCompletion(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(2, testCreds(), "", "")
+
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		fs.RegisterNode(testNode("node1", "10.0.0.1"))
+		fs.RegisterNode(testNode("node2", "10.0.0.2"))
+	}()
+
+	err := fs.WaitForCompletion(5 * time.Second)
+	require.NoError(t, err)
+	assert.True(t, fs.IsComplete())
+}
+
+func TestWaitForCompletionTimeout(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "")
+
+	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
+
+	err := fs.WaitForCompletion(100 * time.Millisecond)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "timed out")
+}
+
+func testServer(t *testing.T, fs *FormationServer) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /formation/join", fs.handleJoin)
+	mux.HandleFunc("GET /formation/status", fs.handleStatus)
+	mux.HandleFunc("GET /formation/health", fs.handleHealth)
+	return httptest.NewServer(mux)
+}
+
+func TestJoinEndpoint(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "")
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	body, _ := json.Marshal(JoinRequest{NodeInfo: testNode("node1", "10.0.0.1")})
+	resp, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	var jr JoinResponse
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&jr))
+	assert.True(t, jr.Success)
+	assert.Equal(t, 1, jr.Joined)
+	assert.Equal(t, 3, jr.Expected)
+}
+
+func TestJoinEndpointDuplicate(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "", "")
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	body, _ := json.Marshal(JoinRequest{NodeInfo: testNode("node1", "10.0.0.1")})
+
+	resp1, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body))
+	require.NoError(t, err)
+	resp1.Body.Close()
+	assert.Equal(t, http.StatusOK, resp1.StatusCode)
+
+	resp2, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body))
+	require.NoError(t, err)
+	defer resp2.Body.Close()
+	assert.Equal(t, http.StatusConflict, resp2.StatusCode)
+}
+
+func TestStatusEndpointIncomplete(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key")
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
+
+	resp, err := http.Get(ts.URL + "/formation/status")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var sr StatusResponse
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&sr))
+	assert.False(t, sr.Complete)
+	assert.Equal(t, 1, sr.Joined)
+	assert.Equal(t, 3, sr.Expected)
+	assert.Nil(t, sr.Credentials)
+	assert.Empty(t, sr.CACert)
+	assert.Nil(t, sr.Nodes)
+}
+
+func TestStatusEndpointComplete(t *testing.T) {
+	t.Parallel()
+	creds := testCreds()
+	fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data")
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
+	require.NoError(t, fs.RegisterNode(testNode("node2", "10.0.0.2")))
+
+	resp, err := http.Get(ts.URL + "/formation/status")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var sr StatusResponse
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&sr))
+	assert.True(t, sr.Complete)
+	assert.Equal(t, 2, sr.Joined)
+	assert.Equal(t, 2, sr.Expected)
+	assert.Len(t, sr.Nodes, 2)
+	require.NotNil(t, sr.Credentials)
+	assert.Equal(t, creds.AccessKey, sr.Credentials.AccessKey)
+	assert.Equal(t, "ca-cert-data", sr.CACert)
+	assert.Equal(t, "ca-key-data", sr.CAKey)
+}
+
+func TestHealthEndpoint(t *testing.T) {
+	t.Parallel()
+	fs := NewFormationServer(1, testCreds(), "", "")
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL + "/formation/health")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+}
+
+func TestBuildClusterRoutes(t *testing.T) {
+	t.Parallel()
+	nodes := map[string]NodeInfo{
+		"node3": {Name: "node3", BindIP: "10.0.0.3"},
+		"node1": {Name: "node1", BindIP: "10.0.0.1"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2"},
+	}
+
+	routes := BuildClusterRoutes(nodes)
+	assert.Equal(t, []string{"10.0.0.1:4248", "10.0.0.2:4248", "10.0.0.3:4248"}, routes)
+}
+
+func TestBuildClusterRoutesUsesClusterIP(t *testing.T) {
+	t.Parallel()
+	nodes := map[string]NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1", ClusterIP: "192.168.1.1"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2"}, // no cluster IP, fallback to bind
+	}
+
+	routes := BuildClusterRoutes(nodes)
+	assert.Equal(t, []string{"192.168.1.1:4248", "10.0.0.2:4248"}, routes)
+}
+
+func TestBuildPredastoreNodes(t *testing.T) {
+	t.Parallel()
+	nodes := map[string]NodeInfo{
+		"node3": {Name: "node3", BindIP: "10.0.0.3"},
+		"node1": {Name: "node1", BindIP: "10.0.0.1"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2"},
+	}
+
+	pnodes := BuildPredastoreNodes(nodes)
+	require.Len(t, pnodes, 3)
+	assert.Equal(t, 1, pnodes[0].ID)
+	assert.Equal(t, "10.0.0.1", pnodes[0].Host)
+	assert.Equal(t, 2, pnodes[1].ID)
+	assert.Equal(t, "10.0.0.2", pnodes[1].Host)
+	assert.Equal(t, 3, pnodes[2].ID)
+	assert.Equal(t, "10.0.0.3", pnodes[2].Host)
+}
+
+func TestFullFormationFlow(t *testing.T) {
+	t.Parallel()
+	creds := testCreds()
+	fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----")
+	ts := testServer(t, fs)
+	defer ts.Close()
+
+	// 3 nodes join sequentially
+	for i, n := range []struct{ name, ip string }{
+		{"node1", "10.0.0.1"},
+		{"node2", "10.0.0.2"},
+		{"node3", "10.0.0.3"},
+	} {
+		body, _ := json.Marshal(JoinRequest{NodeInfo: testNode(n.name, n.ip)})
+		resp, err := http.Post(ts.URL+"/formation/join", "application/json", bytes.NewReader(body))
+		require.NoError(t, err)
+
+		var jr JoinResponse
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&jr))
+		resp.Body.Close()
+
+		assert.True(t, jr.Success)
+		assert.Equal(t, i+1, jr.Joined)
+	}
+
+	// All nodes poll status — should be complete
+	resp, err := http.Get(ts.URL + "/formation/status")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var sr StatusResponse
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&sr))
+	assert.True(t, sr.Complete)
+	assert.Equal(t, 3, sr.Joined)
+	assert.Len(t, sr.Nodes, 3)
+	require.NotNil(t, sr.Credentials)
+	assert.Equal(t, creds.AccessKey, sr.Credentials.AccessKey)
+	assert.Equal(t, creds.SecretKey, sr.Credentials.SecretKey)
+	assert.Equal(t, creds.NatsToken, sr.Credentials.NatsToken)
+	assert.NotEmpty(t, sr.CACert)
+	assert.NotEmpty(t, sr.CAKey)
+
+	// Verify helper outputs
+	routes := BuildClusterRoutes(sr.Nodes)
+	assert.Len(t, routes, 3)
+
+	pnodes := BuildPredastoreNodes(sr.Nodes)
+	assert.Len(t, pnodes, 3)
+	assert.Equal(t, 1, pnodes[0].ID)
+}

hive/formation/helpers.go

@@ -0,0 +1,81 @@
+package formation
+
+import (
+	"sort"
+
+	"github.com/mulgadc/hive/hive/admin"
+)
+
+// hasService reports whether the node's service list includes name.
+// An empty list means all services (backward compat).
+func hasService(services []string, name string) bool {
+	if len(services) == 0 {
+		return true // backward compat: empty = all services
+	}
+	for _, s := range services {
+		if s == name {
+			return true
+		}
+	}
+	return false
+}
+
+// BuildClusterRoutes returns a sorted list of "IP:4248" NATS cluster routes
+// for nodes that run the "nats" service. If ClusterIP is set it is used,
+// otherwise BindIP. Nodes are sorted by name for deterministic ordering.
+func BuildClusterRoutes(nodes map[string]NodeInfo) []string {
+	// Filter to nodes that run NATS
+	natsNodes := make(map[string]NodeInfo)
+	for k, n := range nodes {
+		if hasService(n.Services, "nats") {
+			natsNodes[k] = n
+		}
+	}
+	sorted := sortedNodes(natsNodes)
+	routes := make([]string, len(sorted))
+	for i, n := range sorted {
+		ip := n.ClusterIP
+		if ip == "" {
+			ip = n.BindIP
+		}
+		routes[i] = ip + ":4248"
+	}
+	return routes
+}
+
+// BuildPredastoreNodes returns a sorted list of PredastoreNodeConfig with
+// 1-based IDs for nodes that run the "predastore" service.
+// Nodes are sorted by name for deterministic ordering.
+func BuildPredastoreNodes(nodes map[string]NodeInfo) []admin.PredastoreNodeConfig {
+	// Filter to nodes that run Predastore
+	predaNodes := make(map[string]NodeInfo)
+	for k, n := range nodes {
+		if hasService(n.Services, "predastore") {
+			predaNodes[k] = n
+		}
+	}
+	sorted := sortedNodes(predaNodes)
+	out := make([]admin.PredastoreNodeConfig, len(sorted))
+	for i, n := range sorted {
+		out[i] = admin.PredastoreNodeConfig{
+			ID:   i + 1,
+			Host: n.BindIP,
+		}
+	}
+	return out
+}
+
+// sortedNodes returns nodes sorted by name.
+func sortedNodes(nodes map[string]NodeInfo) []NodeInfo {
+	names := make([]string, 0, len(nodes))
+	for name := range nodes {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	sorted := make([]NodeInfo, len(names))
+	for i, name := range names {
+		sorted[i] = nodes[name]
+	}
+	return sorted
+}

hive/formation/helpers_test.go

@@ -0,0 +1,87 @@
+package formation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHasService_EmptyListMeansAll(t *testing.T) {
+	t.Parallel()
+	assert.True(t, hasService(nil, "nats"))
+	assert.True(t, hasService([]string{}, "predastore"))
+}
+
+func TestHasService_ExplicitList(t *testing.T) {
+	t.Parallel()
+	services := []string{"nats", "daemon"}
+	assert.True(t, hasService(services, "nats"))
+	assert.True(t, hasService(services, "daemon"))
+	assert.False(t, hasService(services, "predastore"))
+}
+
+func TestBuildClusterRoutes_FiltersNonNATSNodes(t *testing.T) {
+	t.Parallel()
+	nodes := map[string]NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1", Services: []string{"nats", "daemon"}},
+		"node2": {Name: "node2", BindIP: "10.0.0.2", Services: []string{"daemon"}}, // no NATS
+		"node3": {Name: "node3", BindIP: "10.0.0.3", Services: []string{"nats", "predastore"}},
+	}
+
+	routes := BuildClusterRoutes(nodes)
+	assert.Equal(t, []string{"10.0.0.1:4248", "10.0.0.3:4248"}, routes)
+}
+
+func TestBuildClusterRoutes_EmptyServicesIncludesAll(t *testing.T) {
+	t.Parallel()
+	// Nodes with empty services (backward compat) should all be included
+	nodes := map[string]NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2"},
+	}
+
+	routes := BuildClusterRoutes(nodes)
+	assert.Equal(t, []string{"10.0.0.1:4248", "10.0.0.2:4248"}, routes)
+}
+
+func TestBuildPredastoreNodes_FiltersByPredastore(t *testing.T) {
+	t.Parallel()
+	nodes := map[string]NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1", Services: []string{"nats", "predastore"}},
+		"node2": {Name: "node2", BindIP: "10.0.0.2", Services: []string{"daemon"}}, // no predastore
+		"node3": {Name: "node3", BindIP: "10.0.0.3", Services: []string{"predastore"}},
+	}
+
+	pnodes := BuildPredastoreNodes(nodes)
+	require.Len(t, pnodes, 2)
+	assert.Equal(t, 1, pnodes[0].ID)
+	assert.Equal(t, "10.0.0.1", pnodes[0].Host)
+	assert.Equal(t, 2, pnodes[1].ID)
+	assert.Equal(t, "10.0.0.3", pnodes[1].Host)
+}
+
+func TestBuildPredastoreNodes_EmptyServicesIncludesAll(t *testing.T) {
+	t.Parallel()
+	nodes := map[string]NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2"},
+		"node3": {Name: "node3", BindIP: "10.0.0.3"},
+	}
+
+	pnodes := BuildPredastoreNodes(nodes)
+	require.Len(t, pnodes, 3)
+}
+
+func TestBuildClusterRoutes_MixedServicesAndEmpty(t *testing.T) {
+	t.Parallel()
+	nodes := map[string]NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1"},                               // empty = all
+		"node2": {Name: "node2", BindIP: "10.0.0.2", Services: []string{"daemon"}}, // no nats
+		"node3": {Name: "node3", BindIP: "10.0.0.3", Services: []string{"nats"}},   // has nats
+	}
+
+	routes := BuildClusterRoutes(nodes)
+	// node1 (empty=all, included), node2 (no nats, excluded), node3 (has nats, included)
+	assert.Equal(t, []string{"10.0.0.1:4248", "10.0.0.3:4248"}, routes)
+}

hive/gateway/auth.go

@@ -0,0 +1,262 @@
+package gateway
+
+import (
+	"crypto/subtle"
+	"fmt"
+	"log/slog"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/google/uuid"
+	"github.com/mulgadc/hive/hive/awserrors"
+	"github.com/mulgadc/predastore/auth"
+)
+
+const (
+	// Maximum allowed clock skew for signature validation (5 minutes)
+	maxClockSkew = 5 * time.Minute
+)
+
+// SigV4AuthMiddleware returns a Fiber middleware that validates AWS Signature V4 authentication.
+// It verifies the Authorization header against the configured credentials.
+func (gw *GatewayConfig) SigV4AuthMiddleware() fiber.Handler {
+	return func(c *fiber.Ctx) error {
+		// Skip OPTIONS requests (CORS preflight)
+		if c.Method() == "OPTIONS" {
+			return c.Next()
+		}
+
+		authHeader := c.Get("Authorization")
+		if authHeader == "" {
+			return gw.writeSigV4Error(c, awserrors.ErrorMissingAuthenticationToken)
+		}
+
+		// Parse the Authorization header
+		// Format: AWS4-HMAC-SHA256 Credential=ACCESS_KEY/DATE/REGION/SERVICE/aws4_request, SignedHeaders=..., Signature=...
+		if !strings.HasPrefix(authHeader, "AWS4-HMAC-SHA256 ") {
+			return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+		}
+
+		// Extract components from the Authorization header
+		parts := strings.Split(authHeader[len("AWS4-HMAC-SHA256 "):], ", ")
+		if len(parts) != 3 {
+			return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+		}
+
+		// Parse Credential
+		var accessKey, date, region, service string
+		if after, ok := strings.CutPrefix(parts[0], "Credential="); ok {
+			credParts := strings.Split(after, "/")
+			if len(credParts) != 5 || credParts[4] != "aws4_request" {
+				return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+			}
+			accessKey = credParts[0]
+			date = credParts[1]
+			region = credParts[2]
+			service = credParts[3]
+		} else {
+			return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+		}
+
+		// Parse SignedHeaders
+		var signedHeaders string
+		if after, ok := strings.CutPrefix(parts[1], "SignedHeaders="); ok {
+			signedHeaders = after
+		} else {
+			return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+		}
+
+		// Parse Signature
+		var providedSignature string
+		if after, ok := strings.CutPrefix(parts[2], "Signature="); ok {
+			providedSignature = after
+		} else {
+			return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+		}
+
+		// Verify access key matches configured credentials
+		fmt.Println("accessKey", accessKey, "gw.AccessKey", gw.AccessKey)
+		if accessKey != gw.AccessKey {
+			slog.Debug("Invalid access key", "provided", accessKey, "expected", gw.AccessKey)
+			return gw.writeSigV4Error(c, awserrors.ErrorInvalidClientTokenId)
+		}
+
+		// Get timestamp from X-Amz-Date header
+		timestamp := c.Get("X-Amz-Date")
+		if timestamp == "" {
+			return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+		}
+
+		// Validate timestamp is within acceptable bounds to prevent replay attacks
+		parsedTime, err := time.Parse("20060102T150405Z", timestamp)
+		if err != nil {
+			slog.Debug("Invalid timestamp format", "timestamp", timestamp)
+			return gw.writeSigV4Error(c, awserrors.ErrorIncompleteSignature)
+		}
+		if time.Since(parsedTime).Abs() > maxClockSkew {
+			slog.Debug("Signature expired", "timestamp", timestamp, "skew", time.Since(parsedTime))
+			return gw.writeSigV4Error(c, awserrors.ErrorSignatureDoesNotMatch)
+		}
+
+		// Compute expected signature
+		expectedSignature := gw.computeSignature(c, date, timestamp, region, service, signedHeaders)
+
+		// Compare signatures using constant-time comparison to prevent timing attacks
+		if subtle.ConstantTimeCompare([]byte(expectedSignature), []byte(providedSignature)) != 1 {
+			slog.Debug("Signature mismatch",
+				"expected", expectedSignature,
+				"provided", providedSignature,
+			)
+			return gw.writeSigV4Error(c, awserrors.ErrorSignatureDoesNotMatch)
+		}
+
+		// Store parsed auth data in context for downstream handlers
+		c.Locals("sigv4.service", service)
+		c.Locals("sigv4.region", region)
+		c.Locals("sigv4.accessKey", accessKey)
+
+		slog.Debug("SigV4 authentication successful", "accessKey", accessKey)
+		return c.Next()
+	}
+}
+
+// computeSignature builds the canonical request and computes the AWS Signature V4 signature.
+func (gw *GatewayConfig) computeSignature(c *fiber.Ctx, date, timestamp, region, service, signedHeaders string) string {
+	// Build canonical URI (URI-encoded path)
+	path := c.Path()
+	if path == "" {
+		path = "/"
+	}
+	canonicalURI := auth.UriEncode(path, false)
+	if canonicalURI == "" {
+		canonicalURI = "/"
+	}
+
+	// Build canonical query string (sorted, encoded)
+	canonicalQueryString := buildCanonicalQueryString(string(c.Request().URI().QueryString()))
+
+	// Build canonical headers from SignedHeaders list
+	headersList := strings.Split(signedHeaders, ";")
+	sort.Strings(headersList)
+
+	var canonicalHeaders strings.Builder
+	for _, header := range headersList {
+		header = strings.ToLower(strings.TrimSpace(header))
+		var value string
+		if header == "host" {
+			value = string(c.Request().Host())
+		} else {
+			value = c.Get(canonicalHeaderName(header))
+		}
+		// Trim leading/trailing whitespace and collapse multiple spaces
+		value = strings.TrimSpace(value)
+		canonicalHeaders.WriteString(header)
+		canonicalHeaders.WriteString(":")
+		canonicalHeaders.WriteString(value)
+		canonicalHeaders.WriteString("\n")
+	}
+
+	// Hash payload body with SHA256
+	payloadHash := auth.HashSHA256(string(c.Body()))
+
+	// Build canonical request
+	canonicalRequest := fmt.Sprintf(
+		"%s\n%s\n%s\n%s\n%s\n%s",
+		c.Method(),
+		canonicalURI,
+		canonicalQueryString,
+		canonicalHeaders.String(),
+		signedHeaders,
+		payloadHash,
+	)
+
+	hashedCanonicalRequest := auth.HashSHA256(canonicalRequest)
+
+	// Build string-to-sign
+	scope := fmt.Sprintf("%s/%s/%s/aws4_request", date, region, service)
+	stringToSign := fmt.Sprintf(
+		"AWS4-HMAC-SHA256\n%s\n%s\n%s",
+		timestamp,
+		scope,
+		hashedCanonicalRequest,
+	)
+
+	// Derive signing key and compute signature
+	signingKey := auth.GetSigningKey(gw.SecretKey, date, region, service)
+	signature := auth.HmacSHA256Hex(signingKey, stringToSign)
+
+	return signature
+}
+
+// buildCanonicalQueryString creates the canonical query string according to AWS specs.
+func buildCanonicalQueryString(queryString string) string {
+	if queryString == "" {
+		return ""
+	}
+
+	// Parse query parameters
+	params := make(map[string][]string)
+	pairs := strings.SplitSeq(queryString, "&")
+	for pair := range pairs {
+		if pair == "" {
+			continue
+		}
+		kv := strings.SplitN(pair, "=", 2)
+		key := kv[0]
+		value := ""
+		if len(kv) == 2 {
+			value = kv[1]
+		}
+		params[key] = append(params[key], value)
+	}
+
+	// Sort keys
+	keys := make([]string, 0, len(params))
+	for k := range params {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	// Build canonical query string
+	var result []string
+	for _, key := range keys {
+		values := params[key]
+		sort.Strings(values)
+		encodedKey := auth.UriEncode(key, true)
+		for _, v := range values {
+			encodedValue := auth.UriEncode(v, true)
+			result = append(result, fmt.Sprintf("%s=%s", encodedKey, encodedValue))
+		}
+	}
+
+	return strings.Join(result, "&")
+}
+
+// canonicalHeaderName converts a lowercase header name to the canonical form for lookup.
+func canonicalHeaderName(header string) string {
+	// Convert header names like "x-amz-date" to "X-Amz-Date" for Fiber's Get method
+	parts := strings.Split(header, "-")
+	for i, part := range parts {
+		if len(part) > 0 {
+			parts[i] = strings.ToUpper(part[:1]) + part[1:]
+		}
+	}
+	return strings.Join(parts, "-")
+}
+
+// writeSigV4Error writes an EC2-compatible XML error response for authentication failures.
+func (gw *GatewayConfig) writeSigV4Error(c *fiber.Ctx, errorCode string) error {
+	requestID := c.Get("x-amz-request-id", uuid.NewString())
+
+	errorMsg, exists := awserrors.ErrorLookup[errorCode]
+	if !exists {
+		errorMsg = awserrors.ErrorMessage{HTTPCode: 500, Message: "Internal error"}
+	}
+
+	xmlError := GenerateEC2ErrorResponse(errorCode, errorMsg.Message, requestID)
+
+	c.Set("Content-Type", "application/xml")
+	return c.Status(errorMsg.HTTPCode).Send(xmlError)
+}

hive/gateway/auth_test.go

@@ -0,0 +1,381 @@
+package gateway
+
+import (
+	"fmt"
+	"io"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/mulgadc/predastore/auth"
+)
+
+const (
+	testAccessKey = "AKIAIOSFODNN7EXAMPLE"
+	testSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	testRegion    = "us-east-1"
+	testService   = "ec2"
+)
+
+// generateTestAuthHeader creates a valid AWS SigV4 Authorization header for testing.
+func generateTestAuthHeader(method, path, queryString, body, accessKey, secretKey, region, service string) (authHeader, timestamp string) {
+	now := time.Now().UTC()
+	timestamp = now.Format(auth.TimeFormat)
+	date := now.Format(auth.ShortTimeFormat)
+
+	// Build canonical URI
+	canonicalURI := auth.UriEncode(path, false)
+	if canonicalURI == "" {
+		canonicalURI = "/"
+	}
+
+	// Build canonical query string
+	canonicalQueryString := buildCanonicalQueryString(queryString)
+
+	// Build canonical headers
+	host := "localhost:9999"
+	canonicalHeaders := fmt.Sprintf("host:%s\nx-amz-date:%s\n", host, timestamp)
+	signedHeaders := "host;x-amz-date"
+
+	// Hash payload
+	payloadHash := auth.HashSHA256(body)
+
+	// Build canonical request
+	canonicalRequest := fmt.Sprintf(
+		"%s\n%s\n%s\n%s\n%s\n%s",
+		method,
+		canonicalURI,
+		canonicalQueryString,
+		canonicalHeaders,
+		signedHeaders,
+		payloadHash,
+	)
+
+	hashedCanonicalRequest := auth.HashSHA256(canonicalRequest)
+
+	// Build string-to-sign
+	scope := fmt.Sprintf("%s/%s/%s/aws4_request", date, region, service)
+	stringToSign := fmt.Sprintf(
+		"AWS4-HMAC-SHA256\n%s\n%s\n%s",
+		timestamp,
+		scope,
+		hashedCanonicalRequest,
+	)
+
+	// Derive signing key and compute signature
+	signingKey := auth.GetSigningKey(secretKey, date, region, service)
+	signature := auth.HmacSHA256Hex(signingKey, stringToSign)
+
+	// Build Authorization header
+	authHeader = fmt.Sprintf(
+		"AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/aws4_request, SignedHeaders=%s, Signature=%s",
+		accessKey,
+		date,
+		region,
+		service,
+		signedHeaders,
+		signature,
+	)
+
+	return authHeader, timestamp
+}
+
+func setupTestApp(accessKey, secretKey string) *fiber.App {
+	gw := &GatewayConfig{
+		DisableLogging: true,
+		AccessKey:      accessKey,
+		SecretKey:      secretKey,
+		Region:         testRegion,
+	}
+
+	app := fiber.New()
+	app.Use(gw.SigV4AuthMiddleware())
+
+	// Simple test handler
+	app.All("/*", func(c *fiber.Ctx) error {
+		return c.SendString("OK")
+	})
+
+	return app
+}
+
+func TestSigV4Auth_NoAuthorizationHeader(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	req := httptest.NewRequest("GET", "/", nil)
+	req.Host = "localhost:9999"
+
+	resp, err := app.Test(req)
+	if err != nil {
+		t.Fatalf("Failed to test request: %v", err)
+	}
+
+	if resp.StatusCode != 403 {
+		t.Errorf("Expected status 403, got %d", resp.StatusCode)
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	if !strings.Contains(string(body), "MissingAuthenticationToken") {
+		t.Errorf("Expected MissingAuthenticationToken error, got: %s", string(body))
+	}
+}
+
+func TestSigV4Auth_MalformedHeader(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	testCases := []struct {
+		name       string
+		authHeader string
+	}{
+		{"empty prefix", "InvalidPrefix Credential=test"},
+		{"missing parts", "AWS4-HMAC-SHA256 Credential=test"},
+		{"invalid credential format", "AWS4-HMAC-SHA256 Credential=a/b/c, SignedHeaders=host, Signature=sig"},
+		{"missing SignedHeaders prefix", "AWS4-HMAC-SHA256 Credential=a/b/c/d/aws4_request, Headers=host, Signature=sig"},
+		{"missing Signature prefix", "AWS4-HMAC-SHA256 Credential=a/b/c/d/aws4_request, SignedHeaders=host, Sig=abc"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest("GET", "/", nil)
+			req.Host = "localhost:9999"
+			req.Header.Set("Authorization", tc.authHeader)
+			req.Header.Set("X-Amz-Date", time.Now().UTC().Format(auth.TimeFormat))
+
+			resp, err := app.Test(req)
+			if err != nil {
+				t.Fatalf("Failed to test request: %v", err)
+			}
+
+			if resp.StatusCode != 400 {
+				t.Errorf("Expected status 400, got %d", resp.StatusCode)
+			}
+
+			body, _ := io.ReadAll(resp.Body)
+			if !strings.Contains(string(body), "IncompleteSignature") {
+				t.Errorf("Expected IncompleteSignature error, got: %s", string(body))
+			}
+		})
+	}
+}
+
+func TestSigV4Auth_InvalidAccessKey(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	authHeader, timestamp := generateTestAuthHeader(
+		"GET", "/", "", "",
+		"INVALID_ACCESS_KEY", testSecretKey, testRegion, testService,
+	)
+
+	req := httptest.NewRequest("GET", "/", nil)
+	req.Host = "localhost:9999"
+	req.Header.Set("Authorization", authHeader)
+	req.Header.Set("X-Amz-Date", timestamp)
+
+	resp, err := app.Test(req)
+	if err != nil {
+		t.Fatalf("Failed to test request: %v", err)
+	}
+
+	if resp.StatusCode != 403 {
+		t.Errorf("Expected status 403, got %d", resp.StatusCode)
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	if !strings.Contains(string(body), "InvalidClientTokenId") {
+		t.Errorf("Expected InvalidClientTokenId error, got: %s", string(body))
+	}
+}
+
+func TestSigV4Auth_InvalidSignature(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	// Generate auth header with wrong secret key
+	authHeader, timestamp := generateTestAuthHeader(
+		"GET", "/", "", "",
+		testAccessKey, "WRONG_SECRET_KEY", testRegion, testService,
+	)
+
+	req := httptest.NewRequest("GET", "/", nil)
+	req.Host = "localhost:9999"
+	req.Header.Set("Authorization", authHeader)
+	req.Header.Set("X-Amz-Date", timestamp)
+
+	resp, err := app.Test(req)
+	if err != nil {
+		t.Fatalf("Failed to test request: %v", err)
+	}
+
+	if resp.StatusCode != 403 {
+		t.Errorf("Expected status 403, got %d", resp.StatusCode)
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	if !strings.Contains(string(body), "SignatureDoesNotMatch") {
+		t.Errorf("Expected SignatureDoesNotMatch error, got: %s", string(body))
+	}
+}
+
+func TestSigV4Auth_ValidSignature(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	authHeader, timestamp := generateTestAuthHeader(
+		"GET", "/", "", "",
+		testAccessKey, testSecretKey, testRegion, testService,
+	)
+
+	req := httptest.NewRequest("GET", "/", nil)
+	req.Host = "localhost:9999"
+	req.Header.Set("Authorization", authHeader)
+	req.Header.Set("X-Amz-Date", timestamp)
+
+	resp, err := app.Test(req)
+	if err != nil {
+		t.Fatalf("Failed to test request: %v", err)
+	}
+
+	if resp.StatusCode != 200 {
+		body, _ := io.ReadAll(resp.Body)
+		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
+	}
+}
+
+func TestSigV4Auth_OptionsSkipsAuth(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	req := httptest.NewRequest("OPTIONS", "/", nil)
+	req.Host = "localhost:9999"
+	// No Authorization header
+
+	resp, err := app.Test(req)
+	if err != nil {
+		t.Fatalf("Failed to test request: %v", err)
+	}
+
+	if resp.StatusCode != 200 {
+		t.Errorf("Expected status 200 for OPTIONS, got %d", resp.StatusCode)
+	}
+}
+
+func TestSigV4Auth_ValidSignatureWithBody(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	body := "Action=DescribeInstances&Version=2016-11-15"
+	authHeader, timestamp := generateTestAuthHeader(
+		"POST", "/", "", body,
+		testAccessKey, testSecretKey, testRegion, testService,
+	)
+
+	req := httptest.NewRequest("POST", "/", strings.NewReader(body))
+	req.Host = "localhost:9999"
+	req.Header.Set("Authorization", authHeader)
+	req.Header.Set("X-Amz-Date", timestamp)
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := app.Test(req)
+	if err != nil {
+		t.Fatalf("Failed to test request: %v", err)
+	}
+
+	if resp.StatusCode != 200 {
+		respBody, _ := io.ReadAll(resp.Body)
+		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(respBody))
+	}
+}
+
+func TestSigV4Auth_ValidSignatureWithQueryString(t *testing.T) {
+	app := setupTestApp(testAccessKey, testSecretKey)
+
+	queryString := "Action=DescribeInstances&Version=2016-11-15"
+	authHeader, timestamp := generateTestAuthHeader(
+		"GET", "/", queryString, "",
+		testAccessKey, testSecretKey, testRegion, testService,
+	)
+
+	req := httptest.NewRequest("GET", "/?"+queryString, nil)
+	req.Host = "localhost:9999"
+	req.Header.Set("Authorization", authHeader)
+	req.Header.Set("X-Amz-Date", timestamp)
+
+	resp, err := app.Test(req)
+	if err != nil {
+		t.Fatalf("Failed to test request: %v", err)
+	}
+
+	if resp.StatusCode != 200 {
+		respBody, _ := io.ReadAll(resp.Body)
+		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(respBody))
+	}
+}
+
+func TestParseAWSQueryArgs_URLDecoding(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    string
+		key      string
+		expected string
+	}{
+		{"plain value", "Action=DescribeInstances", "Action", "DescribeInstances"},
+		{"encoded slashes", "Device=%2Fdev%2Fsdf", "Device", "/dev/sdf"},
+		{"encoded spaces", "Name=my%20volume", "Name", "my volume"},
+		{"encoded plus as space", "Name=my+volume", "Name", "my volume"},
+		{"no encoding needed", "VolumeId=vol-abc123", "VolumeId", "vol-abc123"},
+		{"multiple params", "VolumeId=vol-abc&Device=%2Fdev%2Fsdg", "Device", "/dev/sdg"},
+		{"encoded key dot", "Tag%2EKey=Name", "Tag.Key", "Name"},
+		{"encoded key and value", "Filter%2E1%2EName=instance-id&Filter%2E1%2EValue=i-abc", "Filter.1.Name", "instance-id"},
+		{"key-only encoded", "Tag%2EKey=", "Tag.Key", ""},
+		{"key without value", "Tag%2EKey", "Tag.Key", ""},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := ParseAWSQueryArgs(tc.input)
+			if result[tc.key] != tc.expected {
+				t.Errorf("Expected %q for key %q, got %q", tc.expected, tc.key, result[tc.key])
+			}
+		})
+	}
+}
+
+func TestBuildCanonicalQueryString(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"empty", "", ""},
+		{"single param", "Action=Test", "Action=Test"},
+		{"multiple params sorted", "Version=1&Action=Test", "Action=Test&Version=1"},
+		{"encoded values", "Name=Hello World", "Name=Hello%20World"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := buildCanonicalQueryString(tc.input)
+			if result != tc.expected {
+				t.Errorf("Expected %q, got %q", tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestCanonicalHeaderName(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{"host", "Host"},
+		{"x-amz-date", "X-Amz-Date"},
+		{"content-type", "Content-Type"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			result := canonicalHeaderName(tc.input)
+			if result != tc.expected {
+				t.Errorf("Expected %q, got %q", tc.expected, result)
+			}
+		})
+	}
+}

hive/gateway/ec2.go

@@ -1,385 +1,202 @@
+// Package gateway provides the AWS Gateway for the Hive platform.
+// It handles the incoming requests from the AWS SDK and delegates to the appropriate gateway functions (which calls the NATS handlers).
 package gateway
 
 import (
 	"errors"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/gofiber/fiber/v2"
 	"github.com/mulgadc/hive/hive/awsec2query"
+	"github.com/mulgadc/hive/hive/awserrors"
+	gateway_ec2_account "github.com/mulgadc/hive/hive/gateway/ec2/account"
+	gateway_ec2_eigw "github.com/mulgadc/hive/hive/gateway/ec2/eigw"
 	gateway_ec2_image "github.com/mulgadc/hive/hive/gateway/ec2/image"
 	gateway_ec2_instance "github.com/mulgadc/hive/hive/gateway/ec2/instance"
 	gateway_ec2_key "github.com/mulgadc/hive/hive/gateway/ec2/key"
+	gateway_ec2_snapshot "github.com/mulgadc/hive/hive/gateway/ec2/snapshot"
+	gateway_ec2_tags "github.com/mulgadc/hive/hive/gateway/ec2/tags"
+	gateway_ec2_volume "github.com/mulgadc/hive/hive/gateway/ec2/volume"
+	gateway_ec2_zone "github.com/mulgadc/hive/hive/gateway/ec2/zone"
 	"github.com/mulgadc/hive/hive/utils"
 )
 
-func (gw *GatewayConfig) EC2_Request(ctx *fiber.Ctx) error {
+// EC2Handler processes parsed query args and returns XML response bytes.
+// The action parameter is the EC2 API action name, passed from the map key.
+type EC2Handler func(action string, q map[string]string, gw *GatewayConfig) ([]byte, error)
 
+// ec2Handler creates a type-safe EC2Handler that allocates the typed input struct,
+// parses query params into it, calls the handler, and marshals the output to XML.
+func ec2Handler[In any](handler func(*In, *GatewayConfig) (any, error)) EC2Handler {
+	return func(action string, q map[string]string, gw *GatewayConfig) ([]byte, error) {
+		input := new(In)
+		if err := awsec2query.QueryParamsToStruct(q, input); err != nil {
+			return nil, err
+		}
+		output, err := handler(input, gw)
+		if err != nil {
+			return nil, err
+		}
+		payload := utils.GenerateXMLPayload(action+"Response", output)
+		xmlOutput, err := utils.MarshalToXML(payload)
+		if err != nil {
+			return nil, errors.New("failed to marshal response to XML")
+		}
+		return xmlOutput, nil
+	}
+}
+
+var ec2Actions = map[string]EC2Handler{
+	"DescribeInstances": ec2Handler(func(input *ec2.DescribeInstancesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_instance.DescribeInstances(input, gw.NATSConn, gw.DiscoverActiveNodes())
+	}),
+	"RunInstances": ec2Handler(func(input *ec2.RunInstancesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_instance.RunInstances(input, gw.NATSConn)
+	}),
+	"StartInstances": ec2Handler(func(input *ec2.StartInstancesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_instance.StartInstances(input, gw.NATSConn)
+	}),
+	"StopInstances": ec2Handler(func(input *ec2.StopInstancesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_instance.StopInstances(input, gw.NATSConn)
+	}),
+	"TerminateInstances": ec2Handler(func(input *ec2.TerminateInstancesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_instance.TerminateInstances(input, gw.NATSConn)
+	}),
+	"DescribeInstanceTypes": ec2Handler(func(input *ec2.DescribeInstanceTypesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_instance.DescribeInstanceTypes(input, gw.NATSConn, gw.ExpectedNodes)
+	}),
+	"GetConsoleOutput": ec2Handler(func(input *ec2.GetConsoleOutputInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_instance.GetConsoleOutput(input, gw.NATSConn)
+	}),
+	"CreateKeyPair": ec2Handler(func(input *ec2.CreateKeyPairInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_key.CreateKeyPair(input, gw.NATSConn)
+	}),
+	"DeleteKeyPair": ec2Handler(func(input *ec2.DeleteKeyPairInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_key.DeleteKeyPair(input, gw.NATSConn)
+	}),
+	"DescribeKeyPairs": ec2Handler(func(input *ec2.DescribeKeyPairsInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_key.DescribeKeyPairs(input, gw.NATSConn)
+	}),
+	"ImportKeyPair": func(action string, q map[string]string, gw *GatewayConfig) ([]byte, error) {
+		// Workaround: parser leaves Base64 padding URL-encoded
+		if strings.HasSuffix(q["PublicKeyMaterial"], "%3D%3D") {
+			q["PublicKeyMaterial"] = strings.Replace(q["PublicKeyMaterial"], "%3D%3D", "==", 1)
+		}
+		return ec2Handler(func(input *ec2.ImportKeyPairInput, gw *GatewayConfig) (any, error) {
+			return gateway_ec2_key.ImportKeyPair(input, gw.NATSConn)
+		})(action, q, gw)
+	},
+	"DescribeImages": ec2Handler(func(input *ec2.DescribeImagesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_image.DescribeImages(input, gw.NATSConn)
+	}),
+	"CreateImage": ec2Handler(func(input *ec2.CreateImageInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_image.CreateImage(input, gw.NATSConn)
+	}),
+	"DescribeRegions": ec2Handler(func(input *ec2.DescribeRegionsInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_zone.DescribeRegions(input, gw.Region)
+	}),
+	"DescribeAvailabilityZones": ec2Handler(func(input *ec2.DescribeAvailabilityZonesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_zone.DescribeAvailabilityZones(input, gw.Region, gw.AZ)
+	}),
+	"DescribeVolumes": ec2Handler(func(input *ec2.DescribeVolumesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_volume.DescribeVolumes(input, gw.NATSConn)
+	}),
+	"ModifyVolume": ec2Handler(func(input *ec2.ModifyVolumeInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_volume.ModifyVolume(input, gw.NATSConn)
+	}),
+	"CreateVolume": ec2Handler(func(input *ec2.CreateVolumeInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_volume.CreateVolume(input, gw.NATSConn)
+	}),
+	"DeleteVolume": ec2Handler(func(input *ec2.DeleteVolumeInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_volume.DeleteVolume(input, gw.NATSConn)
+	}),
+	"AttachVolume": ec2Handler(func(input *ec2.AttachVolumeInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_volume.AttachVolume(input, gw.NATSConn)
+	}),
+	"DescribeVolumeStatus": ec2Handler(func(input *ec2.DescribeVolumeStatusInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_volume.DescribeVolumeStatus(input, gw.NATSConn)
+	}),
+	"DetachVolume": ec2Handler(func(input *ec2.DetachVolumeInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_volume.DetachVolume(input, gw.NATSConn)
+	}),
+	"DescribeAccountAttributes": ec2Handler(func(input *ec2.DescribeAccountAttributesInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_account.DescribeAccountAttributes(input)
+	}),
+	"EnableEbsEncryptionByDefault": ec2Handler(func(input *ec2.EnableEbsEncryptionByDefaultInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_account.EnableEbsEncryptionByDefault(input, gw.NATSConn)
+	}),
+	"DisableEbsEncryptionByDefault": ec2Handler(func(input *ec2.DisableEbsEncryptionByDefaultInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_account.DisableEbsEncryptionByDefault(input, gw.NATSConn)
+	}),
+	"GetEbsEncryptionByDefault": ec2Handler(func(input *ec2.GetEbsEncryptionByDefaultInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_account.GetEbsEncryptionByDefault(input, gw.NATSConn)
+	}),
+	"GetSerialConsoleAccessStatus": ec2Handler(func(input *ec2.GetSerialConsoleAccessStatusInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_account.GetSerialConsoleAccessStatus(input, gw.NATSConn)
+	}),
+	"EnableSerialConsoleAccess": ec2Handler(func(input *ec2.EnableSerialConsoleAccessInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_account.EnableSerialConsoleAccess(input, gw.NATSConn)
+	}),
+	"DisableSerialConsoleAccess": ec2Handler(func(input *ec2.DisableSerialConsoleAccessInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_account.DisableSerialConsoleAccess(input, gw.NATSConn)
+	}),
+	"CreateTags": ec2Handler(func(input *ec2.CreateTagsInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_tags.CreateTags(input, gw.NATSConn)
+	}),
+	"DeleteTags": ec2Handler(func(input *ec2.DeleteTagsInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_tags.DeleteTags(input, gw.NATSConn)
+	}),
+	"DescribeTags": ec2Handler(func(input *ec2.DescribeTagsInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_tags.DescribeTags(input, gw.NATSConn)
+	}),
+	"CreateSnapshot": ec2Handler(func(input *ec2.CreateSnapshotInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_snapshot.CreateSnapshot(input, gw.NATSConn)
+	}),
+	"DeleteSnapshot": ec2Handler(func(input *ec2.DeleteSnapshotInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_snapshot.DeleteSnapshot(input, gw.NATSConn)
+	}),
+	"DescribeSnapshots": ec2Handler(func(input *ec2.DescribeSnapshotsInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_snapshot.DescribeSnapshots(input, gw.NATSConn)
+	}),
+	"CopySnapshot": ec2Handler(func(input *ec2.CopySnapshotInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_snapshot.CopySnapshot(input, gw.NATSConn)
+	}),
+	"CreateEgressOnlyInternetGateway": ec2Handler(func(input *ec2.CreateEgressOnlyInternetGatewayInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_eigw.CreateEgressOnlyInternetGateway(input, gw.NATSConn)
+	}),
+	"DeleteEgressOnlyInternetGateway": ec2Handler(func(input *ec2.DeleteEgressOnlyInternetGatewayInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_eigw.DeleteEgressOnlyInternetGateway(input, gw.NATSConn)
+	}),
+	"DescribeEgressOnlyInternetGateways": ec2Handler(func(input *ec2.DescribeEgressOnlyInternetGatewaysInput, gw *GatewayConfig) (any, error) {
+		return gateway_ec2_eigw.DescribeEgressOnlyInternetGateways(input, gw.NATSConn)
+	}),
+}
+
+// ec2LocalActions are actions that don't require a NATS connection.
+var ec2LocalActions = map[string]bool{
+	"DescribeRegions":           true,
+	"DescribeAvailabilityZones": true,
+	"DescribeAccountAttributes": true,
+}
+
+func (gw *GatewayConfig) EC2_Request(ctx *fiber.Ctx) error {
 	queryArgs := ParseAWSQueryArgs(string(ctx.Body()))
 
-	var xmlOutput []byte
-	var err error
-
-	// Run the action
-	// TODO: Generate for each action, unit test each, and invalid action
-	switch queryArgs["Action"] {
-
-	case "DescribeInstances":
-		var input = &ec2.DescribeInstancesInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_instance.DescribeInstances(input, gw.NATSConn, gw.ExpectedNodes)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("DescribeInstancesResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-	case "RunInstances":
-
-		var input = &ec2.RunInstancesInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_instance.RunInstances(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("RunInstanceResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "StartInstances":
-
-		var input = &ec2.StartInstancesInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_instance.StartInstances(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("StartInstancesResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "StopInstances":
-
-		var input = &ec2.StopInstancesInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_instance.StopInstances(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("StopInstancesResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "TerminateInstances":
-
-		var input = &ec2.TerminateInstancesInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_instance.TerminateInstances(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("TerminateInstancesResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "CreateKeyPair":
-
-		var input = &ec2.CreateKeyPairInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_key.CreateKeyPair(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("CreateKeyPairResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "DeleteKeyPair":
-
-		var input = &ec2.DeleteKeyPairInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_key.DeleteKeyPair(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("DeleteKeyPairResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "DescribeKeyPairs":
-
-		var input = &ec2.DescribeKeyPairsInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_key.DescribeKeyPairs(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("DescribeKeyPairsResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "ImportKeyPair":
-
-		var input = &ec2.ImportKeyPairInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_key.ImportKeyPair(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("ImportKeyPairResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	case "DescribeImages":
-
-		var input = &ec2.DescribeImagesInput{}
-		err = awsec2query.QueryParamsToStruct(queryArgs, input)
-
-		if err != nil {
-			return err
-		}
-
-		output, err := gateway_ec2_image.DescribeImages(input, gw.NATSConn)
-
-		if err != nil {
-			return err
-		}
-
-		// Convert to XML
-		payload := utils.GenerateXMLPayload("DescribeImagesResponse", output)
-		xmlOutput, err = utils.MarshalToXML(payload)
-
-		if err != nil {
-			return errors.New("failed to marshal response to XML")
-		}
-
-	default:
-		err = errors.New("InvalidAction")
+	action := queryArgs["Action"]
+	handler, ok := ec2Actions[action]
+	if !ok {
+		return errors.New(awserrors.ErrorInvalidAction)
 	}
 
-	// Return an error XML
+	if gw.NATSConn == nil && !ec2LocalActions[action] {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	xmlOutput, err := handler(action, queryArgs, gw)
 	if err != nil {
 		return err
 	}
 
-	ctx.Status(fiber.StatusOK).Type("text/xml").Send(xmlOutput)
-
-	return nil
-
+	return ctx.Status(fiber.StatusOK).Type("text/xml").Send(xmlOutput)
 }
-
-/*
-func (gw *GatewayConfig) EC2_DescribeInstances(ctx *fiber.Ctx, args map[string]string) error {
-
-	slog.Info("EC2 DescribeInstances called")
-
-	// Return a dummy response
-	response := `<?xml version="1.0" encoding="UTF-8"?>
-<DescribeInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
-  <requestId>2b7ac2f1-9acd-4d73-b0a1-1b2e5607f9ab</requestId>
-  <reservationSet>
-    <item>
-      <reservationId>r-0f6b3c4d5e6789abc</reservationId>
-      <ownerId>123456789012</ownerId>
-      <groupSet/>
-      <instancesSet>
-        <item>
-          <instanceId>i-0123456789abcdef0</instanceId>
-          <imageId>ami-08d4ac5b634553e16</imageId>
-          <instanceState>
-            <code>16</code>
-            <name>running</name>
-          </instanceState>
-          <privateDnsName>ip-10-0-1-25.ap-southeast-2.compute.internal</privateDnsName>
-          <dnsName>ec2-3-26-14-112.ap-southeast-2.compute.amazonaws.com</dnsName>
-          <reason/>
-          <keyName>my-keypair</keyName>
-          <amiLaunchIndex>0</amiLaunchIndex>
-          <productCodes/>
-          <instanceType>t3.medium</instanceType>
-          <launchTime>2025-02-18T04:12:07.000Z</launchTime>
-          <placement>
-            <availabilityZone>ap-southeast-2a</availabilityZone>
-            <groupName/>
-            <tenancy>default</tenancy>
-          </placement>
-          <monitoring>
-            <state>disabled</state>
-          </monitoring>
-          <privateIpAddress>10.0.1.25</privateIpAddress>
-          <ipAddress>3.26.14.112</ipAddress>
-          <subnetId>subnet-0ab1c2d3e4f567890</subnetId>
-          <vpcId>vpc-0123abcd4567efgh8</vpcId>
-          <sourceDestCheck>true</sourceDestCheck>
-          <groupSet>
-            <item>
-              <groupId>sg-0abc1234def567890</groupId>
-              <groupName>default</groupName>
-            </item>
-          </groupSet>
-          <stateReason/>
-          <architecture>x86_64</architecture>
-          <rootDeviceType>ebs</rootDeviceType>
-          <rootDeviceName>/dev/xvda</rootDeviceName>
-          <blockDeviceMapping>
-            <item>
-              <deviceName>/dev/xvda</deviceName>
-              <ebs>
-                <volumeId>vol-06f5e4d3c2b1a0f98</volumeId>
-                <status>attached</status>
-                <attachTime>2025-02-18T04:12:09.000Z</attachTime>
-                <deleteOnTermination>true</deleteOnTermination>
-              </ebs>
-            </item>
-          </blockDeviceMapping>
-          <virtualizationType>hvm</virtualizationType>
-          <tagSet>
-            <item>
-              <key>Name</key>
-              <value>web-01</value>
-            </item>
-          </tagSet>
-          <hypervisor>xen</hypervisor>
-          <networkInterfaceSet>
-            <item>
-              <networkInterfaceId>eni-0a12b34c56d78ef90</networkInterfaceId>
-              <subnetId>subnet-0ab1c2d3e4f567890</subnetId>
-              <vpcId>vpc-0123abcd4567efgh8</vpcId>
-              <description/>
-              <ownerId>123456789012</ownerId>
-              <status>in-use</status>
-              <macAddress>0a:1b:2c:3d:4e:5f</macAddress>
-              <privateIpAddress>10.0.1.25</privateIpAddress>
-              <privateDnsName>ip-10-0-1-25.ap-southeast-2.compute.internal</privateDnsName>
-              <sourceDestCheck>true</sourceDestCheck>
-              <groupSet>
-                <item>
-                  <groupId>sg-0abc1234def567890</groupId>
-                  <groupName>default</groupName>
-                </item>
-              </groupSet>
-              <attachment>
-                <attachmentId>eni-attach-0abc123def4567890</attachmentId>
-                <deviceIndex>0</deviceIndex>
-                <status>attached</status>
-                <attachTime>2025-02-18T04:12:07.000Z</attachTime>
-                <deleteOnTermination>true</deleteOnTermination>
-              </attachment>
-              <privateIpAddressesSet>
-                <item>
-                  <privateIpAddress>10.0.1.25</privateIpAddress>
-                  <privateDnsName>ip-10-0-1-25.ap-southeast-2.compute.internal</privateDnsName>
-                  <primary>true</primary>
-                </item>
-              </privateIpAddressesSet>
-            </item>
-          </networkInterfaceSet>
-        </item>
-      </instancesSet>
-    </item>
-  </reservationSet>
-</DescribeInstancesResponse>
-`
-
-	return ctx.SendString(response)
-}
-
-*/

hive/gateway/ec2/account/DescribeAccountAttributes.go

@@ -0,0 +1,93 @@
+package gateway_ec2_account
+
+import (
+	"log/slog"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// DescribeAccountAttributes returns static account attributes for the Hive platform
+func DescribeAccountAttributes(input *ec2.DescribeAccountAttributesInput) (*ec2.DescribeAccountAttributesOutput, error) {
+	slog.Debug("DescribeAccountAttributes called")
+
+	// Build set of requested attribute names for filtering
+	requestedAttrs := make(map[string]bool)
+	for _, name := range input.AttributeNames {
+		if name != nil {
+			requestedAttrs[*name] = true
+		}
+	}
+
+	// If no specific attributes requested, return all
+	returnAll := len(requestedAttrs) == 0
+
+	var accountAttributes []*ec2.AccountAttribute
+
+	// supported-platforms - VPC only (no EC2-Classic)
+	if returnAll || requestedAttrs["supported-platforms"] {
+		accountAttributes = append(accountAttributes, &ec2.AccountAttribute{
+			AttributeName: aws.String("supported-platforms"),
+			AttributeValues: []*ec2.AccountAttributeValue{
+				{AttributeValue: aws.String("VPC")},
+			},
+		})
+	}
+
+	// default-vpc
+	if returnAll || requestedAttrs["default-vpc"] {
+		accountAttributes = append(accountAttributes, &ec2.AccountAttribute{
+			AttributeName: aws.String("default-vpc"),
+			AttributeValues: []*ec2.AccountAttributeValue{
+				{AttributeValue: aws.String("none")},
+			},
+		})
+	}
+
+	// max-instances
+	if returnAll || requestedAttrs["max-instances"] {
+		accountAttributes = append(accountAttributes, &ec2.AccountAttribute{
+			AttributeName: aws.String("max-instances"),
+			AttributeValues: []*ec2.AccountAttributeValue{
+				{AttributeValue: aws.String("100")},
+			},
+		})
+	}
+
+	// vpc-max-security-groups-per-interface
+	if returnAll || requestedAttrs["vpc-max-security-groups-per-interface"] {
+		accountAttributes = append(accountAttributes, &ec2.AccountAttribute{
+			AttributeName: aws.String("vpc-max-security-groups-per-interface"),
+			AttributeValues: []*ec2.AccountAttributeValue{
+				{AttributeValue: aws.String("5")},
+			},
+		})
+	}
+
+	// max-elastic-ips
+	if returnAll || requestedAttrs["max-elastic-ips"] {
+		accountAttributes = append(accountAttributes, &ec2.AccountAttribute{
+			AttributeName: aws.String("max-elastic-ips"),
+			AttributeValues: []*ec2.AccountAttributeValue{
+				{AttributeValue: aws.String("5")},
+			},
+		})
+	}
+
+	// vpc-max-elastic-ips
+	if returnAll || requestedAttrs["vpc-max-elastic-ips"] {
+		accountAttributes = append(accountAttributes, &ec2.AccountAttribute{
+			AttributeName: aws.String("vpc-max-elastic-ips"),
+			AttributeValues: []*ec2.AccountAttributeValue{
+				{AttributeValue: aws.String("20")},
+			},
+		})
+	}
+
+	output := &ec2.DescribeAccountAttributesOutput{
+		AccountAttributes: accountAttributes,
+	}
+
+	slog.Info("DescribeAccountAttributes completed", "attributeCount", len(accountAttributes))
+	return output, nil
+}

hive/gateway/ec2/account/DisableEbsEncryptionByDefault.go

@@ -0,0 +1,33 @@
+package gateway_ec2_account
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_account "github.com/mulgadc/hive/hive/handlers/ec2/account"
+	"github.com/nats-io/nats.go"
+)
+
+func ValidateDisableEbsEncryptionByDefaultInput(input *ec2.DisableEbsEncryptionByDefaultInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return nil
+}
+
+func DisableEbsEncryptionByDefault(input *ec2.DisableEbsEncryptionByDefaultInput, natsConn *nats.Conn) (ec2.DisableEbsEncryptionByDefaultOutput, error) {
+	var output ec2.DisableEbsEncryptionByDefaultOutput
+
+	if err := ValidateDisableEbsEncryptionByDefaultInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_account.NewNATSAccountSettingsService(natsConn)
+	result, err := svc.DisableEbsEncryptionByDefault(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/account/DisableSerialConsoleAccess.go

@@ -0,0 +1,33 @@
+package gateway_ec2_account
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_account "github.com/mulgadc/hive/hive/handlers/ec2/account"
+	"github.com/nats-io/nats.go"
+)
+
+func ValidateDisableSerialConsoleAccessInput(input *ec2.DisableSerialConsoleAccessInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return nil
+}
+
+func DisableSerialConsoleAccess(input *ec2.DisableSerialConsoleAccessInput, natsConn *nats.Conn) (ec2.DisableSerialConsoleAccessOutput, error) {
+	var output ec2.DisableSerialConsoleAccessOutput
+
+	if err := ValidateDisableSerialConsoleAccessInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_account.NewNATSAccountSettingsService(natsConn)
+	result, err := svc.DisableSerialConsoleAccess(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/account/EnableEbsEncryptionByDefault.go

@@ -0,0 +1,33 @@
+package gateway_ec2_account
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_account "github.com/mulgadc/hive/hive/handlers/ec2/account"
+	"github.com/nats-io/nats.go"
+)
+
+func ValidateEnableEbsEncryptionByDefaultInput(input *ec2.EnableEbsEncryptionByDefaultInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return nil
+}
+
+func EnableEbsEncryptionByDefault(input *ec2.EnableEbsEncryptionByDefaultInput, natsConn *nats.Conn) (ec2.EnableEbsEncryptionByDefaultOutput, error) {
+	var output ec2.EnableEbsEncryptionByDefaultOutput
+
+	if err := ValidateEnableEbsEncryptionByDefaultInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_account.NewNATSAccountSettingsService(natsConn)
+	result, err := svc.EnableEbsEncryptionByDefault(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/account/EnableSerialConsoleAccess.go

@@ -0,0 +1,33 @@
+package gateway_ec2_account
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_account "github.com/mulgadc/hive/hive/handlers/ec2/account"
+	"github.com/nats-io/nats.go"
+)
+
+func ValidateEnableSerialConsoleAccessInput(input *ec2.EnableSerialConsoleAccessInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return nil
+}
+
+func EnableSerialConsoleAccess(input *ec2.EnableSerialConsoleAccessInput, natsConn *nats.Conn) (ec2.EnableSerialConsoleAccessOutput, error) {
+	var output ec2.EnableSerialConsoleAccessOutput
+
+	if err := ValidateEnableSerialConsoleAccessInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_account.NewNATSAccountSettingsService(natsConn)
+	result, err := svc.EnableSerialConsoleAccess(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/account/GetEbsEncryptionByDefault.go

@@ -0,0 +1,33 @@
+package gateway_ec2_account
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_account "github.com/mulgadc/hive/hive/handlers/ec2/account"
+	"github.com/nats-io/nats.go"
+)
+
+func ValidateGetEbsEncryptionByDefaultInput(input *ec2.GetEbsEncryptionByDefaultInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return nil
+}
+
+func GetEbsEncryptionByDefault(input *ec2.GetEbsEncryptionByDefaultInput, natsConn *nats.Conn) (ec2.GetEbsEncryptionByDefaultOutput, error) {
+	var output ec2.GetEbsEncryptionByDefaultOutput
+
+	if err := ValidateGetEbsEncryptionByDefaultInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_account.NewNATSAccountSettingsService(natsConn)
+	result, err := svc.GetEbsEncryptionByDefault(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/account/GetSerialConsoleAccessStatus.go

@@ -0,0 +1,33 @@
+package gateway_ec2_account
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_account "github.com/mulgadc/hive/hive/handlers/ec2/account"
+	"github.com/nats-io/nats.go"
+)
+
+func ValidateGetSerialConsoleAccessStatusInput(input *ec2.GetSerialConsoleAccessStatusInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return nil
+}
+
+func GetSerialConsoleAccessStatus(input *ec2.GetSerialConsoleAccessStatusInput, natsConn *nats.Conn) (ec2.GetSerialConsoleAccessStatusOutput, error) {
+	var output ec2.GetSerialConsoleAccessStatusOutput
+
+	if err := ValidateGetSerialConsoleAccessStatusInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_account.NewNATSAccountSettingsService(natsConn)
+	result, err := svc.GetSerialConsoleAccessStatus(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/account/account_test.go

@@ -0,0 +1,280 @@
+package gateway_ec2_account
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDescribeAccountAttributes_AllAttributes(t *testing.T) {
+	input := &ec2.DescribeAccountAttributesInput{}
+
+	output, err := DescribeAccountAttributes(input)
+	require.NoError(t, err)
+	require.NotNil(t, output)
+
+	assert.Len(t, output.AccountAttributes, 6)
+
+	attrMap := make(map[string]string)
+	for _, attr := range output.AccountAttributes {
+		attrMap[*attr.AttributeName] = *attr.AttributeValues[0].AttributeValue
+	}
+
+	assert.Equal(t, "VPC", attrMap["supported-platforms"])
+	assert.Equal(t, "none", attrMap["default-vpc"])
+	assert.Equal(t, "100", attrMap["max-instances"])
+	assert.Equal(t, "5", attrMap["vpc-max-security-groups-per-interface"])
+	assert.Equal(t, "5", attrMap["max-elastic-ips"])
+	assert.Equal(t, "20", attrMap["vpc-max-elastic-ips"])
+}
+
+func TestDescribeAccountAttributes_FilterSingle(t *testing.T) {
+	input := &ec2.DescribeAccountAttributesInput{
+		AttributeNames: []*string{aws.String("max-instances")},
+	}
+
+	output, err := DescribeAccountAttributes(input)
+	require.NoError(t, err)
+	require.NotNil(t, output)
+
+	assert.Len(t, output.AccountAttributes, 1)
+	assert.Equal(t, "max-instances", *output.AccountAttributes[0].AttributeName)
+	assert.Equal(t, "100", *output.AccountAttributes[0].AttributeValues[0].AttributeValue)
+}
+
+func TestDescribeAccountAttributes_FilterMultiple(t *testing.T) {
+	input := &ec2.DescribeAccountAttributesInput{
+		AttributeNames: []*string{
+			aws.String("supported-platforms"),
+			aws.String("default-vpc"),
+		},
+	}
+
+	output, err := DescribeAccountAttributes(input)
+	require.NoError(t, err)
+	require.NotNil(t, output)
+
+	assert.Len(t, output.AccountAttributes, 2)
+
+	names := make(map[string]bool)
+	for _, attr := range output.AccountAttributes {
+		names[*attr.AttributeName] = true
+	}
+	assert.True(t, names["supported-platforms"])
+	assert.True(t, names["default-vpc"])
+}
+
+func TestDescribeAccountAttributes_FilterNonExistent(t *testing.T) {
+	input := &ec2.DescribeAccountAttributesInput{
+		AttributeNames: []*string{aws.String("nonexistent-attribute")},
+	}
+
+	output, err := DescribeAccountAttributes(input)
+	require.NoError(t, err)
+	require.NotNil(t, output)
+
+	assert.Len(t, output.AccountAttributes, 0)
+}
+
+func TestDescribeAccountAttributes_EmptyAttributeNames(t *testing.T) {
+	input := &ec2.DescribeAccountAttributesInput{
+		AttributeNames: []*string{},
+	}
+
+	output, err := DescribeAccountAttributes(input)
+	require.NoError(t, err)
+	require.NotNil(t, output)
+
+	// Empty slice means return all
+	assert.Len(t, output.AccountAttributes, 6)
+}
+
+func TestValidateEnableEbsEncryptionByDefaultInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.EnableEbsEncryptionByDefaultInput
+		wantErr bool
+	}{
+		{
+			name:    "NilInput",
+			input:   nil,
+			wantErr: true,
+		},
+		{
+			name:    "ValidInput",
+			input:   &ec2.EnableEbsEncryptionByDefaultInput{},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateEnableEbsEncryptionByDefaultInput(tt.input)
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateDisableEbsEncryptionByDefaultInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.DisableEbsEncryptionByDefaultInput
+		wantErr bool
+	}{
+		{
+			name:    "NilInput",
+			input:   nil,
+			wantErr: true,
+		},
+		{
+			name:    "ValidInput",
+			input:   &ec2.DisableEbsEncryptionByDefaultInput{},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateDisableEbsEncryptionByDefaultInput(tt.input)
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateGetEbsEncryptionByDefaultInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.GetEbsEncryptionByDefaultInput
+		wantErr bool
+	}{
+		{
+			name:    "NilInput",
+			input:   nil,
+			wantErr: true,
+		},
+		{
+			name:    "ValidInput",
+			input:   &ec2.GetEbsEncryptionByDefaultInput{},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateGetEbsEncryptionByDefaultInput(tt.input)
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateEnableSerialConsoleAccessInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.EnableSerialConsoleAccessInput
+		wantErr bool
+	}{
+		{
+			name:    "NilInput",
+			input:   nil,
+			wantErr: true,
+		},
+		{
+			name:    "ValidInput",
+			input:   &ec2.EnableSerialConsoleAccessInput{},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateEnableSerialConsoleAccessInput(tt.input)
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateDisableSerialConsoleAccessInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.DisableSerialConsoleAccessInput
+		wantErr bool
+	}{
+		{
+			name:    "NilInput",
+			input:   nil,
+			wantErr: true,
+		},
+		{
+			name:    "ValidInput",
+			input:   &ec2.DisableSerialConsoleAccessInput{},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateDisableSerialConsoleAccessInput(tt.input)
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateGetSerialConsoleAccessStatusInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.GetSerialConsoleAccessStatusInput
+		wantErr bool
+	}{
+		{
+			name:    "NilInput",
+			input:   nil,
+			wantErr: true,
+		},
+		{
+			name:    "ValidInput",
+			input:   &ec2.GetSerialConsoleAccessStatusInput{},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateGetSerialConsoleAccessStatusInput(tt.input)
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

hive/gateway/ec2/eigw/CreateEgressOnlyInternetGateway.go

@@ -0,0 +1,40 @@
+package gateway_ec2_eigw
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_eigw "github.com/mulgadc/hive/hive/handlers/ec2/eigw"
+	"github.com/nats-io/nats.go"
+)
+
+// ValidateCreateEgressOnlyInternetGatewayInput validates the input parameters
+func ValidateCreateEgressOnlyInternetGatewayInput(input *ec2.CreateEgressOnlyInternetGatewayInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+
+	if input.VpcId == nil || *input.VpcId == "" {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+
+	return nil
+}
+
+// CreateEgressOnlyInternetGateway handles the EC2 CreateEgressOnlyInternetGateway API call
+func CreateEgressOnlyInternetGateway(input *ec2.CreateEgressOnlyInternetGatewayInput, natsConn *nats.Conn) (ec2.CreateEgressOnlyInternetGatewayOutput, error) {
+	var output ec2.CreateEgressOnlyInternetGatewayOutput
+
+	if err := ValidateCreateEgressOnlyInternetGatewayInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_eigw.NewNATSEgressOnlyIGWService(natsConn)
+	result, err := svc.CreateEgressOnlyInternetGateway(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/eigw/DeleteEgressOnlyInternetGateway.go

@@ -0,0 +1,40 @@
+package gateway_ec2_eigw
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_eigw "github.com/mulgadc/hive/hive/handlers/ec2/eigw"
+	"github.com/nats-io/nats.go"
+)
+
+// ValidateDeleteEgressOnlyInternetGatewayInput validates the input parameters
+func ValidateDeleteEgressOnlyInternetGatewayInput(input *ec2.DeleteEgressOnlyInternetGatewayInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+
+	if input.EgressOnlyInternetGatewayId == nil || *input.EgressOnlyInternetGatewayId == "" {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+
+	return nil
+}
+
+// DeleteEgressOnlyInternetGateway handles the EC2 DeleteEgressOnlyInternetGateway API call
+func DeleteEgressOnlyInternetGateway(input *ec2.DeleteEgressOnlyInternetGatewayInput, natsConn *nats.Conn) (ec2.DeleteEgressOnlyInternetGatewayOutput, error) {
+	var output ec2.DeleteEgressOnlyInternetGatewayOutput
+
+	if err := ValidateDeleteEgressOnlyInternetGatewayInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_eigw.NewNATSEgressOnlyIGWService(natsConn)
+	result, err := svc.DeleteEgressOnlyInternetGateway(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/eigw/DescribeEgressOnlyInternetGateways.go

@@ -0,0 +1,36 @@
+package gateway_ec2_eigw
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_eigw "github.com/mulgadc/hive/hive/handlers/ec2/eigw"
+	"github.com/nats-io/nats.go"
+)
+
+// ValidateDescribeEgressOnlyInternetGatewaysInput validates the input parameters
+func ValidateDescribeEgressOnlyInternetGatewaysInput(input *ec2.DescribeEgressOnlyInternetGatewaysInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+
+	return nil
+}
+
+// DescribeEgressOnlyInternetGateways handles the EC2 DescribeEgressOnlyInternetGateways API call
+func DescribeEgressOnlyInternetGateways(input *ec2.DescribeEgressOnlyInternetGatewaysInput, natsConn *nats.Conn) (ec2.DescribeEgressOnlyInternetGatewaysOutput, error) {
+	var output ec2.DescribeEgressOnlyInternetGatewaysOutput
+
+	if err := ValidateDescribeEgressOnlyInternetGatewaysInput(input); err != nil {
+		return output, err
+	}
+
+	svc := handlers_ec2_eigw.NewNATSEgressOnlyIGWService(natsConn)
+	result, err := svc.DescribeEgressOnlyInternetGateways(input)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

hive/gateway/ec2/eigw/eigw_test.go

@@ -0,0 +1,127 @@
+package gateway_ec2_eigw
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateCreateEgressOnlyInternetGatewayInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.CreateEgressOnlyInternetGatewayInput
+		wantErr string
+	}{
+		{
+			name:    "nil input",
+			input:   nil,
+			wantErr: awserrors.ErrorInvalidParameterValue,
+		},
+		{
+			name:    "missing VpcId",
+			input:   &ec2.CreateEgressOnlyInternetGatewayInput{},
+			wantErr: awserrors.ErrorMissingParameter,
+		},
+		{
+			name:    "empty VpcId",
+			input:   &ec2.CreateEgressOnlyInternetGatewayInput{VpcId: aws.String("")},
+			wantErr: awserrors.ErrorMissingParameter,
+		},
+		{
+			name:    "valid input",
+			input:   &ec2.CreateEgressOnlyInternetGatewayInput{VpcId: aws.String("vpc-123")},
+			wantErr: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateCreateEgressOnlyInternetGatewayInput(tt.input)
+			if tt.wantErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateDeleteEgressOnlyInternetGatewayInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.DeleteEgressOnlyInternetGatewayInput
+		wantErr string
+	}{
+		{
+			name:    "nil input",
+			input:   nil,
+			wantErr: awserrors.ErrorInvalidParameterValue,
+		},
+		{
+			name:    "missing ID",
+			input:   &ec2.DeleteEgressOnlyInternetGatewayInput{},
+			wantErr: awserrors.ErrorMissingParameter,
+		},
+		{
+			name:    "empty ID",
+			input:   &ec2.DeleteEgressOnlyInternetGatewayInput{EgressOnlyInternetGatewayId: aws.String("")},
+			wantErr: awserrors.ErrorMissingParameter,
+		},
+		{
+			name:    "valid input",
+			input:   &ec2.DeleteEgressOnlyInternetGatewayInput{EgressOnlyInternetGatewayId: aws.String("eigw-abc123")},
+			wantErr: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateDeleteEgressOnlyInternetGatewayInput(tt.input)
+			if tt.wantErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateDescribeEgressOnlyInternetGatewaysInput(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   *ec2.DescribeEgressOnlyInternetGatewaysInput
+		wantErr string
+	}{
+		{
+			name:    "nil input",
+			input:   nil,
+			wantErr: awserrors.ErrorInvalidParameterValue,
+		},
+		{
+			name:    "empty input",
+			input:   &ec2.DescribeEgressOnlyInternetGatewaysInput{},
+			wantErr: "",
+		},
+		{
+			name: "with filter IDs",
+			input: &ec2.DescribeEgressOnlyInternetGatewaysInput{
+				EgressOnlyInternetGatewayIds: []*string{aws.String("eigw-abc")},
+			},
+			wantErr: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateDescribeEgressOnlyInternetGatewaysInput(tt.input)
+			if tt.wantErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, tt.wantErr)
+			}
+		})
+	}
+}

hive/gateway/ec2/image/CopyImage.go

@@ -1,143 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
-)
-
-/*
-Sample JSON:
-
-    "CopyImage":{
-      "name":"CopyImage",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"CopyImageRequest"},
-      "output":{"shape":"CopyImageResult"}
-    },
-
-    "CopyImageRequest":{
-      "type":"structure",
-      "required":[
-        "Name",
-        "SourceImageId",
-        "SourceRegion"
-      ],
-      "members":{
-        "ClientToken":{"shape":"String"},
-        "Description":{"shape":"String"},
-        "Encrypted":{
-          "shape":"Boolean",
-          "locationName":"encrypted"
-        },
-        "KmsKeyId":{
-          "shape":"KmsKeyId",
-          "locationName":"kmsKeyId"
-        },
-        "Name":{"shape":"String"},
-        "SourceImageId":{"shape":"String"},
-        "SourceRegion":{"shape":"String"},
-        "DestinationOutpostArn":{"shape":"String"},
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        },
-        "CopyImageTags":{"shape":"Boolean"},
-        "TagSpecifications":{
-          "shape":"TagSpecificationList",
-          "locationName":"TagSpecification"
-        }
-      }
-    },
-
-    "CopyImageResult":{
-      "type":"structure",
-      "members":{
-        "ImageId":{
-          "shape":"ImageId",
-          "locationName":"imageId"
-        }
-      }
-    }
-*/
-
-func ValidateCopyImageInput(input *ec2.CopyImageInput) (err error) {
-
-	// Check required args from JSON above
-	// required:[
-	//   "Name",
-	//   "SourceImageId",
-	//   "SourceRegion"
-	// ]
-
-	if input == nil {
-		return errors.New("MissingParameter")
-	}
-
-	if input.Name == nil || *input.Name == "" {
-		return errors.New("MissingParameter")
-	}
-
-	if input.SourceImageId == nil || *input.SourceImageId == "" {
-		return errors.New("MissingParameter")
-	}
-
-	if input.SourceRegion == nil || *input.SourceRegion == "" {
-		return errors.New("MissingParameter")
-	}
-
-	// Validate SourceImageId format
-	if !strings.HasPrefix(*input.SourceImageId, "ami-") {
-		return errors.New("InvalidAMIID.Malformed")
-	}
-
-	return
-}
-
-func CopyImage(input *ec2.CopyImageInput) (output ec2.CopyImageOutput, err error) {
-
-	// Validate input
-	err = ValidateCopyImageInput(input)
-
-	if err != nil {
-		return output, err
-	}
-
-	// Marshal to JSON, to send over the wire for processing (NATS)
-	jsonData, err := json.Marshal(input)
-	if err != nil {
-		return output, fmt.Errorf("failed to marshal input to JSON: %w", err)
-	}
-
-	// Run the simulated JSON request via handler, which will return a JSON response
-	handler := handlers_ec2_image.NewCopyImageHandler(handlers_ec2_image.NewMockImageService())
-	jsonResp := handler.Process(jsonData)
-
-	// Validate if the response is successful or an error
-	responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-	if err != nil {
-		if responseError.Code != nil {
-			slog.Error("CopyImage failed", "err", responseError.Code)
-			return output, errors.New(*responseError.Code)
-		}
-		return output, err
-	}
-
-	// Unmarshal the JSON response back into output struct
-	err = json.Unmarshal(jsonResp, &output)
-	if err != nil {
-		return output, fmt.Errorf("failed to unmarshal JSON response to CopyImageOutput: %w", err)
-	}
-
-	return output, nil
-}

hive/gateway/ec2/image/CopyImage_test.go

@@ -1,78 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"errors"
-	"strings"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestValidateCopyImageInput(t *testing.T) {
-	tests := []struct {
-		name  string
-		input *ec2.CopyImageInput
-		want  error
-	}{
-		{
-			name:  "NilInput",
-			input: nil,
-			want:  errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingName",
-			input: &ec2.CopyImageInput{
-				SourceImageId: aws.String("ami-0123456789abcdef0"),
-				SourceRegion:  aws.String("us-east-1"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingSourceImageId",
-			input: &ec2.CopyImageInput{
-				Name:         aws.String("test-copy"),
-				SourceRegion: aws.String("us-east-1"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingSourceRegion",
-			input: &ec2.CopyImageInput{
-				Name:          aws.String("test-copy"),
-				SourceImageId: aws.String("ami-0123456789abcdef0"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "InvalidSourceImageId",
-			input: &ec2.CopyImageInput{
-				Name:          aws.String("test-copy"),
-				SourceImageId: aws.String("invalid-id"),
-				SourceRegion:  aws.String("us-east-1"),
-			},
-			want: errors.New("InvalidAMIID.Malformed"),
-		},
-		{
-			name: "ValidInput",
-			input: &ec2.CopyImageInput{
-				Name:          aws.String("test-copy"),
-				SourceImageId: aws.String("ami-0123456789abcdef0"),
-				SourceRegion:  aws.String("us-east-1"),
-			},
-			want: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result, err := CopyImage(tt.input)
-			assert.Equal(t, tt.want, err)
-			if err == nil {
-				assert.NotEmpty(t, result.ImageId)
-				assert.True(t, strings.HasPrefix(*result.ImageId, "ami-"))
-			}
-		})
-	}
-}

hive/gateway/ec2/image/CreateImage.go

@@ -1,142 +1,52 @@
 package gateway_ec2_image
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
-	"log/slog"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_image "github.com/mulgadc/hive/hive/handlers/ec2/image"
+	"github.com/nats-io/nats.go"
 )
 
-/*
-Sample JSON:
-
-    "CreateImage":{
-      "name":"CreateImage",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"CreateImageRequest"},
-      "output":{"shape":"CreateImageResult"}
-    },
-
-    "CreateImageRequest":{
-      "type":"structure",
-      "required":[
-        "InstanceId",
-        "Name"
-      ],
-      "members":{
-        "BlockDeviceMappings":{
-          "shape":"BlockDeviceMappingRequestList",
-          "locationName":"blockDeviceMapping"
-        },
-        "Description":{
-          "shape":"String",
-          "locationName":"description"
-        },
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        },
-        "InstanceId":{
-          "shape":"InstanceId",
-          "locationName":"instanceId"
-        },
-        "Name":{
-          "shape":"String",
-          "locationName":"name"
-        },
-        "NoReboot":{
-          "shape":"Boolean",
-          "locationName":"noReboot"
-        },
-        "TagSpecifications":{
-          "shape":"TagSpecificationList",
-          "locationName":"TagSpecification"
-        }
-      }
-    },
-
-    "CreateImageResult":{
-      "type":"structure",
-      "members":{
-        "ImageId":{
-          "shape":"ImageId",
-          "locationName":"imageId"
-        }
-      }
-    }
-*/
-
-func ValidateCreateImageInput(input *ec2.CreateImageInput) (err error) {
-
-	// Check required args from JSON above
-	// required:[
-	//   "InstanceId",
-	//   "Name"
-	// ]
-
+// ValidateCreateImageInput validates the input parameters for CreateImage
+func ValidateCreateImageInput(input *ec2.CreateImageInput) error {
 	if input == nil {
-		return errors.New("MissingParameter")
+		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
 	if input.InstanceId == nil || *input.InstanceId == "" {
-		return errors.New("MissingParameter")
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+
+	if !strings.HasPrefix(*input.InstanceId, "i-") {
+		return errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
 	}
 
 	if input.Name == nil || *input.Name == "" {
-		return errors.New("MissingParameter")
+		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	// Validate InstanceId format
-	if !strings.HasPrefix(*input.InstanceId, "i-") {
-		return errors.New("InvalidInstanceID.Malformed")
-	}
-
-	return
+	return nil
 }
 
-func CreateImage(input *ec2.CreateImageInput) (output ec2.CreateImageOutput, err error) {
+// CreateImage handles the EC2 CreateImage API call
+func CreateImage(input *ec2.CreateImageInput, natsConn *nats.Conn) (ec2.CreateImageOutput, error) {
+	var output ec2.CreateImageOutput
 
-	// Validate input
-	err = ValidateCreateImageInput(input)
-
-	if err != nil {
+	if err := ValidateCreateImageInput(input); err != nil {
 		return output, err
 	}
 
-	// Marshal to JSON, to send over the wire for processing (NATS)
-	jsonData, err := json.Marshal(input)
+	svc := handlers_ec2_image.NewNATSImageService(natsConn)
+	result, err := svc.CreateImage(input)
 	if err != nil {
-		return output, fmt.Errorf("failed to marshal input to JSON: %w", err)
-	}
-
-	// Run the simulated JSON request via handler, which will return a JSON response
-	handler := handlers_ec2_image.NewCreateImageHandler(handlers_ec2_image.NewMockImageService())
-	jsonResp := handler.Process(jsonData)
-
-	// Validate if the response is successful or an error
-	responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-	if err != nil {
-		if responseError.Code != nil {
-			slog.Error("CreateImage failed", "err", responseError.Code)
-			return output, errors.New(*responseError.Code)
-		}
 		return output, err
 	}
-
-	// Unmarshal the JSON response back into output struct
-	err = json.Unmarshal(jsonResp, &output)
-	if err != nil {
-		return output, fmt.Errorf("failed to unmarshal JSON response to CreateImageOutput: %w", err)
+	if result == nil {
+		return output, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	return output, nil
+	return *result, nil
 }

hive/gateway/ec2/image/CreateImage_test.go

@@ -1,141 +1,97 @@
 package gateway_ec2_image
 
 import (
-	"encoding/json"
-	"errors"
-	"strings"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
+	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestValidateCreateImageInput(t *testing.T) {
 	tests := []struct {
-		name  string
-		input *ec2.CreateImageInput
-		want  error
+		name    string
+		input   *ec2.CreateImageInput
+		wantErr bool
+		errMsg  string
 	}{
 		{
-			name:  "NilInput",
-			input: nil,
-			want:  errors.New("MissingParameter"),
+			name:    "NilInput",
+			input:   nil,
+			wantErr: true,
+			errMsg:  awserrors.ErrorInvalidParameterValue,
 		},
 		{
-			name: "MissingInstanceId",
-			input: &ec2.CreateImageInput{
-				Name: aws.String("test-image"),
-			},
-			want: errors.New("MissingParameter"),
+			name:    "EmptyInput",
+			input:   &ec2.CreateImageInput{},
+			wantErr: true,
+			errMsg:  awserrors.ErrorMissingParameter,
 		},
 		{
 			name: "MissingName",
 			input: &ec2.CreateImageInput{
-				InstanceId: aws.String("i-0123456789abcdef0"),
+				InstanceId: aws.String("i-1234567890abcdef0"),
 			},
-			want: errors.New("MissingParameter"),
+			wantErr: true,
+			errMsg:  awserrors.ErrorMissingParameter,
 		},
 		{
-			name: "InvalidInstanceId",
+			name: "InvalidInstanceIdFormat",
 			input: &ec2.CreateImageInput{
 				InstanceId: aws.String("invalid-id"),
-				Name:       aws.String("test-image"),
+				Name:       aws.String("my-image"),
 			},
-			want: errors.New("InvalidInstanceID.Malformed"),
+			wantErr: true,
+			errMsg:  awserrors.ErrorInvalidInstanceIDMalformed,
+		},
+		{
+			name: "EmptyInstanceId",
+			input: &ec2.CreateImageInput{
+				InstanceId: aws.String(""),
+				Name:       aws.String("my-image"),
+			},
+			wantErr: true,
+			errMsg:  awserrors.ErrorMissingParameter,
+		},
+		{
+			name: "EmptyName",
+			input: &ec2.CreateImageInput{
+				InstanceId: aws.String("i-1234567890abcdef0"),
+				Name:       aws.String(""),
+			},
+			wantErr: true,
+			errMsg:  awserrors.ErrorMissingParameter,
 		},
 		{
 			name: "ValidInput",
 			input: &ec2.CreateImageInput{
-				InstanceId: aws.String("i-0123456789abcdef0"),
-				Name:       aws.String("test-image"),
+				InstanceId: aws.String("i-1234567890abcdef0"),
+				Name:       aws.String("my-image"),
 			},
-			want: nil,
+			wantErr: false,
+		},
+		{
+			name: "ValidInputWithDescription",
+			input: &ec2.CreateImageInput{
+				InstanceId:  aws.String("i-1234567890abcdef0"),
+				Name:        aws.String("my-image"),
+				Description: aws.String("A snapshot image"),
+			},
+			wantErr: false,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result, err := CreateImage(tt.input)
-			assert.Equal(t, tt.want, err)
-			if err != nil {
-				assert.Empty(t, result.ImageId)
+			err := ValidateCreateImageInput(tt.input)
+
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, tt.errMsg, err.Error())
 			} else {
-				assert.NotEmpty(t, result.ImageId)
-				assert.True(t, strings.HasPrefix(*result.ImageId, "ami-"))
-			}
-		})
-	}
-}
-
-func TestEC2ProcessCreateImage(t *testing.T) {
-	tests := []struct {
-		name              string
-		payload           interface{}
-		rawJSON           []byte
-		wantValidationErr bool
-		wantErrCode       string
-		assertFn          func(t *testing.T, output ec2.CreateImageOutput)
-	}{
-		{
-			name:              "InvalidPayload",
-			payload:           &ec2.DescribeImagesInput{},
-			wantValidationErr: true,
-			wantErrCode:       "ValidationError",
-		},
-		{
-			name: "MissingRequiredFields",
-			payload: &ec2.CreateImageInput{
-				Name: aws.String("test-image"),
-			},
-			wantValidationErr: true,
-			wantErrCode:       "MissingParameter",
-		},
-		{
-			name: "ValidCreateImage",
-			payload: &ec2.CreateImageInput{
-				InstanceId: aws.String("i-0123456789abcdef0"),
-				Name:       aws.String("test-image"),
-			},
-			assertFn: func(t *testing.T, output ec2.CreateImageOutput) {
-				assert.NotNil(t, output.ImageId)
-				assert.True(t, strings.HasPrefix(*output.ImageId, "ami-"))
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var jsonData []byte
-			var err error
-
-			if tt.rawJSON != nil {
-				jsonData = tt.rawJSON
-			} else {
-				jsonData, err = json.Marshal(tt.payload)
 				assert.NoError(t, err)
 			}
-
-			handler := handlers_ec2_image.NewCreateImageHandler(handlers_ec2_image.NewMockImageService())
-			jsonResp := handler.Process(jsonData)
-			responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-			if tt.wantValidationErr {
-				assert.Error(t, err)
-				assert.Equal(t, tt.wantErrCode, aws.StringValue(responseError.Code))
-				return
-			}
-
-			assert.NoError(t, err)
-
-			var output ec2.CreateImageOutput
-			assert.NoError(t, json.Unmarshal(jsonResp, &output))
-
-			if tt.assertFn != nil {
-				tt.assertFn(t, output)
-			}
 		})
 	}
 }

hive/gateway/ec2/image/DeregisterImage.go

@@ -1,98 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
-)
-
-/*
-Sample JSON:
-
-    "DeregisterImage":{
-      "name":"DeregisterImage",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"DeregisterImageRequest"}
-    },
-
-    "DeregisterImageRequest":{
-      "type":"structure",
-      "required":["ImageId"],
-      "members":{
-        "ImageId":{"shape":"ImageId"},
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        }
-      }
-    }
-*/
-
-func ValidateDeregisterImageInput(input *ec2.DeregisterImageInput) (err error) {
-
-	// Check required args from JSON above
-	// required:["ImageId"]
-
-	if input == nil {
-		return errors.New("MissingParameter")
-	}
-
-	if input.ImageId == nil || *input.ImageId == "" {
-		return errors.New("MissingParameter")
-	}
-
-	// Validate ImageId format
-	if !strings.HasPrefix(*input.ImageId, "ami-") {
-		return errors.New("InvalidAMIID.Malformed")
-	}
-
-	return
-}
-
-func DeregisterImage(input *ec2.DeregisterImageInput) (output ec2.DeregisterImageOutput, err error) {
-
-	// Validate input
-	err = ValidateDeregisterImageInput(input)
-
-	if err != nil {
-		return output, err
-	}
-
-	// Marshal to JSON, to send over the wire for processing (NATS)
-	jsonData, err := json.Marshal(input)
-	if err != nil {
-		return output, fmt.Errorf("failed to marshal input to JSON: %w", err)
-	}
-
-	// Run the simulated JSON request via handler, which will return a JSON response
-	handler := handlers_ec2_image.NewDeregisterImageHandler(handlers_ec2_image.NewMockImageService())
-	jsonResp := handler.Process(jsonData)
-
-	// Validate if the response is successful or an error
-	responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-	if err != nil {
-		if responseError.Code != nil {
-			slog.Error("DeregisterImage failed", "err", responseError.Code)
-			return output, errors.New(*responseError.Code)
-		}
-		return output, err
-	}
-
-	// Unmarshal the JSON response back into output struct
-	err = json.Unmarshal(jsonResp, &output)
-	if err != nil {
-		return output, fmt.Errorf("failed to unmarshal JSON response to DeregisterImageOutput: %w", err)
-	}
-
-	return output, nil
-}

hive/gateway/ec2/image/DeregisterImage_test.go

@@ -1,52 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestValidateDeregisterImageInput(t *testing.T) {
-	tests := []struct {
-		name  string
-		input *ec2.DeregisterImageInput
-		want  error
-	}{
-		{
-			name:  "NilInput",
-			input: nil,
-			want:  errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingImageId",
-			input: &ec2.DeregisterImageInput{
-				ImageId: aws.String(""),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "InvalidImageId",
-			input: &ec2.DeregisterImageInput{
-				ImageId: aws.String("invalid-id"),
-			},
-			want: errors.New("InvalidAMIID.Malformed"),
-		},
-		{
-			name: "ValidInput",
-			input: &ec2.DeregisterImageInput{
-				ImageId: aws.String("ami-0123456789abcdef0"),
-			},
-			want: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			_, err := DeregisterImage(tt.input)
-			assert.Equal(t, tt.want, err)
-		})
-	}
-}

hive/gateway/ec2/image/DescribeImageAttribute.go

@@ -1,142 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
-)
-
-/*
-Sample JSON:
-
-    "DescribeImageAttribute":{
-      "name":"DescribeImageAttribute",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"DescribeImageAttributeRequest"},
-      "output":{"shape":"ImageAttribute"}
-    },
-
-    "DescribeImageAttributeRequest":{
-      "type":"structure",
-      "required":[
-        "Attribute",
-        "ImageId"
-      ],
-      "members":{
-        "Attribute":{"shape":"ImageAttributeName"},
-        "ImageId":{"shape":"ImageId"},
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        }
-      }
-    },
-
-    "ImageAttribute":{
-      "type":"structure",
-      "members":{
-        "BlockDeviceMappings":{
-          "shape":"BlockDeviceMappingList",
-          "locationName":"blockDeviceMapping"
-        },
-        "ImageId":{
-          "shape":"ImageId",
-          "locationName":"imageId"
-        },
-        "LaunchPermissions":{
-          "shape":"LaunchPermissionList",
-          "locationName":"launchPermission"
-        },
-        "ProductCodes":{
-          "shape":"ProductCodeList",
-          "locationName":"productCodes"
-        },
-        "Description":{"shape":"AttributeValue"},
-        "KernelId":{"shape":"AttributeValue"},
-        "RamdiskId":{"shape":"AttributeValue"},
-        "SriovNetSupport":{"shape":"AttributeValue"},
-        "BootMode":{"shape":"AttributeValue"},
-        "TpmSupport":{"shape":"AttributeValue"},
-        "UefiData":{"shape":"AttributeValue"},
-        "LastLaunchedTime":{"shape":"AttributeValue"},
-        "ImdsSupport":{"shape":"AttributeValue"},
-        "DeregistrationProtection":{"shape":"AttributeValue"}
-      }
-    }
-*/
-
-func ValidateDescribeImageAttributeInput(input *ec2.DescribeImageAttributeInput) (err error) {
-
-	// Check required args from JSON above
-	// required:[
-	//   "Attribute",
-	//   "ImageId"
-	// ]
-
-	if input == nil {
-		return errors.New("MissingParameter")
-	}
-
-	if input.Attribute == nil || *input.Attribute == "" {
-		return errors.New("MissingParameter")
-	}
-
-	if input.ImageId == nil || *input.ImageId == "" {
-		return errors.New("MissingParameter")
-	}
-
-	// Validate ImageId format
-	if !strings.HasPrefix(*input.ImageId, "ami-") {
-		return errors.New("InvalidAMIID.Malformed")
-	}
-
-	return
-}
-
-func DescribeImageAttribute(input *ec2.DescribeImageAttributeInput) (output ec2.DescribeImageAttributeOutput, err error) {
-
-	// Validate input
-	err = ValidateDescribeImageAttributeInput(input)
-
-	if err != nil {
-		return output, err
-	}
-
-	// Marshal to JSON, to send over the wire for processing (NATS)
-	jsonData, err := json.Marshal(input)
-	if err != nil {
-		return output, fmt.Errorf("failed to marshal input to JSON: %w", err)
-	}
-
-	// Run the simulated JSON request via handler, which will return a JSON response
-	handler := handlers_ec2_image.NewDescribeImageAttributeHandler(handlers_ec2_image.NewMockImageService())
-	jsonResp := handler.Process(jsonData)
-
-	// Validate if the response is successful or an error
-	responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-	if err != nil {
-		if responseError.Code != nil {
-			slog.Error("DescribeImageAttribute failed", "err", responseError.Code)
-			return output, errors.New(*responseError.Code)
-		}
-		return output, err
-	}
-
-	// Unmarshal the JSON response back into output struct
-	err = json.Unmarshal(jsonResp, &output)
-	if err != nil {
-		return output, fmt.Errorf("failed to unmarshal JSON response to DescribeImageAttributeOutput: %w", err)
-	}
-
-	return output, nil
-}

hive/gateway/ec2/image/DescribeImageAttribute_test.go

@@ -1,64 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestValidateDescribeImageAttributeInput(t *testing.T) {
-	tests := []struct {
-		name  string
-		input *ec2.DescribeImageAttributeInput
-		want  error
-	}{
-		{
-			name:  "NilInput",
-			input: nil,
-			want:  errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingAttribute",
-			input: &ec2.DescribeImageAttributeInput{
-				ImageId: aws.String("ami-0123456789abcdef0"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingImageId",
-			input: &ec2.DescribeImageAttributeInput{
-				Attribute: aws.String("description"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "InvalidImageId",
-			input: &ec2.DescribeImageAttributeInput{
-				ImageId:   aws.String("invalid-id"),
-				Attribute: aws.String("description"),
-			},
-			want: errors.New("InvalidAMIID.Malformed"),
-		},
-		{
-			name: "ValidInput",
-			input: &ec2.DescribeImageAttributeInput{
-				ImageId:   aws.String("ami-0123456789abcdef0"),
-				Attribute: aws.String("description"),
-			},
-			want: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result, err := DescribeImageAttribute(tt.input)
-			assert.Equal(t, tt.want, err)
-			if err == nil {
-				assert.NotEmpty(t, result.ImageId)
-			}
-		})
-	}
-}

hive/gateway/ec2/image/DescribeImages.go

@@ -5,73 +5,12 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
+	"github.com/mulgadc/hive/hive/awserrors"
+	handlers_ec2_image "github.com/mulgadc/hive/hive/handlers/ec2/image"
 	"github.com/nats-io/nats.go"
 )
 
-/*
-Sample JSON:
-
-    "DescribeImages":{
-      "name":"DescribeImages",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"DescribeImagesRequest"},
-      "output":{"shape":"DescribeImagesResult"}
-    },
-
-    "DescribeImagesRequest":{
-      "type":"structure",
-      "members":{
-        "ExecutableUsers":{
-          "shape":"ExecutableByStringList",
-          "locationName":"ExecutableBy"
-        },
-        "Filters":{
-          "shape":"FilterList",
-          "locationName":"Filter"
-        },
-        "ImageIds":{
-          "shape":"ImageIdStringList",
-          "locationName":"ImageId"
-        },
-        "Owners":{
-          "shape":"OwnerStringList",
-          "locationName":"Owner"
-        },
-        "IncludeDeprecated":{"shape":"Boolean"},
-        "IncludeDisabled":{"shape":"Boolean"},
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        },
-        "MaxResults":{"shape":"Integer"},
-        "NextToken":{"shape":"String"}
-      }
-    },
-
-    "DescribeImagesResult":{
-      "type":"structure",
-      "members":{
-        "Images":{
-          "shape":"ImageList",
-          "locationName":"imagesSet"
-        },
-        "NextToken":{
-          "shape":"String",
-          "locationName":"nextToken"
-        }
-      }
-    }
-*/
-
 func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
-
-	// No required fields for DescribeImages
-	// All parameters are optional
-
 	if input == nil {
 		return nil
 	}
@@ -80,7 +19,7 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 	if input.ImageIds != nil {
 		for _, imageId := range input.ImageIds {
 			if imageId != nil && !strings.HasPrefix(*imageId, "ami-") {
-				return errors.New("InvalidAMIID.Malformed")
+				return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
 			}
 		}
 	}

hive/gateway/ec2/image/DescribeImages_test.go

@@ -1,54 +1,95 @@
 package gateway_ec2_image
 
 import (
-	"errors"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestValidateDescribeImagesInput(t *testing.T) {
 	tests := []struct {
-		name  string
-		input *ec2.DescribeImagesInput
-		want  error
+		name    string
+		input   *ec2.DescribeImagesInput
+		wantErr bool
+		errMsg  string
 	}{
 		{
-			name:  "NilInput",
-			input: nil,
-			want:  nil,
+			name:    "NilInput",
+			input:   nil,
+			wantErr: false,
 		},
 		{
-			name:  "EmptyInput",
-			input: &ec2.DescribeImagesInput{},
-			want:  nil,
+			name:    "EmptyInput",
+			input:   &ec2.DescribeImagesInput{},
+			wantErr: false,
 		},
 		{
-			name: "InvalidImageId",
-			input: &ec2.DescribeImagesInput{
-				ImageIds: []*string{aws.String("invalid-id")},
-			},
-			want: errors.New("InvalidAMIID.Malformed"),
-		},
-		{
-			name: "ValidInput",
+			name: "ValidImageId",
 			input: &ec2.DescribeImagesInput{
 				ImageIds: []*string{aws.String("ami-0123456789abcdef0")},
 			},
-			want: nil,
+			wantErr: false,
+		},
+		{
+			name: "InvalidImageId_NoPrefix",
+			input: &ec2.DescribeImagesInput{
+				ImageIds: []*string{aws.String("invalid-id")},
+			},
+			wantErr: true,
+			errMsg:  awserrors.ErrorInvalidAMIIDMalformed,
+		},
+		{
+			name: "InvalidImageId_WrongPrefix",
+			input: &ec2.DescribeImagesInput{
+				ImageIds: []*string{aws.String("vol-0123456789abcdef0")},
+			},
+			wantErr: true,
+			errMsg:  awserrors.ErrorInvalidAMIIDMalformed,
+		},
+		{
+			name: "MultipleValidImageIds",
+			input: &ec2.DescribeImagesInput{
+				ImageIds: []*string{
+					aws.String("ami-111"),
+					aws.String("ami-222"),
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "MixedValidAndInvalidImageIds",
+			input: &ec2.DescribeImagesInput{
+				ImageIds: []*string{
+					aws.String("ami-valid"),
+					aws.String("invalid-id"),
+				},
+			},
+			wantErr: true,
+			errMsg:  awserrors.ErrorInvalidAMIIDMalformed,
+		},
+		{
+			name: "EmptyImageId",
+			input: &ec2.DescribeImagesInput{
+				ImageIds: []*string{aws.String("")},
+			},
+			wantErr: true,
+			errMsg:  awserrors.ErrorInvalidAMIIDMalformed,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			// Skip full DescribeImages test - it now requires NATS connection
-			// Only test validation logic
-			t.Skip("Skipping - DescribeImages now requires NATS connection. Use ValidateDescribeImagesInput directly for validation tests.")
-
 			err := ValidateDescribeImagesInput(tt.input)
-			assert.Equal(t, tt.want, err)
+
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Equal(t, tt.errMsg, err.Error())
+			} else {
+				assert.NoError(t, err)
+			}
 		})
 	}
 }

hive/gateway/ec2/image/ImportImage.go

@@ -1,27 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"fmt"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-)
-
-func ImportImage() {
-
-	var test = ec2.ImportImageInput{}
-
-	fmt.Println(test)
-
-	var t2 = ec2.ImportImageOutput{}
-
-	fmt.Println(t2)
-
-	var t3 = ec2.DescribeImagesInput{}
-
-	fmt.Println(t3)
-
-	var t4 = ec2.DescribeImagesOutput{}
-
-	fmt.Println(t4)
-
-}

hive/gateway/ec2/image/ModifyImageAttribute.go

@@ -1,124 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
-)
-
-/*
-Sample JSON:
-
-    "ModifyImageAttribute":{
-      "name":"ModifyImageAttribute",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"ModifyImageAttributeRequest"}
-    },
-
-    "ModifyImageAttributeRequest":{
-      "type":"structure",
-      "required":["ImageId"],
-      "members":{
-        "Attribute":{"shape":"String"},
-        "Description":{"shape":"AttributeValue"},
-        "ImageId":{"shape":"ImageId"},
-        "LaunchPermission":{"shape":"LaunchPermissionModifications"},
-        "OperationType":{"shape":"OperationType"},
-        "ProductCodes":{
-          "shape":"ProductCodeStringList",
-          "locationName":"ProductCode"
-        },
-        "UserGroups":{
-          "shape":"UserGroupStringList",
-          "locationName":"UserGroup"
-        },
-        "UserIds":{
-          "shape":"UserIdStringList",
-          "locationName":"UserId"
-        },
-        "Value":{"shape":"String"},
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        },
-        "OrganizationArns":{
-          "shape":"OrganizationArnStringList",
-          "locationName":"OrganizationArn"
-        },
-        "OrganizationalUnitArns":{
-          "shape":"OrganizationalUnitArnStringList",
-          "locationName":"OrganizationalUnitArn"
-        },
-        "ImdsSupport":{"shape":"AttributeValue"}
-      }
-    }
-*/
-
-func ValidateModifyImageAttributeInput(input *ec2.ModifyImageAttributeInput) (err error) {
-
-	// Check required args from JSON above
-	// required:["ImageId"]
-
-	if input == nil {
-		return errors.New("MissingParameter")
-	}
-
-	if input.ImageId == nil || *input.ImageId == "" {
-		return errors.New("MissingParameter")
-	}
-
-	// Validate ImageId format
-	if !strings.HasPrefix(*input.ImageId, "ami-") {
-		return errors.New("InvalidAMIID.Malformed")
-	}
-
-	return
-}
-
-func ModifyImageAttribute(input *ec2.ModifyImageAttributeInput) (output ec2.ModifyImageAttributeOutput, err error) {
-
-	// Validate input
-	err = ValidateModifyImageAttributeInput(input)
-
-	if err != nil {
-		return output, err
-	}
-
-	// Marshal to JSON, to send over the wire for processing (NATS)
-	jsonData, err := json.Marshal(input)
-	if err != nil {
-		return output, fmt.Errorf("failed to marshal input to JSON: %w", err)
-	}
-
-	// Run the simulated JSON request via handler, which will return a JSON response
-	handler := handlers_ec2_image.NewModifyImageAttributeHandler(handlers_ec2_image.NewMockImageService())
-	jsonResp := handler.Process(jsonData)
-
-	// Validate if the response is successful or an error
-	responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-	if err != nil {
-		if responseError.Code != nil {
-			slog.Error("ModifyImageAttribute failed", "err", responseError.Code)
-			return output, errors.New(*responseError.Code)
-		}
-		return output, err
-	}
-
-	// Unmarshal the JSON response back into output struct
-	err = json.Unmarshal(jsonResp, &output)
-	if err != nil {
-		return output, fmt.Errorf("failed to unmarshal JSON response to ModifyImageAttributeOutput: %w", err)
-	}
-
-	return output, nil
-}

hive/gateway/ec2/image/ModifyImageAttribute_test.go

@@ -1,53 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestValidateModifyImageAttributeInput(t *testing.T) {
-	tests := []struct {
-		name  string
-		input *ec2.ModifyImageAttributeInput
-		want  error
-	}{
-		{
-			name:  "NilInput",
-			input: nil,
-			want:  errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingImageId",
-			input: &ec2.ModifyImageAttributeInput{
-				Description: &ec2.AttributeValue{Value: aws.String("test")},
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "InvalidImageId",
-			input: &ec2.ModifyImageAttributeInput{
-				ImageId: aws.String("invalid-id"),
-			},
-			want: errors.New("InvalidAMIID.Malformed"),
-		},
-		{
-			name: "ValidInput",
-			input: &ec2.ModifyImageAttributeInput{
-				ImageId:     aws.String("ami-0123456789abcdef0"),
-				Description: &ec2.AttributeValue{Value: aws.String("test")},
-			},
-			want: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			_, err := ModifyImageAttribute(tt.input)
-			assert.Equal(t, tt.want, err)
-		})
-	}
-}

hive/gateway/ec2/image/RegisterImage.go

@@ -1,162 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
-)
-
-/*
-Sample JSON:
-
-    "RegisterImage":{
-      "name":"RegisterImage",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"RegisterImageRequest"},
-      "output":{"shape":"RegisterImageResult"}
-    },
-
-    "RegisterImageRequest":{
-      "type":"structure",
-      "required":["Name"],
-      "members":{
-        "ImageLocation":{"shape":"String"},
-        "Architecture":{
-          "shape":"ArchitectureValues",
-          "locationName":"architecture"
-        },
-        "BlockDeviceMappings":{
-          "shape":"BlockDeviceMappingRequestList",
-          "locationName":"BlockDeviceMapping"
-        },
-        "Description":{
-          "shape":"String",
-          "locationName":"description"
-        },
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        },
-        "EnaSupport":{
-          "shape":"Boolean",
-          "locationName":"enaSupport"
-        },
-        "KernelId":{
-          "shape":"KernelId",
-          "locationName":"kernelId"
-        },
-        "Name":{
-          "shape":"String",
-          "locationName":"name"
-        },
-        "BillingProducts":{
-          "shape":"BillingProductList",
-          "locationName":"BillingProduct"
-        },
-        "RamdiskId":{
-          "shape":"RamdiskId",
-          "locationName":"ramdiskId"
-        },
-        "RootDeviceName":{
-          "shape":"String",
-          "locationName":"rootDeviceName"
-        },
-        "SriovNetSupport":{
-          "shape":"String",
-          "locationName":"sriovNetSupport"
-        },
-        "VirtualizationType":{
-          "shape":"String",
-          "locationName":"virtualizationType"
-        },
-        "BootMode":{
-          "shape":"BootModeValues",
-          "locationName":"bootMode"
-        },
-        "TpmSupport":{
-          "shape":"TpmSupportValues"
-        },
-        "UefiData":{"shape":"StringType"},
-        "ImdsSupport":{
-          "shape":"ImdsSupportValues"
-        },
-        "TagSpecifications":{
-          "shape":"TagSpecificationList",
-          "locationName":"TagSpecification"
-        }
-      }
-    },
-
-    "RegisterImageResult":{
-      "type":"structure",
-      "members":{
-        "ImageId":{
-          "shape":"ImageId",
-          "locationName":"imageId"
-        }
-      }
-    }
-*/
-
-func ValidateRegisterImageInput(input *ec2.RegisterImageInput) (err error) {
-
-	// Check required args from JSON above
-	// required:["Name"]
-
-	if input == nil {
-		return errors.New("MissingParameter")
-	}
-
-	if input.Name == nil || *input.Name == "" {
-		return errors.New("MissingParameter")
-	}
-
-	return
-}
-
-func RegisterImage(input *ec2.RegisterImageInput) (output ec2.RegisterImageOutput, err error) {
-
-	// Validate input
-	err = ValidateRegisterImageInput(input)
-
-	if err != nil {
-		return output, err
-	}
-
-	// Marshal to JSON, to send over the wire for processing (NATS)
-	jsonData, err := json.Marshal(input)
-	if err != nil {
-		return output, fmt.Errorf("failed to marshal input to JSON: %w", err)
-	}
-
-	// Run the simulated JSON request via handler, which will return a JSON response
-	handler := handlers_ec2_image.NewRegisterImageHandler(handlers_ec2_image.NewMockImageService())
-	jsonResp := handler.Process(jsonData)
-
-	// Validate if the response is successful or an error
-	responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-	if err != nil {
-		if responseError.Code != nil {
-			slog.Error("RegisterImage failed", "err", responseError.Code)
-			return output, errors.New(*responseError.Code)
-		}
-		return output, err
-	}
-
-	// Unmarshal the JSON response back into output struct
-	err = json.Unmarshal(jsonResp, &output)
-	if err != nil {
-		return output, fmt.Errorf("failed to unmarshal JSON response to RegisterImageOutput: %w", err)
-	}
-
-	return output, nil
-}

hive/gateway/ec2/image/RegisterImage_test.go

@@ -1,51 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"errors"
-	"strings"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestValidateRegisterImageInput(t *testing.T) {
-	tests := []struct {
-		name  string
-		input *ec2.RegisterImageInput
-		want  error
-	}{
-		{
-			name:  "NilInput",
-			input: nil,
-			want:  errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingName",
-			input: &ec2.RegisterImageInput{
-				ImageLocation: aws.String("s3://bucket/image"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "ValidInput",
-			input: &ec2.RegisterImageInput{
-				Name:          aws.String("test-image"),
-				ImageLocation: aws.String("s3://bucket/image"),
-			},
-			want: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result, err := RegisterImage(tt.input)
-			assert.Equal(t, tt.want, err)
-			if err == nil {
-				assert.NotEmpty(t, result.ImageId)
-				assert.True(t, strings.HasPrefix(*result.ImageId, "ami-"))
-			}
-		})
-	}
-}

hive/gateway/ec2/image/ResetImageAttribute.go

@@ -1,109 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/image"
-	"github.com/mulgadc/hive/hive/utils"
-)
-
-/*
-Sample JSON:
-
-    "ResetImageAttribute":{
-      "name":"ResetImageAttribute",
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"ResetImageAttributeRequest"}
-    },
-
-    "ResetImageAttributeRequest":{
-      "type":"structure",
-      "required":[
-        "Attribute",
-        "ImageId"
-      ],
-      "members":{
-        "Attribute":{"shape":"ResetImageAttributeName"},
-        "ImageId":{"shape":"ImageId"},
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        }
-      }
-    }
-*/
-
-func ValidateResetImageAttributeInput(input *ec2.ResetImageAttributeInput) (err error) {
-
-	// Check required args from JSON above
-	// required:[
-	//   "Attribute",
-	//   "ImageId"
-	// ]
-
-	if input == nil {
-		return errors.New("MissingParameter")
-	}
-
-	if input.Attribute == nil || *input.Attribute == "" {
-		return errors.New("MissingParameter")
-	}
-
-	if input.ImageId == nil || *input.ImageId == "" {
-		return errors.New("MissingParameter")
-	}
-
-	// Validate ImageId format
-	if !strings.HasPrefix(*input.ImageId, "ami-") {
-		return errors.New("InvalidAMIID.Malformed")
-	}
-
-	return
-}
-
-func ResetImageAttribute(input *ec2.ResetImageAttributeInput) (output ec2.ResetImageAttributeOutput, err error) {
-
-	// Validate input
-	err = ValidateResetImageAttributeInput(input)
-
-	if err != nil {
-		return output, err
-	}
-
-	// Marshal to JSON, to send over the wire for processing (NATS)
-	jsonData, err := json.Marshal(input)
-	if err != nil {
-		return output, fmt.Errorf("failed to marshal input to JSON: %w", err)
-	}
-
-	// Run the simulated JSON request via handler, which will return a JSON response
-	handler := handlers_ec2_image.NewResetImageAttributeHandler(handlers_ec2_image.NewMockImageService())
-	jsonResp := handler.Process(jsonData)
-
-	// Validate if the response is successful or an error
-	responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-	if err != nil {
-		if responseError.Code != nil {
-			slog.Error("ResetImageAttribute failed", "err", responseError.Code)
-			return output, errors.New(*responseError.Code)
-		}
-		return output, err
-	}
-
-	// Unmarshal the JSON response back into output struct
-	err = json.Unmarshal(jsonResp, &output)
-	if err != nil {
-		return output, fmt.Errorf("failed to unmarshal JSON response to ResetImageAttributeOutput: %w", err)
-	}
-
-	return output, nil
-}

hive/gateway/ec2/image/ResetImageAttribute_test.go

@@ -1,61 +0,0 @@
-package gateway_ec2_image
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestValidateResetImageAttributeInput(t *testing.T) {
-	tests := []struct {
-		name  string
-		input *ec2.ResetImageAttributeInput
-		want  error
-	}{
-		{
-			name:  "NilInput",
-			input: nil,
-			want:  errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingAttribute",
-			input: &ec2.ResetImageAttributeInput{
-				ImageId: aws.String("ami-0123456789abcdef0"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "MissingImageId",
-			input: &ec2.ResetImageAttributeInput{
-				Attribute: aws.String("launchPermission"),
-			},
-			want: errors.New("MissingParameter"),
-		},
-		{
-			name: "InvalidImageId",
-			input: &ec2.ResetImageAttributeInput{
-				ImageId:   aws.String("invalid-id"),
-				Attribute: aws.String("launchPermission"),
-			},
-			want: errors.New("InvalidAMIID.Malformed"),
-		},
-		{
-			name: "ValidInput",
-			input: &ec2.ResetImageAttributeInput{
-				ImageId:   aws.String("ami-0123456789abcdef0"),
-				Attribute: aws.String("launchPermission"),
-			},
-			want: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			_, err := ResetImageAttribute(tt.input)
-			assert.Equal(t, tt.want, err)
-		})
-	}
-}

hive/gateway/ec2/instance/DescribeInstanceTypes.go

@@ -0,0 +1,138 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats.go"
+)
+
+// DescribeInstanceTypes queries all hive nodes for their instance types via NATS
+func DescribeInstanceTypes(input *ec2.DescribeInstanceTypesInput, natsConn *nats.Conn, expectedNodes int) (*ec2.DescribeInstanceTypesOutput, error) {
+	// Marshal input to JSON
+	jsonData, err := json.Marshal(input)
+	if err != nil {
+		slog.Error("DescribeInstanceTypes: Failed to marshal input", "err", err)
+		return nil, fmt.Errorf("failed to marshal input: %w", err)
+	}
+
+	// Create an inbox for collecting responses from all nodes
+	inbox := nats.NewInbox()
+	sub, err := natsConn.SubscribeSync(inbox)
+	if err != nil {
+		slog.Error("DescribeInstanceTypes: Failed to create inbox subscription", "err", err)
+		return nil, fmt.Errorf("failed to create inbox: %w", err)
+	}
+	defer sub.Unsubscribe()
+
+	// Publish request to all nodes (no queue group, so all daemons receive it)
+	err = natsConn.PublishRequest("ec2.DescribeInstanceTypes", inbox, jsonData)
+	if err != nil {
+		slog.Error("DescribeInstanceTypes: Failed to publish request", "err", err)
+		return nil, fmt.Errorf("failed to publish request: %w", err)
+	}
+
+	// Collect responses from all nodes
+	// Timeout serves as a safety mechanism in case some nodes don't respond
+	timeout := 3 * time.Second
+	deadline := time.Now().Add(timeout)
+
+	var allInstanceTypes []*ec2.InstanceTypeInfo
+	responsesReceived := 0
+
+	if expectedNodes <= 0 {
+		expectedNodes = -1 // Disable early exit
+		slog.Warn("DescribeInstanceTypes: ExpectedNodes not configured, using timeout-only collection")
+	}
+
+	for time.Now().Before(deadline) {
+		// Check if we've received responses from all expected nodes
+		if expectedNodes > 0 && responsesReceived >= expectedNodes {
+			slog.Info("DescribeInstanceTypes: Received responses from all expected nodes", "expected", expectedNodes, "received", responsesReceived)
+			break
+		}
+
+		// Calculate remaining timeout
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+
+		// Wait for next message with remaining timeout
+		msg, err := sub.NextMsg(remaining)
+		if err != nil {
+			if err == nats.ErrTimeout {
+				// Timeout reached, no more messages
+				break
+			}
+			slog.Error("DescribeInstanceTypes: Error receiving message", "err", err)
+			break
+		}
+
+		// Increment response counter (even for errors, as we heard from the node)
+		responsesReceived++
+
+		// Check if response is an error
+		responseError, err := utils.ValidateErrorPayload(msg.Data)
+		if err != nil {
+			// Response is an error payload - log but continue collecting from other nodes
+			slog.Warn("DescribeInstanceTypes: Received error from node", "code", responseError.Code, "responses_received", responsesReceived)
+			continue
+		}
+
+		// Parse the DescribeInstanceTypesOutput from this node
+		var nodeOutput ec2.DescribeInstanceTypesOutput
+		err = json.Unmarshal(msg.Data, &nodeOutput)
+		if err != nil {
+			slog.Error("DescribeInstanceTypes: Failed to unmarshal node response", "err", err)
+			continue
+		}
+
+		// Aggregate instance types from this node
+		if nodeOutput.InstanceTypes != nil {
+			allInstanceTypes = append(allInstanceTypes, nodeOutput.InstanceTypes...)
+			slog.Info("DescribeInstanceTypes: Collected instance types from node", "count", len(nodeOutput.InstanceTypes), "responses_received", responsesReceived)
+		}
+	}
+
+	// By default, deduplicate instance types.
+	// If the "capacity" filter is set to "true", show all available slots (duplicates).
+	showCapacity := false
+	for _, f := range input.Filters {
+		if f.Name != nil && *f.Name == "capacity" {
+			for _, v := range f.Values {
+				if v != nil && *v == "true" {
+					showCapacity = true
+					break
+				}
+			}
+		}
+	}
+
+	var finalInstanceTypes []*ec2.InstanceTypeInfo
+	if showCapacity {
+		finalInstanceTypes = allInstanceTypes
+	} else {
+		seen := make(map[string]bool)
+		for _, it := range allInstanceTypes {
+			if it != nil && it.InstanceType != nil {
+				if !seen[*it.InstanceType] {
+					seen[*it.InstanceType] = true
+					finalInstanceTypes = append(finalInstanceTypes, it)
+				}
+			}
+		}
+	}
+
+	// Build final aggregated response
+	output := &ec2.DescribeInstanceTypesOutput{
+		InstanceTypes: finalInstanceTypes,
+	}
+
+	slog.Info("DescribeInstanceTypes: Aggregated response", "total_instance_types", len(finalInstanceTypes), "show_capacity", showCapacity)
+	return output, nil
+}

hive/gateway/ec2/instance/DescribeInstanceTypes_test.go

@@ -0,0 +1,239 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDescribeInstanceTypes_SingleNode(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
+			InstanceTypes: []*ec2.InstanceTypeInfo{
+				{InstanceType: aws.String("t3.micro")},
+				{InstanceType: aws.String("t3.small")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	input := &ec2.DescribeInstanceTypesInput{}
+	output, err := DescribeInstanceTypes(input, nc, 1)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Len(t, output.InstanceTypes, 2)
+	assert.Equal(t, "t3.micro", *output.InstanceTypes[0].InstanceType)
+	assert.Equal(t, "t3.small", *output.InstanceTypes[1].InstanceType)
+}
+
+func TestDescribeInstanceTypes_DeduplicatesAcrossNodes(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Node 1
+	nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
+			InstanceTypes: []*ec2.InstanceTypeInfo{
+				{InstanceType: aws.String("t3.micro")},
+				{InstanceType: aws.String("t3.small")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	// Node 2 reports same types
+	nc2, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	defer nc2.Close()
+
+	nc2.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
+			InstanceTypes: []*ec2.InstanceTypeInfo{
+				{InstanceType: aws.String("t3.micro")},
+				{InstanceType: aws.String("m5.large")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	nc.Flush()
+	nc2.Flush()
+
+	input := &ec2.DescribeInstanceTypesInput{}
+	output, err := DescribeInstanceTypes(input, nc, 2)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	// t3.micro should appear once, t3.small once, m5.large once = 3 unique
+	assert.Len(t, output.InstanceTypes, 3)
+
+	seen := make(map[string]int)
+	for _, it := range output.InstanceTypes {
+		seen[*it.InstanceType]++
+	}
+	assert.Equal(t, 1, seen["t3.micro"])
+	assert.Equal(t, 1, seen["t3.small"])
+	assert.Equal(t, 1, seen["m5.large"])
+}
+
+func TestDescribeInstanceTypes_CapacityFilterShowsDuplicates(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Node 1
+	nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
+			InstanceTypes: []*ec2.InstanceTypeInfo{
+				{InstanceType: aws.String("t3.micro")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	// Node 2 reports same type
+	nc2, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	defer nc2.Close()
+
+	nc2.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
+			InstanceTypes: []*ec2.InstanceTypeInfo{
+				{InstanceType: aws.String("t3.micro")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	nc.Flush()
+	nc2.Flush()
+
+	// With capacity filter = true, duplicates should be preserved
+	input := &ec2.DescribeInstanceTypesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("capacity"),
+				Values: []*string{aws.String("true")},
+			},
+		},
+	}
+	output, err := DescribeInstanceTypes(input, nc, 2)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	// Both slots should appear since capacity filter is enabled
+	assert.Len(t, output.InstanceTypes, 2)
+}
+
+func TestDescribeInstanceTypes_CapacityFilterFalseDeduplicates(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
+			InstanceTypes: []*ec2.InstanceTypeInfo{
+				{InstanceType: aws.String("t3.micro")},
+				{InstanceType: aws.String("t3.micro")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	// capacity=false should still deduplicate
+	input := &ec2.DescribeInstanceTypesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("capacity"),
+				Values: []*string{aws.String("false")},
+			},
+		},
+	}
+	output, err := DescribeInstanceTypes(input, nc, 1)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Len(t, output.InstanceTypes, 1)
+}
+
+func TestDescribeInstanceTypes_NoSubscribers(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	input := &ec2.DescribeInstanceTypesInput{}
+	output, err := DescribeInstanceTypes(input, nc, 0)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Empty(t, output.InstanceTypes)
+}
+
+func TestDescribeInstanceTypes_NodeReturnsError(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		errorPayload := utils.GenerateErrorPayload("InternalError")
+		msg.Respond(errorPayload)
+	})
+
+	input := &ec2.DescribeInstanceTypesInput{}
+	output, err := DescribeInstanceTypes(input, nc, 1)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Empty(t, output.InstanceTypes)
+}
+
+func TestDescribeInstanceTypes_MalformedJSON(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		msg.Respond([]byte(`not valid json`))
+	})
+
+	input := &ec2.DescribeInstanceTypesInput{}
+	output, err := DescribeInstanceTypes(input, nc, 1)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Empty(t, output.InstanceTypes)
+}
+
+func TestDescribeInstanceTypes_NilInstanceTypeSkipped(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
+			InstanceTypes: []*ec2.InstanceTypeInfo{
+				{InstanceType: aws.String("t3.micro")},
+				{InstanceType: nil}, // nil InstanceType should be skipped in dedup
+				nil,                 // nil entry should be skipped
+			},
+		})
+		msg.Respond(data)
+	})
+
+	input := &ec2.DescribeInstanceTypesInput{}
+	output, err := DescribeInstanceTypes(input, nc, 1)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Len(t, output.InstanceTypes, 1)
+	assert.Equal(t, "t3.micro", *output.InstanceTypes[0].InstanceType)
+}
+
+func TestDescribeInstanceTypes_ClosedConnection(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	closedNC, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	closedNC.Close()
+
+	input := &ec2.DescribeInstanceTypesInput{}
+	_, err = DescribeInstanceTypes(input, closedNC, 1)
+
+	require.Error(t, err)
+}

hive/gateway/ec2/instance/DescribeInstances.go

@@ -101,6 +101,25 @@ func DescribeInstances(input *ec2.DescribeInstancesInput, natsConn *nats.Conn, e
 		}
 	}
 
+	// Query stopped instances from shared KV (via queue group — only one daemon handles it)
+	stoppedMsg, err := natsConn.Request("ec2.DescribeStoppedInstances", jsonData, 3*time.Second)
+	if err != nil {
+		slog.Warn("DescribeInstances: Failed to query stopped instances (may not be available)", "err", err)
+	} else {
+		responseError, parseErr := utils.ValidateErrorPayload(stoppedMsg.Data)
+		if parseErr != nil {
+			slog.Warn("DescribeInstances: Stopped instances query returned error", "code", responseError.Code)
+		} else {
+			var stoppedOutput ec2.DescribeInstancesOutput
+			if err := json.Unmarshal(stoppedMsg.Data, &stoppedOutput); err != nil {
+				slog.Error("DescribeInstances: Failed to unmarshal stopped instances response", "err", err)
+			} else if stoppedOutput.Reservations != nil {
+				allReservations = append(allReservations, stoppedOutput.Reservations...)
+				slog.Info("DescribeInstances: Collected stopped instance reservations", "count", len(stoppedOutput.Reservations))
+			}
+		}
+	}
+
 	// Build final aggregated response
 	output := &ec2.DescribeInstancesOutput{
 		Reservations: allReservations,

hive/gateway/ec2/instance/DescribeInstances_test.go

@@ -0,0 +1,256 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDescribeInstances_SingleNode(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	reservation := &ec2.Reservation{
+		ReservationId: aws.String("r-abc123"),
+		Instances: []*ec2.Instance{
+			{
+				InstanceId:   aws.String("i-001"),
+				InstanceType: aws.String("t3.micro"),
+				State:        &ec2.InstanceState{Code: aws.Int64(16), Name: aws.String("running")},
+			},
+		},
+	}
+
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{reservation},
+		})
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 1)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	require.Len(t, output.Reservations, 1)
+	assert.Equal(t, "r-abc123", *output.Reservations[0].ReservationId)
+	assert.Equal(t, "i-001", *output.Reservations[0].Instances[0].InstanceId)
+}
+
+func TestDescribeInstances_MultipleNodes(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Two nodes each return different instances
+	nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{
+				{
+					ReservationId: aws.String("r-node1"),
+					Instances: []*ec2.Instance{
+						{InstanceId: aws.String("i-node1-001")},
+					},
+				},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	nc2, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	defer nc2.Close()
+
+	nc2.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{
+				{
+					ReservationId: aws.String("r-node2"),
+					Instances: []*ec2.Instance{
+						{InstanceId: aws.String("i-node2-001")},
+						{InstanceId: aws.String("i-node2-002")},
+					},
+				},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	// Wait for subscriptions to propagate
+	nc.Flush()
+	nc2.Flush()
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 2)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Len(t, output.Reservations, 2)
+}
+
+func TestDescribeInstances_NoSubscribers(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 0)
+
+	// No subscribers means timeout — function returns empty reservations, no error
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Empty(t, output.Reservations)
+}
+
+func TestDescribeInstances_NodeReturnsError(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		errorPayload := utils.GenerateErrorPayload("InternalError")
+		msg.Respond(errorPayload)
+	})
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 1)
+
+	// Error responses from nodes are logged but don't fail the overall call
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Empty(t, output.Reservations)
+}
+
+func TestDescribeInstances_MixedResponses(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Node 1: returns valid data
+	nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{
+				{
+					ReservationId: aws.String("r-good"),
+					Instances:     []*ec2.Instance{{InstanceId: aws.String("i-good")}},
+				},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	// Node 2: returns error
+	nc2, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	defer nc2.Close()
+
+	nc2.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		errorPayload := utils.GenerateErrorPayload("InternalError")
+		msg.Respond(errorPayload)
+	})
+
+	nc.Flush()
+	nc2.Flush()
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 2)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	// Only the valid node's reservation should appear
+	assert.Len(t, output.Reservations, 1)
+	assert.Equal(t, "r-good", *output.Reservations[0].ReservationId)
+}
+
+func TestDescribeInstances_MalformedJSON(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		msg.Respond([]byte(`{invalid json`))
+	})
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 1)
+
+	// Malformed JSON from a node is skipped, not fatal
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Empty(t, output.Reservations)
+}
+
+func TestDescribeInstances_EmptyReservations(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstancesOutput{
+			Reservations: nil,
+		})
+		msg.Respond(data)
+	})
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 1)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Empty(t, output.Reservations)
+}
+
+func TestDescribeInstances_TimeoutCollection(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// Node responds after a delay (but within timeout)
+	nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.Sleep(500 * time.Millisecond)
+		data, _ := json.Marshal(&ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{
+				{ReservationId: aws.String("r-delayed")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	input := &ec2.DescribeInstancesInput{}
+	output, err := DescribeInstances(input, nc, 0) // 0 = timeout-based collection
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Len(t, output.Reservations, 1)
+}
+
+func TestDescribeInstances_EarlyExitWithExpectedNodes(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		data, _ := json.Marshal(&ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{
+				{ReservationId: aws.String("r-fast")},
+			},
+		})
+		msg.Respond(data)
+	})
+
+	input := &ec2.DescribeInstancesInput{}
+	start := time.Now()
+	output, err := DescribeInstances(input, nc, 1)
+	duration := time.Since(start)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Len(t, output.Reservations, 1)
+	// Should exit early, well before the 3-second timeout
+	assert.Less(t, duration, 2*time.Second)
+}
+
+func TestDescribeInstances_ClosedConnection(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	closedNC, err := nats.Connect(nc.ConnectedUrl())
+	require.NoError(t, err)
+	closedNC.Close()
+
+	input := &ec2.DescribeInstancesInput{}
+	_, err = DescribeInstances(input, closedNC, 1)
+
+	require.Error(t, err)
+}

hive/gateway/ec2/instance/GetConsoleOutput.go

@@ -0,0 +1,41 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/nats-io/nats.go"
+)
+
+// GetConsoleOutput retrieves console output for a specific instance via NATS.
+// Routes directly to the node running the instance via ec2.{instanceID}.GetConsoleOutput.
+func GetConsoleOutput(input *ec2.GetConsoleOutputInput, natsConn *nats.Conn) (*ec2.GetConsoleOutputOutput, error) {
+	if input.InstanceId == nil || *input.InstanceId == "" {
+		return nil, fmt.Errorf("InstanceId is required")
+	}
+
+	jsonData, err := json.Marshal(input)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal input: %w", err)
+	}
+
+	topic := fmt.Sprintf("ec2.%s.GetConsoleOutput", *input.InstanceId)
+	msg, err := natsConn.Request(topic, jsonData, 5*time.Second)
+	if err != nil {
+		if err == nats.ErrNoResponders || err == nats.ErrTimeout {
+			return nil, fmt.Errorf("instance %s not found or not running", *input.InstanceId)
+		}
+		return nil, fmt.Errorf("failed to get console output: %w", err)
+	}
+
+	var output ec2.GetConsoleOutputOutput
+	if err := json.Unmarshal(msg.Data, &output); err != nil {
+		slog.Error("GetConsoleOutput: Failed to unmarshal response", "err", err)
+		return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+	}
+
+	return &output, nil
+}

hive/gateway/ec2/instance/RunInstances.go

@@ -3,8 +3,12 @@ package gateway_ec2_instance
 import (
 	"errors"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	"github.com/mulgadc/hive/hive/utils"
+
 	handlers_ec2_instance "github.com/mulgadc/hive/hive/handlers/ec2/instance"
 	"github.com/nats-io/nats.go"
 )
@@ -13,159 +17,38 @@ type RunInstancesResponse struct {
 	Reservation *ec2.Reservation `locationName:"RunInstancesResponse"`
 }
 
-/*
-Sample JSON:
-
-    "RunInstances":{
-      "name":"RunInstances", // Function name
-      "http":{
-        "method":"POST",
-        "requestUri":"/"
-      },
-      "input":{"shape":"RunInstancesRequest"}, // Input shape from AWS API
-      "output":{"shape":"Reservation"} // Output shape from AWS API
-    },
-
-    "RunInstancesRequest":{
-      "type":"structure",
-      "required":[
-        "MaxCount",
-        "MinCount"
-      ],
-      "members":{
-        "BlockDeviceMappings":{
-          "shape":"BlockDeviceMappingRequestList",
-          "locationName":"BlockDeviceMapping"
-        },
-        "ImageId":{"shape":"ImageId"},
-        "InstanceType":{"shape":"InstanceType"},
-        "Ipv6AddressCount":{"shape":"Integer"},
-        "Ipv6Addresses":{
-          "shape":"InstanceIpv6AddressList",
-          "locationName":"Ipv6Address"
-        },
-        "KernelId":{"shape":"KernelId"},
-        "KeyName":{"shape":"KeyPairName"},
-        "MaxCount":{"shape":"Integer"},
-        "MinCount":{"shape":"Integer"},
-        "Monitoring":{"shape":"RunInstancesMonitoringEnabled"},
-        "Placement":{"shape":"Placement"},
-        "RamdiskId":{"shape":"RamdiskId"},
-        "SecurityGroupIds":{
-          "shape":"SecurityGroupIdStringList",
-          "locationName":"SecurityGroupId"
-        },
-        "SecurityGroups":{
-          "shape":"SecurityGroupStringList",
-          "locationName":"SecurityGroup"
-        },
-        "SubnetId":{"shape":"SubnetId"},
-        "UserData":{"shape":"RunInstancesUserData"},
-        "AdditionalInfo":{
-          "shape":"String",
-          "locationName":"additionalInfo"
-        },
-        "ClientToken":{
-          "shape":"String",
-          "idempotencyToken":true,
-          "locationName":"clientToken"
-        },
-        "DisableApiTermination":{
-          "shape":"Boolean",
-          "locationName":"disableApiTermination"
-        },
-        "DryRun":{
-          "shape":"Boolean",
-          "locationName":"dryRun"
-        },
-        "EbsOptimized":{
-          "shape":"Boolean",
-          "locationName":"ebsOptimized"
-        },
-        "IamInstanceProfile":{
-          "shape":"IamInstanceProfileSpecification",
-          "locationName":"iamInstanceProfile"
-        },
-        "InstanceInitiatedShutdownBehavior":{
-          "shape":"ShutdownBehavior",
-          "locationName":"instanceInitiatedShutdownBehavior"
-        },
-        "NetworkInterfaces":{
-          "shape":"InstanceNetworkInterfaceSpecificationList",
-          "locationName":"networkInterface"
-        },
-        "PrivateIpAddress":{
-          "shape":"String",
-          "locationName":"privateIpAddress"
-        },
-        "ElasticGpuSpecification":{"shape":"ElasticGpuSpecifications"},
-        "ElasticInferenceAccelerators":{
-          "shape":"ElasticInferenceAccelerators",
-          "locationName":"ElasticInferenceAccelerator"
-        },
-        "TagSpecifications":{
-          "shape":"TagSpecificationList",
-          "locationName":"TagSpecification"
-        },
-        "LaunchTemplate":{"shape":"LaunchTemplateSpecification"},
-        "InstanceMarketOptions":{"shape":"InstanceMarketOptionsRequest"},
-        "CreditSpecification":{"shape":"CreditSpecificationRequest"},
-        "CpuOptions":{"shape":"CpuOptionsRequest"},
-        "CapacityReservationSpecification":{"shape":"CapacityReservationSpecification"},
-        "HibernationOptions":{"shape":"HibernationOptionsRequest"},
-        "LicenseSpecifications":{
-          "shape":"LicenseSpecificationListRequest",
-          "locationName":"LicenseSpecification"
-        },
-        "MetadataOptions":{"shape":"InstanceMetadataOptionsRequest"},
-        "EnclaveOptions":{"shape":"EnclaveOptionsRequest"},
-        "PrivateDnsNameOptions":{"shape":"PrivateDnsNameOptionsRequest"},
-        "MaintenanceOptions":{"shape":"InstanceMaintenanceOptionsRequest"},
-        "DisableApiStop":{"shape":"Boolean"},
-        "EnablePrimaryIpv6":{"shape":"Boolean"}
-      }
-    },
-*/
-
 func ValidateRunInstancesInput(input *ec2.RunInstancesInput) (err error) {
-
-	// Check required args from JSON above
-	// required:[
-	//   "MaxCount",
-	//   "MinCount"
-	// ]
-
 	if input == nil {
-		return errors.New("MissingParameter")
+		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
 	if input.MinCount == nil {
-		return errors.New("MissingParameter")
+		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
 	if *input.MinCount == 0 {
-		return errors.New("InvalidParameterValue")
+		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
 	if *input.MaxCount == 0 {
-		return errors.New("InvalidParameterValue")
+		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
 	// Additional validation from EC2 spec
 	if *input.MinCount > *input.MaxCount {
-		return errors.New("InvalidParameterValue")
+		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
 	if input.ImageId == nil || *input.ImageId == "" {
-		return errors.New("MissingParameter")
+		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
 	if input.InstanceType == nil || *input.InstanceType == "" {
-		return errors.New("MissingParameter")
+		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
 	if !strings.HasPrefix(*input.ImageId, "ami-") {
-		return errors.New("InvalidAMIID.Malformed")
+		return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
 
 	}
 
@@ -188,9 +71,33 @@ func RunInstances(input *ec2.RunInstancesInput, natsConn *nats.Conn) (reservatio
 	// Call the service directly (no need for JSON marshaling/unmarshaling in same process)
 	reservationPtr, err := service.RunInstances(input)
 	if err != nil {
+		if errors.Is(err, nats.ErrNoResponders) {
+			// ErrNoResponders means no daemon subscribes to ec2.RunInstances.{type}.
+			// This happens when either: (a) the type is unknown, or (b) all nodes
+			// are at capacity. Query DescribeInstanceTypes to differentiate.
+			if !isKnownInstanceType(natsConn, *input.InstanceType) {
+				return reservation, errors.New(awserrors.ErrorInvalidInstanceType)
+			}
+			return reservation, errors.New(awserrors.ErrorInsufficientInstanceCapacity)
+		}
 		return reservation, err
 	}
 
 	// Dereference pointer to return value
 	return *reservationPtr, nil
 }
+
+// isKnownInstanceType checks whether any daemon recognizes the given instance type.
+func isKnownInstanceType(natsConn *nats.Conn, instanceType string) bool {
+	result, err := utils.NATSRequest[ec2.DescribeInstanceTypesOutput](
+		natsConn, "ec2.DescribeInstanceTypes", &ec2.DescribeInstanceTypesInput{}, 3*time.Second)
+	if err != nil || result == nil {
+		return false
+	}
+	for _, t := range result.InstanceTypes {
+		if t.InstanceType != nil && *t.InstanceType == instanceType {
+			return true
+		}
+	}
+	return false
+}

hive/gateway/ec2/instance/RunInstances_test.go

@@ -1,47 +1,15 @@
 package gateway_ec2_instance
 
 import (
-	"encoding/json"
 	"errors"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/handlers/ec2/instance"
-	"github.com/mulgadc/hive/hive/utils"
-	"github.com/nats-io/nats-server/v2/server"
+	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
-// startTestNATSServer starts an embedded NATS server for testing
-func startTestNATSServer(t *testing.T) (*server.Server, string) {
-	opts := &server.Options{
-		Host:      "127.0.0.1",
-		Port:      -1, // Auto-allocate available port
-		JetStream: false,
-		NoLog:     true,
-		NoSigs:    true,
-	}
-
-	ns, err := server.NewServer(opts)
-	require.NoError(t, err, "Failed to create NATS server")
-
-	// Start server in goroutine
-	go ns.Start()
-
-	// Wait for server to be ready
-	if !ns.ReadyForConnections(5 * time.Second) {
-		t.Fatal("NATS server failed to start")
-	}
-
-	url := ns.ClientURL()
-	t.Logf("Test NATS server started at: %s", url)
-
-	return ns, url
-}
-
 var defaults = ec2.RunInstancesInput{
 	ImageId:      aws.String("ami-0abcdef1234567890"),
 	InstanceType: aws.String("t2.micro"),
@@ -74,7 +42,7 @@ func TestParseRunInstances(t *testing.T) {
 				SecurityGroupIds: defaults.SecurityGroupIds,
 				SubnetId:         defaults.SubnetId,
 			},
-			want: errors.New("InvalidParameterValue"),
+			want: errors.New(awserrors.ErrorInvalidParameterValue),
 		},
 
 		{
@@ -88,7 +56,7 @@ func TestParseRunInstances(t *testing.T) {
 				SecurityGroupIds: defaults.SecurityGroupIds,
 				SubnetId:         defaults.SubnetId,
 			},
-			want: errors.New("InvalidParameterValue"),
+			want: errors.New(awserrors.ErrorInvalidParameterValue),
 		},
 
 		{
@@ -102,7 +70,7 @@ func TestParseRunInstances(t *testing.T) {
 				SecurityGroupIds: defaults.SecurityGroupIds,
 				SubnetId:         defaults.SubnetId,
 			},
-			want: errors.New("InvalidParameterValue"),
+			want: errors.New(awserrors.ErrorInvalidParameterValue),
 		},
 
 		{
@@ -116,7 +84,7 @@ func TestParseRunInstances(t *testing.T) {
 				SecurityGroupIds: defaults.SecurityGroupIds,
 				SubnetId:         defaults.SubnetId,
 			},
-			want: errors.New("MissingParameter"),
+			want: errors.New(awserrors.ErrorMissingParameter),
 		},
 
 		{
@@ -130,7 +98,7 @@ func TestParseRunInstances(t *testing.T) {
 				SecurityGroupIds: defaults.SecurityGroupIds,
 				SubnetId:         defaults.SubnetId,
 			},
-			want: errors.New("MissingParameter"),
+			want: errors.New(awserrors.ErrorMissingParameter),
 		},
 
 		{
@@ -144,7 +112,69 @@ func TestParseRunInstances(t *testing.T) {
 				SecurityGroupIds: defaults.SecurityGroupIds,
 				SubnetId:         defaults.SubnetId,
 			},
-			want: errors.New("InvalidAMIID.Malformed"),
+			want: errors.New(awserrors.ErrorInvalidAMIIDMalformed),
+		},
+
+		{
+			name:  "NilInput",
+			input: nil,
+			want:  errors.New(awserrors.ErrorMissingParameter),
+		},
+
+		{
+			name: "NilMinCount",
+			input: &ec2.RunInstancesInput{
+				ImageId:          defaults.ImageId,
+				InstanceType:     defaults.InstanceType,
+				MinCount:         nil,
+				MaxCount:         aws.Int64(1),
+				KeyName:          defaults.KeyName,
+				SecurityGroupIds: defaults.SecurityGroupIds,
+				SubnetId:         defaults.SubnetId,
+			},
+			want: errors.New(awserrors.ErrorMissingParameter),
+		},
+
+		{
+			name: "MinCountGreaterThanMaxCount",
+			input: &ec2.RunInstancesInput{
+				ImageId:          defaults.ImageId,
+				InstanceType:     defaults.InstanceType,
+				MinCount:         aws.Int64(5),
+				MaxCount:         aws.Int64(2),
+				KeyName:          defaults.KeyName,
+				SecurityGroupIds: defaults.SecurityGroupIds,
+				SubnetId:         defaults.SubnetId,
+			},
+			want: errors.New(awserrors.ErrorInvalidParameterValue),
+		},
+
+		{
+			name: "NilImageId",
+			input: &ec2.RunInstancesInput{
+				ImageId:          nil,
+				InstanceType:     defaults.InstanceType,
+				MinCount:         aws.Int64(1),
+				MaxCount:         aws.Int64(1),
+				KeyName:          defaults.KeyName,
+				SecurityGroupIds: defaults.SecurityGroupIds,
+				SubnetId:         defaults.SubnetId,
+			},
+			want: errors.New(awserrors.ErrorMissingParameter),
+		},
+
+		{
+			name: "NilInstanceType",
+			input: &ec2.RunInstancesInput{
+				ImageId:          defaults.ImageId,
+				InstanceType:     nil,
+				MinCount:         aws.Int64(1),
+				MaxCount:         aws.Int64(1),
+				KeyName:          defaults.KeyName,
+				SecurityGroupIds: defaults.SecurityGroupIds,
+				SubnetId:         defaults.SubnetId,
+			},
+			want: errors.New(awserrors.ErrorMissingParameter),
 		},
 
 		// Successful test
@@ -186,107 +216,3 @@ func TestParseRunInstances(t *testing.T) {
 	// Additional test
 
 }
-
-func TestEC2ProcessRunInstances(t *testing.T) {
-	makeValidInput := func() *ec2.RunInstancesInput {
-		return &ec2.RunInstancesInput{
-			ImageId:          aws.String(*defaults.ImageId),
-			InstanceType:     aws.String(*defaults.InstanceType),
-			MinCount:         aws.Int64(1),
-			MaxCount:         aws.Int64(1),
-			KeyName:          aws.String(*defaults.KeyName),
-			SecurityGroupIds: []*string{aws.String(*defaults.SecurityGroupIds[0])},
-			SubnetId:         aws.String(*defaults.SubnetId),
-		}
-	}
-
-	tests := []struct {
-		name              string
-		payload           interface{}
-		rawJSON           []byte
-		wantValidationErr bool
-		wantErrCode       string
-		assertFn          func(t *testing.T, reservation ec2.Reservation)
-	}{
-		{
-			name:              "MismatchedShape",
-			payload:           &ec2.DescribeInstancesInput{DryRun: aws.Bool(true)},
-			wantValidationErr: true,
-			wantErrCode:       "ValidationError",
-		},
-		{
-			name: "MissingRequiredFields",
-			payload: &ec2.RunInstancesInput{
-				ImageId:  aws.String("ami-0abcdef1234567890"),
-				MaxCount: aws.Int64(1),
-			},
-			wantValidationErr: true,
-			wantErrCode:       "MissingParameter",
-		},
-		{
-			name: "UnknownFieldInPayload",
-			rawJSON: []byte(`{
-				"ImageId":"ami-0abcdef1234567890",
-				"InstanceType":"t2.micro",
-				"MinCount":1,
-				"MaxCount":1,
-				"Unexpected":true
-			}`),
-			wantValidationErr: true,
-			wantErrCode:       "ValidationError",
-		},
-		{
-			name:    "ValidRunInstances",
-			payload: makeValidInput(),
-			assertFn: func(t *testing.T, reservation ec2.Reservation) {
-				assert.Len(t, reservation.Instances, 1)
-				if len(reservation.Instances) == 0 {
-					return
-				}
-				instance := reservation.Instances[0]
-				assert.Equal(t, defaults.ImageId, instance.ImageId)
-				assert.Equal(t, defaults.InstanceType, instance.InstanceType)
-				assert.Equal(t, defaults.KeyName, instance.KeyName)
-				assert.Equal(t, aws.Int64(16), instance.State.Code)
-				assert.Equal(t, aws.String("running"), instance.State.Name)
-				assert.Equal(t, defaults.SubnetId, instance.SubnetId)
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var (
-				jsonData []byte
-				err      error
-			)
-
-			if tt.rawJSON != nil {
-				jsonData = tt.rawJSON
-			} else {
-				jsonData, err = json.Marshal(tt.payload)
-				assert.NoError(t, err)
-			}
-
-			handler := handlers_ec2_instance.NewRunInstancesHandler(handlers_ec2_instance.NewMockInstanceService())
-			jsonResp := handler.Process(jsonData)
-
-			responseError, err := utils.ValidateErrorPayload(jsonResp)
-
-			if tt.wantValidationErr {
-				assert.Error(t, err)
-				assert.Equal(t, tt.wantErrCode, aws.StringValue(responseError.Code))
-				return
-			}
-
-			assert.NoError(t, err)
-
-			var reservation ec2.Reservation
-			assert.NoError(t, json.Unmarshal(jsonResp, &reservation))
-
-			if tt.assertFn != nil {
-				tt.assertFn(t, reservation)
-			}
-		})
-	}
-}

hive/gateway/ec2/instance/StartInstances.go

@@ -7,13 +7,19 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 )
 
-// StartInstances sends start commands to specified instances via NATS
+// startStoppedInstanceRequest is the payload sent to the ec2.start topic
+type startStoppedInstanceRequest struct {
+	InstanceID string `json:"instance_id"`
+}
+
+// StartInstances sends start requests to the ec2.start queue group topic.
+// Any available daemon can pick up the request and launch the stopped instance.
 func StartInstances(input *ec2.StartInstancesInput, natsConn *nats.Conn) (*ec2.StartInstancesOutput, error) {
-	if input.InstanceIds == nil || len(input.InstanceIds) == 0 {
+	if len(input.InstanceIds) == 0 {
 		return nil, fmt.Errorf("no instance IDs provided")
 	}
 
@@ -21,80 +27,37 @@ func StartInstances(input *ec2.StartInstancesInput, natsConn *nats.Conn) (*ec2.S
 
 	var stateChanges []*ec2.InstanceStateChange
 
-	// Process each instance
 	for _, instanceIDPtr := range input.InstanceIds {
 		if instanceIDPtr == nil {
 			continue
 		}
 		instanceID := *instanceIDPtr
 
-		// Build the QMP command to resume the instance
-		command := qmp.Command{
-			ID: instanceID,
-			QMPCommand: qmp.QMPCommand{
-				Execute:   "cont",
-				Arguments: map[string]interface{}{},
-			},
-			Attributes: qmp.Attributes{
-				StopInstance:      false,
-				TerminateInstance: false,
-				StartInstance:     true,
-			},
-		}
-
-		// Marshal the command
-		jsonData, err := json.Marshal(command)
+		req := startStoppedInstanceRequest{InstanceID: instanceID}
+		jsonData, err := json.Marshal(req)
 		if err != nil {
-			slog.Error("StartInstances: Failed to marshal command", "instance_id", instanceID, "err", err)
+			slog.Error("StartInstances: Failed to marshal request", "instance_id", instanceID, "err", err)
 			continue
 		}
 
-		// Send NATS request to the specific instance topic
-		subject := fmt.Sprintf("ec2.cmd.%s", instanceID)
+		slog.Info("StartInstances: Sending NATS request", "subject", "ec2.start", "instance_id", instanceID)
 
-		// TODO: Improve, stop/termination may take significant time
-		msg, err := natsConn.Request(subject, jsonData, 30*time.Second)
+		msg, err := natsConn.Request("ec2.start", jsonData, 30*time.Second)
 		if err != nil {
-			slog.Error("StartInstances: Failed to send command", "instance_id", instanceID, "err", err)
-			// Add failed state change
-			stateChange := &ec2.InstanceStateChange{
-				InstanceId: &instanceID,
-				CurrentState: &ec2.InstanceState{
-					Code: new(int64),
-					Name: new(string),
-				},
-				PreviousState: &ec2.InstanceState{
-					Code: new(int64),
-					Name: new(string),
-				},
-			}
-			*stateChange.CurrentState.Code = 80 // stopped
-			*stateChange.CurrentState.Name = "stopped"
-			*stateChange.PreviousState.Code = 80
-			*stateChange.PreviousState.Name = "stopped"
-			stateChanges = append(stateChanges, stateChange)
+			slog.Error("StartInstances: Failed to send start request", "instance_id", instanceID, "err", err)
+			stateChanges = append(stateChanges, newStateChange(instanceID, 80, "stopped", 80, "stopped"))
+			continue
+		}
+
+		// Check if the daemon returned an error response
+		if responseError, parseErr := utils.ValidateErrorPayload(msg.Data); parseErr != nil {
+			slog.Error("StartInstances: Daemon returned error", "instance_id", instanceID, "code", responseError.Code)
+			stateChanges = append(stateChanges, newStateChange(instanceID, 80, "stopped", 80, "stopped"))
 			continue
 		}
 
 		slog.Info("StartInstances: Command sent successfully", "instance_id", instanceID, "response", string(msg.Data))
-
-		// Build state change response (stopped -> pending)
-		stateChange := &ec2.InstanceStateChange{
-			InstanceId: &instanceID,
-			CurrentState: &ec2.InstanceState{
-				Code: new(int64),
-				Name: new(string),
-			},
-			PreviousState: &ec2.InstanceState{
-				Code: new(int64),
-				Name: new(string),
-			},
-		}
-		*stateChange.CurrentState.Code = 0 // pending
-		*stateChange.CurrentState.Name = "pending"
-		*stateChange.PreviousState.Code = 80 // stopped
-		*stateChange.PreviousState.Name = "stopped"
-		stateChanges = append(stateChanges, stateChange)
+		stateChanges = append(stateChanges, newStateChange(instanceID, 0, "pending", 80, "stopped"))
 	}
 
 	output := &ec2.StartInstancesOutput{

hive/gateway/ec2/instance/StartInstances_test.go

@@ -0,0 +1,169 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStartInstances_Success(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-0123456789abcdef0"
+
+	// Mock subscriber for the ec2.start queue group topic
+	nc.QueueSubscribe("ec2.start", "hive-workers", func(msg *nats.Msg) {
+		var req startStoppedInstanceRequest
+		err := json.Unmarshal(msg.Data, &req)
+		require.NoError(t, err)
+
+		assert.Equal(t, instanceID, req.InstanceID)
+
+		msg.Respond([]byte(`{"status":"running","instanceId":"` + instanceID + `"}`))
+	})
+
+	input := &ec2.StartInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	output, err := StartInstances(input, nc)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	require.Len(t, output.StartingInstances, 1)
+
+	sc := output.StartingInstances[0]
+	assert.Equal(t, instanceID, *sc.InstanceId)
+	assert.Equal(t, int64(0), *sc.CurrentState.Code)
+	assert.Equal(t, "pending", *sc.CurrentState.Name)
+	assert.Equal(t, int64(80), *sc.PreviousState.Code)
+	assert.Equal(t, "stopped", *sc.PreviousState.Name)
+}
+
+func TestStartInstances_MultipleInstances(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	ids := []string{"i-001", "i-002", "i-003"}
+
+	// Single subscriber handles all start requests via queue group
+	nc.QueueSubscribe("ec2.start", "hive-workers", func(msg *nats.Msg) {
+		var req startStoppedInstanceRequest
+		json.Unmarshal(msg.Data, &req)
+		msg.Respond([]byte(`{"status":"running","instanceId":"` + req.InstanceID + `"}`))
+	})
+
+	input := &ec2.StartInstancesInput{
+		InstanceIds: []*string{aws.String(ids[0]), aws.String(ids[1]), aws.String(ids[2])},
+	}
+
+	output, err := StartInstances(input, nc)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Len(t, output.StartingInstances, 3)
+
+	for i, sc := range output.StartingInstances {
+		assert.Equal(t, ids[i], *sc.InstanceId)
+		assert.Equal(t, "pending", *sc.CurrentState.Name)
+	}
+}
+
+func TestStartInstances_EmptyInstanceIds(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	input := &ec2.StartInstancesInput{
+		InstanceIds: []*string{},
+	}
+
+	_, err := StartInstances(input, nc)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "no instance IDs provided")
+}
+
+func TestStartInstances_NilInstanceIdSkipped(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-valid"
+	nc.QueueSubscribe("ec2.start", "hive-workers", func(msg *nats.Msg) {
+		msg.Respond([]byte(`{"status":"running","instanceId":"` + instanceID + `"}`))
+	})
+
+	input := &ec2.StartInstancesInput{
+		InstanceIds: []*string{nil, aws.String(instanceID), nil},
+	}
+
+	output, err := StartInstances(input, nc)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	// Only the valid ID should produce a state change
+	assert.Len(t, output.StartingInstances, 1)
+	assert.Equal(t, instanceID, *output.StartingInstances[0].InstanceId)
+}
+
+func TestStartInstances_NATSRequestFails(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	// No subscriber for ec2.start, so NATS request will fail
+	instanceID := "i-nosubscriber"
+
+	input := &ec2.StartInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	output, err := StartInstances(input, nc)
+
+	require.NoError(t, err) // Function itself doesn't error, it records state change
+	require.NotNil(t, output)
+	require.Len(t, output.StartingInstances, 1)
+
+	// On NATS failure, state should reflect "still stopped"
+	sc := output.StartingInstances[0]
+	assert.Equal(t, instanceID, *sc.InstanceId)
+	assert.Equal(t, int64(80), *sc.CurrentState.Code)
+	assert.Equal(t, "stopped", *sc.CurrentState.Name)
+	assert.Equal(t, int64(80), *sc.PreviousState.Code)
+	assert.Equal(t, "stopped", *sc.PreviousState.Name)
+}
+
+func TestStartInstances_MixedSuccessAndFailure(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	goodID := "i-good"
+	badID := "i-bad"
+
+	// Subscribe to ec2.start but only respond for goodID
+	nc.QueueSubscribe("ec2.start", "hive-workers", func(msg *nats.Msg) {
+		var req startStoppedInstanceRequest
+		json.Unmarshal(msg.Data, &req)
+		if req.InstanceID == goodID {
+			msg.Respond([]byte(`{"status":"running","instanceId":"` + goodID + `"}`))
+		} else {
+			// Simulate error for bad instance
+			msg.Respond([]byte(`{"Code":"InvalidInstanceID.NotFound"}`))
+		}
+	})
+
+	input := &ec2.StartInstancesInput{
+		InstanceIds: []*string{aws.String(goodID), aws.String(badID)},
+	}
+
+	output, err := StartInstances(input, nc)
+
+	require.NoError(t, err)
+	require.Len(t, output.StartingInstances, 2)
+
+	// First instance succeeded: state → pending
+	assert.Equal(t, goodID, *output.StartingInstances[0].InstanceId)
+	assert.Equal(t, "pending", *output.StartingInstances[0].CurrentState.Name)
+
+	// Second instance: daemon returned an error, state remains stopped
+	assert.Equal(t, badID, *output.StartingInstances[1].InstanceId)
+	assert.Equal(t, int64(80), *output.StartingInstances[1].CurrentState.Code)
+	assert.Equal(t, "stopped", *output.StartingInstances[1].CurrentState.Name)
+}

hive/gateway/ec2/instance/StopInstances.go

@@ -12,9 +12,9 @@ import (
 )
 
 // StopInstances sends stop commands to specified instances via NATS
-// Uses system_powerdown without stop_instance attribute to allow restart
+// Uses system_powerdown with stop_instance attribute to prevent auto-restart on daemon boot
 func StopInstances(input *ec2.StopInstancesInput, natsConn *nats.Conn) (*ec2.StopInstancesOutput, error) {
-	if input.InstanceIds == nil || len(input.InstanceIds) == 0 {
+	if len(input.InstanceIds) == 0 {
 		return nil, fmt.Errorf("no instance IDs provided")
 	}
 
@@ -30,15 +30,15 @@ func StopInstances(input *ec2.StopInstancesInput, natsConn *nats.Conn) (*ec2.Sto
 		instanceID := *instanceIDPtr
 
 		// Build the QMP command to stop the instance
-		// Note: system_powerdown without stop_instance allows restart
+		// Note: system_powerdown with stop_instance=true prevents auto-restart on daemon boot
 		command := qmp.Command{
 			ID: instanceID,
 			QMPCommand: qmp.QMPCommand{
 				Execute:   "system_powerdown",
-				Arguments: map[string]interface{}{},
+				Arguments: map[string]any{},
 			},
 			Attributes: qmp.Attributes{
-				StopInstance:      false, // Allow restart on daemon/node restart
+				StopInstance:      true, // Don't auto-restart on daemon boot
 				TerminateInstance: false,
 			},
 		}
@@ -55,45 +55,13 @@ func StopInstances(input *ec2.StopInstancesInput, natsConn *nats.Conn) (*ec2.Sto
 		msg, err := natsConn.Request(subject, jsonData, 5*time.Second)
 		if err != nil {
 			slog.Error("StopInstances: Failed to send command", "instance_id", instanceID, "err", err)
-			// Add failed state change
-			stateChange := &ec2.InstanceStateChange{
-				InstanceId: &instanceID,
-				CurrentState: &ec2.InstanceState{
-					Code: new(int64),
-					Name: new(string),
-				},
-				PreviousState: &ec2.InstanceState{
-					Code: new(int64),
-					Name: new(string),
-				},
-			}
-			*stateChange.CurrentState.Code = 16 // running
-			*stateChange.CurrentState.Name = "running"
-			*stateChange.PreviousState.Code = 16
-			*stateChange.PreviousState.Name = "running"
-			stateChanges = append(stateChanges, stateChange)
+			stateChanges = append(stateChanges, newStateChange(instanceID, 16, "running", 16, "running"))
 			continue
 		}
 
 		slog.Info("StopInstances: Command sent successfully", "instance_id", instanceID, "response", string(msg.Data))
 
-		// Build state change response (running -> stopping)
-		stateChange := &ec2.InstanceStateChange{
-			InstanceId: &instanceID,
-			CurrentState: &ec2.InstanceState{
-				Code: new(int64),
-				Name: new(string),
-			},
-			PreviousState: &ec2.InstanceState{
-				Code: new(int64),
-				Name: new(string),
-			},
-		}
-		*stateChange.CurrentState.Code = 64 // stopping
-		*stateChange.CurrentState.Name = "stopping"
-		*stateChange.PreviousState.Code = 16 // running
-		*stateChange.PreviousState.Name = "running"
-		stateChanges = append(stateChanges, stateChange)
+		stateChanges = append(stateChanges, newStateChange(instanceID, 64, "stopping", 16, "running"))
 	}
 
 	output := &ec2.StopInstancesOutput{

hive/gateway/ec2/instance/StopInstances_test.go

@@ -0,0 +1,159 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStopInstances_Success(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-0123456789abcdef0"
+
+	nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
+		var cmd qmp.Command
+		err := json.Unmarshal(msg.Data, &cmd)
+		require.NoError(t, err)
+
+		assert.Equal(t, instanceID, cmd.ID)
+		assert.Equal(t, "system_powerdown", cmd.QMPCommand.Execute)
+		assert.True(t, cmd.Attributes.StopInstance)
+		assert.False(t, cmd.Attributes.TerminateInstance)
+
+		msg.Respond([]byte(`{"return":{}}`))
+	})
+
+	input := &ec2.StopInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	output, err := StopInstances(input, nc)
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	require.Len(t, output.StoppingInstances, 1)
+
+	sc := output.StoppingInstances[0]
+	assert.Equal(t, instanceID, *sc.InstanceId)
+	assert.Equal(t, int64(64), *sc.CurrentState.Code)
+	assert.Equal(t, "stopping", *sc.CurrentState.Name)
+	assert.Equal(t, int64(16), *sc.PreviousState.Code)
+	assert.Equal(t, "running", *sc.PreviousState.Name)
+}
+
+func TestStopInstances_MultipleInstances(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	ids := []string{"i-001", "i-002"}
+
+	for _, id := range ids {
+		id := id
+		nc.Subscribe("ec2.cmd."+id, func(msg *nats.Msg) {
+			msg.Respond([]byte(`{"return":{}}`))
+		})
+	}
+
+	input := &ec2.StopInstancesInput{
+		InstanceIds: []*string{aws.String(ids[0]), aws.String(ids[1])},
+	}
+
+	output, err := StopInstances(input, nc)
+
+	require.NoError(t, err)
+	require.Len(t, output.StoppingInstances, 2)
+
+	for i, sc := range output.StoppingInstances {
+		assert.Equal(t, ids[i], *sc.InstanceId)
+		assert.Equal(t, "stopping", *sc.CurrentState.Name)
+		assert.Equal(t, "running", *sc.PreviousState.Name)
+	}
+}
+
+func TestStopInstances_EmptyInstanceIds(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	input := &ec2.StopInstancesInput{
+		InstanceIds: []*string{},
+	}
+
+	_, err := StopInstances(input, nc)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "no instance IDs provided")
+}
+
+func TestStopInstances_NilInstanceIdSkipped(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-valid"
+	nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
+		msg.Respond([]byte(`{"return":{}}`))
+	})
+
+	input := &ec2.StopInstancesInput{
+		InstanceIds: []*string{nil, aws.String(instanceID)},
+	}
+
+	output, err := StopInstances(input, nc)
+
+	require.NoError(t, err)
+	assert.Len(t, output.StoppingInstances, 1)
+	assert.Equal(t, instanceID, *output.StoppingInstances[0].InstanceId)
+}
+
+func TestStopInstances_NATSRequestFails(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-nosubscriber"
+
+	input := &ec2.StopInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	output, err := StopInstances(input, nc)
+
+	require.NoError(t, err)
+	require.Len(t, output.StoppingInstances, 1)
+
+	// On NATS failure, state should reflect "still running"
+	sc := output.StoppingInstances[0]
+	assert.Equal(t, instanceID, *sc.InstanceId)
+	assert.Equal(t, int64(16), *sc.CurrentState.Code)
+	assert.Equal(t, "running", *sc.CurrentState.Name)
+	assert.Equal(t, int64(16), *sc.PreviousState.Code)
+	assert.Equal(t, "running", *sc.PreviousState.Name)
+}
+
+func TestStopInstances_MixedSuccessAndFailure(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	goodID := "i-good"
+	badID := "i-bad"
+
+	nc.Subscribe("ec2.cmd."+goodID, func(msg *nats.Msg) {
+		msg.Respond([]byte(`{"return":{}}`))
+	})
+
+	input := &ec2.StopInstancesInput{
+		InstanceIds: []*string{aws.String(goodID), aws.String(badID)},
+	}
+
+	output, err := StopInstances(input, nc)
+
+	require.NoError(t, err)
+	require.Len(t, output.StoppingInstances, 2)
+
+	// First: success → stopping
+	assert.Equal(t, goodID, *output.StoppingInstances[0].InstanceId)
+	assert.Equal(t, "stopping", *output.StoppingInstances[0].CurrentState.Name)
+
+	// Second: failure → still running
+	assert.Equal(t, badID, *output.StoppingInstances[1].InstanceId)
+	assert.Equal(t, "running", *output.StoppingInstances[1].CurrentState.Name)
+}