New doc links

Same pattern as H-5 — crypto/rand failure caused a panic. Now returns
(string, error) so callers can handle it gracefully.

.github/workflows/e2e.yml

@@ -1,4 +1,4 @@
-name: Hive E2E Test
+name: Hive E2E Tests
 
 on:
   push:
@@ -11,70 +11,186 @@ on:
       - go.sum
       - Makefile
       - tests/e2e/**
-      - scripts/**
       - .github/workflows/e2e.yml
   workflow_dispatch:
 
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+
 jobs:
-  build_e2e_image:
-    name: Build E2E Image
-    runs-on: self-hosted
-
+  e2e_single:
+    name: Single-Node E2E
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 25
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
     steps:
-      - name: Checkout Code
-        uses: actions/checkout@v6
+      - name: Clean workspace
+        run: rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v2
+        id: app-token
         with:
-          path: hive
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
 
-      - name: Clone Dependencies
-        run: ./hive/scripts/clone-deps.sh
+      - uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
 
-      - name: Build E2E Image
+      - name: Provision VM
+        working-directory: mulga/scripts/tofu-cluster
         run: |
-          docker build \
-            -t hive-e2e:latest \
-            -f hive/tests/e2e/Dockerfile.e2e \
-            .
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          tofu init -input=false >> "$TOFU_LOG" 2>&1
+          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
+          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
+          tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
-  hive_e2e_test:
-    name: Hive Single-Node E2E Test
-    runs-on: self-hosted
-    needs: build_e2e_image
+      - name: Bootstrap Cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
 
-    steps:
       - name: Run E2E Tests
+        working-directory: mulga/scripts/tofu-cluster
         run: |
-          docker run --privileged --rm \
-            -v /dev/kvm:/dev/kvm \
-            --name hive-e2e-test \
-            hive-e2e:latest
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            'cd ~/Development/mulga/hive && ./tests/e2e/run-e2e.sh'
 
-  hive_multinode_e2e_test:
-    name: Hive Multi-Node E2E Test
-    runs-on: self-hosted
-    needs: build_e2e_image
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
 
+  e2e_pseudo_multi:
+    name: Pseudo Multi-Node E2E
+    runs-on: [self-hosted, ci-pseudo]
+    timeout-minutes: 25
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
     steps:
-      - name: Run Multi-Node E2E Tests
+      - name: Clean workspace
+        run: rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+
+          path: mulga
+
+      - name: Provision VM
+        working-directory: mulga/scripts/tofu-cluster
         run: |
-          docker run --privileged --rm \
-            -v /dev/kvm:/dev/kvm \
-            --cap-add=NET_ADMIN \
-            --name hive-multinode-e2e-test \
-            hive-e2e:latest \
-            ./tests/e2e/run-multinode-e2e.sh
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          tofu init -input=false >> "$TOFU_LOG" 2>&1
+          tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
+          ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
+          tofu apply -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
-  cleanup:
-    name: Cleanup
-    runs-on: self-hosted
-    needs: [hive_e2e_test, hive_multinode_e2e_test]
-    if: always()
+      - name: Bootstrap Cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1
 
+      - name: Run E2E Tests
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            'cd ~/Development/mulga/hive && ./tests/e2e/run-pseudo-multinode-e2e.sh'
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
+
+  e2e_real_multi:
+    name: Real Multi-Node E2E
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 25
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
     steps:
-      - name: Remove Stale Containers
-        run: |
-          docker rm -f hive-e2e-test 2>/dev/null || true
-          docker rm -f hive-multinode-e2e-test 2>/dev/null || true
+      - name: Clean workspace
+        run: rm -rf "$GITHUB_WORKSPACE"/*
 
-      - name: Prune Docker Images
-        run: docker image prune -f
+      - uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          tofu init -input=false >> "$TOFU_LOG" 2>&1
+          tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
+          ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
+          tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+
+      - name: Bootstrap Cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
+
+      - name: Run E2E Tests
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+
+          # Build node IP list
+          NODE_IPS=""
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
+          done
+
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "cd ~/Development/mulga/hive && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true

.github/workflows/hive.yml

@@ -37,15 +37,15 @@ jobs:
           repository: mulgadc/predastore
           path: predastore
 
-      - name: Setup Go Workspace
-        run: go work init ./hive ./viperblock ./predastore
-
       - name: Setup Go
         uses: actions/setup-go@v6
         with:
           cache-dependency-path: "hive/go.sum"
           go-version-file: "hive/go.mod"
 
+      - name: Setup Go Workspace
+        run: go work init ./hive ./viperblock ./predastore
+
       - name: Build Hive
         run: make -C hive build
 
@@ -77,15 +77,15 @@ jobs:
           repository: mulgadc/predastore
           path: predastore
 
-      - name: Setup Go Workspace
-        run: go work init ./hive ./viperblock ./predastore
-
       - name: Setup Go
         uses: actions/setup-go@v6
         with:
           cache-dependency-path: "hive/go.sum"
           go-version-file: "hive/go.mod"
 
+      - name: Setup Go Workspace
+        run: go work init ./hive ./viperblock ./predastore
+
       - name: Run Tests with Race Detector
         run: make -C hive test-race
 
@@ -111,15 +111,15 @@ jobs:
           repository: mulgadc/predastore
           path: predastore
 
-      - name: Setup Go Workspace
-        run: go work init ./hive ./viperblock ./predastore
-
       - name: Setup Go
         uses: actions/setup-go@v6
         with:
           cache-dependency-path: "hive/go.sum"
           go-version-file: "hive/go.mod"
 
+      - name: Setup Go Workspace
+        run: go work init ./hive ./viperblock ./predastore
+
       - name: Check Format
         run: make -C hive check-format
 

.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: Release
+
+on:
+  push:
+    tags: ['v*']
+
+permissions:
+  contents: write
+
+jobs:
+  distro:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: hive
+
+      - uses: actions/checkout@v4
+        with:
+          repository: mulgadc/predastore
+          path: predastore
+
+      - uses: actions/checkout@v4
+        with:
+          repository: mulgadc/viperblock
+          path: viperblock
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up QEMU (for arm64 cross-build)
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build distribution tarballs
+        run: cd hive && make distro VERSION=${{ github.ref_name }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            hive/dist/*.tar.gz
+            hive/dist/*.sha256
+            hive/scripts/setup.sh
+          generate_release_notes: true

.github/workflows/ui.yml

@@ -42,5 +42,8 @@ jobs:
       - name: Run Biome Check
         run: pnpm check
 
+      - name: Run Tests with Coverage
+        run: pnpm test:coverage
+
       - name: Build Frontend
         run: pnpm build

.gitignore

@@ -13,6 +13,7 @@ go.work.sum
 
 # Build outputs
 bin/
+dist/
 tmp/
 *.test
 *.out

Makefile

@@ -55,9 +55,13 @@ build-ui:
 	cd hive/services/hiveui/frontend && pnpm build
 
 # GO commands
+VERSION ?= $(shell git describe --tags --always --dirty)
+COMMIT  ?= $(shell git rev-parse --short HEAD)
+LDFLAGS := -s -w -X github.com/mulgadc/hive/cmd/hive/cmd.Version=$(VERSION) -X github.com/mulgadc/hive/cmd/hive/cmd.Commit=$(COMMIT)
+
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	go build $(GO_BUILD_MOD) -ldflags "-s -w" -o ./bin/$(GO_PROJECT_NAME) cmd/hive/main.go
+	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/hive/main.go
 
 go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
@@ -115,9 +119,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.0 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.1 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.1.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -193,39 +197,38 @@ security-check:
 	$(_Q)set -o pipefail && go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./... $(_SECQ) tests/staticcheck-report.txt $(if $(QUIET),|| { cat tests/staticcheck-report.txt; exit 1; })
 	@echo "  staticcheck ok"
 
-# Docker E2E tests (mirrors GitHub Actions e2e.yml)
-# Usage: make test-docker              # both suites
-#        make test-docker-single       # single-node only
-#        make test-docker-multi        # multi-node only
-PARENT_DIR := $(shell cd .. && pwd)
-E2E_IMAGE := hive-e2e:latest
+# Build release tarballs for both architectures via Docker
+distro:
+	@echo "Building hive $(VERSION) distribution tarballs..."
+	@mkdir -p dist/
+	@echo "Building linux/amd64..."
+	docker buildx build \
+		--platform linux/amd64 \
+		--build-arg VERSION=$(VERSION) \
+		-f build/Dockerfile.distro \
+		--output type=local,dest=dist/amd64/ \
+		../
+	tar -czf dist/hive-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
+	sha256sum dist/hive-$(VERSION)-linux-amd64.tar.gz > dist/hive-$(VERSION)-linux-amd64.tar.gz.sha256
+	@echo "Building linux/arm64..."
+	docker buildx build \
+		--platform linux/arm64 \
+		--build-arg VERSION=$(VERSION) \
+		-f build/Dockerfile.distro \
+		--output type=local,dest=dist/arm64/ \
+		../
+	tar -czf dist/hive-$(VERSION)-linux-arm64.tar.gz -C dist/arm64 .
+	sha256sum dist/hive-$(VERSION)-linux-arm64.tar.gz > dist/hive-$(VERSION)-linux-arm64.tar.gz.sha256
+	@echo ""
+	@echo "Distribution tarballs:"
+	@ls -lh dist/*.tar.gz
+	@echo ""
+	@cat dist/*.sha256
 
-test-docker-build:
-	@echo -e "\n....Building E2E Docker image...."
-	@for dep in viperblock predastore; do \
-		if [ ! -d "$(PARENT_DIR)/$$dep" ]; then \
-			echo "Missing sibling repo $$dep — running clone-deps.sh"; \
-			./scripts/clone-deps.sh; \
-			break; \
-		fi; \
-	done
-	docker build -t $(E2E_IMAGE) -f tests/e2e/Dockerfile.e2e $(PARENT_DIR)
-
-test-docker-single: test-docker-build
-	@echo -e "\n....Running Single-Node E2E Tests...."
-	docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
-
-test-docker-multi: test-docker-build
-	@echo -e "\n....Running Multi-Node E2E Tests...."
-	docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
-
-test-docker: test-docker-build
-	@echo -e "\n....Running Single-Node E2E Tests...."
-	docker run --privileged --rm -v /dev/kvm:/dev/kvm $(E2E_IMAGE)
-	@echo -e "\n....Running Multi-Node E2E Tests...."
-	docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
+distro-clean:
+	rm -rf dist/
 
 .PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
 	install-system install-go install-aws quickinstall \
 	format check-format modernize check-modernize vet security-check \
-	test-docker-build test-docker-single test-docker-multi test-docker
+	distro distro-clean

README.md

@@ -52,23 +52,30 @@ AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, o
 - Works Offline – No reliance on centralized cloud services or external networks.
 - Open Source – Licensed under Apache 2.0. Fork it, extend it, or deploy it as-is.
 
-## Development Setup
+## Installation
 
-### For Developers
+Installation requires an Ubuntu / Debian system. See the detailed documentation at [https://docs.mulgadc.com](https://docs.mulgadc.com/) for maintaining and installing Hive.
 
-Get started with Hive development in minutes:
+### Single node install
+
+The installation is straightforward to set up and running on a single node for testing purposes.
 
 ```bash
-# Clone the repository
-git clone https://github.com/mulgadc/hive.git
-cd hive
-
-# Setup dependencies and development environment
-./scripts/clone-deps.sh    # Clone viperblock + predastore repositories
-./scripts/dev-setup.sh     # Setup complete development environment
+export AWS_PROFILE=hive
+export AWS_REGION=ap-southeast-2-sc
 ```
 
-Next, follow the detailed [Installation Guide](./INSTALL.md) to bootstrap a new environment with SSH keys, AMI (OS images) and how to spin up new EC2 instances using Hive.
+```bash
+curl https://install.mulgadc.com/ | bash
+
+hive admin init --region $AWS_REGION
+
+aws ec2 describe-instance-types
+```
+
+### Development Setup
+
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/source-install/) documentation
 
 ### Component Repositories
 
@@ -79,12 +86,6 @@ Hive coordinates these independent components:
 
 Each component can be developed independently. See component-specific documentation for focused development guides.
 
-## Detailed Installation Method
-
-See [Installation Guide](INSTALL.md) on how to setup a Hive node to get started.
-
-Then configure your AWS CLI to point to Hive's endpoints and start launching VMs, buckets, and volumes using familiar commands and Terraform scripts.
-
 ## Development Philosophy
 
 ### Built by Engineers, For Engineers

build/Dockerfile.distro

@@ -0,0 +1,48 @@
+ARG VERSION=dev
+
+FROM golang:1.26-bookworm AS builder
+
+ARG VERSION
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    nbdkit nbdkit-plugin-dev pkg-config gcc make \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+
+# Copy go.work first (lives in hive/)
+COPY hive/go.work ./hive/
+
+# Copy module files for dependency caching
+COPY hive/go.mod hive/go.sum ./hive/
+COPY predastore/go.mod predastore/go.sum ./predastore/
+COPY viperblock/go.mod viperblock/go.sum ./viperblock/
+COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
+COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
+
+# Download dependencies (cached layer)
+RUN cd hive && go mod download
+
+# Copy full source
+COPY hive/ ./hive/
+COPY predastore/ ./predastore/
+COPY viperblock/ ./viperblock/
+
+# Build hive binary
+RUN COMMIT=$(cd hive && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \
+    cd hive && go build -ldflags "-s -w \
+    -X github.com/mulgadc/hive/cmd/hive/cmd.Version=${VERSION} \
+    -X github.com/mulgadc/hive/cmd/hive/cmd.Commit=${COMMIT}" \
+    -o /out/hive cmd/hive/main.go
+
+# Build nbdkit viperblock plugin (.so)
+RUN cd viperblock && go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go
+
+# Copy setup scripts and systemd units
+RUN cp hive/scripts/setup.sh /out/setup.sh 2>/dev/null || true
+RUN cp hive/scripts/setup-ovn.sh /out/setup-ovn.sh
+RUN mkdir -p /out/systemd && cp hive/build/systemd/* /out/systemd/
+RUN cp hive/build/logrotate/hive /out/logrotate-hive
+
+FROM scratch AS export
+COPY --from=builder /out/ /

build/logrotate/hive

@@ -0,0 +1,9 @@
+/var/log/hive/*.log {
+    daily
+    rotate 14
+    compress
+    delaycompress
+    missingok
+    notifempty
+    copytruncate
+}

build/systemd/hive-awsgw.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Hive AWS Gateway
+PartOf=hive.target
+After=hive-nats.service hive-daemon.service
+Requires=hive-nats.service
+
+[Service]
+Type=simple
+User=hive
+Group=hive
+ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStart=/usr/local/bin/hive service awsgw start
+Restart=on-failure
+RestartSec=5
+OOMScoreAdjust=-500
+Environment=HIVE_CONFIG_PATH=/etc/hive/hive.toml
+Environment=HIVE_BASE_DIR=/var/lib/hive/
+Environment=HIVE_AWSGW_TLS_CERT=/etc/hive/server.pem
+Environment=HIVE_AWSGW_TLS_KEY=/etc/hive/server.key
+
+[Install]
+WantedBy=hive.target

build/systemd/hive-daemon.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=Hive Daemon (VM Orchestration)
+PartOf=hive.target
+After=hive-nats.service hive-viperblock.service
+Requires=hive-nats.service
+
+[Service]
+Type=simple
+User=hive
+Group=hive
+ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStart=/usr/local/bin/hive service hive start
+Restart=on-failure
+RestartSec=5
+OOMScoreAdjust=-500
+Environment=HIVE_CONFIG_PATH=/etc/hive/hive.toml
+Environment=HIVE_BASE_DIR=/var/lib/hive/hive/
+Environment=HIVE_WAL_DIR=/var/lib/hive/hive/
+
+[Install]
+WantedBy=hive.target

build/systemd/hive-nats.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Hive NATS Message Bus
+PartOf=hive.target
+Before=hive-predastore.service hive-daemon.service hive-awsgw.service hive-vpcd.service
+
+[Service]
+Type=simple
+User=hive
+Group=hive
+ExecStart=/usr/local/bin/hive service nats start
+Restart=on-failure
+RestartSec=5
+OOMScoreAdjust=-500
+Environment=HIVE_CONFIG_PATH=/etc/hive/nats/nats.conf
+
+[Install]
+WantedBy=hive.target

build/systemd/hive-predastore.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Hive Predastore (S3-compatible Object Storage)
+PartOf=hive.target
+After=hive-nats.service
+Requires=hive-nats.service
+
+[Service]
+Type=simple
+User=hive
+Group=hive
+ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStart=/usr/local/bin/hive service predastore start
+Restart=on-failure
+RestartSec=5
+OOMScoreAdjust=-500
+Environment=HIVE_PREDASTORE_CONFIG_PATH=/etc/hive/predastore/predastore.toml
+Environment=HIVE_PREDASTORE_BASE_PATH=/var/lib/hive/predastore/
+Environment=HIVE_PREDASTORE_TLS_CERT=/etc/hive/server.pem
+Environment=HIVE_PREDASTORE_TLS_KEY=/etc/hive/server.key
+
+[Install]
+WantedBy=hive.target

build/systemd/hive-ui.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Hive Web UI
+PartOf=hive.target
+After=hive-awsgw.service
+
+[Service]
+Type=simple
+User=hive
+Group=hive
+ExecStart=/usr/local/bin/hive service hive-ui start
+Restart=on-failure
+RestartSec=5
+OOMScoreAdjust=-500
+
+[Install]
+WantedBy=hive.target

build/systemd/hive-viperblock.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=Hive Viperblock (Block Storage)
+PartOf=hive.target
+After=hive-nats.service hive-predastore.service
+Requires=hive-nats.service
+
+[Service]
+Type=simple
+User=hive
+Group=hive
+ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStart=/usr/local/bin/hive service viperblock start
+Restart=on-failure
+RestartSec=5
+OOMScoreAdjust=-500
+EnvironmentFile=/etc/hive/systemd.env
+Environment=HIVE_CONFIG_PATH=/etc/hive/hive.toml
+Environment=HIVE_BASE_DIR=/var/lib/hive/viperblock/
+
+[Install]
+WantedBy=hive.target

build/systemd/hive-vpcd.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Hive VPC Daemon (OVN Networking)
+PartOf=hive.target
+After=hive-nats.service hive-daemon.service
+Requires=hive-nats.service
+
+[Service]
+Type=simple
+User=hive
+Group=hive
+ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStart=/usr/local/bin/hive service vpcd start
+Restart=on-failure
+RestartSec=5
+OOMScoreAdjust=-500
+Environment=HIVE_CONFIG_PATH=/etc/hive/hive.toml
+
+[Install]
+WantedBy=hive.target

build/systemd/hive.target

@@ -0,0 +1,8 @@
+[Unit]
+Description=Hive Infrastructure Platform
+After=network-online.target
+Wants=network-online.target
+Wants=hive-nats.service hive-predastore.service hive-viperblock.service hive-daemon.service hive-awsgw.service hive-vpcd.service hive-ui.service
+
+[Install]
+WantedBy=multi-user.target

cmd/hive/cmd/admin.go

@@ -37,11 +37,13 @@ import (
 	"github.com/mulgadc/hive/hive/admin"
 	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/formation"
+	handlers_ec2_vpc "github.com/mulgadc/hive/hive/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/hive/hive/handlers/iam"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/mulgadc/viperblock/viperblock/backends/s3"
 	"github.com/mulgadc/viperblock/viperblock/v_utils"
+	"github.com/nats-io/nats.go"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 )
@@ -194,12 +196,8 @@ func init() {
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
 	accountCreateCmd.MarkFlagRequired("name")
 
-	homeDir, _ := os.UserHomeDir()
-	configDir := fmt.Sprintf("%s/hive/config", homeDir)
-	hiveDir := fmt.Sprintf("%s/hive/", homeDir)
-
-	rootCmd.PersistentFlags().String("config-dir", configDir, "Configuration directory")
-	rootCmd.PersistentFlags().String("hive-dir", hiveDir, "Hive base directory")
+	rootCmd.PersistentFlags().String("config-dir", DefaultConfigDir(), "Configuration directory")
+	rootCmd.PersistentFlags().String("hive-dir", DefaultDataDir(), "Hive base directory")
 
 	// Flags for admin init
 	adminInitCmd.Flags().Bool("force", false, "Force re-initialization (overwrites existing config)")
@@ -257,8 +255,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 
 	// Use default config path
 	if cfgFile == "" {
-		homeDir, _ := os.UserHomeDir()
-		cfgFile = fmt.Sprintf("%s/hive/config/hive.toml", homeDir)
+		cfgFile = DefaultConfigFile()
 	}
 
 	//configDir, _ := cmd.Flags().GetString("config-dir")
@@ -564,12 +561,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 	// Default config directory
 	if configDir == "" {
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error getting home directory: %v\n", err)
-			os.Exit(1)
-		}
-		configDir = filepath.Join(homeDir, "hive", "config")
+		configDir = DefaultConfigDir()
 	}
 
 	fmt.Println("🚀 Initializing Hive platform...")
@@ -592,15 +584,18 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 	fmt.Printf("✅ Created config directory: %s\n", configDir)
 
-	// Generate AWS credentials
-	accessKey := admin.GenerateAWSAccessKey()
-	secretKey := admin.GenerateAWSSecretKey()
-	accountID := admin.GenerateAccountID()
-
-	fmt.Println("\n🔑 Generated root credentials (save these — they won't be shown again):")
-	fmt.Printf("   Access Key: %s\n", accessKey)
-	fmt.Printf("   Secret Key: %s\n", secretKey)
-	fmt.Printf("   Account ID: %s\n", accountID)
+	// Generate system credentials (for service-to-service auth in config files)
+	accessKey, err := admin.GenerateAWSAccessKey()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating access key: %v\n", err)
+		os.Exit(1)
+	}
+	secretKey, err := admin.GenerateAWSSecretKey()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating secret key: %v\n", err)
+		os.Exit(1)
+	}
+	accountID := admin.SystemAccountID()
 
 	// Generate IAM master key (AES-256, used to encrypt secrets in NATS KV)
 	masterKey, err := handlers_iam.GenerateMasterKey()
@@ -608,7 +603,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "Error generating IAM master key: %v\n", err)
 		os.Exit(1)
 	}
-	if err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID); err != nil {
+	bootstrapResult, err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID)
+	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
@@ -616,18 +612,27 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
 	fmt.Printf("   Bootstrap: %s\n", filepath.Join(configDir, "bootstrap.json"))
 
+	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
+	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
+	fmt.Printf("   Secret Key:  %s\n", bootstrapResult.AdminSecretKey)
+	fmt.Printf("   Account:     %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID())
+	fmt.Printf("   AWS Profile: hive\n")
+
 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
 	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)
 
 	// Generate NATS token
-	natsToken := admin.GenerateNATSToken()
+	natsToken, err := admin.GenerateNATSToken()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating NATS token: %v\n", err)
+		os.Exit(1)
+	}
 	fmt.Println("\n🔒 Generated NATS authentication token")
 
 	if hiveRoot == "" {
-		// Get home directory for data path
-		homeDir, _ := os.UserHomeDir()
-		hiveRoot = filepath.Join(homeDir, "hive")
+		hiveRoot = DefaultDataDir()
 	}
+	hiveRoot = filepath.Clean(hiveRoot)
 
 	// Determine if this is a multi-node formation
 	isMultiNode := nodes >= 2 && bindIP != "0.0.0.0"
@@ -688,7 +693,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 
 		// Generate multi-node predastore.toml
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -709,6 +714,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		Region:    region,
 		NatsToken: natsToken,
 		DataDir:   hiveRoot,
+		ConfigDir: configDir,
 
 		Node:          node,
 		Az:            az,
@@ -740,9 +746,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Update ~/.aws/credentials and ~/.aws/config
+	// Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system)
 	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP); err != nil {
+	if err := admin.SetupAWSCredentials(bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, certPath, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
 	} else {
 		fmt.Println("✅ AWS credentials configured")
@@ -750,6 +756,16 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 	admin.CreateServiceDirectories(hiveRoot)
 
+	// In production layout (running as root), fix ownership so the service user
+	// can read config and write data. SUDO_USER identifies the operator account.
+	if os.Getuid() == 0 {
+		sudoUser := os.Getenv("SUDO_USER")
+		if sudoUser != "" {
+			admin.ChownRecursive(configDir, sudoUser)
+			admin.ChownRecursive(hiveRoot, sudoUser)
+		}
+	}
+
 	// Print success message
 	fmt.Println("\n🎉 Hive initialization complete!")
 	fmt.Println("\n📋 Next steps:")
@@ -785,7 +801,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		fmt.Fprintf(os.Stderr, "❌ Error generating IAM master key: %v\n", err)
 		os.Exit(1)
 	}
-	if err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID); err != nil {
+	bootstrapResult, err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID)
+	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
@@ -805,12 +822,14 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	}
 
 	creds := &formation.SharedCredentials{
-		AccessKey:   accessKey,
-		SecretKey:   secretKey,
-		AccountID:   accountID,
-		NatsToken:   natsToken,
-		ClusterName: clusterName,
-		Region:      region,
+		AccessKey:      accessKey,
+		SecretKey:      secretKey,
+		AccountID:      accountID,
+		NatsToken:      natsToken,
+		ClusterName:    clusterName,
+		Region:         region,
+		AdminAccessKey: bootstrapResult.AdminAccessKey,
+		AdminSecretKey: bootstrapResult.AdminSecretKey,
 	}
 
 	fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData))
@@ -877,7 +896,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	// Generate multi-node predastore config
 	var predastoreNodeID int
 	if len(predastoreNodes) >= 3 {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -902,6 +921,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		Region:    region,
 		NatsToken: natsToken,
 		DataDir:   hiveRoot,
+		ConfigDir: configDir,
 
 		Node:          node,
 		Az:            az,
@@ -934,9 +954,9 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
-	// Update ~/.aws/credentials and ~/.aws/config
+	// Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system)
 	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP); err != nil {
+	if err := admin.SetupAWSCredentials(bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, certPath, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
 	} else {
 		fmt.Println("✅ AWS credentials configured")
@@ -1006,12 +1026,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	// Set default data directory
 	if dataDir == "" {
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "❌ Error getting home directory: %v\n", err)
-			os.Exit(1)
-		}
-		dataDir = filepath.Join(homeDir, "hive")
+		dataDir = DefaultDataDir()
 	}
 
 	fmt.Println("🚀 Joining Hive cluster...")
@@ -1112,12 +1127,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	// Set up config directory
 	if configDir == "" {
-		homeDir, err := os.UserHomeDir()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error getting home directory: %v\n", err)
-			os.Exit(1)
-		}
-		configDir = filepath.Join(homeDir, "hive", "config")
+		configDir = DefaultConfigDir()
 	}
 
 	if err := os.MkdirAll(configDir, 0700); err != nil {
@@ -1154,7 +1164,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "❌ Error decoding master key: %v\n", err)
 		os.Exit(1)
 	}
-	if err := writeBootstrapFiles(configDir, masterKeyBytes, creds.AccessKey, creds.SecretKey, creds.AccountID); err != nil {
+	if err := writeBootstrapFilesWithAdmin(configDir, masterKeyBytes, creds.AccessKey, creds.SecretKey, creds.AccountID, creds.AdminAccessKey, creds.AdminSecretKey); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
@@ -1194,7 +1204,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	hasPredastoreConfig := len(predastoreNodes) >= 3
 
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1223,6 +1233,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		Region:    creds.Region,
 		NatsToken: creds.NatsToken,
 		DataDir:   dataDir,
+		ConfigDir: configDir,
 
 		Node:          node,
 		Az:            az,
@@ -1256,9 +1267,9 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Update ~/.aws/credentials and ~/.aws/config
+	// Update ~/.aws/credentials and ~/.aws/config with admin credentials (not system)
 	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(creds.AccessKey, creds.SecretKey, creds.Region, caCertPath, bindIP); err != nil {
+	if err := admin.SetupAWSCredentials(creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, caCertPath, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
 	} else {
 		fmt.Println("✅ AWS credentials configured")
@@ -1304,26 +1315,26 @@ func buildRemoteNodes(allNodes map[string]formation.NodeInfo, localNode string)
 
 // initIAMServiceFromConfig loads config, connects to NATS, loads the master
 // key, and returns an initialised IAMServiceImpl. Callers must defer nc.Close().
-func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterConfig, func(), error) {
+func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterConfig, *nats.Conn, func(), error) {
 	cfg, nc, err := loadConfigAndConnect()
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("connect to cluster: %w", err)
+		return nil, nil, nil, nil, fmt.Errorf("connect to cluster: %w", err)
 	}
 
 	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
 	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
 	if err != nil {
 		nc.Close()
-		return nil, nil, nil, fmt.Errorf("load master key: %w", err)
+		return nil, nil, nil, nil, fmt.Errorf("load master key: %w", err)
 	}
 
-	svc, err := handlers_iam.NewIAMServiceImpl(nc, masterKey)
+	svc, err := handlers_iam.NewIAMServiceImpl(nc, masterKey, len(cfg.Nodes))
 	if err != nil {
 		nc.Close()
-		return nil, nil, nil, fmt.Errorf("init IAM service: %w", err)
+		return nil, nil, nil, nil, fmt.Errorf("init IAM service: %w", err)
 	}
 
-	return svc, cfg, func() { nc.Close() }, nil
+	return svc, cfg, nc, func() { nc.Close() }, nil
 }
 
 // adminAccessPolicyDocument is the AdministratorAccess policy document that
@@ -1333,7 +1344,7 @@ const adminAccessPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect
 func runAccountCreate(cmd *cobra.Command, args []string) {
 	name, _ := cmd.Flags().GetString("name")
 
-	svc, cfg, cleanup, err := initIAMServiceFromConfig()
+	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)
@@ -1348,6 +1359,16 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	}
 	accountID := account.AccountID
 
+	// Create default VPC for the new account (belt-and-suspenders: daemon also
+	// does this via iam.account.created event, but daemon may not be running).
+	nodeConfig := cfg.Nodes[cfg.Node]
+	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(&nodeConfig, nc)
+	if vpcErr != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC service: %v\n", vpcErr)
+	} else if vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC: %v\n", vpcErr)
+	}
+
 	// 2. Create admin user
 	_, err = svc.CreateUser(accountID, &iam.CreateUserInput{
 		UserName: aws.String("admin"),
@@ -1393,7 +1414,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 
 	endpointHost := "localhost"
 	certPath := filepath.Join(cfg.NodeBaseDir(), "config", "ca.pem")
-	nodeConfig := cfg.Nodes[cfg.Node]
+	nodeConfig = cfg.Nodes[cfg.Node]
 	if h, _, err := net.SplitHostPort(nodeConfig.AWSGW.Host); err == nil {
 		if h != "" && h != "0.0.0.0" {
 			endpointHost = h
@@ -1410,8 +1431,12 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	}); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not update AWS credentials: %v\n", err)
 	}
+	region := cfg.Nodes[cfg.Node].Region
+	if region == "" {
+		region = "ap-southeast-2"
+	}
 	if err := admin.UpdateAWSINIFile(configPath, "profile "+profileName, map[string]string{
-		"region":       "us-east-1",
+		"region":       region,
 		"endpoint_url": endpointURL,
 		"ca_bundle":    certPath,
 		"output":       "json",
@@ -1432,7 +1457,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 }
 
 func runAccountList(cmd *cobra.Command, args []string) {
-	svc, _, cleanup, err := initIAMServiceFromConfig()
+	svc, _, _, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)
@@ -1461,18 +1486,61 @@ func runAccountList(cmd *cobra.Command, args []string) {
 	}
 }
 
-func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKey, accountID string) error {
+// writeBootstrapResult holds the admin credentials so callers can
+// write them to ~/.aws/credentials instead of the system credentials.
+type writeBootstrapResult struct {
+	AdminAccessKey string
+	AdminSecretKey string
+}
+
+// writeBootstrapFiles generates new admin credentials and writes the bootstrap
+// files (master.key + bootstrap.json). Used by init flows (single and multi-node).
+func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKey, accountID string) (*writeBootstrapResult, error) {
+	adminAccessKey, err := admin.GenerateAWSAccessKey()
+	if err != nil {
+		return nil, fmt.Errorf("generate admin access key: %w", err)
+	}
+	adminSecretKey, err := admin.GenerateAWSSecretKey()
+	if err != nil {
+		return nil, fmt.Errorf("generate admin secret key: %w", err)
+	}
+	if err := writeBootstrapFilesWithAdmin(configDir, masterKey, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey); err != nil {
+		return nil, err
+	}
+	return &writeBootstrapResult{
+		AdminAccessKey: adminAccessKey,
+		AdminSecretKey: adminSecretKey,
+	}, nil
+}
+
+// writeBootstrapFilesWithAdmin writes the bootstrap files using the provided
+// admin credentials. Used by join flows where admin creds come from the leader.
+func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string) error {
 	if err := handlers_iam.SaveMasterKey(filepath.Join(configDir, "master.key"), masterKey); err != nil {
 		return fmt.Errorf("saving master key: %w", err)
 	}
 	encryptedSecret, err := handlers_iam.EncryptSecret(secretKey, masterKey)
 	if err != nil {
-		return fmt.Errorf("encrypting root secret: %w", err)
+		return fmt.Errorf("encrypting system secret: %w", err)
 	}
-	return handlers_iam.SaveBootstrapData(
-		filepath.Join(configDir, "bootstrap.json"),
-		&handlers_iam.BootstrapData{
-			AccessKeyID: accessKey, EncryptedSecret: encryptedSecret, AccountID: accountID,
+
+	adminEncryptedSecret, err := handlers_iam.EncryptSecret(adminSecretKey, masterKey)
+	if err != nil {
+		return fmt.Errorf("encrypting admin secret: %w", err)
+	}
+
+	bd := &handlers_iam.BootstrapData{
+		AccessKeyID:     accessKey,
+		EncryptedSecret: encryptedSecret,
+		AccountID:       accountID,
+		Admin: &handlers_iam.AdminBootstrapData{
+			AccountID:       admin.DefaultAccountID(),
+			AccountName:     admin.DefaultAccountName(),
+			UserName:        "admin",
+			AccessKeyID:     adminAccessKey,
+			EncryptedSecret: adminEncryptedSecret,
 		},
-	)
+	}
+
+	return handlers_iam.SaveBootstrapData(filepath.Join(configDir, "bootstrap.json"), bd)
 }

cmd/hive/cmd/get.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 	"github.com/pterm/pterm"
@@ -65,8 +66,7 @@ func init() {
 func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) {
 	cfgPath := viper.GetString("config")
 	if cfgPath == "" {
-		homeDir, _ := os.UserHomeDir()
-		cfgPath = fmt.Sprintf("%s/hive/config/hive.toml", homeDir)
+		cfgPath = DefaultConfigFile()
 	}
 
 	cfg, err := config.LoadConfig(cfgPath)
@@ -161,9 +161,9 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 	}
 
 	// Parse responses into a map by node name
-	respondedNodes := make(map[string]config.NodeStatusResponse)
+	respondedNodes := make(map[string]types.NodeStatusResponse)
 	for _, data := range responses {
-		var resp config.NodeStatusResponse
+		var resp types.NodeStatusResponse
 		if err := json.Unmarshal(data, &resp); err != nil {
 			continue
 		}
@@ -235,14 +235,14 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 	}
 
 	type vmRow struct {
-		config.VMInfo
+		types.VMInfo
 		Node string
 		Host string
 	}
 
 	var allVMs []vmRow
 	for _, data := range responses {
-		var resp config.NodeVMsResponse
+		var resp types.NodeVMsResponse
 		if err := json.Unmarshal(data, &resp); err != nil {
 			continue
 		}

cmd/hive/cmd/paths.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// DefaultConfigDir returns the default configuration directory.
+// Production: /etc/hive (when running as root or /etc/hive exists)
+// Development: ~/hive/config
+func DefaultConfigDir() string {
+	if isProductionLayout() {
+		return "/etc/hive"
+	}
+	homeDir, _ := os.UserHomeDir()
+	return filepath.Join(homeDir, "hive", "config")
+}
+
+// DefaultDataDir returns the default data directory.
+// Production: /var/lib/hive
+// Development: ~/hive
+func DefaultDataDir() string {
+	if isProductionLayout() {
+		return "/var/lib/hive"
+	}
+	homeDir, _ := os.UserHomeDir()
+	return filepath.Join(homeDir, "hive")
+}
+
+// DefaultConfigFile returns the default path to hive.toml.
+func DefaultConfigFile() string {
+	return filepath.Join(DefaultConfigDir(), "hive.toml")
+}
+
+// isProductionLayout returns true when running in a production install.
+// Detected by: running as root, or /etc/hive directory exists.
+func isProductionLayout() bool {
+	if os.Getuid() == 0 {
+		if info, err := os.Stat("/etc/hive"); err == nil && info.IsDir() {
+			return true
+		}
+	}
+	return false
+}

cmd/hive/cmd/paths_test.go

@@ -0,0 +1,57 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestDefaultConfigDir_Dev(t *testing.T) {
+	// Non-root user without /etc/hive should get dev paths
+	if os.Getuid() == 0 {
+		t.Skip("test requires non-root user")
+	}
+
+	homeDir, _ := os.UserHomeDir()
+	expected := filepath.Join(homeDir, "hive", "config")
+	got := DefaultConfigDir()
+	if got != expected {
+		t.Errorf("DefaultConfigDir() = %q, want %q", got, expected)
+	}
+}
+
+func TestDefaultDataDir_Dev(t *testing.T) {
+	if os.Getuid() == 0 {
+		t.Skip("test requires non-root user")
+	}
+
+	homeDir, _ := os.UserHomeDir()
+	expected := filepath.Join(homeDir, "hive")
+	got := DefaultDataDir()
+	if got != expected {
+		t.Errorf("DefaultDataDir() = %q, want %q", got, expected)
+	}
+}
+
+func TestDefaultConfigFile(t *testing.T) {
+	if os.Getuid() == 0 {
+		t.Skip("test requires non-root user")
+	}
+
+	got := DefaultConfigFile()
+	homeDir, _ := os.UserHomeDir()
+	expected := filepath.Join(homeDir, "hive", "config", "hive.toml")
+	if got != expected {
+		t.Errorf("DefaultConfigFile() = %q, want %q", got, expected)
+	}
+}
+
+func TestIsProductionLayout_NonRoot(t *testing.T) {
+	if os.Getuid() == 0 {
+		t.Skip("test requires non-root user")
+	}
+
+	if isProductionLayout() {
+		t.Error("isProductionLayout() = true for non-root user, want false")
+	}
+}

cmd/hive/cmd/root.go

@@ -116,7 +116,11 @@ func initConfig() {
 	appConfig, err = config.LoadConfig(cfgFile)
 
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Warning: %v\n", err)
-		fmt.Fprintln(os.Stderr, "Continuing with environment variables and defaults...")
+		// If a config file was explicitly provided, treat load failure as fatal
+		if cfgFile != "" {
+			fmt.Fprintf(os.Stderr, "Error: failed to load config %s: %v\n", cfgFile, err)
+			os.Exit(1)
+		}
+		// No config specified — continue with env/defaults (e.g., hive --help)
 	}
 }

cmd/hive/cmd/service.go

@@ -148,7 +148,10 @@ var predastoreStartCmd = &cobra.Command{
 			return
 		}
 
-		service.Start()
+		if _, err := service.Start(); err != nil {
+			fmt.Println("Error starting predastore service:", err)
+			os.Exit(1)
+		}
 
 		fmt.Println("Predastore service started", service)
 	},
@@ -167,7 +170,10 @@ var predastoreStopCmd = &cobra.Command{
 			return
 		}
 
-		service.Stop()
+		if err = service.Stop(); err != nil {
+			fmt.Println("Error stopping predastore service:", err)
+			os.Exit(1)
+		}
 
 		fmt.Println("Predastore service stopped")
 
@@ -201,12 +207,11 @@ var viperblockStartCmd = &cobra.Command{
 
 		// TODO: Support ENV vars, CLI, otherwise revert to config.LoadConfig()
 		clusterConfig, err := config.LoadConfig(cfgFile)
-		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
-
 		if err != nil {
 			fmt.Println("Error loading config file:", err)
 			return
 		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		natsHost := viper.GetString("nats-host")
 
@@ -320,7 +325,10 @@ var viperblockStopCmd = &cobra.Command{
 			return
 		}
 
-		service.Stop()
+		if err = service.Stop(); err != nil {
+			fmt.Println("Error stopping viperblock service:", err)
+			os.Exit(1)
+		}
 
 		fmt.Println("Viperblock service stopped")
 
@@ -364,7 +372,10 @@ var natsStartCmd = &cobra.Command{
 			return
 		}
 
-		service.Start()
+		if _, err = service.Start(); err != nil {
+			fmt.Println("Error starting nats service:", err)
+			os.Exit(1)
+		}
 		fmt.Println("NATS service started")
 	},
 }
@@ -382,7 +393,10 @@ var natsStopCmd = &cobra.Command{
 			return
 		}
 
-		service.Stop()
+		if err = service.Stop(); err != nil {
+			fmt.Println("Error stopping nats service:", err)
+			os.Exit(1)
+		}
 
 		fmt.Println("Nats service stopped")
 	},
@@ -412,12 +426,11 @@ var hiveStartCmd = &cobra.Command{
 
 		// TODO: Support ENV vars, CLI, otherwise revert to config.LoadConfig()
 		clusterConfig, err := config.LoadConfig(cfgFile)
-		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
-
 		if err != nil {
 			fmt.Println("Error loading config file:", err)
 			return
 		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		// Overwrite defaults (CLI first, config second, env third)
 		baseDir := viper.GetString("base-dir")
@@ -450,7 +463,10 @@ var hiveStartCmd = &cobra.Command{
 			hiveSvc.SetConfigPath(cfgFile)
 		}
 
-		svc.Start()
+		if _, err = svc.Start(); err != nil {
+			fmt.Println("Error starting hive service:", err)
+			os.Exit(1)
+		}
 		fmt.Println("HIVE service started")
 	},
 }
@@ -468,7 +484,10 @@ var hiveStopCmd = &cobra.Command{
 			return
 		}
 
-		service.Stop()
+		if err = service.Stop(); err != nil {
+			fmt.Println("Error stopping hive service:", err)
+			os.Exit(1)
+		}
 
 		fmt.Println("Hive service stopped")
 	},
@@ -501,12 +520,11 @@ var awsgwStartCmd = &cobra.Command{
 
 		// TODO: Support ENV vars, CLI, otherwise revert to config.LoadConfig()
 		clusterConfig, err := config.LoadConfig(cfgFile)
-		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
-
 		if err != nil {
 			fmt.Println("Error loading config file:", err)
 			return
 		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		// Overwrite defaults (CLI first, config second, env third)
 		awsgwHost := viper.GetString("host")
@@ -545,7 +563,10 @@ var awsgwStartCmd = &cobra.Command{
 			return
 		}
 
-		service.Start()
+		if _, err = service.Start(); err != nil {
+			fmt.Println("Error starting awsgw service:", err)
+			os.Exit(1)
+		}
 		fmt.Println("AWSGW service started")
 	},
 }
@@ -563,7 +584,10 @@ var awsgwStopCmd = &cobra.Command{
 			return
 		}
 
-		service.Stop()
+		if err = service.Stop(); err != nil {
+			fmt.Println("Error stopping awsgw service:", err)
+			os.Exit(1)
+		}
 
 		fmt.Println("AWSGW service stopped")
 	},
@@ -600,7 +624,10 @@ var hiveUIStartCmd = &cobra.Command{
 			return
 		}
 
-		svc.Start()
+		if _, err = svc.Start(); err != nil {
+			fmt.Println("Error starting hive-ui service:", err)
+			os.Exit(1)
+		}
 		fmt.Println("hive-ui service started")
 	},
 }
@@ -618,7 +645,10 @@ var hiveUIStopCmd = &cobra.Command{
 			return
 		}
 
-		svc.Stop()
+		if err = svc.Stop(); err != nil {
+			fmt.Println("Error stopping hive-ui service:", err)
+			os.Exit(1)
+		}
 		fmt.Println("hive-ui service stopped")
 	},
 }
@@ -677,7 +707,10 @@ var vpcdStartCmd = &cobra.Command{
 			return
 		}
 
-		svc.Start()
+		if _, err = svc.Start(); err != nil {
+			fmt.Println("Error starting vpcd service:", err)
+			os.Exit(1)
+		}
 		fmt.Println("vpcd service started")
 	},
 }
@@ -694,7 +727,10 @@ var vpcdStopCmd = &cobra.Command{
 			return
 		}
 
-		svc.Stop()
+		if err = svc.Stop(); err != nil {
+			fmt.Println("Error stopping vpcd service:", err)
+			os.Exit(1)
+		}
 		fmt.Println("vpcd service stopped")
 	},
 }

cmd/hive/cmd/templates/nats.conf

@@ -2,14 +2,14 @@
 # Generated by Hive admin init
 
 # Server configuration
-{{ if .BindIP }}
+{{- if .BindIP }}
 listen: {{.BindIP}}:4222
-{{ else }}
+{{- else }}
 listen: 0.0.0.0:4222
-{{ end }}
+{{- end }}
 
+{{- if and .ClusterBindIP .ClusterRoutes }}
 
-{{ if and .ClusterBindIP .ClusterRoutes }}
 cluster {
   name: {{ .ClusterName }}
   listen: {{.ClusterBindIP}}:4248
@@ -20,8 +20,7 @@ cluster {
 {{- end }}
   ]
 }
-{{ end }}
-
+{{- end }}
 
 server_name: hive-nats-{{.Node}}
 
@@ -49,9 +48,9 @@ max_payload: 1MB
 max_pending: 64MB
 
 # Monitoring
-{{ if .BindIP }}
+{{- if .BindIP }}
 http: {{.BindIP}}:8222
-{{ else }}
+{{- else }}
 http_port: 8222
-{{ end }}
+{{- end }}
 

cmd/hive/cmd/templates/predastore-multinode.toml

@@ -11,7 +11,8 @@ debug = false
 [rs]
 data = 2
 parity = 1
-{{range .Nodes}}
+{{- range .Nodes}}
+
 [[db]]
 id = {{.ID}}
 host = "{{.Host}}"
@@ -22,14 +23,16 @@ secret_access_key = "{{$.SecretKey}}"
 {{- if eq .ID 1}}
 leader = true
 {{- end}}
-{{end}}
+{{- end}}
 {{- range .Nodes}}
+
 [[nodes]]
 id = {{.ID}}
 host = "{{.Host}}"
 port = 9991
 path = "distributed/nodes/node-{{.ID}}/"
-{{end}}
+{{- end}}
+
 [[buckets]]
 name = "predastore"
 region = "{{.Region}}"
@@ -43,3 +46,10 @@ secret_access_key = "{{.SecretKey}}"
 policy = [
   { bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
 ]
+
+# IAM authentication via NATS KV (enables multi-account S3 access)
+[iam]
+nats_url = "nats://{{.BindIP}}:4222"
+nats_token = "{{.NatsToken}}"
+master_key_path = "{{.ConfigDir}}/master.key"
+access_keys_bucket = "hive-iam-access-keys"

cmd/hive/cmd/templates/predastore.toml

@@ -79,4 +79,11 @@ access_key_id = "{{.AccessKey}}"
 secret_access_key = "{{.SecretKey}}"
 policy = [
   { bucket = "predastore", actions = ["s3:ListBucket",  "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
-]
+]
+
+# IAM authentication via NATS KV (enables multi-account S3 access)
+[iam]
+nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}}:4222"
+nats_token = "{{.NatsToken}}"
+master_key_path = "{{.ConfigDir}}/master.key"
+access_keys_bucket = "hive-iam-access-keys"

cmd/hive/cmd/top.go

@@ -22,7 +22,7 @@ import (
 	"sort"
 	"time"
 
-	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 )
@@ -62,9 +62,9 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	respondedNodes := make(map[string]config.NodeStatusResponse)
+	respondedNodes := make(map[string]types.NodeStatusResponse)
 	for _, data := range responses {
-		var resp config.NodeStatusResponse
+		var resp types.NodeStatusResponse
 		if err := json.Unmarshal(data, &resp); err != nil {
 			continue
 		}

cmd/hive/cmd/version.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/spf13/cobra"
+)
+
+// Version variables set via ldflags at build time.
+// Example: go build -ldflags "-X github.com/mulgadc/hive/cmd/hive/cmd.Version=v1.0.0"
+var (
+	Version = "dev"
+	Commit  = "unknown"
+)
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print the Hive version",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Printf("hive %s (%s) %s/%s\n", Version, Commit, runtime.GOOS, runtime.GOARCH)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}

cmd/hive/cmd/version_test.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"testing"
+)
+
+func TestVersionVarsExist(t *testing.T) {
+	// Verify the version variables are set (defaults when not injected via ldflags)
+	if Version == "" {
+		t.Error("Version should not be empty")
+	}
+	if Commit == "" {
+		t.Error("Commit should not be empty")
+	}
+}
+
+func TestVersionCommand(t *testing.T) {
+	// Verify the version command is registered
+	found := false
+	for _, cmd := range rootCmd.Commands() {
+		if cmd.Use == "version" {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Error("version command not registered on rootCmd")
+	}
+}

docs/README.md

@@ -0,0 +1,3 @@
+# Hive Documentation
+
+This directory contains the Hive documentation. This is available in individual markdown files, and a generated website to aid navigation and browing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)

docs/creating-accounts/README.md

@@ -0,0 +1,61 @@
+---
+title: "Creating Accounts"
+description: "Create and manage isolated user accounts with their own resources."
+category: "Administration"
+tags:
+  - accounts
+  - iam
+  - admin
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+---
+
+# Creating Accounts
+
+> Create and manage isolated user accounts with their own resources.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Hive supports multi-tenant account isolation. Each account gets its own IAM credentials, AWS CLI profile, and isolated resource namespace.
+
+## Instructions
+
+## Create Account
+
+```bash
+./bin/hive admin account create --name myteam
+export AWS_PROFILE=hive-myteam
+```
+
+## Verify
+
+```bash
+aws sts get-caller-identity
+```
+
+## Troubleshooting
+
+## Credentials not working
+
+Verify the AWS CLI configuration files exist and contain the correct profile:
+
+```bash
+cat ~/.aws/config
+cat ~/.aws/credentials
+```
+
+Ensure the `AWS_PROFILE` environment variable matches the profile name:
+
+```bash
+echo $AWS_PROFILE
+export AWS_PROFILE=hive-myteam
+```

docs/creating-an-environment/README.md

@@ -0,0 +1,119 @@
+---
+title: "Creating an Environment"
+description: "End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances."
+category: "Environments"
+tags:
+  - environment
+  - vpc
+  - ec2
+  - ssh
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "Debian Cloud Images"
+    url: "https://cloud.debian.org/images/cloud/bookworm/latest/"
+---
+
+# Creating an Environment
+
+> End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+This guide walks through the complete workflow for creating a production-ready Hive environment.
+
+**Steps:**
+1. Create a user account
+2. Import or create SSH keys
+3. Create a VPC and subnet
+4. Select and import an AMI
+5. Launch EC2 instances
+6. Connect via SSH
+7. Monitor instance health
+
+## Instructions
+
+## 1. Import SSH Key
+
+```bash
+aws ec2 import-key-pair --key-name "hive-key" --public-key-material fileb://~/.ssh/id_rsa.pub
+```
+
+## 2. Create VPC
+
+```bash
+aws ec2 create-vpc --cidr-block 10.200.0.0/16
+export HIVE_VPC="vpc-XXX"
+
+aws ec2 create-subnet --vpc-id $HIVE_VPC --cidr-block 10.200.1.0/24
+export HIVE_SUBNET="subnet-XXX"
+```
+
+## 3. Import AMI
+
+```bash
+./bin/hive admin images list
+./bin/hive admin images import --name debian-12-x86_64
+export HIVE_AMI="ami-XXX"
+```
+
+## 4. Launch Instance
+
+```bash
+aws ec2 run-instances \
+  --image-id $HIVE_AMI \
+  --instance-type t3.small \
+  --key-name hive-key \
+  --subnet-id $HIVE_SUBNET \
+  --count 1
+```
+
+## 5. Connect via SSH
+
+```bash
+ps auxw | grep $INSTANCE_ID
+# Look for: hostfwd=tcp:127.0.0.1:<port>-:22
+
+ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p <port>
+```
+
+## Troubleshooting
+
+## Instance stuck in pending
+
+Check the Hive daemon and QEMU logs for boot errors:
+
+```bash
+ls ~/hive/logs/
+cat ~/hive/logs/daemon.log
+```
+
+Verify the AMI exists and architecture matches your host:
+
+```bash
+aws ec2 describe-images --image-ids $HIVE_AMI
+```
+
+## SSH connection refused
+
+cloud-init takes 30-60 seconds to configure networking and SSH after boot. Wait and retry.
+
+Verify the SSH port forwarding is active:
+
+```bash
+ps auxw | grep hostfwd
+```
+
+Ensure you're using the correct key and port:
+
+```bash
+ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p <port>
+```

docs/hive-admin-cli/README.md

@@ -0,0 +1,97 @@
+---
+title: "Hive Admin CLI"
+description: "Complete reference for the Hive administration CLI. Manage accounts, nodes, VMs, and services."
+category: "Administration"
+tags:
+  - cli
+  - admin
+  - reference
+badge: cli
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+---
+
+# Hive Admin CLI
+
+> Complete reference for the Hive administration CLI.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+The `hive` binary is the central administration tool for managing your Hive infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control.
+
+**Binary location:** `~/Development/mulga/hive/bin/hive`
+
+All services in the Hive platform are managed through this single binary.
+
+## Instructions
+
+## Account Management
+
+```bash
+./bin/hive admin account create --name myteam
+export AWS_PROFILE=hive-myteam
+```
+
+## Node Management
+
+```bash
+./bin/hive get nodes
+```
+
+```
+NAME    STATUS    IP              REGION           AZ               UPTIME   VMs
+node1   Ready     127.0.0.1       ap-southeast-2   ap-southeast-2a  2m       0
+node2   Ready     127.0.0.2       ap-southeast-2   ap-southeast-2a  2m       0
+node3   Ready     127.0.0.3       ap-southeast-2   ap-southeast-2a  2m       0
+```
+
+## Monitor Resources
+
+```bash
+./bin/hive top nodes
+```
+
+## Image Management
+
+```bash
+./bin/hive admin images list
+./bin/hive admin images import --name debian-12-arm64
+```
+
+## Cluster Shutdown
+
+```bash
+./bin/hive admin cluster shutdown
+```
+
+## Troubleshooting
+
+## Permission denied running hive
+
+The binary may not be executable. Fix permissions:
+
+```bash
+chmod +x ./bin/hive
+```
+
+If you get permission errors during operations, ensure you're running with appropriate privileges. Some OVN and networking commands require `sudo`.
+
+## Services fail to start
+
+Check the daemon logs for specific errors:
+
+```bash
+ls ~/hive/logs/
+cat ~/hive/logs/daemon.log
+```
+
+Common causes include port conflicts, missing OVN configuration, or untrusted CA certificates.

docs/hybrid-sync/README.md

@@ -0,0 +1,74 @@
+---
+title: "Hybrid Sync"
+description: "Synchronize data between Hive and AWS when network connectivity is available."
+category: "Migration"
+tags:
+  - hybrid
+  - sync
+  - s3
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+---
+
+# Hybrid Sync
+
+> Synchronize data between Hive and AWS when network connectivity is available.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Hive's hybrid mode enables bidirectional data synchronization between local infrastructure and AWS cloud services. Ideal for intermittent connectivity environments.
+
+## Instructions
+
+## Push Local to Cloud
+
+```bash
+aws s3 sync s3://local-bucket/ s3://cloud-bucket/ \
+  --source-region hive --region us-east-1
+```
+
+## Pull Cloud to Local
+
+```bash
+aws s3 sync s3://cloud-bucket/ s3://local-bucket/ \
+  --source-region us-east-1 --region hive
+```
+
+## EBS Volume Backup
+
+```bash
+rsync -avz /data/ user@cloud-server:/backup/hive-data/
+```
+
+## Troubleshooting
+
+## Sync fails mid-transfer
+
+S3 sync is idempotent — re-run the same command to resume where it left off:
+
+```bash
+aws s3 sync s3://local-bucket/ s3://cloud-bucket/ \
+  --source-region hive --region us-east-1
+```
+
+Only changed or missing files will be transferred on subsequent runs.
+
+## Permissions errors during sync
+
+Verify your AWS credentials are configured for both the source and destination:
+
+```bash
+aws sts get-caller-identity --profile hive
+aws sts get-caller-identity --profile default
+```
+
+Ensure the IAM user has `s3:GetObject`, `s3:PutObject`, and `s3:ListBucket` permissions on both buckets.

docs/installing-hive-airgapped/README.md

@@ -0,0 +1,132 @@
+---
+title: "Air-Gapped Installation"
+description: "Deploy Hive in environments without internet connectivity. Covers offline package caching, USB deployment, and package verification."
+category: "Getting Started"
+tags:
+  - install
+  - air-gapped
+  - offline
+badge: advanced
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "apt-cacher-ng"
+    url: "https://wiki.debian.org/AptCacherNg"
+---
+
+# Air-Gapped Installation
+
+> Deploy Hive in environments without internet connectivity.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+In air-gapped or disconnected environments, Hive can be deployed without internet access. This guide covers preparing offline packages on a connected machine, creating USB deployment media, and installing on the target server with package verification.
+
+**Deployment methods:**
+- **apt-cacher-ng** — Local APT cache for Debian/Ubuntu packages
+- **Go module cache** — Pre-downloaded Go dependencies
+- **USB deployment** — Portable installation media with all dependencies
+- **GPG verification** — Cryptographic package integrity checks
+
+## Instructions
+
+## Step 1. Prepare packages (on a connected machine)
+
+Set up a local APT cache to collect all required packages:
+
+```bash
+sudo apt install apt-cacher-ng
+sudo systemctl enable --now apt-cacher-ng
+```
+
+Pre-download all Hive dependencies:
+
+```bash
+sudo apt install --download-only nbdkit nbdkit-plugin-dev pkg-config \
+  qemu-system qemu-utils qemu-kvm libvirt-daemon-system \
+  libvirt-clients libvirt-dev make gcc unzip xz-utils file \
+  ovn-central ovn-host openvswitch-switch
+```
+
+## Step 2. Create USB deployment media
+
+```bash
+mkdir -p /media/hive-deploy/{apt-packages,go-cache,hive-source}
+cp /var/cache/apt/archives/*.deb /media/hive-deploy/apt-packages/
+cp -r ~/go/pkg/mod/cache/ /media/hive-deploy/go-cache/
+cp -r ~/Development/mulga/hive /media/hive-deploy/hive-source/
+```
+
+## Step 3. Verify package integrity
+
+Sign and verify packages before transferring to the target:
+
+```bash
+gpg --import /media/hive-deploy/mulga-signing-key.asc
+gpg --verify hive-v1.0.tar.gz.sig hive-v1.0.tar.gz
+sha256sum -c /media/hive-deploy/checksums.sha256
+```
+
+## Step 4. Install on the target server
+
+Mount the USB and install:
+
+```bash
+sudo mount /dev/sdb1 /mnt/usb
+sudo dpkg -i /mnt/usb/apt-packages/*.deb
+cp -r /mnt/usb/hive-source ~/Development/mulga/hive
+cd ~/Development/mulga/hive && make build
+```
+
+Then follow the [Source Install](/docs/source-install) guide from Step 3 (Setup OVN) onwards.
+
+## Troubleshooting
+
+## Missing dependencies after dpkg install
+
+Some packages may have unresolved dependencies. Fix with:
+
+```bash
+sudo dpkg -i /mnt/usb/apt-packages/*.deb
+sudo apt-get install -f
+```
+
+The `-f` flag tells apt to fix broken dependencies using what's available locally.
+
+## GPG verification fails
+
+The signing key may not match or the download was corrupted. Re-import the key and verify:
+
+```bash
+gpg --import /media/hive-deploy/mulga-signing-key.asc
+gpg --verify hive-v1.0.tar.gz.sig hive-v1.0.tar.gz
+```
+
+If verification still fails, re-download the package on the connected machine and compare checksums:
+
+```bash
+sha256sum hive-v1.0.tar.gz
+```
+
+## Go module cache errors
+
+The Go module cache may have been incompletely copied. On the connected machine, re-export the full cache:
+
+```bash
+cp -r ~/go/pkg/mod/cache/ /media/hive-deploy/go-cache/
+```
+
+On the target, restore it:
+
+```bash
+mkdir -p ~/go/pkg/mod/
+cp -r /mnt/usb/go-cache ~/go/pkg/mod/cache
+```

docs/installing-hive-multi-node/README.md

@@ -0,0 +1,285 @@
+---
+title: "Multi-Node Installation"
+description: "Deploy Hive across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
+category: "Getting Started"
+tags:
+  - install
+  - multi node
+  - cluster
+badge: availability-zone
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "Viperblock (EBS)"
+    url: "https://github.com/mulgadc/viperblock"
+---
+
+# Multi-Node Installation
+
+> Deploy Hive across multiple servers to create an availability zone.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+A Hive cluster operates as a fully distributed infrastructure region — similar to an AWS Region — where multiple nodes provide high availability, data durability, and fault tolerance.
+
+**Services distributed across nodes:**
+- **NATS** — Clustered message bus for request routing and JetStream replication
+- **Predastore** — Raft consensus with Reed-Solomon erasure coding (RS 2+1)
+- **Viperblock** — Block storage co-located with compute nodes
+- **Hive Daemon** — Independent AWS API request serving per node
+
+Cluster formation is automatic. When initializing with `--nodes 3`, the init node starts a formation server and waits for peers to join. Once all nodes register, each receives the full cluster topology — credentials, CA certificates, NATS routes, Predastore peer lists — no manual configuration synchronization required.
+
+**Network requirements:**
+- Minimum 1 NIC per server
+- 2 NICs recommended for production (management + overlay)
+- UDP port 6081 open between hosts (Geneve tunnels)
+
+## Instructions
+
+## Step 1. Install Hive on each server
+
+Run the binary installer on **every server** in the cluster:
+
+```bash
+curl https://install.mulgadc.com/ | bash
+```
+
+## Step 2. Set node IP variables
+
+On **each server**, export the IPs for all nodes (replace with your actual IPs):
+
+```bash
+export HIVE_NODE1=192.168.1.10
+export HIVE_NODE2=192.168.1.11
+export HIVE_NODE3=192.168.1.12
+```
+
+To find your server's IP:
+
+```bash
+ip addr show | grep "inet "
+```
+
+## Step 3. Setup OVN networking
+
+OVN must be configured on every server before forming the cluster. Server 1 runs OVN central (NB and SB databases) and must be set up first.
+
+**Server 1 — OVN central + compute (run first):**
+
+```bash
+hive setup-ovn --management --encap-ip=$HIVE_NODE1
+```
+
+Verify OVN central is ready before proceeding:
+
+```bash
+sudo ovn-sbctl show
+```
+
+**Server 2 — Compute node (after server 1 is ready):**
+
+```bash
+hive setup-ovn --ovn-remote=tcp:$HIVE_NODE1:6642 --encap-ip=$HIVE_NODE2
+```
+
+**Server 3 — Compute node (after server 1 is ready):**
+
+```bash
+hive setup-ovn --ovn-remote=tcp:$HIVE_NODE1:6642 --encap-ip=$HIVE_NODE3
+```
+
+Verify all chassis have registered (from server 1):
+
+```bash
+sudo ovn-sbctl show
+```
+
+You should see 3 chassis entries, one per server, each with a Geneve encap IP.
+
+## Step 4. Form the cluster
+
+The formation process requires running init and join commands concurrently. The init node blocks until all expected nodes have joined.
+
+**Server 1 — Initialize the cluster:**
+
+```bash
+hive admin init \
+  --node node1 \
+  --nodes 3 \
+  --bind $HIVE_NODE1 \
+  --cluster-bind $HIVE_NODE1 \
+  --port 4432 \
+  --hive-dir ~/hive/ \
+  --config-dir ~/hive/config/ \
+  --region us-east-1 \
+  --az us-east-1a
+```
+
+**Server 2 — Join the cluster (while init is running):**
+
+```bash
+hive admin join \
+  --node node2 \
+  --bind $HIVE_NODE2 \
+  --cluster-bind $HIVE_NODE2 \
+  --host $HIVE_NODE1:4432 \
+  --data-dir ~/hive/ \
+  --config-dir ~/hive/config/ \
+  --region us-east-1 \
+  --az us-east-1a
+```
+
+**Server 3 — Join the cluster (while init is running):**
+
+```bash
+hive admin join \
+  --node node3 \
+  --bind $HIVE_NODE3 \
+  --cluster-bind $HIVE_NODE3 \
+  --host $HIVE_NODE1:4432 \
+  --data-dir ~/hive/ \
+  --config-dir ~/hive/config/ \
+  --region us-east-1 \
+  --az us-east-1a
+```
+
+All three processes will exit with a cluster summary once formation is complete.
+
+## Step 5. Start services and verify
+
+Start services on **all servers**:
+
+```bash
+hive start
+```
+
+From any node, verify the cluster:
+
+```bash
+hive get nodes
+```
+
+```
+NAME    STATUS    IP              REGION      AZ           UPTIME   VMs
+node1   Ready     192.168.1.10    us-east-1   us-east-1a   2m       0
+node2   Ready     192.168.1.11    us-east-1   us-east-1a   2m       0
+node3   Ready     192.168.1.12    us-east-1   us-east-1a   2m       0
+```
+
+## Step 6. Trust the CA certificate
+
+On **each server**, trust the CA generated during init:
+
+```bash
+sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
+sudo update-ca-certificates
+export AWS_PROFILE=hive
+```
+
+## Step 7. Launch instances across the cluster
+
+Connect to any node's AWS Gateway — all nodes serve the same cluster state:
+
+```bash
+aws ec2 run-instances \
+  --image-id $HIVE_AMI \
+  --instance-type t3.small \
+  --key-name hive-key \
+  --subnet-id $HIVE_SUBNET \
+  --count 3
+```
+
+Hive distributes instances across nodes based on available capacity. Check which node each instance landed on:
+
+```bash
+hive get vms
+```
+
+## Shutdown
+
+A graceful shutdown coordinates all nodes — drains VMs, flushes storage, and stops services:
+
+```bash
+hive admin cluster shutdown
+```
+
+## Troubleshooting
+
+## Nodes not joining the cluster
+
+The init command must still be running when you execute join on other servers. If init has already exited, re-run it.
+
+Check that all servers can reach the init node on the formation port:
+
+```bash
+curl -s http://$HIVE_NODE1:4432/health
+```
+
+## OVN chassis not registering
+
+Compute nodes need to reach OVN central on server 1. Verify from server 1:
+
+```bash
+sudo ovn-sbctl show
+```
+
+If chassis are missing, check firewall rules for port 6642:
+
+```bash
+sudo ss -tlnp | grep 6642
+```
+
+Re-run the OVN setup on the affected compute node.
+
+## Cross-host VMs cannot communicate
+
+This indicates a Geneve tunnel issue. Verify tunnels are established:
+
+```bash
+sudo ovs-vsctl show | grep -i geneve
+```
+
+Ensure UDP port 6081 is open between all hosts:
+
+```bash
+sudo ss -ulnp | grep 6081
+```
+
+If tunnels are missing, verify `--encap-ip` was set correctly during OVN setup.
+
+## Health check fails on a node
+
+Check if the daemon is running and listening on the correct IP:
+
+```bash
+curl -s http://$HIVE_NODE1:4432/health
+```
+
+Review the daemon logs for errors:
+
+```bash
+cat ~/hive/logs/daemon.log
+```
+
+Verify `--bind` was set to the correct IP during cluster formation.
+
+## NATS cluster routing issues
+
+Check NATS logs for connection errors between nodes:
+
+```bash
+grep -i "route\|cluster" ~/hive/logs/nats.log
+```
+
+Ensure `--cluster-bind` IPs are reachable between all servers.

docs/installing-hive/README.md

@@ -0,0 +1,198 @@
+---
+title: "Installing Hive"
+description: "Install Hive on a single server using the binary installer. Get a working region with EC2, VPC, EBS, and S3 in minutes."
+category: "Getting Started"
+tags:
+  - install
+  - single node
+  - quickstart
+badge: quickstart
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "Viperblock (EBS)"
+    url: "https://github.com/mulgadc/viperblock"
+---
+
+# Installing Hive
+
+> Install Hive on a single server using the binary installer.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Hive is an open-source infrastructure platform that brings core AWS services — EC2, VPC, EBS, and S3 — to bare-metal, edge, and on-prem environments.
+
+This guide installs Hive on a single server using the binary installer. For multi-server clusters, see [Multi-Node Installation](/docs/installing-hive-multi-node). To build from source, see [Source Install](/docs/source-install).
+
+**Supported Operating Systems:**
+- Ubuntu 22.04 / 24.04 / 25.10
+- Debian 12.13
+
+**What gets installed:**
+- Hive daemon and CLI
+- QEMU/KVM for virtual machine management
+- OVN/Open vSwitch for VPC networking
+- Predastore (S3-compatible object storage)
+- Viperblock (EBS-compatible block storage)
+- AWS CLI v2
+
+## Instructions
+
+## Step 1. Install Hive
+
+```bash
+curl https://install.mulgadc.com/ | bash
+```
+
+The installer downloads the Hive binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI).
+
+## Step 2. Initialize a region
+
+Create your first region and availability zone:
+
+```bash
+hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+```
+
+## Step 3. Trust the CA certificate
+
+Hive generates a local CA during initialization. Add it to your system trust store:
+
+```bash
+sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
+sudo update-ca-certificates
+```
+
+## Step 4. Start services
+
+```bash
+hive start
+```
+
+Set the AWS profile to use the default admin account:
+
+```bash
+export AWS_PROFILE=hive
+```
+
+## Step 5. Import an AMI
+
+List available images and import one matching your architecture:
+
+```bash
+hive admin images list
+hive admin images import --name debian-12-arm64
+```
+
+## Step 6. Create a VPC and launch an instance
+
+```bash
+aws ec2 create-vpc --cidr-block 10.200.0.0/16
+export HIVE_VPC="vpc-XXX"
+
+aws ec2 create-subnet --vpc-id $HIVE_VPC --cidr-block 10.200.1.0/24
+export HIVE_SUBNET="subnet-XXX"
+
+aws ec2 import-key-pair --key-name "hive-key" --public-key-material fileb://~/.ssh/id_rsa.pub
+
+aws ec2 run-instances \
+  --image-id $HIVE_AMI \
+  --instance-type t3.small \
+  --key-name hive-key \
+  --subnet-id $HIVE_SUBNET \
+  --count 1
+```
+
+## Step 7. Connect via SSH
+
+Find the forwarded SSH port and connect:
+
+```bash
+ps auxw | grep hostfwd
+# Look for: hostfwd=tcp:127.0.0.1:<port>-:22
+
+ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p <port>
+```
+
+## Troubleshooting
+
+## hive command not found
+
+The binary is not in your PATH. Run the installer again or add the install directory to your PATH:
+
+```bash
+export PATH=$PATH:~/hive/bin/
+```
+
+Add this to your `~/.bashrc` or `~/.zshrc` for persistence.
+
+## OVN services not starting
+
+Check if the OVN controller is active:
+
+```bash
+sudo systemctl is-active ovn-controller
+sudo ovn-sbctl show
+```
+
+If inactive, re-run the OVN setup that the installer performs. Check system logs for details:
+
+```bash
+journalctl -u ovn-controller --no-pager -n 20
+```
+
+## CA certificate not trusted
+
+AWS CLI will reject HTTPS connections if the Hive CA is not trusted. Re-add the certificate:
+
+```bash
+sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
+sudo update-ca-certificates
+```
+
+Verify it was added:
+
+```bash
+ls -la /usr/local/share/ca-certificates/hive-ca.crt
+```
+
+## Instance stuck in pending
+
+Check the Hive daemon logs for errors:
+
+```bash
+ls ~/hive/logs/
+cat ~/hive/logs/daemon.log
+```
+
+Verify the AMI was imported successfully:
+
+```bash
+aws ec2 describe-images
+```
+
+## SSH connection refused
+
+cloud-init takes 30-60 seconds to configure the instance after boot. Wait and retry.
+
+Verify the SSH port forwarding is active:
+
+```bash
+ps auxw | grep hostfwd
+```
+
+If no `hostfwd` entry appears, the instance may not have started correctly. Check instance state:
+
+```bash
+aws ec2 describe-instances --instance-ids $INSTANCE_ID
+```

docs/launching-instances/README.md

@@ -0,0 +1,97 @@
+---
+title: "Launching Instances"
+description: "Launch, manage, and connect to EC2-compatible virtual machines on Hive."
+category: "Environments"
+tags:
+  - ec2
+  - instances
+  - vm
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "AWS EC2 CLI"
+    url: "https://docs.aws.amazon.com/cli/latest/reference/ec2/"
+---
+
+# Launching Instances
+
+> Launch, manage, and connect to EC2-compatible virtual machines on Hive.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Hive provides EC2-compatible VM management built on QEMU/KVM. Instances support cloud-init, SSH key injection, VPC networking, and standard AWS lifecycle operations.
+
+**Supported operations:**
+- `run-instances` — Launch new VMs
+- `describe-instances` — Query state
+- `stop-instances` / `start-instances` — Lifecycle
+- `terminate-instances` — Permanent removal
+
+## Instructions
+
+## Launch
+
+```bash
+aws ec2 run-instances \
+  --image-id $HIVE_AMI \
+  --instance-type t3.small \
+  --key-name hive-key \
+  --subnet-id $HIVE_SUBNET \
+  --count 1
+
+export INSTANCE_ID="i-XXX"
+```
+
+## Manage
+
+```bash
+aws ec2 describe-instances --instance-ids $INSTANCE_ID
+aws ec2 stop-instances --instance-ids $INSTANCE_ID
+aws ec2 start-instances --instance-ids $INSTANCE_ID
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+```
+
+## SSH (Development)
+
+```bash
+ps auxw | grep $INSTANCE_ID
+ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p <port>
+```
+
+## Troubleshooting
+
+## Instance fails to boot
+
+Check QEMU logs for the instance and verify the AMI architecture matches your host:
+
+```bash
+cat ~/hive/logs/daemon.log
+aws ec2 describe-images --image-ids $HIVE_AMI
+```
+
+If the AMI is for a different architecture (e.g. arm64 on an x86_64 host), import the correct image:
+
+```bash
+./bin/hive admin images list
+./bin/hive admin images import --name debian-12-x86_64
+```
+
+## Cannot SSH into instance
+
+cloud-init needs time to configure the instance after boot. Wait 30-60 seconds and retry.
+
+Verify the SSH key was specified correctly when launching:
+
+```bash
+aws ec2 describe-instances --instance-ids $INSTANCE_ID
+```
+
+Check the `KeyName` field matches the key you're using to connect.

docs/moving-aws-workload/README.md

@@ -0,0 +1,102 @@
+---
+title: "Moving an AWS Workload to Mulga"
+description: "Migrate existing AWS workloads to Hive using compatible APIs, SDKs, and Terraform."
+category: "Migration"
+tags:
+  - migration
+  - aws
+  - terraform
+badge: new
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "AWS CLI Reference"
+    url: "https://docs.aws.amazon.com/cli/latest/reference/"
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+---
+
+# Moving an AWS Workload to Mulga
+
+> Migrate existing AWS workloads to Hive using compatible APIs, SDKs, and Terraform.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Hive provides drop-in compatibility with AWS APIs, making it possible to migrate existing workloads with minimal changes.
+
+**Supported Services:** VPC, EC2, EBS, S3, IAM
+
+**Compatible Tools:** AWS CLI, AWS SDKs, Terraform, any S3-compatible client
+
+## Instructions
+
+## Configure AWS CLI
+
+```bash
+export AWS_PROFILE=hive
+aws ec2 describe-instances
+aws s3 ls
+```
+
+## Terraform
+
+```hcl
+provider "aws" {
+  region     = "ap-southeast-2"
+  access_key = "your-hive-access-key"
+  secret_key = "your-hive-secret-key"
+
+  endpoints {
+    ec2 = "https://localhost:9999"
+    s3  = "https://localhost:8443"
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+}
+```
+
+## Hybrid S3 Sync
+
+```bash
+aws s3 sync s3://local-bucket/ s3://cloud-bucket/ --source-region hive --region us-east-1
+```
+
+## Troubleshooting
+
+## Terraform provider errors
+
+Ensure all three skip flags are set in your provider configuration:
+
+```hcl
+skip_credentials_validation = true
+skip_metadata_api_check     = true
+skip_requesting_account_id  = true
+```
+
+Without these, Terraform will try to validate credentials and metadata against real AWS endpoints.
+
+## S3 signature errors
+
+Hive uses AWS Signature V4. Ensure your AWS CLI is version 2.0 or higher:
+
+```bash
+aws --version
+```
+
+If using an older version, upgrade:
+
+```bash
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+sudo ./aws/install --update
+```

docs/source-install/README.md

@@ -0,0 +1,194 @@
+---
+title: "Source Install"
+description: "Build Hive from source for development, custom builds, or contributing. Includes dependency setup, compilation, and OVN configuration."
+category: "Getting Started"
+tags:
+  - install
+  - source
+  - development
+badge: source
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "Viperblock (EBS)"
+    url: "https://github.com/mulgadc/viperblock"
+  - title: "Go Downloads"
+    url: "https://go.dev/dl/"
+  - title: "AWS CLI Install"
+    url: "https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
+---
+
+# Source Install
+
+> Build Hive from source for development, custom builds, or contributing.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+This guide covers building Hive from source code. Use this if you want to contribute to Hive, build custom versions, or inspect the codebase. For production deployments, the [binary installer](/docs/installing-hive) is recommended.
+
+**Supported Operating Systems:**
+- Ubuntu 22.04 / 24.04 / 25.10
+- Debian 12.13
+
+**Requirements:**
+- Go 1.26.1+
+- GCC, make, pkg-config
+- QEMU/KVM
+- OVN/Open vSwitch
+- AWS CLI v2
+
+## Instructions
+
+## Step 1. Install system dependencies
+
+### Quick Install
+
+Bootstrap all dependencies in one step:
+
+```bash
+sudo make -C hive quickinstall
+```
+
+Ensure Go is in your PATH:
+
+```bash
+export PATH=$PATH:/usr/local/go/bin/
+```
+
+### Manual Install
+
+Alternatively, install packages individually:
+
+```bash
+sudo add-apt-repository universe
+sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file ovn-central ovn-host openvswitch-switch
+```
+
+Install Go 1.26.1+ from [https://go.dev/dl/](https://go.dev/dl/) and verify:
+
+```bash
+go version
+```
+
+Install AWS CLI v2:
+
+```bash
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+sudo ./aws/install
+aws --version
+```
+
+## Step 2. Clone and build
+
+```bash
+mkdir -p ~/Development/mulga/
+cd ~/Development/mulga/
+git clone https://github.com/mulgadc/hive.git
+cd hive
+./scripts/clone-deps.sh    # Clone viperblock + predastore
+./scripts/dev-setup.sh     # Setup complete dev environment
+```
+
+Confirm `./bin/hive` exists and is executable.
+
+## Step 3. Setup OVN
+
+For a single-node dev environment:
+
+```bash
+./scripts/setup-ovn.sh --management
+```
+
+This creates the `br-int` integration bridge, starts `ovn-controller`, configures Geneve tunnel endpoints, and enables IP forwarding.
+
+## Step 4. Initialize
+
+Create a region and availability zone:
+
+```bash
+./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+```
+
+## Step 5. Trust the CA certificate
+
+```bash
+sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
+sudo update-ca-certificates
+```
+
+## Step 6. Start services
+
+```bash
+./scripts/start-dev.sh
+export AWS_PROFILE=hive
+```
+
+## Troubleshooting
+
+## Go not found in PATH
+
+After installing Go, it may not be in your current shell session. Add it:
+
+```bash
+export PATH=$PATH:/usr/local/go/bin/
+```
+
+Add this to your `~/.bashrc` or `~/.zshrc` for persistence. Verify with:
+
+```bash
+go version
+```
+
+## ./bin/hive missing after build
+
+The build did not complete successfully. Re-run the setup:
+
+```bash
+./scripts/dev-setup.sh
+```
+
+Check the output for compilation errors. Common causes are missing Go version or missing system packages.
+
+## OVN services not starting
+
+Verify the setup script was run with `--management`:
+
+```bash
+./scripts/setup-ovn.sh --management
+```
+
+Check OVN controller status:
+
+```bash
+sudo systemctl is-active ovn-controller
+sudo ovn-sbctl show
+```
+
+## CA certificate not trusted
+
+AWS CLI will reject HTTPS connections to Hive services. Re-add the certificate:
+
+```bash
+sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt
+sudo update-ca-certificates
+```
+
+## clone-deps.sh fails
+
+Ensure you have access to the GitHub repositories. If using SSH and it fails, try HTTPS:
+
+```bash
+git config --global url."https://github.com/".insteadOf "git@github.com:"
+./scripts/clone-deps.sh
+```

docs/vpc-networking/README.md

@@ -0,0 +1,97 @@
+---
+title: "VPC Networking"
+description: "Create and manage Virtual Private Clouds with subnets, DHCP, and Geneve overlay networking."
+category: "Environments"
+tags:
+  - vpc
+  - networking
+  - ovn
+resources:
+  - title: "Hive Repository"
+    url: "https://github.com/mulgadc/hive"
+  - title: "OVN Architecture"
+    url: "https://www.ovn.org/en/"
+---
+
+# VPC Networking
+
+> Create and manage Virtual Private Clouds with subnets, DHCP, and Geneve overlay networking.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Every EC2 instance runs inside a VPC with an isolated virtual network. Hive uses OVN (Open Virtual Network) to provide the networking layer.
+
+**How it works:**
+- VPC becomes an OVN logical router
+- Subnet becomes an OVN logical switch with DHCP
+- Each instance gets an OVN port with automatic IP assignment
+- Cross-host traffic uses Geneve tunnels (UDP 6081)
+
+## Instructions
+
+## Create VPC
+
+```bash
+aws ec2 create-vpc --cidr-block 10.200.0.0/16
+export HIVE_VPC="vpc-XXX"
+```
+
+## Create Subnet
+
+```bash
+aws ec2 create-subnet --vpc-id $HIVE_VPC --cidr-block 10.200.1.0/24
+export HIVE_SUBNET="subnet-XXX"
+```
+
+## Verify
+
+```bash
+aws ec2 describe-vpcs
+aws ec2 describe-subnets
+sudo ovn-nbctl lr-list
+sudo ovn-nbctl ls-list
+```
+
+## Troubleshooting
+
+## VPC creation fails
+
+Ensure OVN services are running and the vpcd daemon is active:
+
+```bash
+sudo systemctl is-active ovn-controller
+```
+
+Check the vpcd logs for errors:
+
+```bash
+cat ~/hive/logs/vpcd.log
+```
+
+## Instances cannot reach each other
+
+This typically means Geneve tunnels are not established between hosts. Verify tunnel configuration:
+
+```bash
+sudo ovs-vsctl show | grep -i geneve
+```
+
+Ensure UDP port 6081 is open between all hosts:
+
+```bash
+sudo ss -ulnp | grep 6081
+```
+
+From inside a VM, check that the private IP was assigned via DHCP:
+
+```bash
+ip addr show
+```

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/hive
 
-go 1.26.0
+go 1.26.1
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
@@ -8,8 +8,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.0.2
-	github.com/mulgadc/viperblock v1.0.16
+	github.com/mulgadc/predastore v1.0.3
+	github.com/mulgadc/viperblock v1.0.17
 	github.com/nats-io/nats-server/v2 v2.12.4
 	github.com/nats-io/nats.go v1.49.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1

go.sum

@@ -291,10 +291,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mulgadc/predastore v1.0.2 h1:7BRE5a0geTShZGHWaXAdkrH3MfhcJ5KnRmhY5V8P5Hw=
-github.com/mulgadc/predastore v1.0.2/go.mod h1:RQyt9BROgyPfqC6TU69PuZ7RlHRDzZr11RYHL7eQwgk=
-github.com/mulgadc/viperblock v1.0.16 h1:eMICuMGjsHqmKsBLDBYwdWzoncPBCHCoPFY8f5VtM00=
-github.com/mulgadc/viperblock v1.0.16/go.mod h1:qck/WYWiXjGRJzoNM99JsFX/z00sZOmf9PM7umKYq3w=
+github.com/mulgadc/predastore v1.0.3 h1:qkuOEUH9B98QzVByn5osKeXm7SSPgk+ardQ3MibPSok=
+github.com/mulgadc/predastore v1.0.3/go.mod h1:x3Z6Ts/PONpmVx/R6t/hkHDJ3+0xRieYS3BuuVaNgyA=
+github.com/mulgadc/viperblock v1.0.17 h1:bGRVlXgcqnmH+Xxon03W/ZsbXqu/EE5+/VG3J4E3EBU=
+github.com/mulgadc/viperblock v1.0.17/go.mod h1:w9V0DjUH5hMXbXsiLbDr9lpqzXBGhEiYB1BeV6TYBkw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

hive/admin/admin.go

@@ -8,11 +8,14 @@ import (
 	"encoding/base64"
 	"encoding/pem"
 	"fmt"
+	"io/fs"
 	"log/slog"
 	"math/big"
 	"net"
 	"os"
+	"os/user"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -37,6 +40,7 @@ type ConfigSettings struct {
 	Region    string
 	NatsToken string
 	DataDir   string
+	ConfigDir string
 
 	// Add more fields as needed
 	Node   string
@@ -204,6 +208,42 @@ func FileExists(path string) bool {
 	return err == nil
 }
 
+// ChownRecursive changes ownership of a path and all its contents to the
+// specified username. Used after init to hand production directories back
+// to the service user when init runs as root via sudo.
+// Best-effort: errors are logged but do not halt the operation.
+func ChownRecursive(path, username string) {
+	u, err := user.Lookup(username)
+	if err != nil {
+		return
+	}
+	uid, _ := strconv.Atoi(u.Uid)
+	gid, _ := strconv.Atoi(u.Gid)
+
+	// Use os.Root to scope filesystem operations and avoid symlink TOCTOU races.
+	// Falls back to direct chown if os.Root is not available.
+	root, rootErr := os.OpenRoot(path)
+	if rootErr != nil {
+		// Fallback: direct chown on the top-level path only
+		if chownErr := os.Chown(path, uid, gid); chownErr != nil { // #nosec G122
+			slog.Debug("chown failed", "path", path, "err", chownErr)
+		}
+		return
+	}
+	defer root.Close()
+
+	_ = fs.WalkDir(root.FS(), ".", func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil
+		}
+		fullPath := filepath.Join(path, p)
+		if chownErr := os.Lchown(fullPath, uid, gid); chownErr != nil {
+			slog.Debug("chown failed", "path", fullPath, "err", chownErr)
+		}
+		return nil
+	})
+}
+
 // updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs
 func UpdateAWSINIFile(path, section string, values map[string]string) error {
 	var cfg *ini.File
@@ -240,44 +280,57 @@ func UpdateAWSINIFile(path, section string, values map[string]string) error {
 
 // generateAWSAccessKey generates an AWS-style access key
 // Format: AKIA + 16 random uppercase alphanumeric characters
-func GenerateAWSAccessKey() string {
+func GenerateAWSAccessKey() (string, error) {
 	const prefix = "AKIA"
 	const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 	const length = 16
 
 	result := make([]byte, length)
 	for i := range result {
-		num, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charset))))
+		num, err := rand.Int(rand.Reader, big.NewInt(int64(len(charset))))
+		if err != nil {
+			return "", fmt.Errorf("crypto/rand failure: %w", err)
+		}
 		result[i] = charset[num.Int64()]
 	}
 
-	return prefix + string(result)
+	return prefix + string(result), nil
 }
 
 // generateAWSSecretKey generates an AWS-style secret key
 // 40 character base64-encoded string
-func GenerateAWSSecretKey() string {
+func GenerateAWSSecretKey() (string, error) {
 	bytes := make([]byte, 30) // 30 bytes = 40 chars in base64
 	if _, err := rand.Read(bytes); err != nil {
-		panic(err)
+		return "", fmt.Errorf("crypto/rand failure: %w", err)
 	}
-	return base64.StdEncoding.EncodeToString(bytes)
+	return base64.StdEncoding.EncodeToString(bytes), nil
 }
 
-// GenerateAccountID returns the global platform account ID (000000000000).
-// This is used during bootstrap (hive admin init) for the root/platform account.
-// Customer accounts are created via IAMService.CreateAccount() with sequential IDs.
-func GenerateAccountID() string {
+// SystemAccountID returns the system/root account ID (000000000000).
+// Used for service-to-service auth credentials baked into config files.
+func SystemAccountID() string {
 	return "000000000000"
 }
 
-// generateNATSToken generates a secure random token for NATS
-func GenerateNATSToken() string {
+// DefaultAccountID returns the default admin account ID (000000000001).
+// This is the first human-facing account created during bootstrap.
+func DefaultAccountID() string {
+	return "000000000001"
+}
+
+// DefaultAccountName returns the default admin account name ("hive").
+func DefaultAccountName() string {
+	return "hive"
+}
+
+// GenerateNATSToken generates a secure random token for NATS
+func GenerateNATSToken() (string, error) {
 	bytes := make([]byte, 32)
 	if _, err := rand.Read(bytes); err != nil {
-		panic(err)
+		return "", fmt.Errorf("crypto/rand failure: %w", err)
 	}
-	return "nats_" + base64.URLEncoding.EncodeToString(bytes)[:32]
+	return "nats_" + base64.URLEncoding.EncodeToString(bytes)[:32], nil
 }
 
 // GenerateCACert generates a Certificate Authority certificate and key
@@ -600,7 +653,7 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string)
 // GenerateMultiNodePredastoreConfig produces a complete predastore.toml for a
 // multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
 // and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
-func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region string) (string, error) {
+func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, bindIP string) (string, error) {
 	if len(nodes) < 3 {
 		return "", fmt.Errorf("multi-node predastore requires at least 3 nodes, got %d", len(nodes))
 	}
@@ -610,7 +663,10 @@ func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNod
 		AccessKey string
 		SecretKey string
 		Region    string
-	}{nodes, accessKey, secretKey, region}
+		NatsToken string
+		ConfigDir string
+		BindIP    string
+	}{nodes, accessKey, secretKey, region, natsToken, configDir, bindIP}
 
 	tmpl, err := template.New("predastore-multinode").Parse(templateStr)
 	if err != nil {

hive/admin/admin_test.go

@@ -18,7 +18,8 @@ import (
 // --- Key / Token generation ---
 
 func TestGenerateAWSAccessKey_Format(t *testing.T) {
-	key := GenerateAWSAccessKey()
+	key, err := GenerateAWSAccessKey()
+	assert.NoError(t, err)
 	assert.Len(t, key, 20)
 	assert.True(t, strings.HasPrefix(key, "AKIA"))
 	for _, c := range key[4:] {
@@ -28,32 +29,44 @@ func TestGenerateAWSAccessKey_Format(t *testing.T) {
 }
 
 func TestGenerateAWSAccessKey_Uniqueness(t *testing.T) {
-	assert.NotEqual(t, GenerateAWSAccessKey(), GenerateAWSAccessKey())
+	k1, err := GenerateAWSAccessKey()
+	assert.NoError(t, err)
+	k2, err := GenerateAWSAccessKey()
+	assert.NoError(t, err)
+	assert.NotEqual(t, k1, k2)
 }
 
 func TestGenerateAWSSecretKey_Format(t *testing.T) {
-	key := GenerateAWSSecretKey()
+	key, err := GenerateAWSSecretKey()
+	assert.NoError(t, err)
 	assert.Len(t, key, 40)
-	_, err := base64.StdEncoding.DecodeString(key)
+	_, err = base64.StdEncoding.DecodeString(key)
 	assert.NoError(t, err, "secret key should be valid base64")
 }
 
 func TestGenerateAWSSecretKey_Uniqueness(t *testing.T) {
-	assert.NotEqual(t, GenerateAWSSecretKey(), GenerateAWSSecretKey())
+	k1, err := GenerateAWSSecretKey()
+	assert.NoError(t, err)
+	k2, err := GenerateAWSSecretKey()
+	assert.NoError(t, err)
+	assert.NotEqual(t, k1, k2)
 }
 
-func TestGenerateAccountID_ReturnsGlobalID(t *testing.T) {
-	id := GenerateAccountID()
+func TestSystemAccountID(t *testing.T) {
+	id := SystemAccountID()
 	assert.Equal(t, "000000000000", id)
 	assert.Len(t, id, 12)
 }
 
-func TestGenerateAccountID_Deterministic(t *testing.T) {
-	assert.Equal(t, GenerateAccountID(), GenerateAccountID())
+func TestDefaultAccountID(t *testing.T) {
+	id := DefaultAccountID()
+	assert.Equal(t, "000000000001", id)
+	assert.Len(t, id, 12)
 }
 
 func TestGenerateNATSToken_Format(t *testing.T) {
-	token := GenerateNATSToken()
+	token, err := GenerateNATSToken()
+	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(token, "nats_"))
 	assert.Len(t, token, 37) // 5 prefix + 32 random
 	// URL-safe base64: no '+' or '/'
@@ -62,7 +75,11 @@ func TestGenerateNATSToken_Format(t *testing.T) {
 }
 
 func TestGenerateNATSToken_Uniqueness(t *testing.T) {
-	assert.NotEqual(t, GenerateNATSToken(), GenerateNATSToken())
+	t1, err := GenerateNATSToken()
+	assert.NoError(t, err)
+	t2, err := GenerateNATSToken()
+	assert.NoError(t, err)
+	assert.NotEqual(t, t1, t2)
 }
 
 // --- Config file generation ---
@@ -523,7 +540,7 @@ host = "{{.Host}}"
 		{ID: 3, Host: "10.0.0.3"},
 	}
 
-	result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1")
+	result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
 	require.NoError(t, err)
 	assert.Contains(t, result, `host = "10.0.0.1"`)
 	assert.Contains(t, result, `host = "10.0.0.3"`)
@@ -535,7 +552,7 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
 	_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
 		{ID: 1, Host: "10.0.0.1"},
 		{ID: 2, Host: "10.0.0.2"},
-	}, "AK", "SK", "us-east-1")
+	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "at least 3 nodes")
 }
@@ -543,7 +560,7 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
 func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
 	_, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []PredastoreNodeConfig{
 		{ID: 1, Host: "a"}, {ID: 2, Host: "b"}, {ID: 3, Host: "c"},
-	}, "AK", "SK", "us-east-1")
+	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "failed to parse")
 }
@@ -615,3 +632,46 @@ secretkey = "{{.SecretKey}}"
 	assert.Contains(t, content, `region = "us-east-1"`)
 	assert.Contains(t, content, fmt.Sprintf(`accesskey = "%s"`, settings.AccessKey))
 }
+
+func TestChownRecursive_InvalidUser(t *testing.T) {
+	// ChownRecursive should silently return on invalid username
+	tmpDir := t.TempDir()
+	testFile := filepath.Join(tmpDir, "test.txt")
+	os.WriteFile(testFile, []byte("test"), 0644)
+
+	// Should not panic or error with a non-existent user
+	ChownRecursive(tmpDir, "nonexistent-user-that-does-not-exist-12345")
+
+	// File should still exist and be readable
+	_, err := os.ReadFile(testFile)
+	assert.NoError(t, err)
+}
+
+func TestChownRecursive_CurrentUser(t *testing.T) {
+	tmpDir := t.TempDir()
+	subDir := filepath.Join(tmpDir, "sub")
+	os.MkdirAll(subDir, 0755)
+	testFile := filepath.Join(subDir, "test.txt")
+	os.WriteFile(testFile, []byte("test"), 0644)
+
+	// Chown to current user should succeed (no-op effectively)
+	currentUser := os.Getenv("USER")
+	if currentUser == "" {
+		t.Skip("USER env not set")
+	}
+
+	ChownRecursive(tmpDir, currentUser)
+
+	// Verify files are still accessible
+	_, err := os.ReadFile(testFile)
+	assert.NoError(t, err)
+}
+
+func TestChownRecursive_NonExistentPath(t *testing.T) {
+	// Should not panic on a path that doesn't exist
+	currentUser := os.Getenv("USER")
+	if currentUser == "" {
+		t.Skip("USER env not set")
+	}
+	ChownRecursive("/tmp/nonexistent-path-12345", currentUser)
+}

hive/config/config.go

@@ -1,14 +1,11 @@
 package config
 
 import (
-	"encoding/json"
 	"fmt"
 	"log/slog"
 	"os"
 	"slices"
-	"sync"
 
-	"github.com/nats-io/nats.go"
 	"github.com/spf13/viper"
 )
 
@@ -40,45 +37,6 @@ type Config struct {
 	WalDir  string `mapstructure:"wal_dir"`
 }
 
-// NodeBaseDir returns the BaseDir for the current node, or "" if the config
-// is nil, the node name is unset, or the node is not found in the Nodes map.
-func (cc *ClusterConfig) NodeBaseDir() string {
-	if cc == nil || cc.Node == "" {
-		slog.Warn("NodeBaseDir: no config or node name set, using global PID path")
-		return ""
-	}
-	node, ok := cc.Nodes[cc.Node]
-	if !ok {
-		slog.Error("NodeBaseDir: node not found in config", "node", cc.Node)
-		return ""
-	}
-	if node.BaseDir == "" {
-		slog.Warn("NodeBaseDir: BaseDir is empty for node, using global PID path", "node", cc.Node)
-	}
-	return node.BaseDir
-}
-
-// AllServices is the default service list when Services is empty (backward compat).
-var AllServices = []string{"nats", "predastore", "viperblock", "daemon", "awsgw", "vpcd", "ui"}
-
-// HasService reports whether the node runs the named service.
-// An empty Services list means all services (backward compat).
-func (c Config) HasService(name string) bool {
-	services := c.Services
-	if len(services) == 0 {
-		services = AllServices
-	}
-	return slices.Contains(services, name)
-}
-
-// GetServices returns the configured service list, defaulting to AllServices.
-func (c Config) GetServices() []string {
-	if len(c.Services) == 0 {
-		return AllServices
-	}
-	return c.Services
-}
-
 type AWSGWConfig struct {
 	Host    string `mapstructure:"host"`
 	TLSKey  string `mapstructure:"tlskey"`
@@ -134,180 +92,43 @@ type NATSSub struct {
 	Subject string `mapstructure:"subject"`
 }
 
-type EBSRequests struct {
-	Requests []EBSRequest `mapstructure:"ebs_requests"`
-	Mu       sync.Mutex   `json:"-"`
-}
-
-type EBSRequest struct {
-	Name                string
-	VolType             string
-	Boot                bool
-	EFI                 bool
-	CloudInit           bool
-	DeleteOnTermination bool
-	NBDURI              string // NBD URI - socket path (nbd:unix:/path.sock) or TCP (nbd://host:port)
-	DeviceName          string // AWS API device name (e.g. /dev/sdf) for hot-plugged volumes
-}
-
-// NBDTransport defines the transport type for NBD connections
-type NBDTransport string
-
-const (
-	// NBDTransportSocket uses Unix domain sockets (faster, local only)
-	NBDTransportSocket NBDTransport = "socket"
-	// NBDTransportTCP uses TCP connections (required for remote/DPU scenarios)
-	NBDTransportTCP NBDTransport = "tcp"
-)
-
-type EBSMountResponse struct {
-	URI     string
-	Mounted bool
-	Error   string
-}
-
-type EBSUnMountResponse struct {
-	Volume  string
-	Mounted bool
-	Error   string
-}
-
-type EBSSyncRequest struct {
-	Volume string
-}
-
-type EBSSyncResponse struct {
-	Volume string
-	Synced bool
-	Error  string
-}
-
-type EBSDeleteRequest struct {
-	Volume string
-}
-
-type EBSDeleteResponse struct {
-	Volume  string
-	Success bool
-	Error   string
-}
-
-type EBSSnapshotRequest struct {
-	Volume     string
-	SnapshotID string
-}
-
-type EBSSnapshotResponse struct {
-	SnapshotID string
-	Success    bool
-	Error      string
-}
-
-// EC2, TODO: Move to vm.go or more applicable place
-type EC2StartInstancesRequest struct {
-	InstanceID string
-}
-
-type EC2StartInstancesResponse struct {
-	InstanceID string
-	Status     string
-	Error      string
-}
-
-// TODO: Make a generic function for the response
-func (ec2StartInstanceResponse EC2StartInstancesResponse) Respond(msg *nats.Msg) {
-
-	response, err := json.Marshal(ec2StartInstanceResponse)
-	if err != nil {
-		slog.Error("Failed to marshal response", "err", err)
-		return
+// NodeBaseDir returns the BaseDir for the current node, or "" if the config
+// is nil, the node name is unset, or the node is not found in the Nodes map.
+func (cc *ClusterConfig) NodeBaseDir() string {
+	if cc == nil || cc.Node == "" {
+		slog.Warn("NodeBaseDir: no config or node name set, using global PID path")
+		return ""
 	}
-
-	if err := msg.Respond(response); err != nil {
-		slog.Error("Failed to respond to NATS request", "err", err)
+	node, ok := cc.Nodes[cc.Node]
+	if !ok {
+		slog.Error("NodeBaseDir: node not found in config", "node", cc.Node)
+		return ""
 	}
-
+	if node.BaseDir == "" {
+		slog.Warn("NodeBaseDir: BaseDir is empty for node, using global PID path", "node", cc.Node)
+	}
+	return node.BaseDir
 }
 
-// Cluster manager API types
-type NodeJoinRequest struct {
-	Node       string `json:"node"`
-	Region     string `json:"region"`
-	AZ         string `json:"az"`
-	DataDir    string `json:"data_dir"`
-	DaemonHost string `json:"daemon_host"` // Host:port where this node's daemon is accessible
+// AllServices is the default service list when Services is empty (backward compat).
+var AllServices = []string{"nats", "predastore", "viperblock", "daemon", "awsgw", "vpcd", "ui"}
+
+// HasService reports whether the node runs the named service.
+// An empty Services list means all services (backward compat).
+func (c Config) HasService(name string) bool {
+	services := c.Services
+	if len(services) == 0 {
+		services = AllServices
+	}
+	return slices.Contains(services, name)
 }
 
-// SharedClusterData contains only the shared cluster information (no node-specific top-level fields)
-type SharedClusterData struct {
-	Epoch   uint64            `json:"epoch" toml:"epoch"`
-	Version string            `json:"version" toml:"version"`
-	Nodes   map[string]Config `json:"nodes" toml:"nodes"`
-}
-
-type NodeJoinResponse struct {
-	Success     bool               `json:"success"`
-	Message     string             `json:"message"`
-	SharedData  *SharedClusterData `json:"shared_data,omitempty"`
-	ConfigHash  string             `json:"config_hash,omitempty"`
-	JoiningNode string             `json:"joining_node,omitempty"` // The node name that is joining
-	// CA distribution for per-node certificate generation
-	CACert string `json:"ca_cert,omitempty"` // PEM-encoded CA certificate
-	CAKey  string `json:"ca_key,omitempty"`  // PEM-encoded CA private key
-	// Predastore config distribution for multi-node clusters
-	PredastoreConfig string `json:"predastore_config,omitempty"` // Full predastore.toml content
-}
-
-type NodeHealthResponse struct {
-	Node          string            `json:"node"`
-	Status        string            `json:"status"`
-	ConfigHash    string            `json:"config_hash"`
-	Epoch         uint64            `json:"epoch"`
-	Uptime        int64             `json:"uptime"`
-	Services      []string          `json:"services"`
-	ServiceHealth map[string]string `json:"service_health,omitempty"`
-}
-
-// NodeStatusResponse is returned by the hive.node.status NATS topic (fan-out).
-type NodeStatusResponse struct {
-	Node          string            `json:"node"`
-	Status        string            `json:"status"`
-	Host          string            `json:"host"`
-	Region        string            `json:"region"`
-	AZ            string            `json:"az"`
-	Uptime        int64             `json:"uptime"`
-	Services      []string          `json:"services"`
-	TotalVCPU     int               `json:"total_vcpu"`
-	TotalMemGB    float64           `json:"total_mem_gb"`
-	AllocVCPU     int               `json:"alloc_vcpu"`
-	AllocMemGB    float64           `json:"alloc_mem_gb"`
-	VMCount       int               `json:"vm_count"`
-	InstanceTypes []InstanceTypeCap `json:"instance_types"`
-}
-
-// InstanceTypeCap describes available capacity for one instance type on a node.
-type InstanceTypeCap struct {
-	Name      string  `json:"name"`
-	VCPU      int     `json:"vcpu"`
-	MemoryGB  float64 `json:"memory_gb"`
-	Available int     `json:"available"`
-}
-
-// VMInfo describes a single VM for the cluster stats CLI.
-type VMInfo struct {
-	InstanceID   string  `json:"instance_id"`
-	Status       string  `json:"status"`
-	InstanceType string  `json:"instance_type"`
-	VCPU         int     `json:"vcpu"`
-	MemoryGB     float64 `json:"memory_gb"`
-	LaunchTime   int64   `json:"launch_time"`
-}
-
-// NodeVMsResponse is returned by the hive.node.vms NATS topic (fan-out).
-type NodeVMsResponse struct {
-	Node string   `json:"node"`
-	Host string   `json:"host"`
-	VMs  []VMInfo `json:"vms"`
+// GetServices returns the configured service list, defaulting to AllServices.
+func (c Config) GetServices() []string {
+	if len(c.Services) == 0 {
+		return AllServices
+	}
+	return c.Services
 }
 
 // LoadConfig loads the configuration from file and environment variables

hive/config/config_test.go

@@ -1,14 +1,10 @@
 package config
 
 import (
-	"encoding/json"
 	"os"
 	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -304,13 +300,6 @@ func TestNodeBaseDir_EmptyBaseDir(t *testing.T) {
 	assert.Equal(t, "", cc.NodeBaseDir())
 }
 
-// Tests for type constants
-
-func TestNBDTransportConstants(t *testing.T) {
-	assert.Equal(t, NBDTransport("socket"), NBDTransportSocket)
-	assert.Equal(t, NBDTransport("tcp"), NBDTransportTCP)
-}
-
 // Tests for ViperblockConfig
 
 func TestLoadConfig_ViperblockShardWAL_Explicit(t *testing.T) {
@@ -380,77 +369,3 @@ shardwal = true
 	require.NotNil(t, n.Viperblock.ShardWAL)
 	assert.True(t, *n.Viperblock.ShardWAL)
 }
-
-// TestEC2StartInstancesResponseRespond verifies Respond marshals and sends the response via NATS.
-func TestEC2StartInstancesResponseRespond(t *testing.T) {
-	ns, err := server.NewServer(&server.Options{
-		Host:   "127.0.0.1",
-		Port:   -1,
-		NoLog:  true,
-		NoSigs: true,
-	})
-	require.NoError(t, err)
-	go ns.Start()
-	require.True(t, ns.ReadyForConnections(5*time.Second))
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(ns.ClientURL())
-	require.NoError(t, err)
-	defer nc.Close()
-
-	tests := []struct {
-		name string
-		resp EC2StartInstancesResponse
-	}{
-		{
-			name: "successful start",
-			resp: EC2StartInstancesResponse{
-				InstanceID: "i-abc123",
-				Status:     "pending",
-			},
-		},
-		{
-			name: "start with error",
-			resp: EC2StartInstancesResponse{
-				InstanceID: "i-def456",
-				Status:     "failed",
-				Error:      "instance not found",
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			subject := "test.ec2.start"
-
-			sub, err := nc.SubscribeSync(subject)
-			require.NoError(t, err)
-			defer sub.Unsubscribe()
-
-			inbox := nc.NewRespInbox()
-			replySub, err := nc.SubscribeSync(inbox)
-			require.NoError(t, err)
-			defer replySub.Unsubscribe()
-			require.NoError(t, nc.Flush())
-
-			err = nc.PublishRequest(subject, inbox, []byte("{}"))
-			require.NoError(t, err)
-
-			msg, err := sub.NextMsg(2 * time.Second)
-			require.NoError(t, err)
-
-			tt.resp.Respond(msg)
-
-			reply, err := replySub.NextMsg(2 * time.Second)
-			require.NoError(t, err)
-
-			var decoded EC2StartInstancesResponse
-			err = json.Unmarshal(reply.Data, &decoded)
-			require.NoError(t, err)
-
-			assert.Equal(t, tt.resp.InstanceID, decoded.InstanceID)
-			assert.Equal(t, tt.resp.Status, decoded.Status)
-			assert.Equal(t, tt.resp.Error, decoded.Error)
-		})
-	}
-}

hive/daemon/daemon.go

@@ -15,6 +15,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"sync"
@@ -22,6 +23,8 @@ import (
 	"syscall"
 	"time"
 
+	"net/http"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/go-chi/chi/v5"
@@ -40,12 +43,12 @@ import (
 	"github.com/mulgadc/hive/hive/instancetypes"
 	"github.com/mulgadc/hive/hive/objectstore"
 	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/nats-io/nats.go"
 	"github.com/pelletier/go-toml/v2"
-	"net/http"
 )
 
 type BlockDeviceMapping struct {
@@ -130,8 +133,9 @@ type Daemon struct {
 	// NetworkPlumber handles tap device lifecycle for VPC networking
 	networkPlumber NetworkPlumber
 
-	// shuttingDown is set to true during coordinated cluster shutdown (GATE phase).
-	// When true, the daemon rejects new work and setupShutdown skips VM stops.
+	// shuttingDown is set to true during coordinated cluster shutdown (GATE phase)
+	// or during SIGTERM-based shutdown. When true, the daemon rejects new work,
+	// crash handlers bail out, and setupShutdown skips redundant VM stops.
 	shuttingDown atomic.Bool
 
 	// ready is set to true once NATS connection, JetStream, and all services
@@ -284,7 +288,7 @@ func (rm *ResourceManager) GetAvailableInstanceTypeInfos(showCapacity bool) []*e
 }
 
 // GetResourceStats returns current resource allocation stats for the node status response.
-func (rm *ResourceManager) GetResourceStats() (totalVCPU int, totalMemGB float64, allocVCPU int, allocMemGB float64, caps []config.InstanceTypeCap) {
+func (rm *ResourceManager) GetResourceStats() (totalVCPU int, totalMemGB float64, allocVCPU int, allocMemGB float64, caps []types.InstanceTypeCap) {
 	rm.mu.RLock()
 	defer rm.mu.RUnlock()
 
@@ -388,6 +392,7 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.start", d.handleEC2StartStoppedInstance, "hive-workers"},
 		{"ec2.terminate", d.handleEC2TerminateStoppedInstance, "hive-workers"},
 		{"ec2.DescribeStoppedInstances", d.handleEC2DescribeStoppedInstances, "hive-workers"},
+		{"ec2.DescribeTerminatedInstances", d.handleEC2DescribeTerminatedInstances, "hive-workers"},
 		// these 2 fan out to all nodes and gateway aggregates the results
 		{"ec2.DescribeInstances", d.handleEC2DescribeInstances, ""},
 		{"ec2.DescribeInstanceTypes", d.handleEC2DescribeInstanceTypes, ""},
@@ -401,6 +406,8 @@ func (d *Daemon) subscribeAll() error {
 		{"hive.nodes.discover", d.handleNodeDiscover, ""},
 		{"hive.node.status", d.handleNodeStatus, ""},
 		{"hive.node.vms", d.handleNodeVMs, ""},
+		// Account creation → create default VPC for new account
+		{"iam.account.created", d.handleAccountCreated, "hive-workers"},
 		// Coordinated cluster shutdown phases (fan-out, no queue group)
 		{"hive.cluster.shutdown.gate", d.handleShutdownGate, ""},
 		{"hive.cluster.shutdown.drain", d.handleShutdownDrain, ""},
@@ -522,6 +529,7 @@ func (d *Daemon) Start() error {
 	}
 
 	d.waitForClusterReady()
+	d.upgradeJetStreamReplicas()
 	d.restoreInstances()
 
 	if err := d.subscribeAll(); err != nil {
@@ -603,6 +611,10 @@ func (d *Daemon) initJetStream() error {
 			err = d.jsManager.InitClusterStateBucket()
 		}
 
+		if err == nil {
+			err = d.jsManager.InitTerminatedInstanceBucket()
+		}
+
 		if err == nil {
 			slog.Info("JetStream KV stores initialized successfully", "replicas", 1, "attempts", attempt, "elapsed", time.Since(start).Round(time.Second))
 			break
@@ -618,17 +630,26 @@ func (d *Daemon) initJetStream() error {
 		retryDelay = min(retryDelay*2, 10*time.Second)
 	}
 
-	// Upgrade replicas if cluster has more than one node
-	clusterSize := len(d.clusterConfig.Nodes)
-	if clusterSize > 1 {
-		if err := d.jsManager.UpdateReplicas(clusterSize); err != nil {
-			slog.Warn("Failed to upgrade JetStream replicas on startup (other NATS nodes may not be ready)", "targetReplicas", clusterSize, "error", err)
-		}
-	}
+	// Replica upgrade is deferred to after all services have created their
+	// KV buckets and the cluster is ready (see upgradeJetStreamReplicas).
 
 	return nil
 }
 
+// upgradeJetStreamReplicas bumps the replication factor on ALL KV_* streams
+// to match the cluster size. This runs after all services have created their
+// buckets and after waitForClusterReady so that enough NATS peers are online
+// to accept the new replica count.
+func (d *Daemon) upgradeJetStreamReplicas() {
+	clusterSize := len(d.clusterConfig.Nodes)
+	if clusterSize <= 1 || d.jsManager == nil {
+		return
+	}
+	if err := d.jsManager.UpdateReplicas(clusterSize); err != nil {
+		slog.Warn("Failed to upgrade JetStream replicas", "targetReplicas", clusterSize, "error", err)
+	}
+}
+
 // initServiceWithRetry initializes a service using the provided init function,
 // retrying with exponential backoff (500ms→10s) for up to 5 minutes. During
 // cluster restarts, JetStream KV may be temporarily unavailable while NATS
@@ -721,23 +742,31 @@ func (d *Daemon) checkPredastoreReady() bool {
 	return true
 }
 
-// migrateStoppedToSharedKV writes a stopped instance to shared KV and removes
+// migrateInstanceToKV writes an instance to the given KV write function and removes
 // it from the local instance map. Returns true if migration succeeded.
-func (d *Daemon) migrateStoppedToSharedKV(instance *vm.VM) bool {
+func (d *Daemon) migrateInstanceToKV(instance *vm.VM, writeFn func(string, *vm.VM) error, label string) bool {
 	if d.jsManager == nil {
 		return false
 	}
 	instance.LastNode = d.node
-	if err := d.jsManager.WriteStoppedInstance(instance.ID, instance); err != nil {
-		slog.Error("Failed to migrate stopped instance to shared KV",
-			"instance", instance.ID, "err", err)
+	if err := writeFn(instance.ID, instance); err != nil {
+		slog.Error("Failed to migrate instance to KV",
+			"instance", instance.ID, "bucket", label, "err", err)
 		return false
 	}
 	delete(d.Instances.VMS, instance.ID)
-	slog.Info("Migrated stopped instance to shared KV", "instance", instance.ID)
+	slog.Info("Migrated instance to KV", "instance", instance.ID, "bucket", label)
 	return true
 }
 
+func (d *Daemon) migrateStoppedToSharedKV(instance *vm.VM) bool {
+	return d.migrateInstanceToKV(instance, d.jsManager.WriteStoppedInstance, "stopped")
+}
+
+func (d *Daemon) migrateTerminatedToKV(instance *vm.VM) bool {
+	return d.migrateInstanceToKV(instance, d.jsManager.WriteTerminatedInstance, "terminated")
+}
+
 // maxConcurrentRecovery limits how many VMs are relaunched in parallel during recovery.
 const maxConcurrentRecovery = 2
 
@@ -782,7 +811,14 @@ func (d *Daemon) restoreInstances() {
 		instance := d.Instances.VMS[i]
 
 		if instance.Status == vm.StateTerminated {
-			slog.Info("Instance state is terminated, skipping", "instance", instance.ID)
+			if !d.migrateTerminatedToKV(instance) {
+				// KV write failed — keep in local state so the next restart
+				// retries the migration. Deleting here would create a "void":
+				// the instance disappears from both local state and the
+				// terminated KV, making it invisible to DescribeInstances.
+				slog.Warn("Terminated instance KV migration failed, will retry on next restart",
+					"instance", instance.ID)
+			}
 			continue
 		}
 
@@ -838,6 +874,10 @@ func (d *Daemon) restoreInstances() {
 				continue
 			}
 
+			if instance.Status == vm.StateTerminated && d.migrateTerminatedToKV(instance) {
+				continue
+			}
+
 			if err := d.WriteState(); err != nil {
 				slog.Error("Failed to persist state, will retry on next restart",
 					"instance", instance.ID, "error", err)
@@ -872,6 +912,11 @@ func (d *Daemon) restoreInstances() {
 			go func(inst *vm.VM) {
 				defer wg.Done()
 				defer func() { <-sem }() // release
+				defer func() {
+					if r := recover(); r != nil {
+						slog.Error("Panic during instance recovery", "instanceId", inst.ID, "panic", r, "stack", string(debug.Stack()))
+					}
+				}()
 				slog.Info("Launching instance (recovery)", "instance", inst.ID)
 				if err := d.LaunchInstance(inst); err != nil {
 					slog.Error("Failed to launch instance", "instanceId", inst.ID, "err", err)
@@ -951,7 +996,7 @@ func (d *Daemon) awaitShutdown() {
 // computeConfigHash computes SHA256 hash of the shared cluster config (excluding node-specific fields)
 func (d *Daemon) computeConfigHash() (string, error) {
 	// Only hash the shared cluster data, not the node-specific top-level field
-	sharedData := config.SharedClusterData{
+	sharedData := types.SharedClusterData{
 		Epoch:   d.clusterConfig.Epoch,
 		Version: d.clusterConfig.Version,
 		Nodes:   d.clusterConfig.Nodes,
@@ -1038,7 +1083,7 @@ func (d *Daemon) ClusterManager() error {
 			status = "starting"
 		}
 
-		response := config.NodeHealthResponse{
+		response := types.NodeHealthResponse{
 			Node:          d.node,
 			Status:        status,
 			ConfigHash:    configHash,
@@ -1049,19 +1094,23 @@ func (d *Daemon) ClusterManager() error {
 		}
 
 		w.Header().Set("Content-Type", "application/json")
-		_ = json.NewEncoder(w).Encode(response)
+		if err := json.NewEncoder(w).Encode(response); err != nil {
+			slog.Error("Failed to encode health response", "error", err)
+		}
 	})
 
 	// Join endpoint - accepts new nodes joining the cluster
 	r.Post("/join", func(w http.ResponseWriter, r *http.Request) {
-		var req config.NodeJoinRequest
+		var req types.NodeJoinRequest
 		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 			w.Header().Set("Content-Type", "application/json")
 			w.WriteHeader(400)
-			_ = json.NewEncoder(w).Encode(config.NodeJoinResponse{
+			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: "invalid request body",
-			})
+			}); err != nil {
+				slog.Error("Failed to encode join error response", "error", err)
+			}
 			return
 		}
 
@@ -1071,10 +1120,12 @@ func (d *Daemon) ClusterManager() error {
 		if req.Node == "" || req.Region == "" || req.AZ == "" {
 			w.Header().Set("Content-Type", "application/json")
 			w.WriteHeader(400)
-			_ = json.NewEncoder(w).Encode(config.NodeJoinResponse{
+			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: "node, region, and az are required",
-			})
+			}); err != nil {
+				slog.Error("Failed to encode join validation response", "error", err)
+			}
 			return
 		}
 
@@ -1082,10 +1133,12 @@ func (d *Daemon) ClusterManager() error {
 		if _, exists := d.clusterConfig.Nodes[req.Node]; exists {
 			w.Header().Set("Content-Type", "application/json")
 			w.WriteHeader(409)
-			_ = json.NewEncoder(w).Encode(config.NodeJoinResponse{
+			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: fmt.Sprintf("node %s already exists in cluster", req.Node),
-			})
+			}); err != nil {
+				slog.Error("Failed to encode join conflict response", "error", err)
+			}
 			return
 		}
 
@@ -1115,10 +1168,12 @@ func (d *Daemon) ClusterManager() error {
 			slog.Error("Failed to save cluster config", "error", err)
 			w.Header().Set("Content-Type", "application/json")
 			w.WriteHeader(500)
-			_ = json.NewEncoder(w).Encode(config.NodeJoinResponse{
+			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: "failed to save cluster config",
-			})
+			}); err != nil {
+				slog.Error("Failed to encode join save-error response", "error", err)
+			}
 			return
 		}
 
@@ -1137,7 +1192,7 @@ func (d *Daemon) ClusterManager() error {
 		}
 
 		// Send only shared cluster data (exclude node-specific top-level fields)
-		sharedData := &config.SharedClusterData{
+		sharedData := &types.SharedClusterData{
 			Epoch:   d.clusterConfig.Epoch,
 			Version: d.clusterConfig.Version,
 			Nodes:   d.clusterConfig.Nodes,
@@ -1171,7 +1226,7 @@ func (d *Daemon) ClusterManager() error {
 		}
 
 		w.Header().Set("Content-Type", "application/json")
-		_ = json.NewEncoder(w).Encode(config.NodeJoinResponse{
+		if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 			Success:          true,
 			Message:          fmt.Sprintf("node %s successfully joined cluster", req.Node),
 			SharedData:       sharedData,
@@ -1180,7 +1235,9 @@ func (d *Daemon) ClusterManager() error {
 			CACert:           caCert,
 			CAKey:            caKey,
 			PredastoreConfig: predastoreConfig,
-		})
+		}); err != nil {
+			slog.Error("Failed to encode join success response", "error", err)
+		}
 	})
 
 	// Get cluster config endpoint
@@ -1188,10 +1245,12 @@ func (d *Daemon) ClusterManager() error {
 		configHash, _ := d.computeConfigHash()
 
 		w.Header().Set("Content-Type", "application/json")
-		_ = json.NewEncoder(w).Encode(map[string]any{
+		if err := json.NewEncoder(w).Encode(map[string]any{
 			"config":      d.clusterConfig,
 			"config_hash": configHash,
-		})
+		}); err != nil {
+			slog.Error("Failed to encode config response", "error", err)
+		}
 	})
 
 	// Start HTTP server in goroutine
@@ -1251,6 +1310,12 @@ func (d *Daemon) SendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceId
 	q.Mu.Lock()
 	defer q.Mu.Unlock()
 
+	// Set a read deadline so we don't block forever on a hung QEMU process
+	if err := q.Conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
+		return nil, fmt.Errorf("set read deadline: %w", err)
+	}
+	defer q.Conn.SetReadDeadline(time.Time{}) // clear deadline after command
+
 	if err := q.Encoder.Encode(cmd); err != nil {
 		return nil, fmt.Errorf("encode error: %w", err)
 	}
@@ -1266,6 +1331,10 @@ func (d *Daemon) SendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceId
 			// by the command handlers that initiate the action, avoiding races
 			// between event-driven and command-driven transitions.
 			slog.Info("QMP event", "event", msg["event"], "instanceId", instanceId)
+			// Extend deadline after receiving an event (QEMU is alive, just chatty)
+			if err := q.Conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
+				return nil, fmt.Errorf("set read deadline: %w", err)
+			}
 			continue
 		}
 		if errObj, ok := msg["error"].(map[string]any); ok {
@@ -1352,7 +1421,7 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 					// to stop viperblockd processes. S3 data cleanup happens via DeleteVolume
 					// on the parent root volume (which deletes -efi/ and -cloudinit/ prefixes).
 					if ebsRequest.EFI || ebsRequest.CloudInit {
-						ebsDeleteData, err := json.Marshal(config.EBSDeleteRequest{Volume: ebsRequest.Name})
+						ebsDeleteData, err := json.Marshal(types.EBSDeleteRequest{Volume: ebsRequest.Name})
 						if err != nil {
 							slog.Error("Failed to marshal ebs.delete request for internal volume", "name", ebsRequest.Name, "err", err)
 							continue
@@ -1456,10 +1525,13 @@ func (d *Daemon) setupShutdown() {
 		// Cancel context to stop heartbeat and other goroutines
 		d.cancel()
 
-		// If coordinated shutdown already handled VMs (DRAIN phase), skip stopInstance
+		// If coordinated shutdown already handled VMs (DRAIN phase), skip stopInstance.
+		// Otherwise, set the flag now so crash handlers and restart schedulers
+		// know to bail out during SIGTERM-based shutdown.
 		if d.shuttingDown.Load() {
 			slog.Info("Coordinated shutdown in progress, skipping VM stop (already handled by DRAIN phase)")
 		} else {
+			d.shuttingDown.Store(true)
 			// Pass instances to terminate
 			if err := d.stopInstance(d.Instances.VMS, false); err != nil {
 				slog.Error("Failed to stop instances during shutdown", "err", err)
@@ -1516,6 +1588,10 @@ func (d *Daemon) CreateQMPClient(instance *vm.VM) (err error) {
 
 	// Send qmp_capabilities handshake to init
 	_, err = d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{Execute: "qmp_capabilities"}, instance.ID)
+	if err != nil {
+		slog.Error("Failed QMP capabilities handshake", "err", err)
+		return err
+	}
 
 	// Simple heartbeat to confirm QMP and the instance is running / healthy
 	go func() {
@@ -1552,11 +1628,6 @@ func (d *Daemon) CreateQMPClient(instance *vm.VM) (err error) {
 		}
 	}()
 
-	if err != nil {
-		slog.Error("Failed to create QMP client", "err", err)
-		return err
-	}
-
 	return nil
 
 }
@@ -2021,6 +2092,8 @@ func (d *Daemon) ebsTopic(action string) string {
 func (d *Daemon) MountVolumes(instance *vm.VM) error {
 
 	instance.EBSRequests.Mu.Lock()
+	defer instance.EBSRequests.Mu.Unlock()
+
 	for k, v := range instance.EBSRequests.Requests {
 
 		// Send the volume payload as JSON
@@ -2042,7 +2115,7 @@ func (d *Daemon) MountVolumes(instance *vm.VM) error {
 		}
 
 		// Unmarshal the response
-		var ebsMountResponse config.EBSMountResponse
+		var ebsMountResponse types.EBSMountResponse
 		err = json.Unmarshal(reply.Data, &ebsMountResponse)
 
 		if err != nil {
@@ -2064,15 +2137,13 @@ func (d *Daemon) MountVolumes(instance *vm.VM) error {
 
 	}
 
-	instance.EBSRequests.Mu.Unlock()
-
 	return nil
 
 }
 
 // rollbackEBSMount sends an ebs.unmount request to undo a previously successful ebs.mount.
 // Rollback failures are logged but not propagated; callers treat this as best-effort cleanup.
-func (d *Daemon) rollbackEBSMount(req config.EBSRequest) {
+func (d *Daemon) rollbackEBSMount(req types.EBSRequest) {
 	data, err := json.Marshal(req)
 	if err != nil {
 		slog.Error("rollbackEBSMount: failed to marshal unmount request", "volume", req.Name, "err", err)
@@ -2083,7 +2154,7 @@ func (d *Daemon) rollbackEBSMount(req config.EBSRequest) {
 		slog.Error("rollbackEBSMount: ebs.unmount NATS request failed", "volume", req.Name, "err", err)
 		return
 	}
-	var resp config.EBSUnMountResponse
+	var resp types.EBSUnMountResponse
 	if err := json.Unmarshal(msg.Data, &resp); err != nil {
 		slog.Error("rollbackEBSMount: failed to unmarshal response", "volume", req.Name, "err", err)
 		return

hive/daemon/daemon_handlers.go

@@ -8,8 +8,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/hive/hive/awserrors"
-	"github.com/mulgadc/hive/hive/config"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/nats-io/nats.go"
@@ -57,10 +56,10 @@ func handleNATSRequest[I any, O any](msg *nats.Msg, serviceFn func(*I, string) (
 
 // handleEC2Events processes incoming EC2 instance events (start, stop, terminate, attach-volume)
 func (d *Daemon) handleEC2Events(msg *nats.Msg) {
-	var command qmp.Command
+	var command types.EC2InstanceCommand
 
 	if err := json.Unmarshal(msg.Data, &command); err != nil {
-		slog.Error("Error unmarshaling QMP command", "err", err)
+		slog.Error("Error unmarshaling EC2 instance command", "err", err)
 		respondWithError(msg, awserrors.ErrorServerInternal)
 		return
 	}
@@ -89,52 +88,13 @@ func (d *Daemon) handleEC2Events(msg *nats.Msg) {
 		d.handleDetachVolume(msg, command, instance)
 	case command.Attributes.StartInstance:
 		d.handleStartInstance(msg, command, instance)
+	case command.Attributes.RebootInstance:
+		d.handleRebootInstance(msg, command, instance)
 	case command.Attributes.StopInstance, command.Attributes.TerminateInstance:
 		d.handleStopOrTerminateInstance(msg, command, instance)
 	default:
-		d.handleQMPCommand(msg, command, instance)
-	}
-}
-
-func (d *Daemon) handleQMPCommand(msg *nats.Msg, command qmp.Command, instance *vm.VM) {
-	resp, err := d.SendQMPCommand(instance.QMPClient, command.QMPCommand, instance.ID)
-	if err != nil {
-		slog.Error("Failed to send QMP command", "err", err)
+		slog.Warn("Unhandled EC2 instance command", "id", command.ID, "attributes", command.Attributes)
 		respondWithError(msg, awserrors.ErrorServerInternal)
-		return
-	}
-
-	slog.Debug("RAW QMP Response", "resp", string(resp.Return))
-
-	// Unmarshal the response
-	target, ok := qmp.CommandResponseTypes[command.QMPCommand.Execute]
-	if !ok {
-		slog.Warn("Unhandled QMP command", "cmd", command.QMPCommand.Execute)
-		if err := msg.Respond(resp.Return); err != nil {
-			slog.Error("Failed to respond to NATS request", "err", err)
-		}
-		return
-	}
-
-	if err := json.Unmarshal(resp.Return, target); err != nil {
-		slog.Error("Failed to unmarshal QMP response", "cmd", command.QMPCommand.Execute, "err", err)
-		if err := msg.Respond(resp.Return); err != nil {
-			slog.Error("Failed to respond to NATS request", "err", err)
-		}
-		return
-	}
-
-	// Update attributes and respond
-	d.Instances.Mu.Lock()
-	instance.Attributes = command.Attributes
-	d.Instances.Mu.Unlock()
-
-	if err := d.WriteState(); err != nil {
-		slog.Error("Failed to write state to disk", "err", err)
-	}
-
-	if err := msg.Respond(resp.Return); err != nil {
-		slog.Error("Failed to respond to NATS request", "err", err)
 	}
 }
 
@@ -153,7 +113,7 @@ func (d *Daemon) handleHealthCheck(msg *nats.Msg) {
 		status = "starting"
 	}
 
-	response := config.NodeHealthResponse{
+	response := types.NodeHealthResponse{
 		Node:       d.node,
 		Status:     status,
 		ConfigHash: configHash,
@@ -165,15 +125,10 @@ func (d *Daemon) handleHealthCheck(msg *nats.Msg) {
 	slog.Debug("Health check responded", "node", d.node, "epoch", d.clusterConfig.Epoch)
 }
 
-// NodeDiscoverResponse is the response for node discovery requests
-type NodeDiscoverResponse struct {
-	Node string `json:"node"`
-}
-
 // handleNodeDiscover responds to node discovery requests with this node's ID
 // Used by the gateway to dynamically discover active hive nodes in the cluster
 func (d *Daemon) handleNodeDiscover(msg *nats.Msg) {
-	response := NodeDiscoverResponse{
+	response := types.NodeDiscoverResponse{
 		Node: d.node,
 	}
 
@@ -204,7 +159,7 @@ func (d *Daemon) handleNodeStatus(msg *nats.Msg) {
 	}
 	d.Instances.Mu.Unlock()
 
-	resp := config.NodeStatusResponse{
+	resp := types.NodeStatusResponse{
 		Node:          d.node,
 		Status:        "Ready",
 		Host:          d.daemonIP(),
@@ -227,9 +182,9 @@ func (d *Daemon) handleNodeStatus(msg *nats.Msg) {
 // Used by the CLI: hive get vms.
 func (d *Daemon) handleNodeVMs(msg *nats.Msg) {
 	d.Instances.Mu.Lock()
-	vms := make([]config.VMInfo, 0, len(d.Instances.VMS))
+	vms := make([]types.VMInfo, 0, len(d.Instances.VMS))
 	for _, v := range d.Instances.VMS {
-		info := config.VMInfo{
+		info := types.VMInfo{
 			InstanceID:   v.ID,
 			Status:       string(v.Status),
 			InstanceType: v.InstanceType,
@@ -246,7 +201,7 @@ func (d *Daemon) handleNodeVMs(msg *nats.Msg) {
 	}
 	d.Instances.Mu.Unlock()
 
-	resp := config.NodeVMsResponse{
+	resp := types.NodeVMsResponse{
 		Node: d.node,
 		Host: d.daemonIP(),
 		VMs:  vms,

hive/daemon/daemon_handlers_instance.go

@@ -11,8 +11,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/hive/hive/awserrors"
-	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/nats-io/nats.go"
@@ -265,6 +265,12 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 	// Launch all instances (volumes and VMs)
 	var successCount int
 	for _, instance := range instances {
+		// Pre-compute dev MAC so cloud-init can generate per-interface netplan
+		// that suppresses the default route on the dev/hostfwd NIC.
+		if d.config.Daemon.DevNetworking && instance.ENIId != "" {
+			instance.DevMAC = generateDevMAC(instance.ID)
+		}
+
 		// Prepare the root volume, cloud-init, EFI drives via NBD (AMI clone to new volume)
 		volumeInfos, err := d.instanceService.GenerateVolumes(runInstancesInput, instance)
 		if err != nil {
@@ -306,7 +312,34 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 	slog.Info("handleEC2RunInstances completed", "requested", launchCount, "created", len(instances), "launched", successCount)
 }
 
-func (d *Daemon) handleStartInstance(msg *nats.Msg, command qmp.Command, instance *vm.VM) {
+func (d *Daemon) handleRebootInstance(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
+	slog.Info("Rebooting instance", "id", command.ID)
+
+	d.Instances.Mu.Lock()
+	status := instance.Status
+	d.Instances.Mu.Unlock()
+
+	if status != vm.StateRunning {
+		slog.Error("RebootInstance: instance not in running state", "instanceId", command.ID, "status", status)
+		respondWithError(msg, awserrors.ErrorIncorrectInstanceState)
+		return
+	}
+
+	_, err := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{Execute: "system_reset"}, command.ID)
+	if err != nil {
+		slog.Error("RebootInstance: QMP system_reset failed", "instanceId", command.ID, "err", err)
+		respondWithError(msg, awserrors.ErrorServerInternal)
+		return
+	}
+
+	slog.Info("Instance rebooted", "instanceId", command.ID)
+
+	if err := msg.Respond([]byte(`{}`)); err != nil {
+		slog.Error("Failed to respond to NATS request", "err", err)
+	}
+}
+
+func (d *Daemon) handleStartInstance(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
 	slog.Info("Starting instance", "id", command.ID)
 
 	// Validate instance is in stopped state
@@ -360,7 +393,7 @@ func (d *Daemon) handleStartInstance(msg *nats.Msg, command qmp.Command, instanc
 	}
 }
 
-func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command qmp.Command, instance *vm.VM) {
+func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
 	isTerminate := command.Attributes.TerminateInstance
 	action := "Stopping"
 	initialState := vm.StateStopping
@@ -399,7 +432,7 @@ func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command qmp.Comman
 	}
 
 	// Run cleanup in goroutine to not block NATS
-	go func(inst *vm.VM, attrs qmp.Attributes) {
+	go func(inst *vm.VM, attrs types.EC2CommandAttributes) {
 		stopErr := d.stopInstance(map[string]*vm.VM{inst.ID: inst}, isTerminate)
 
 		if stopErr != nil {
@@ -418,44 +451,67 @@ func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command qmp.Comman
 			}
 			slog.Info("Instance "+string(finalState), "id", inst.ID)
 
-			// For stop (not terminate): release ownership to shared KV so any
-			// daemon can pick up the next StartInstance request.
-			if !isTerminate && d.jsManager != nil {
-
-				// Write to shared KV first — if daemon crashes after this but
-				// before local cleanup, restoreInstances handles the overlap.
-				if err := d.jsManager.WriteStoppedInstance(inst.ID, inst); err != nil {
-					slog.Error("Failed to write stopped instance to shared KV, keeping local ownership",
-						"instanceId", inst.ID, "err", err)
-				} else {
-					// Unsubscribe from per-instance NATS topic
-					d.mu.Lock()
-					if sub, ok := d.natsSubscriptions[inst.ID]; ok {
-						if err := sub.Unsubscribe(); err != nil {
-							slog.Error("Failed to unsubscribe stopped instance", "instanceId", inst.ID, "err", err)
-						}
-						delete(d.natsSubscriptions, inst.ID)
-					}
-					d.mu.Unlock()
-
-					// Remove from local instance map
-					d.Instances.Mu.Lock()
-					delete(d.Instances.VMS, inst.ID)
-					d.Instances.Mu.Unlock()
-
-					// Persist local state without the stopped instance
-					if err := d.WriteState(); err != nil {
-						slog.Error("Failed to persist state after releasing stopped instance, re-adding to local map for consistency",
+			if d.jsManager != nil {
+				if isTerminate {
+					// Write to terminated KV bucket (auto-expires after 1 hour via TTL).
+					// If this fails, keep the instance in local state so DescribeInstances
+					// still sees it and restoreInstances can retry the KV migration.
+					if err := d.jsManager.WriteTerminatedInstance(inst.ID, inst); err != nil {
+						slog.Error("Failed to write terminated instance to KV, keeping in local state for retry",
 							"instanceId", inst.ID, "err", err)
-						// Re-add to local map so in-memory state matches on-disk state.
-						// On next restart, restoreInstances will retry the migration.
-						d.Instances.Mu.Lock()
-						d.Instances.VMS[inst.ID] = inst
-						d.Instances.Mu.Unlock()
-					} else {
-						slog.Info("Released stopped instance ownership to shared KV",
-							"instanceId", inst.ID, "lastNode", d.node)
+						return
 					}
+				} else {
+					// Write to shared KV first — if daemon crashes after this but
+					// before local cleanup, restoreInstances handles the overlap.
+					if err := d.jsManager.WriteStoppedInstance(inst.ID, inst); err != nil {
+						slog.Error("Failed to write stopped instance to shared KV, keeping local ownership",
+							"instanceId", inst.ID, "err", err)
+						return
+					}
+				}
+
+				// Guard + delete must be atomic under the same lock hold.
+				// A concurrent ec2.start handler may have loaded the instance
+				// from stopped KV, re-added it to VMS with a new pointer, and
+				// launched it. Deleting here would destroy the running instance's
+				// state — creating a "ghost instance" visible nowhere.
+				d.Instances.Mu.Lock()
+				current, exists := d.Instances.VMS[inst.ID]
+				if !exists || current != inst {
+					d.Instances.Mu.Unlock()
+					slog.Info("Instance was reclaimed by another handler, skipping local cleanup",
+						"instanceId", inst.ID, "state", string(finalState))
+					return
+				}
+				delete(d.Instances.VMS, inst.ID)
+				d.Instances.Mu.Unlock()
+
+				// Unsubscribe from per-instance NATS topic. Safe to do after
+				// the delete — LaunchInstance already unsubscribes stale entries
+				// before creating new ones (daemon.go:1658-1664).
+				d.mu.Lock()
+				if sub, ok := d.natsSubscriptions[inst.ID]; ok {
+					if err := sub.Unsubscribe(); err != nil {
+						slog.Error("Failed to unsubscribe instance", "instanceId", inst.ID, "err", err)
+					}
+					delete(d.natsSubscriptions, inst.ID)
+				}
+				d.mu.Unlock()
+
+				// Persist local state without the instance
+				if err := d.WriteState(); err != nil {
+					slog.Error("Failed to persist state after releasing instance, re-adding to local map for consistency",
+						"instanceId", inst.ID, "err", err)
+					// Only re-add if another handler hasn't claimed the slot
+					d.Instances.Mu.Lock()
+					if _, occupied := d.Instances.VMS[inst.ID]; !occupied {
+						d.Instances.VMS[inst.ID] = inst
+					}
+					d.Instances.Mu.Unlock()
+				} else {
+					slog.Info("Released instance ownership to KV",
+						"instanceId", inst.ID, "state", string(finalState), "lastNode", d.node)
 				}
 			}
 		}
@@ -685,7 +741,7 @@ func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) {
 	// Add instance to local map and clear stop attribute before launch
 	d.Instances.Mu.Lock()
 	d.Instances.VMS[instance.ID] = instance
-	instance.Attributes = qmp.Attributes{StartInstance: true}
+	instance.Attributes = types.EC2CommandAttributes{StartInstance: true}
 	d.Instances.Mu.Unlock()
 
 	// Launch the instance infrastructure (QEMU, QMP, NATS subscriptions)
@@ -780,7 +836,7 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) {
 	for _, ebsRequest := range instance.EBSRequests.Requests {
 		// Internal volumes (EFI, cloud-init) are always cleaned up via ebs.delete
 		if ebsRequest.EFI || ebsRequest.CloudInit {
-			ebsDeleteData, err := json.Marshal(config.EBSDeleteRequest{Volume: ebsRequest.Name})
+			ebsDeleteData, err := json.Marshal(types.EBSDeleteRequest{Volume: ebsRequest.Name})
 			if err != nil {
 				slog.Error("handleEC2TerminateStoppedInstance: failed to marshal ebs.delete request", "name", ebsRequest.Name, "err", err)
 				continue
@@ -810,13 +866,26 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) {
 	}
 	instance.EBSRequests.Mu.Unlock()
 
-	// Remove from shared KV
-	if err := d.jsManager.DeleteStoppedInstance(req.InstanceID); err != nil {
-		slog.Error("handleEC2TerminateStoppedInstance: failed to delete from shared KV", "instanceId", req.InstanceID, "err", err)
+	// Write to terminated KV bucket FIRST so the instance is visible in DescribeInstances.
+	// If this fails, the instance remains in the stopped bucket (safe to retry).
+	instance.Status = vm.StateTerminated
+	if err := d.jsManager.WriteTerminatedInstance(req.InstanceID, instance); err != nil {
+		slog.Error("handleEC2TerminateStoppedInstance: failed to write to terminated KV, aborting", "instanceId", req.InstanceID, "err", err)
 		respondWithError(msg, awserrors.ErrorServerInternal)
 		return
 	}
 
+	// Now safe to remove from shared stopped KV — instance already exists in terminated bucket.
+	// Retry once on failure to avoid duplicate entries in DescribeInstances.
+	if err := d.jsManager.DeleteStoppedInstance(req.InstanceID); err != nil {
+		slog.Warn("handleEC2TerminateStoppedInstance: first stopped KV delete failed, retrying",
+			"instanceId", req.InstanceID, "err", err)
+		if retryErr := d.jsManager.DeleteStoppedInstance(req.InstanceID); retryErr != nil {
+			slog.Error("handleEC2TerminateStoppedInstance: stopped KV delete failed after retry, instance may appear in both buckets",
+				"instanceId", req.InstanceID, "err", retryErr)
+		}
+	}
+
 	slog.Info("Terminated stopped instance from shared KV", "instanceId", req.InstanceID)
 
 	if err := msg.Respond(fmt.Appendf(nil, `{"status":"terminated","instanceId":"%s"}`, req.InstanceID)); err != nil {
@@ -826,16 +895,24 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) {
 
 // handleEC2DescribeStoppedInstances returns stopped instances from shared KV.
 func (d *Daemon) handleEC2DescribeStoppedInstances(msg *nats.Msg) {
+	d.describeInstancesFromKV(msg, d.jsManager.ListStoppedInstances, 80, "stopped", "handleEC2DescribeStoppedInstances")
+}
 
+// handleEC2DescribeTerminatedInstances returns terminated instances from the terminated KV bucket.
+func (d *Daemon) handleEC2DescribeTerminatedInstances(msg *nats.Msg) {
+	d.describeInstancesFromKV(msg, d.jsManager.ListTerminatedInstances, 48, "terminated", "handleEC2DescribeTerminatedInstances")
+}
+
+// describeInstancesFromKV is a shared helper for DescribeStopped/TerminatedInstances handlers.
+// It lists instances from a KV bucket, filters by account/instance ID, and responds with reservations.
+func (d *Daemon) describeInstancesFromKV(msg *nats.Msg, listFn func() ([]*vm.VM, error), fallbackCode int64, fallbackName, handlerName string) {
 	if d.jsManager == nil {
 		respondWithError(msg, awserrors.ErrorServerInternal)
 		return
 	}
 
-	// Extract account ID from NATS header for scoping
 	accountID := utils.AccountIDFromMsg(msg)
 
-	// Parse optional filters from the request
 	describeInput := &ec2.DescribeInstancesInput{}
 	if len(msg.Data) > 0 {
 		if errResp := utils.UnmarshalJsonPayload(describeInput, msg.Data); errResp != nil {
@@ -846,67 +923,61 @@ func (d *Daemon) handleEC2DescribeStoppedInstances(msg *nats.Msg) {
 		}
 	}
 
-	// Build instance ID filter
 	instanceIDFilter := make(map[string]bool)
-	if len(describeInput.InstanceIds) > 0 {
-		for _, id := range describeInput.InstanceIds {
-			if id != nil {
-				instanceIDFilter[*id] = true
-			}
+	for _, id := range describeInput.InstanceIds {
+		if id != nil {
+			instanceIDFilter[*id] = true
 		}
 	}
 
-	instances, err := d.jsManager.ListStoppedInstances()
+	instances, err := listFn()
 	if err != nil {
-		slog.Error("handleEC2DescribeStoppedInstances: failed to list stopped instances", "err", err)
+		slog.Error(handlerName+": failed to list instances", "err", err)
 		respondWithError(msg, awserrors.ErrorServerInternal)
 		return
 	}
 
-	// Group instances by reservation ID
 	reservationMap := make(map[string]*ec2.Reservation)
 
 	for _, instance := range instances {
-		// Skip instances not owned by the caller's account.
-		// Pre-Phase4 instances (empty AccountID) are only visible to root.
 		if !isInstanceVisible(accountID, instance.AccountID) {
 			continue
 		}
-
-		// Apply instance ID filter
 		if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] {
 			continue
 		}
-
-		if instance.Reservation != nil && instance.Instance != nil {
-			resID := ""
-			if instance.Reservation.ReservationId != nil {
-				resID = *instance.Reservation.ReservationId
-			}
-
-			if _, exists := reservationMap[resID]; !exists {
-				reservation := &ec2.Reservation{}
-				reservation.SetReservationId(resID)
-				if instance.Reservation.OwnerId != nil {
-					reservation.SetOwnerId(*instance.Reservation.OwnerId)
-				}
-				reservation.Instances = []*ec2.Instance{}
-				reservationMap[resID] = reservation
-			}
-
-			// Update the instance state
-			instanceCopy := *instance.Instance
-			instanceCopy.State = &ec2.InstanceState{}
-			if info, ok := vm.EC2StateCodes[instance.Status]; ok {
-				instanceCopy.State.SetCode(info.Code)
-				instanceCopy.State.SetName(info.Name)
-			} else {
-				instanceCopy.State.SetCode(80)
-				instanceCopy.State.SetName("stopped")
-			}
-
-			reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy)
+		if instance.Reservation == nil || instance.Instance == nil {
+			slog.Warn(handlerName+": skipping instance with nil Reservation/Instance (data integrity issue)",
+				"instanceId", instance.ID)
+			continue
 		}
+
+		resID := ""
+		if instance.Reservation.ReservationId != nil {
+			resID = *instance.Reservation.ReservationId
+		}
+
+		if _, exists := reservationMap[resID]; !exists {
+			reservation := &ec2.Reservation{}
+			reservation.SetReservationId(resID)
+			if instance.Reservation.OwnerId != nil {
+				reservation.SetOwnerId(*instance.Reservation.OwnerId)
+			}
+			reservation.Instances = []*ec2.Instance{}
+			reservationMap[resID] = reservation
+		}
+
+		instanceCopy := *instance.Instance
+		instanceCopy.State = &ec2.InstanceState{}
+		if info, ok := vm.EC2StateCodes[instance.Status]; ok {
+			instanceCopy.State.SetCode(info.Code)
+			instanceCopy.State.SetName(info.Name)
+		} else {
+			instanceCopy.State.SetCode(fallbackCode)
+			instanceCopy.State.SetName(fallbackName)
+		}
+
+		reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy)
 	}
 
 	reservations := make([]*ec2.Reservation, 0, len(reservationMap))
@@ -914,12 +985,8 @@ func (d *Daemon) handleEC2DescribeStoppedInstances(msg *nats.Msg) {
 		reservations = append(reservations, reservation)
 	}
 
-	output := &ec2.DescribeInstancesOutput{
-		Reservations: reservations,
-	}
-
-	respondWithJSON(msg, output)
-	slog.Info("handleEC2DescribeStoppedInstances completed", "count", len(reservations))
+	respondWithJSON(msg, &ec2.DescribeInstancesOutput{Reservations: reservations})
+	slog.Info(handlerName+" completed", "count", len(reservations))
 }
 
 // handleEC2ModifyInstanceAttribute modifies attributes of a stopped instance in shared KV.

hive/daemon/daemon_handlers_test.go

@@ -26,6 +26,7 @@ import (
 	handlers_ec2_vpc "github.com/mulgadc/hive/hive/handlers/ec2/vpc"
 	"github.com/mulgadc/hive/hive/objectstore"
 	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/nats-io/nats-server/v2/server"
@@ -80,6 +81,8 @@ func createFullTestDaemonWithJetStream(t *testing.T, natsURL string) *Daemon {
 	require.NoError(t, err)
 	err = daemon.jsManager.InitKVBucket()
 	require.NoError(t, err)
+	err = daemon.jsManager.InitTerminatedInstanceBucket()
+	require.NoError(t, err)
 
 	return daemon
 }
@@ -329,7 +332,7 @@ func TestHandleHealthCheck(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, reply)
 
-	var resp config.NodeHealthResponse
+	var resp types.NodeHealthResponse
 	err = json.Unmarshal(reply.Data, &resp)
 	require.NoError(t, err)
 
@@ -363,7 +366,7 @@ func TestHandleNodeDiscover(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, reply)
 
-	var resp NodeDiscoverResponse
+	var resp types.NodeDiscoverResponse
 	err = json.Unmarshal(reply.Data, &resp)
 	require.NoError(t, err)
 
@@ -552,9 +555,9 @@ func TestHandleEC2Events_StopInstance(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	cmd := qmp.Command{
+	cmd := types.EC2InstanceCommand{
 		ID:         instanceID,
-		Attributes: qmp.Attributes{StopInstance: true},
+		Attributes: types.EC2CommandAttributes{StopInstance: true},
 	}
 	cmdData, _ := json.Marshal(cmd)
 
@@ -598,9 +601,9 @@ func TestHandleEC2Events_TerminateInstance(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	cmd := qmp.Command{
+	cmd := types.EC2InstanceCommand{
 		ID:         instanceID,
-		Attributes: qmp.Attributes{TerminateInstance: true},
+		Attributes: types.EC2CommandAttributes{TerminateInstance: true},
 	}
 	cmdData, _ := json.Marshal(cmd)
 
@@ -620,6 +623,142 @@ func TestHandleEC2Events_TerminateInstance(t *testing.T) {
 	assert.Equal(t, vm.StateShuttingDown, status)
 }
 
+func TestHandleEC2Events_RebootRunningInstance(t *testing.T) {
+	natsURL := sharedNATSURL
+
+	daemon := createTestDaemon(t, natsURL)
+
+	instanceID := "i-test-reboot-001"
+	daemon.Instances.VMS[instanceID] = &vm.VM{
+		ID:           instanceID,
+		InstanceType: getTestInstanceType(),
+		Status:       vm.StateRunning,
+		Instance:     &ec2.Instance{},
+		QMPClient:    &qmp.QMPClient{},
+		AccountID:    testAccountID,
+	}
+
+	sub, err := daemon.natsConn.Subscribe(
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		daemon.handleEC2Events,
+	)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	cmd := types.EC2InstanceCommand{
+		ID:         instanceID,
+		Attributes: types.EC2CommandAttributes{RebootInstance: true},
+	}
+	cmdData, _ := json.Marshal(cmd)
+
+	reply, err := natsRequest(daemon.natsConn,
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		cmdData,
+		5*time.Second,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, reply)
+
+	// With nil QMPClient encoder/decoder, SendQMPCommand returns error,
+	// so we expect an error response (ServerInternal).
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Contains(t, errResp, "Code")
+
+	// Instance should remain in running state (reboot doesn't change state)
+	daemon.Instances.Mu.Lock()
+	status := daemon.Instances.VMS[instanceID].Status
+	daemon.Instances.Mu.Unlock()
+	assert.Equal(t, vm.StateRunning, status)
+}
+
+func TestHandleEC2Events_RebootStoppedInstance(t *testing.T) {
+	natsURL := sharedNATSURL
+
+	daemon := createTestDaemon(t, natsURL)
+
+	instanceID := "i-test-reboot-stopped"
+	daemon.Instances.VMS[instanceID] = &vm.VM{
+		ID:           instanceID,
+		InstanceType: getTestInstanceType(),
+		Status:       vm.StateStopped,
+		Instance:     &ec2.Instance{},
+		QMPClient:    &qmp.QMPClient{},
+		AccountID:    testAccountID,
+	}
+
+	sub, err := daemon.natsConn.Subscribe(
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		daemon.handleEC2Events,
+	)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	cmd := types.EC2InstanceCommand{
+		ID:         instanceID,
+		Attributes: types.EC2CommandAttributes{RebootInstance: true},
+	}
+	cmdData, _ := json.Marshal(cmd)
+
+	reply, err := natsRequest(daemon.natsConn,
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		cmdData,
+		5*time.Second,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, reply)
+
+	// Should get IncorrectInstanceState error
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Contains(t, errResp, "Code")
+}
+
+func TestHandleEC2Events_RebootTerminatedInstance(t *testing.T) {
+	natsURL := sharedNATSURL
+
+	daemon := createTestDaemon(t, natsURL)
+
+	instanceID := "i-test-reboot-terminated"
+	daemon.Instances.VMS[instanceID] = &vm.VM{
+		ID:           instanceID,
+		InstanceType: getTestInstanceType(),
+		Status:       vm.StateTerminated,
+		Instance:     &ec2.Instance{},
+		QMPClient:    &qmp.QMPClient{},
+		AccountID:    testAccountID,
+	}
+
+	sub, err := daemon.natsConn.Subscribe(
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		daemon.handleEC2Events,
+	)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	cmd := types.EC2InstanceCommand{
+		ID:         instanceID,
+		Attributes: types.EC2CommandAttributes{RebootInstance: true},
+	}
+	cmdData, _ := json.Marshal(cmd)
+
+	reply, err := natsRequest(daemon.natsConn,
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		cmdData,
+		5*time.Second,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, reply)
+
+	// Should get IncorrectInstanceState error
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Contains(t, errResp, "Code")
+}
+
 func TestHandleEC2Events_InstanceNotFound(t *testing.T) {
 	natsURL := sharedNATSURL
 
@@ -629,9 +768,9 @@ func TestHandleEC2Events_InstanceNotFound(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	cmd := qmp.Command{
+	cmd := types.EC2InstanceCommand{
 		ID:         "i-nonexistent",
-		Attributes: qmp.Attributes{StopInstance: true},
+		Attributes: types.EC2CommandAttributes{StopInstance: true},
 	}
 	cmdData, _ := json.Marshal(cmd)
 
@@ -1483,12 +1622,12 @@ func TestAttachVolume_ZoneMismatch(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			AttachVolume: true,
 		},
-		AttachVolumeData: &qmp.AttachVolumeData{
+		AttachVolumeData: &types.AttachVolumeData{
 			VolumeID: volumeID,
 		},
 	}
@@ -1940,7 +2079,7 @@ func TestHandleNodeStatus(t *testing.T) {
 	reply, err := daemon.natsConn.Request("hive.node.status.test", nil, 5*time.Second)
 	require.NoError(t, err)
 
-	var resp config.NodeStatusResponse
+	var resp types.NodeStatusResponse
 	err = json.Unmarshal(reply.Data, &resp)
 	require.NoError(t, err)
 
@@ -1966,7 +2105,7 @@ func TestHandleNodeStatus_NoVMs(t *testing.T) {
 	reply, err := daemon.natsConn.Request("hive.node.status.empty", nil, 5*time.Second)
 	require.NoError(t, err)
 
-	var resp config.NodeStatusResponse
+	var resp types.NodeStatusResponse
 	err = json.Unmarshal(reply.Data, &resp)
 	require.NoError(t, err)
 
@@ -2007,7 +2146,7 @@ func TestHandleNodeVMs(t *testing.T) {
 	reply, err := daemon.natsConn.Request("hive.node.vms.test", nil, 5*time.Second)
 	require.NoError(t, err)
 
-	var resp config.NodeVMsResponse
+	var resp types.NodeVMsResponse
 	err = json.Unmarshal(reply.Data, &resp)
 	require.NoError(t, err)
 
@@ -2016,7 +2155,7 @@ func TestHandleNodeVMs(t *testing.T) {
 	assert.Len(t, resp.VMs, 2)
 
 	// Build a lookup by instance ID
-	vmsByID := make(map[string]config.VMInfo)
+	vmsByID := make(map[string]types.VMInfo)
 	for _, v := range resp.VMs {
 		vmsByID[v.InstanceID] = v
 	}
@@ -2044,7 +2183,7 @@ func TestHandleNodeVMs_Empty(t *testing.T) {
 	reply, err := daemon.natsConn.Request("hive.node.vms.empty", nil, 5*time.Second)
 	require.NoError(t, err)
 
-	var resp config.NodeVMsResponse
+	var resp types.NodeVMsResponse
 	err = json.Unmarshal(reply.Data, &resp)
 	require.NoError(t, err)
 
@@ -2071,7 +2210,7 @@ func TestHandleNodeVMs_UnknownInstanceType(t *testing.T) {
 	reply, err := daemon.natsConn.Request("hive.node.vms.unknown", nil, 5*time.Second)
 	require.NoError(t, err)
 
-	var resp config.NodeVMsResponse
+	var resp types.NodeVMsResponse
 	err = json.Unmarshal(reply.Data, &resp)
 	require.NoError(t, err)
 
@@ -2547,7 +2686,7 @@ func TestHandleEC2TerminateStoppedInstance_WithVolumes(t *testing.T) {
 		},
 	}
 	// Add EFI, CloudInit, and a user volume with DeleteOnTermination
-	stoppedVM.EBSRequests.Requests = []config.EBSRequest{
+	stoppedVM.EBSRequests.Requests = []types.EBSRequest{
 		{Name: "vol-efi-001", EFI: true},
 		{Name: "vol-ci-001", CloudInit: true},
 		{Name: "vol-user-001", DeleteOnTermination: true},
@@ -2754,9 +2893,9 @@ func TestAttachVolume_MissingVolumeData(t *testing.T) {
 	defer sub.Unsubscribe()
 
 	// AttachVolume with nil AttachVolumeData
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			AttachVolume: true,
 		},
 		AttachVolumeData: nil,
@@ -2793,12 +2932,12 @@ func TestAttachVolume_InstanceNotRunning(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			AttachVolume: true,
 		},
-		AttachVolumeData: &qmp.AttachVolumeData{
+		AttachVolumeData: &types.AttachVolumeData{
 			VolumeID: "vol-test-123",
 		},
 	}
@@ -2834,12 +2973,12 @@ func TestAttachVolume_VolumeNotFound(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			AttachVolume: true,
 		},
-		AttachVolumeData: &qmp.AttachVolumeData{
+		AttachVolumeData: &types.AttachVolumeData{
 			VolumeID: "vol-nonexistent-999",
 		},
 	}
@@ -2896,12 +3035,12 @@ func TestAttachVolume_VolumeInUse(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			AttachVolume: true,
 		},
-		AttachVolumeData: &qmp.AttachVolumeData{
+		AttachVolumeData: &types.AttachVolumeData{
 			VolumeID: volumeID,
 		},
 	}
@@ -2939,9 +3078,9 @@ func TestDetachVolume_MissingVolumeData(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
 		DetachVolumeData: nil,
@@ -2978,12 +3117,12 @@ func TestDetachVolume_InstanceNotRunning(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: "vol-test-123",
 		},
 	}
@@ -3019,12 +3158,12 @@ func TestDetachVolume_VolumeNotAttached(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: "vol-not-attached-999",
 		},
 	}
@@ -3051,7 +3190,7 @@ func TestDetachVolume_BootVolumeRejected(t *testing.T) {
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
-	instance.EBSRequests.Requests = []config.EBSRequest{
+	instance.EBSRequests.Requests = []types.EBSRequest{
 		{Name: "vol-boot-001", Boot: true, DeviceName: "/dev/sda1"},
 	}
 	daemon.Instances.VMS[instanceID] = instance
@@ -3063,12 +3202,12 @@ func TestDetachVolume_BootVolumeRejected(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: "vol-boot-001",
 		},
 	}
@@ -3095,7 +3234,7 @@ func TestDetachVolume_DeviceMismatch(t *testing.T) {
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
-	instance.EBSRequests.Requests = []config.EBSRequest{
+	instance.EBSRequests.Requests = []types.EBSRequest{
 		{Name: "vol-mismatch-001", DeviceName: "/dev/sdf"},
 	}
 	daemon.Instances.VMS[instanceID] = instance
@@ -3107,12 +3246,12 @@ func TestDetachVolume_DeviceMismatch(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: "vol-mismatch-001",
 			Device:   "/dev/sdg",
 		},
@@ -3211,3 +3350,134 @@ func TestHandleEC2DescribeInstances_MalformedInstanceID(t *testing.T) {
 
 	assert.Contains(t, string(reply.Data), "InvalidInstanceID.Malformed")
 }
+
+// --- Terminated instance handler tests ---
+
+func TestHandleEC2DescribeTerminatedInstances_ReturnsTerminatedInstances(t *testing.T) {
+	daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
+
+	terminatedVM := &vm.VM{
+		ID:           "i-describe-term-001",
+		Status:       vm.StateTerminated,
+		InstanceType: getTestInstanceType(),
+		LastNode:     "node-1",
+		AccountID:    testAccountID,
+		Reservation: &ec2.Reservation{
+			ReservationId: aws.String("r-term-001"),
+			OwnerId:       aws.String("123456789012"),
+		},
+		Instance: &ec2.Instance{
+			InstanceId:   aws.String("i-describe-term-001"),
+			InstanceType: aws.String(getTestInstanceType()),
+		},
+	}
+	err := daemon.jsManager.WriteTerminatedInstance(terminatedVM.ID, terminatedVM)
+	require.NoError(t, err)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeTerminatedInstances", "hive-workers", daemon.handleEC2DescribeTerminatedInstances)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	input := &ec2.DescribeInstancesInput{}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeTerminatedInstances", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstancesOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+
+	found := false
+	for _, res := range output.Reservations {
+		for _, inst := range res.Instances {
+			if inst.InstanceId != nil && *inst.InstanceId == "i-describe-term-001" {
+				found = true
+				assert.Equal(t, "terminated", *inst.State.Name)
+				assert.Equal(t, int64(48), *inst.State.Code)
+			}
+		}
+	}
+	assert.True(t, found, "Should find terminated instance in DescribeTerminatedInstances output")
+
+	_ = daemon.jsManager.DeleteTerminatedInstance(terminatedVM.ID)
+}
+
+func TestHandleEC2DescribeTerminatedInstances_WithFilter(t *testing.T) {
+	daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
+
+	for _, id := range []string{"i-tfilter-001", "i-tfilter-002"} {
+		v := &vm.VM{
+			ID:        id,
+			Status:    vm.StateTerminated,
+			LastNode:  "node-1",
+			AccountID: testAccountID,
+			Reservation: &ec2.Reservation{
+				ReservationId: aws.String("r-tfilter"),
+				OwnerId:       aws.String("123456789012"),
+			},
+			Instance: &ec2.Instance{
+				InstanceId:   aws.String(id),
+				InstanceType: aws.String(getTestInstanceType()),
+			},
+		}
+		require.NoError(t, daemon.jsManager.WriteTerminatedInstance(v.ID, v))
+	}
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeTerminatedInstances", "hive-workers", daemon.handleEC2DescribeTerminatedInstances)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	// Filter for only one instance
+	input := &ec2.DescribeInstancesInput{
+		InstanceIds: []*string{aws.String("i-tfilter-001")},
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeTerminatedInstances", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstancesOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+
+	instanceCount := 0
+	for _, res := range output.Reservations {
+		instanceCount += len(res.Instances)
+	}
+	assert.Equal(t, 1, instanceCount, "Filter should return only the requested instance")
+
+	_ = daemon.jsManager.DeleteTerminatedInstance("i-tfilter-001")
+	_ = daemon.jsManager.DeleteTerminatedInstance("i-tfilter-002")
+}
+
+func TestHandleEC2TerminateStoppedInstance_WritesToTerminatedKV(t *testing.T) {
+	daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
+
+	stoppedVM := &vm.VM{
+		ID:           "i-stop-to-term-001",
+		Status:       vm.StateStopped,
+		InstanceType: getTestInstanceType(),
+		AccountID:    testAccountID,
+	}
+	require.NoError(t, daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM))
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.terminate", "hive-workers", daemon.handleEC2TerminateStoppedInstance)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reqData, _ := json.Marshal(terminateStoppedInstanceRequest{InstanceID: stoppedVM.ID})
+	reply, err := natsRequest(daemon.natsConn, "ec2.terminate", reqData, 30*time.Second)
+	require.NoError(t, err)
+	assert.Contains(t, string(reply.Data), "terminated")
+
+	// Verify instance is in terminated KV
+	terminatedInst, err := daemon.jsManager.LoadTerminatedInstance(stoppedVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, terminatedInst, "terminated instance should exist in terminated KV")
+	assert.Equal(t, vm.StateTerminated, terminatedInst.Status)
+
+	// Verify instance is removed from stopped KV
+	stoppedInst, err := daemon.jsManager.LoadStoppedInstance(stoppedVM.ID)
+	require.NoError(t, err)
+	assert.Nil(t, stoppedInst, "instance should be removed from stopped KV")
+
+	_ = daemon.jsManager.DeleteTerminatedInstance(stoppedVM.ID)
+}

hive/daemon/daemon_handlers_volume.go

@@ -8,8 +8,8 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/hive/hive/awserrors"
-	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/nats-io/nats.go"
@@ -20,7 +20,7 @@ import (
 //	Phase 1: ebs.mount via NATS   (rolls back with ebs.unmount)
 //	Phase 2: QMP blockdev-add     (rolls back Phase 1)
 //	Phase 3: QMP device_add       (rolls back Phase 2 + Phase 1)
-func (d *Daemon) handleAttachVolume(msg *nats.Msg, command qmp.Command, instance *vm.VM) {
+func (d *Daemon) handleAttachVolume(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
 	slog.Info("Attaching volume to instance", "instanceId", command.ID)
 
 	// Validate AttachVolumeData
@@ -89,7 +89,7 @@ func (d *Daemon) handleAttachVolume(msg *nats.Msg, command qmp.Command, instance
 	}
 
 	// Create EBS mount request
-	ebsRequest := config.EBSRequest{
+	ebsRequest := types.EBSRequest{
 		Name:       volumeID,
 		DeviceName: device,
 	}
@@ -108,7 +108,7 @@ func (d *Daemon) handleAttachVolume(msg *nats.Msg, command qmp.Command, instance
 		return
 	}
 
-	var mountResp config.EBSMountResponse
+	var mountResp types.EBSMountResponse
 	if err := json.Unmarshal(mountReply.Data, &mountResp); err != nil {
 		slog.Error("AttachVolume: failed to unmarshal mount response", "err", err)
 		respondWithError(msg, awserrors.ErrorServerInternal)
@@ -294,7 +294,7 @@ func (d *Daemon) handleAttachVolume(msg *nats.Msg, command qmp.Command, instance
 //	Phase 1: QMP device_del    (remove guest device)
 //	Phase 2: QMP blockdev-del  (remove block node)
 //	Phase 3: ebs.unmount NATS  (stop NBD server)
-func (d *Daemon) handleDetachVolume(msg *nats.Msg, command qmp.Command, instance *vm.VM) {
+func (d *Daemon) handleDetachVolume(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
 	slog.Info("Detaching volume from instance", "instanceId", command.ID)
 
 	// Validate DetachVolumeData
@@ -321,7 +321,7 @@ func (d *Daemon) handleDetachVolume(msg *nats.Msg, command qmp.Command, instance
 
 	// Find the volume in EBSRequests
 	instance.EBSRequests.Mu.Lock()
-	var ebsReq config.EBSRequest
+	var ebsReq types.EBSRequest
 	found := false
 	for _, req := range instance.EBSRequests.Requests {
 		if req.Name == volumeID {
@@ -481,7 +481,7 @@ func (d *Daemon) handleEC2ModifyVolume(msg *nats.Msg) {
 
 	// Notify viperblockd to reload state after volume modification (e.g. resize)
 	if modifyVolumeInput.VolumeId != nil {
-		syncData, err := json.Marshal(config.EBSSyncRequest{Volume: *modifyVolumeInput.VolumeId})
+		syncData, err := json.Marshal(types.EBSSyncRequest{Volume: *modifyVolumeInput.VolumeId})
 		if err != nil {
 			slog.Error("failed to marshal ebs.sync request", "volumeId", *modifyVolumeInput.VolumeId, "err", err)
 		} else {

hive/daemon/daemon_handlers_vpc.go

@@ -1,6 +1,9 @@
 package daemon
 
 import (
+	"encoding/json"
+	"log/slog"
+
 	"github.com/nats-io/nats.go"
 )
 
@@ -39,3 +42,22 @@ func (d *Daemon) handleEC2DeleteNetworkInterface(msg *nats.Msg) {
 func (d *Daemon) handleEC2DescribeNetworkInterfaces(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.DescribeNetworkInterfaces)
 }
+
+// handleAccountCreated creates a default VPC for a newly created account.
+func (d *Daemon) handleAccountCreated(msg *nats.Msg) {
+	var evt struct {
+		AccountID string `json:"account_id"`
+	}
+	if err := json.Unmarshal(msg.Data, &evt); err != nil {
+		slog.Error("Failed to unmarshal account creation event", "error", err)
+		return
+	}
+	if evt.AccountID == "" {
+		slog.Error("Account creation event has empty account ID")
+		return
+	}
+	if err := d.vpcService.EnsureDefaultVPC(evt.AccountID); err != nil {
+		slog.Error("Failed to create default VPC for new account",
+			"accountID", evt.AccountID, "error", err)
+	}
+}

hive/daemon/daemon_test.go

@@ -20,6 +20,7 @@ import (
 	handlers_ec2_volume "github.com/mulgadc/hive/hive/handlers/ec2/volume"
 	"github.com/mulgadc/hive/hive/objectstore"
 	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/pelletier/go-toml/v2"
@@ -861,20 +862,20 @@ func TestDaemon_BootAllocation(t *testing.T) {
 			InstanceType: getTestInstanceType(),
 			Status:       vm.StateRunning,
 			AccountID:    testAccountID,
-			Attributes:   qmp.Attributes{StopInstance: false},
+			Attributes:   types.EC2CommandAttributes{StopInstance: false},
 		},
 		"i-stopped": {
 			ID:           "i-stopped",
 			InstanceType: getTestInstanceType(),
 			Status:       vm.StateStopped,
 			AccountID:    testAccountID,
-			Attributes:   qmp.Attributes{StopInstance: true},
+			Attributes:   types.EC2CommandAttributes{StopInstance: true},
 		},
 		"i-terminated": {
 			ID:           "i-terminated",
 			InstanceType: getTestInstanceType(),
 			Status:       vm.StateTerminated,
-			Attributes:   qmp.Attributes{StopInstance: false},
+			Attributes:   types.EC2CommandAttributes{StopInstance: false},
 		},
 	}
 
@@ -1618,7 +1619,7 @@ func TestResourceManager_ConcurrentAccess(t *testing.T) {
 
 // TestEBSRequest_DeleteOnTermination_DefaultFalse verifies the default value
 func TestEBSRequest_DeleteOnTermination_DefaultFalse(t *testing.T) {
-	req := config.EBSRequest{
+	req := types.EBSRequest{
 		Name: "vol-test",
 		Boot: true,
 	}
@@ -1627,7 +1628,7 @@ func TestEBSRequest_DeleteOnTermination_DefaultFalse(t *testing.T) {
 
 // TestEBSRequest_DeleteOnTermination_SetTrue verifies the field can be set
 func TestEBSRequest_DeleteOnTermination_SetTrue(t *testing.T) {
-	req := config.EBSRequest{
+	req := types.EBSRequest{
 		Name:                "vol-test",
 		Boot:                true,
 		DeleteOnTermination: true,
@@ -1697,7 +1698,7 @@ func TestGenerateVolumes_DeleteOnTermination_FromBlockDeviceMapping(t *testing.T
 				"deleteOnTermination should match expected value")
 
 			// Verify the flag is correctly assigned to an EBSRequest
-			ebsReq := config.EBSRequest{
+			ebsReq := types.EBSRequest{
 				Name:                "vol-test",
 				Boot:                true,
 				DeleteOnTermination: deleteOnTermination,
@@ -1722,12 +1723,12 @@ func TestStopInstance_DeleteOnTermination_VolumeDeletion(t *testing.T) {
 
 	// Mock ebs.unmount subscriber
 	unmountSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		var req config.EBSRequest
+		var req types.EBSRequest
 		json.Unmarshal(msg.Data, &req)
 		mu.Lock()
 		unmountedVolumes[req.Name] = true
 		mu.Unlock()
-		resp := config.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		resp := types.EBSUnMountResponse{Volume: req.Name, Mounted: false}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -1736,12 +1737,12 @@ func TestStopInstance_DeleteOnTermination_VolumeDeletion(t *testing.T) {
 
 	// Mock ebs.delete subscriber
 	deleteSub, err := daemon.natsConn.Subscribe("ebs.delete", func(msg *nats.Msg) {
-		var req config.EBSDeleteRequest
+		var req types.EBSDeleteRequest
 		json.Unmarshal(msg.Data, &req)
 		mu.Lock()
 		ebsDeletedVolumes[req.Volume] = true
 		mu.Unlock()
-		resp := config.EBSDeleteResponse{Volume: req.Volume, Success: true}
+		resp := types.EBSDeleteResponse{Volume: req.Volume, Success: true}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -1760,8 +1761,8 @@ func TestStopInstance_DeleteOnTermination_VolumeDeletion(t *testing.T) {
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
 		QMPClient:    &qmp.QMPClient{}, // nil encoder/decoder => QMP will fail, which is fine
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:                "vol-root",
 					Boot:                true,
@@ -1820,12 +1821,12 @@ func TestStopInstance_DeleteOnTermination_False_SkipsVolumeDeletion(t *testing.T
 
 	// Mock ebs.unmount subscriber
 	unmountSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		var req config.EBSRequest
+		var req types.EBSRequest
 		json.Unmarshal(msg.Data, &req)
 		mu.Lock()
 		unmountedVolumes[req.Name] = true
 		mu.Unlock()
-		resp := config.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		resp := types.EBSUnMountResponse{Volume: req.Name, Mounted: false}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -1834,12 +1835,12 @@ func TestStopInstance_DeleteOnTermination_False_SkipsVolumeDeletion(t *testing.T
 
 	// Mock ebs.delete subscriber
 	deleteSub, err := daemon.natsConn.Subscribe("ebs.delete", func(msg *nats.Msg) {
-		var req config.EBSDeleteRequest
+		var req types.EBSDeleteRequest
 		json.Unmarshal(msg.Data, &req)
 		mu.Lock()
 		ebsDeletedVolumes[req.Volume] = true
 		mu.Unlock()
-		resp := config.EBSDeleteResponse{Volume: req.Volume, Success: true}
+		resp := types.EBSDeleteResponse{Volume: req.Volume, Success: true}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -1858,8 +1859,8 @@ func TestStopInstance_DeleteOnTermination_False_SkipsVolumeDeletion(t *testing.T
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
 		QMPClient:    &qmp.QMPClient{},
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:                "vol-keep",
 					Boot:                true,
@@ -1918,7 +1919,7 @@ func TestStopInstance_NoDelete_OnStop(t *testing.T) {
 
 	// Mock ebs.unmount subscriber
 	unmountSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := config.EBSUnMountResponse{Mounted: false}
+		resp := types.EBSUnMountResponse{Mounted: false}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -1927,12 +1928,12 @@ func TestStopInstance_NoDelete_OnStop(t *testing.T) {
 
 	// Mock ebs.delete subscriber - should NOT be called
 	deleteSub, err := daemon.natsConn.Subscribe("ebs.delete", func(msg *nats.Msg) {
-		var req config.EBSDeleteRequest
+		var req types.EBSDeleteRequest
 		json.Unmarshal(msg.Data, &req)
 		mu.Lock()
 		ebsDeletedVolumes[req.Volume] = true
 		mu.Unlock()
-		resp := config.EBSDeleteResponse{Volume: req.Volume, Success: true}
+		resp := types.EBSDeleteResponse{Volume: req.Volume, Success: true}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -1945,8 +1946,8 @@ func TestStopInstance_NoDelete_OnStop(t *testing.T) {
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
 		QMPClient:    &qmp.QMPClient{},
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:                "vol-stop-root",
 					Boot:                true,
@@ -2010,9 +2011,9 @@ func TestHandleEC2Events_AttachVolume(t *testing.T) {
 	defer sub.Unsubscribe()
 
 	t.Run("MissingAttachVolumeData", func(t *testing.T) {
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				AttachVolume: true,
 			},
 			// No AttachVolumeData
@@ -2036,12 +2037,12 @@ func TestHandleEC2Events_AttachVolume(t *testing.T) {
 		instance.Status = vm.StateStopped
 		daemon.Instances.Mu.Unlock()
 
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				AttachVolume: true,
 			},
-			AttachVolumeData: &qmp.AttachVolumeData{
+			AttachVolumeData: &types.AttachVolumeData{
 				VolumeID: volumeID,
 			},
 		}
@@ -2063,12 +2064,12 @@ func TestHandleEC2Events_AttachVolume(t *testing.T) {
 
 	t.Run("VolumeNotFound", func(t *testing.T) {
 		// volumeService.GetVolumeConfig will fail since we have no S3 backend
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				AttachVolume: true,
 			},
-			AttachVolumeData: &qmp.AttachVolumeData{
+			AttachVolumeData: &types.AttachVolumeData{
 				VolumeID: "vol-nonexistent",
 				Device:   "/dev/sdf",
 			},
@@ -2089,7 +2090,7 @@ func TestHandleEC2Events_AttachVolume(t *testing.T) {
 // TestEBSRequest_JSON_Serialization verifies DeleteOnTermination survives JSON round-trip
 // (important for JetStream state persistence)
 func TestEBSRequest_JSON_Serialization(t *testing.T) {
-	original := config.EBSRequest{
+	original := types.EBSRequest{
 		Name:                "vol-test-json",
 		Boot:                true,
 		DeleteOnTermination: true,
@@ -2099,7 +2100,7 @@ func TestEBSRequest_JSON_Serialization(t *testing.T) {
 	data, err := json.Marshal(original)
 	require.NoError(t, err)
 
-	var restored config.EBSRequest
+	var restored types.EBSRequest
 	err = json.Unmarshal(data, &restored)
 	require.NoError(t, err)
 
@@ -2136,8 +2137,8 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 			},
 		},
 		QMPClient: &qmp.QMPClient{}, // nil encoder/decoder
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:       volumeID,
 					DeviceName: "/dev/sdf",
@@ -2156,9 +2157,9 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 	defer sub.Unsubscribe()
 
 	t.Run("MissingDetachVolumeData", func(t *testing.T) {
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
 			// No DetachVolumeData
@@ -2180,12 +2181,12 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 		instance.Status = vm.StateStopped
 		daemon.Instances.Mu.Unlock()
 
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
-			DetachVolumeData: &qmp.DetachVolumeData{
+			DetachVolumeData: &types.DetachVolumeData{
 				VolumeID: volumeID,
 			},
 		}
@@ -2206,12 +2207,12 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 	})
 
 	t.Run("VolumeNotAttached", func(t *testing.T) {
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
-			DetachVolumeData: &qmp.DetachVolumeData{
+			DetachVolumeData: &types.DetachVolumeData{
 				VolumeID: "vol-nonexistent",
 			},
 		}
@@ -2231,18 +2232,18 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 
 		// Add a boot volume to the instance
 		instance.EBSRequests.Mu.Lock()
-		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, config.EBSRequest{
+		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, types.EBSRequest{
 			Name: bootVolumeID,
 			Boot: true,
 		})
 		instance.EBSRequests.Mu.Unlock()
 
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
-			DetachVolumeData: &qmp.DetachVolumeData{
+			DetachVolumeData: &types.DetachVolumeData{
 				VolumeID: bootVolumeID,
 			},
 		}
@@ -2266,18 +2267,18 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 		efiVolumeID := "vol-efi-protected"
 
 		instance.EBSRequests.Mu.Lock()
-		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, config.EBSRequest{
+		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, types.EBSRequest{
 			Name: efiVolumeID,
 			EFI:  true,
 		})
 		instance.EBSRequests.Mu.Unlock()
 
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
-			DetachVolumeData: &qmp.DetachVolumeData{
+			DetachVolumeData: &types.DetachVolumeData{
 				VolumeID: efiVolumeID,
 			},
 		}
@@ -2300,18 +2301,18 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 		ciVolumeID := "vol-cloudinit-protected"
 
 		instance.EBSRequests.Mu.Lock()
-		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, config.EBSRequest{
+		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, types.EBSRequest{
 			Name:      ciVolumeID,
 			CloudInit: true,
 		})
 		instance.EBSRequests.Mu.Unlock()
 
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
-			DetachVolumeData: &qmp.DetachVolumeData{
+			DetachVolumeData: &types.DetachVolumeData{
 				VolumeID: ciVolumeID,
 			},
 		}
@@ -2331,12 +2332,12 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 	})
 
 	t.Run("DeviceMismatch", func(t *testing.T) {
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
-			DetachVolumeData: &qmp.DetachVolumeData{
+			DetachVolumeData: &types.DetachVolumeData{
 				VolumeID: volumeID,
 				Device:   "/dev/sdg", // actual is /dev/sdf
 			},
@@ -2355,12 +2356,12 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 	t.Run("QMPDeviceDelFails_NoForce", func(t *testing.T) {
 		// With nil QMPClient encoder/decoder, SendQMPCommand returns error.
 		// Without force=true, this should return ServerInternal.
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				DetachVolume: true,
 			},
-			DetachVolumeData: &qmp.DetachVolumeData{
+			DetachVolumeData: &types.DetachVolumeData{
 				VolumeID: volumeID,
 				Force:    false,
 			},
@@ -2479,8 +2480,8 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 			},
 		},
 		QMPClient: qmpClient,
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:       "vol-root",
 					Boot:       true,
@@ -2499,10 +2500,10 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 	// Subscribe a mock ebs.unmount handler
 	ebsUnmountCalled := make(chan string, 1)
 	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		var req config.EBSRequest
+		var req types.EBSRequest
 		json.Unmarshal(msg.Data, &req)
 		ebsUnmountCalled <- req.Name
-		resp := config.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		resp := types.EBSUnMountResponse{Volume: req.Name, Mounted: false}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -2516,12 +2517,12 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: volumeID,
 		},
 	}
@@ -2627,8 +2628,8 @@ func TestDetachVolume_ForceFlag(t *testing.T) {
 			},
 		},
 		QMPClient: qmpClient,
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:       volumeID,
 					DeviceName: "/dev/sdf",
@@ -2641,7 +2642,7 @@ func TestDetachVolume_ForceFlag(t *testing.T) {
 
 	// Mock ebs.unmount
 	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := config.EBSUnMountResponse{Mounted: false}
+		resp := types.EBSUnMountResponse{Mounted: false}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -2655,12 +2656,12 @@ func TestDetachVolume_ForceFlag(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: volumeID,
 			Force:    true,
 		},
@@ -2741,8 +2742,8 @@ func TestDetachVolume_BlockdevDelFailure(t *testing.T) {
 			},
 		},
 		QMPClient: qmpClient,
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:       volumeID,
 					DeviceName: "/dev/sdf",
@@ -2760,12 +2761,12 @@ func TestDetachVolume_BlockdevDelFailure(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: volumeID,
 		},
 	}
@@ -2833,8 +2834,8 @@ func TestDetachVolume_SuccessWithDeviceMatch(t *testing.T) {
 			},
 		},
 		QMPClient: qmpClient,
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:       volumeID,
 					DeviceName: "/dev/sdh",
@@ -2846,7 +2847,7 @@ func TestDetachVolume_SuccessWithDeviceMatch(t *testing.T) {
 	daemon.Instances.VMS[instanceID] = instance
 
 	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := config.EBSUnMountResponse{Mounted: false}
+		resp := types.EBSUnMountResponse{Mounted: false}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -2860,12 +2861,12 @@ func TestDetachVolume_SuccessWithDeviceMatch(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: volumeID,
 			Device:   "/dev/sdh", // matches actual device
 		},
@@ -2909,8 +2910,8 @@ func TestAttachVolume_ReplacesStaleEBSRequest(t *testing.T) {
 		AccountID:    testAccountID,
 		Instance:     &ec2.Instance{},
 		QMPClient:    qmpClient,
-		EBSRequests: config.EBSRequests{
-			Requests: []config.EBSRequest{
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
 				{
 					Name:       volumeID,
 					DeviceName: "/dev/sdf", // stale entry from before stop
@@ -2923,7 +2924,7 @@ func TestAttachVolume_ReplacesStaleEBSRequest(t *testing.T) {
 
 	// Mock ebs.mount to return success with a new NBDURI
 	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
-		resp := config.EBSMountResponse{URI: "nbd://new:2222"}
+		resp := types.EBSMountResponse{URI: "nbd://new:2222"}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -2937,12 +2938,12 @@ func TestAttachVolume_ReplacesStaleEBSRequest(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			AttachVolume: true,
 		},
-		AttachVolumeData: &qmp.AttachVolumeData{
+		AttachVolumeData: &types.AttachVolumeData{
 			VolumeID: volumeID,
 			Device:   "/dev/sdg", // new device
 		},
@@ -2966,7 +2967,7 @@ func TestAttachVolume_ReplacesStaleEBSRequest(t *testing.T) {
 
 	// Direct unit test: simulate what the fixed attach handler does
 	instance.EBSRequests.Mu.Lock()
-	newReq := config.EBSRequest{
+	newReq := types.EBSRequest{
 		Name:       volumeID,
 		DeviceName: "/dev/sdg",
 		NBDURI:     "nbd://new:2222",
@@ -3309,17 +3310,17 @@ func TestRollbackEBSMount_Success(t *testing.T) {
 
 	// Mock ebs.unmount subscriber that returns success
 	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		var req config.EBSRequest
+		var req types.EBSRequest
 		json.Unmarshal(msg.Data, &req)
 		unmountCalled <- req.Name
-		resp := config.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		resp := types.EBSUnMountResponse{Volume: req.Name, Mounted: false}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	ebsReq := config.EBSRequest{
+	ebsReq := types.EBSRequest{
 		Name:       "vol-rollback-test",
 		DeviceName: "/dev/sdf",
 	}
@@ -3343,14 +3344,14 @@ func TestRollbackEBSMount_UnmountError(t *testing.T) {
 
 	// Mock ebs.unmount subscriber that returns an error
 	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := config.EBSUnMountResponse{Error: "unmount failed: device busy"}
+		resp := types.EBSUnMountResponse{Error: "unmount failed: device busy"}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	ebsReq := config.EBSRequest{Name: "vol-rollback-err"}
+	ebsReq := types.EBSRequest{Name: "vol-rollback-err"}
 
 	// Should not panic — errors are logged but not propagated
 	daemon.rollbackEBSMount(ebsReq)
@@ -3364,14 +3365,14 @@ func TestRollbackEBSMount_StillMounted(t *testing.T) {
 	daemon := createTestDaemon(t, natsURL)
 
 	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := config.EBSUnMountResponse{Mounted: true} // still mounted
+		resp := types.EBSUnMountResponse{Mounted: true} // still mounted
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
 	require.NoError(t, err)
 	defer sub.Unsubscribe()
 
-	ebsReq := config.EBSRequest{Name: "vol-still-mounted"}
+	ebsReq := types.EBSRequest{Name: "vol-still-mounted"}
 
 	// Should not panic
 	daemon.rollbackEBSMount(ebsReq)
@@ -3385,7 +3386,7 @@ func TestRollbackEBSMount_NATSTimeout(t *testing.T) {
 	daemon := createTestDaemon(t, natsURL)
 
 	// No ebs.unmount subscriber — will timeout
-	ebsReq := config.EBSRequest{Name: "vol-timeout"}
+	ebsReq := types.EBSRequest{Name: "vol-timeout"}
 
 	// Should not panic, just log the timeout
 	daemon.rollbackEBSMount(ebsReq)
@@ -3454,9 +3455,9 @@ func TestStopTerminate_IncorrectInstanceState(t *testing.T) {
 		instance.Status = vm.StateStopped
 		daemon.Instances.Mu.Unlock()
 
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				StopInstance: true,
 			},
 		}
@@ -3477,9 +3478,9 @@ func TestStopTerminate_IncorrectInstanceState(t *testing.T) {
 		instance.Status = vm.StateTerminated
 		daemon.Instances.Mu.Unlock()
 
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				TerminateInstance: true,
 			},
 		}
@@ -3531,8 +3532,8 @@ func TestNextAvailableDevice(t *testing.T) {
 			name: "existing EBSRequests skipped",
 			instance: &vm.VM{
 				Instance: &ec2.Instance{},
-				EBSRequests: config.EBSRequests{
-					Requests: []config.EBSRequest{
+				EBSRequests: types.EBSRequests{
+					Requests: []types.EBSRequest{
 						{Name: "vol-1", DeviceName: "/dev/sdf"},
 					},
 				},
@@ -3547,8 +3548,8 @@ func TestNextAvailableDevice(t *testing.T) {
 						{DeviceName: aws.String("/dev/sdf")},
 					},
 				},
-				EBSRequests: config.EBSRequests{
-					Requests: []config.EBSRequest{
+				EBSRequests: types.EBSRequests{
+					Requests: []types.EBSRequest{
 						{Name: "vol-1", DeviceName: "/dev/sdg"},
 					},
 				},
@@ -3586,8 +3587,8 @@ func TestNextAvailableDevice(t *testing.T) {
 		{
 			name: "empty DeviceName in EBSRequests ignored",
 			instance: &vm.VM{
-				EBSRequests: config.EBSRequests{
-					Requests: []config.EBSRequest{
+				EBSRequests: types.EBSRequests{
+					Requests: []types.EBSRequest{
 						{DeviceName: ""},
 						{DeviceName: "/dev/sdf"},
 					},

hive/daemon/jetstream.go

@@ -3,6 +3,7 @@ package daemon
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log/slog"
 	"strings"
 	"sync"
@@ -21,15 +22,20 @@ const (
 	InstanceStatePrefix = "node."
 	// StoppedInstancePrefix is the key prefix for stopped instances in shared KV
 	StoppedInstancePrefix = "instance."
+	// TerminatedInstanceBucket is the name of the KV bucket for terminated instances (auto-expiry via TTL)
+	TerminatedInstanceBucket = "hive-terminated-instances"
+	// TerminatedInstancePrefix is the key prefix for terminated instances
+	TerminatedInstancePrefix = "terminated."
 )
 
 // JetStreamManager manages JetStream KV store operations for instance state
 type JetStreamManager struct {
-	js        nats.JetStreamContext
-	kv        nats.KeyValue // hive-instance-state
-	clusterKV nats.KeyValue // hive-cluster-state
-	replicas  int
-	kvMu      sync.Mutex // protects kv during recovery
+	js           nats.JetStreamContext
+	kv           nats.KeyValue // hive-instance-state
+	clusterKV    nats.KeyValue // hive-cluster-state
+	terminatedKV nats.KeyValue // hive-terminated-instances
+	replicas     int
+	kvMu         sync.Mutex // protects kv during recovery
 }
 
 // NewJetStreamManager creates a new JetStreamManager from a NATS connection.
@@ -101,6 +107,34 @@ func (m *JetStreamManager) InitClusterStateBucket() error {
 	return nil
 }
 
+// InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
+// JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
+func (m *JetStreamManager) InitTerminatedInstanceBucket() error {
+	kv, err := m.js.KeyValue(TerminatedInstanceBucket)
+	if err != nil {
+		if errors.Is(err, nats.ErrBucketNotFound) {
+			slog.Debug("Creating JetStream KV bucket", "bucket", TerminatedInstanceBucket, "replicas", m.replicas)
+			kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{
+				Bucket:      TerminatedInstanceBucket,
+				Description: "Terminated instances (auto-expire after 1 hour)",
+				History:     1,
+				Replicas:    m.replicas,
+				TTL:         1 * time.Hour,
+			})
+			if err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	} else {
+		slog.Debug("Connected to existing JetStream KV bucket", "bucket", TerminatedInstanceBucket)
+	}
+
+	m.terminatedKV = kv
+	return nil
+}
+
 // isStreamUnavailable checks if an error indicates the underlying JetStream stream
 // was lost or is unreachable. This can happen during NATS cluster formation when
 // streams created with low replication are disrupted by node join/catchup operations.
@@ -120,18 +154,18 @@ func isStreamUnavailable(err error) bool {
 	return strings.Contains(err.Error(), "stream not found")
 }
 
-// recoverKVBucket attempts to reconnect to or re-create the instance-state KV bucket
-// after the underlying JetStream stream was lost during cluster formation.
-// Returns the recovered KV handle directly so callers avoid a racy re-read of m.kv.
-func (m *JetStreamManager) recoverKVBucket() (nats.KeyValue, error) {
+// recoverBucket attempts to reconnect to or re-create a KV bucket after the
+// underlying JetStream stream was lost during cluster formation.
+// Returns the recovered KV handle directly so callers avoid a racy re-read.
+func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.KeyValue) (nats.KeyValue, error) {
 	m.kvMu.Lock()
 	defer m.kvMu.Unlock()
 
 	// Try to reconnect to existing bucket first (another goroutine may have recovered it)
-	kv, err := m.js.KeyValue(InstanceStateBucket)
+	kv, err := m.js.KeyValue(cfg.Bucket)
 	if err == nil {
-		m.kv = kv
-		slog.Info("Reconnected to KV bucket", "bucket", InstanceStateBucket)
+		*field = kv
+		slog.Info("Reconnected to KV bucket", "bucket", cfg.Bucket)
 		return kv, nil
 	}
 
@@ -140,23 +174,35 @@ func (m *JetStreamManager) recoverKVBucket() (nats.KeyValue, error) {
 	}
 
 	// Bucket truly doesn't exist — recreate it
-	slog.Warn("KV bucket stream lost, recreating", "bucket", InstanceStateBucket, "replicas", m.replicas)
-	kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{
-		Bucket:      InstanceStateBucket,
-		Description: "Hive instance state storage",
-		History:     1,
-		Replicas:    m.replicas,
-	})
+	slog.Warn("KV bucket stream lost, recreating", "bucket", cfg.Bucket, "replicas", m.replicas)
+	cfg.History = 1
+	cfg.Replicas = m.replicas
+	kv, err = m.js.CreateKeyValue(cfg)
 	if err != nil {
-		slog.Error("Failed to recreate KV bucket", "bucket", InstanceStateBucket, "err", err)
+		slog.Error("Failed to recreate KV bucket", "bucket", cfg.Bucket, "err", err)
 		return nil, err
 	}
 
-	m.kv = kv
-	slog.Info("KV bucket recreated successfully", "bucket", InstanceStateBucket)
+	*field = kv
+	slog.Info("KV bucket recreated successfully", "bucket", cfg.Bucket)
 	return kv, nil
 }
 
+func (m *JetStreamManager) recoverKVBucket() (nats.KeyValue, error) {
+	return m.recoverBucket(&nats.KeyValueConfig{
+		Bucket:      InstanceStateBucket,
+		Description: "Hive instance state storage",
+	}, &m.kv)
+}
+
+func (m *JetStreamManager) recoverTerminatedKVBucket() (nats.KeyValue, error) {
+	return m.recoverBucket(&nats.KeyValueConfig{
+		Bucket:      TerminatedInstanceBucket,
+		Description: "Terminated instances (auto-expire after 1 hour)",
+		TTL:         1 * time.Hour,
+	}, &m.terminatedKV)
+}
+
 // Heartbeat represents a daemon's periodic health status published to cluster KV.
 type Heartbeat struct {
 	Node          string   `json:"node"`
@@ -256,11 +302,14 @@ func (m *JetStreamManager) WriteShutdownMarker(nodeID string) error {
 	if m.clusterKV == nil {
 		return errors.New("cluster state KV not initialized")
 	}
-	data, _ := json.Marshal(map[string]any{
+	data, err := json.Marshal(map[string]any{
 		"node":      nodeID,
 		"timestamp": time.Now().UTC().Format(time.RFC3339),
 	})
-	_, err := m.clusterKV.Put("shutdown."+nodeID, data)
+	if err != nil {
+		return fmt.Errorf("marshal shutdown marker: %w", err)
+	}
+	_, err = m.clusterKV.Put("shutdown."+nodeID, data)
 	return err
 }
 
@@ -296,14 +345,17 @@ func (m *JetStreamManager) WriteServiceManifest(nodeID string, services []string
 	if m.clusterKV == nil {
 		return errors.New("cluster state KV not initialized")
 	}
-	data, _ := json.Marshal(map[string]any{
+	data, err := json.Marshal(map[string]any{
 		"node":            nodeID,
 		"services":        services,
 		"nats_host":       natsHost,
 		"predastore_host": predastoreHost,
 		"timestamp":       time.Now().UTC().Format(time.RFC3339),
 	})
-	_, err := m.clusterKV.Put("node."+nodeID+".services", data)
+	if err != nil {
+		return fmt.Errorf("marshal service manifest: %w", err)
+	}
+	_, err = m.clusterKV.Put("node."+nodeID+".services", data)
 	return err
 }
 
@@ -427,52 +479,48 @@ func (m *JetStreamManager) DeleteState(nodeID string) error {
 	return nil
 }
 
-// UpdateReplicas updates the replica count for the KV bucket's underlying stream.
+// UpdateReplicas updates the replica count for ALL JetStream KV buckets.
+// It iterates over every KV_* stream and bumps replicas to match the cluster size.
+// This ensures service buckets (IAM, VPC, IGW, etc.) are replicated alongside daemon buckets.
 // This should be called when new nodes join the cluster.
-// Note: Increasing replicas requires the new replica count of NATS servers to be available.
 func (m *JetStreamManager) UpdateReplicas(newReplicas int) error {
 	if m.js == nil {
 		return errors.New("JetStream context not initialized")
 	}
 
-	// KV buckets are backed by streams with name "KV_<bucket>"
-	streamName := "KV_" + InstanceStateBucket
-
-	// Get current stream info
-	streamInfo, err := m.js.StreamInfo(streamName)
-	if err != nil {
-		return err
-	}
-
-	currentReplicas := streamInfo.Config.Replicas
-	if currentReplicas >= newReplicas {
-		slog.Debug("Stream already has sufficient replicas", "current", currentReplicas, "requested", newReplicas)
-		return nil
-	}
-
-	// Update the stream config with new replica count
-	streamInfo.Config.Replicas = newReplicas
-
-	_, err = m.js.UpdateStream(&streamInfo.Config)
-	if err != nil {
-		return err
-	}
-
 	m.replicas = newReplicas
-	slog.Info("Updated JetStream KV bucket replicas", "bucket", InstanceStateBucket, "oldReplicas", currentReplicas, "newReplicas", newReplicas)
 
-	// Also update the cluster-state bucket if it exists
-	if m.clusterKV != nil {
-		clusterStreamName := "KV_" + ClusterStateBucket
-		clusterInfo, err := m.js.StreamInfo(clusterStreamName)
-		if err == nil && clusterInfo.Config.Replicas < newReplicas {
-			clusterInfo.Config.Replicas = newReplicas
-			if _, err := m.js.UpdateStream(&clusterInfo.Config); err != nil {
-				slog.Warn("Failed to update cluster-state bucket replicas", "error", err)
-			} else {
-				slog.Info("Updated cluster-state bucket replicas", "bucket", ClusterStateBucket, "newReplicas", newReplicas)
-			}
+	// Iterate all streams and update any KV-backed stream (prefixed "KV_")
+	updated := 0
+	for name := range m.js.StreamNames() {
+		if !strings.HasPrefix(name, "KV_") {
+			continue
 		}
+
+		info, err := m.js.StreamInfo(name)
+		if err != nil {
+			slog.Warn("Failed to get stream info", "stream", name, "error", err)
+			continue
+		}
+
+		if info.Config.Replicas >= newReplicas {
+			continue
+		}
+
+		oldReplicas := info.Config.Replicas
+		info.Config.Replicas = newReplicas
+		if _, err := m.js.UpdateStream(&info.Config); err != nil {
+			slog.Warn("Failed to update KV bucket replicas", "stream", name, "error", err)
+			continue
+		}
+
+		bucket := strings.TrimPrefix(name, "KV_")
+		slog.Info("Updated KV bucket replicas", "bucket", bucket, "oldReplicas", oldReplicas, "newReplicas", newReplicas)
+		updated++
+	}
+
+	if updated > 0 {
+		slog.Info("KV replication update complete", "bucketsUpdated", updated, "replicas", newReplicas)
 	}
 
 	return nil
@@ -637,3 +685,158 @@ func (m *JetStreamManager) ListStoppedInstances() ([]*vm.VM, error) {
 
 	return instances, nil
 }
+
+// WriteTerminatedInstance writes a terminated instance to the terminated KV bucket.
+// The entry will auto-expire after the bucket's TTL (1 hour).
+func (m *JetStreamManager) WriteTerminatedInstance(instanceID string, instance *vm.VM) error {
+	if m.terminatedKV == nil {
+		return errors.New("terminated instance KV bucket not initialized")
+	}
+
+	jsonData, err := json.Marshal(instance)
+	if err != nil {
+		return err
+	}
+
+	key := TerminatedInstancePrefix + instanceID
+	_, err = m.terminatedKV.Put(key, jsonData)
+	if err != nil {
+		if isStreamUnavailable(err) {
+			slog.Warn("KV stream unavailable, attempting recovery", "operation", "WriteTerminatedInstance", "key", key, "err", err)
+			kv, recoverErr := m.recoverTerminatedKVBucket()
+			if recoverErr != nil {
+				return err
+			}
+			if _, retryErr := kv.Put(key, jsonData); retryErr != nil {
+				return retryErr
+			}
+			slog.Debug("Wrote terminated instance to JetStream KV (after recovery)", "key", key, "instanceId", instanceID)
+			return nil
+		}
+		return err
+	}
+
+	slog.Debug("Wrote terminated instance to JetStream KV", "key", key, "instanceId", instanceID)
+	return nil
+}
+
+// ListTerminatedInstances returns all terminated instances from the terminated KV bucket.
+func (m *JetStreamManager) ListTerminatedInstances() ([]*vm.VM, error) {
+	if m.terminatedKV == nil {
+		return nil, errors.New("terminated instance KV bucket not initialized")
+	}
+
+	keys, err := m.terminatedKV.Keys()
+	if err != nil {
+		if errors.Is(err, nats.ErrNoKeysFound) {
+			return nil, nil
+		}
+		if isStreamUnavailable(err) {
+			slog.Warn("KV stream unavailable, attempting recovery", "operation", "ListTerminatedInstances", "err", err)
+			kv, recoverErr := m.recoverTerminatedKVBucket()
+			if recoverErr != nil {
+				return nil, err
+			}
+			keys, err = kv.Keys()
+			if err != nil {
+				if errors.Is(err, nats.ErrNoKeysFound) {
+					return nil, nil
+				}
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	var instances []*vm.VM
+	for _, key := range keys {
+		if !strings.HasPrefix(key, TerminatedInstancePrefix) {
+			continue
+		}
+
+		entry, err := m.terminatedKV.Get(key)
+		if err != nil {
+			if errors.Is(err, nats.ErrKeyNotFound) {
+				continue
+			}
+			return nil, err
+		}
+
+		var instance vm.VM
+		if err := json.Unmarshal(entry.Value(), &instance); err != nil {
+			slog.Error("Failed to unmarshal terminated instance", "key", key, "err", err)
+			continue
+		}
+
+		instances = append(instances, &instance)
+	}
+
+	return instances, nil
+}
+
+// DeleteTerminatedInstance removes a terminated instance from the terminated KV bucket.
+func (m *JetStreamManager) DeleteTerminatedInstance(instanceID string) error {
+	if m.terminatedKV == nil {
+		return errors.New("terminated instance KV bucket not initialized")
+	}
+
+	key := TerminatedInstancePrefix + instanceID
+	err := m.terminatedKV.Delete(key)
+	if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
+		if isStreamUnavailable(err) {
+			slog.Warn("KV stream unavailable, attempting recovery", "operation", "DeleteTerminatedInstance", "key", key, "err", err)
+			kv, recoverErr := m.recoverTerminatedKVBucket()
+			if recoverErr != nil {
+				return err
+			}
+			if retryErr := kv.Delete(key); retryErr != nil && !errors.Is(retryErr, nats.ErrKeyNotFound) {
+				return retryErr
+			}
+			return nil
+		}
+		return err
+	}
+
+	slog.Debug("Deleted terminated instance from JetStream KV", "key", key)
+	return nil
+}
+
+// LoadTerminatedInstance loads a single terminated instance from the terminated KV bucket.
+// Returns nil, nil if the key does not exist.
+func (m *JetStreamManager) LoadTerminatedInstance(instanceID string) (*vm.VM, error) {
+	if m.terminatedKV == nil {
+		return nil, errors.New("terminated instance KV bucket not initialized")
+	}
+
+	key := TerminatedInstancePrefix + instanceID
+	entry, err := m.terminatedKV.Get(key)
+	if err != nil {
+		if errors.Is(err, nats.ErrKeyNotFound) {
+			return nil, nil
+		}
+		if isStreamUnavailable(err) {
+			slog.Warn("KV stream unavailable, attempting recovery", "operation", "LoadTerminatedInstance", "key", key, "err", err)
+			kv, recoverErr := m.recoverTerminatedKVBucket()
+			if recoverErr != nil {
+				return nil, err
+			}
+			entry, err = kv.Get(key)
+			if err != nil {
+				if errors.Is(err, nats.ErrKeyNotFound) {
+					return nil, nil
+				}
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	var instance vm.VM
+	if err := json.Unmarshal(entry.Value(), &instance); err != nil {
+		return nil, err
+	}
+
+	return &instance, nil
+}

hive/daemon/jetstream_test.go

@@ -962,3 +962,249 @@ func TestIsStreamUnavailable(t *testing.T) {
 	assert.False(t, isStreamUnavailable(errors.New("some other error")))
 	assert.False(t, isStreamUnavailable(nats.ErrKeyNotFound))
 }
+
+// --- Terminated instance KV tests ---
+
+func TestJetStreamManager_WriteAndLoadTerminatedInstance(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	testVM := &vm.VM{
+		ID:           "i-term-001",
+		Status:       vm.StateTerminated,
+		InstanceType: "t3.micro",
+		LastNode:     "node-1",
+	}
+
+	err = jsm.WriteTerminatedInstance(testVM.ID, testVM)
+	require.NoError(t, err)
+
+	loaded, err := jsm.LoadTerminatedInstance(testVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+	assert.Equal(t, testVM.ID, loaded.ID)
+	assert.Equal(t, vm.StateTerminated, loaded.Status)
+	assert.Equal(t, "t3.micro", loaded.InstanceType)
+	assert.Equal(t, "node-1", loaded.LastNode)
+
+	_ = jsm.DeleteTerminatedInstance(testVM.ID)
+}
+
+func TestJetStreamManager_LoadTerminatedInstance_NotFound(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	loaded, err := jsm.LoadTerminatedInstance("i-nonexistent")
+	require.NoError(t, err)
+	assert.Nil(t, loaded)
+}
+
+func TestJetStreamManager_DeleteTerminatedInstance(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	testVM := &vm.VM{
+		ID:     "i-term-del",
+		Status: vm.StateTerminated,
+	}
+
+	err = jsm.WriteTerminatedInstance(testVM.ID, testVM)
+	require.NoError(t, err)
+
+	err = jsm.DeleteTerminatedInstance(testVM.ID)
+	require.NoError(t, err)
+
+	loaded, err := jsm.LoadTerminatedInstance(testVM.ID)
+	require.NoError(t, err)
+	assert.Nil(t, loaded)
+
+	// Delete non-existent should not error
+	err = jsm.DeleteTerminatedInstance("i-does-not-exist")
+	require.NoError(t, err)
+}
+
+func TestJetStreamManager_ListTerminatedInstances(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	vms := []*vm.VM{
+		{ID: "i-tlist-001", Status: vm.StateTerminated, InstanceType: "t3.micro"},
+		{ID: "i-tlist-002", Status: vm.StateTerminated, InstanceType: "t3.small"},
+	}
+
+	for _, v := range vms {
+		require.NoError(t, jsm.WriteTerminatedInstance(v.ID, v))
+	}
+	defer func() {
+		for _, v := range vms {
+			_ = jsm.DeleteTerminatedInstance(v.ID)
+		}
+	}()
+
+	instances, err := jsm.ListTerminatedInstances()
+	require.NoError(t, err)
+	// Use >= because other tests may leave entries in the shared bucket
+	assert.GreaterOrEqual(t, len(instances), 2)
+
+	idSet := map[string]bool{}
+	for _, inst := range instances {
+		idSet[inst.ID] = true
+	}
+	assert.True(t, idSet["i-tlist-001"])
+	assert.True(t, idSet["i-tlist-002"])
+}
+
+func TestJetStreamManager_WriteLoadDeleteTerminatedInstance_RoundTrip(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	testVM := &vm.VM{
+		ID:     "i-trt-001",
+		Status: vm.StateTerminated,
+	}
+
+	// Write
+	require.NoError(t, jsm.WriteTerminatedInstance(testVM.ID, testVM))
+
+	// Load — should find it
+	loaded, err := jsm.LoadTerminatedInstance(testVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+	assert.Equal(t, vm.StateTerminated, loaded.Status)
+
+	// Delete
+	require.NoError(t, jsm.DeleteTerminatedInstance(testVM.ID))
+
+	// Load — should be gone
+	loaded, err = jsm.LoadTerminatedInstance(testVM.ID)
+	require.NoError(t, err)
+	assert.Nil(t, loaded)
+}
+
+func TestJetStreamManager_TerminatedInstance_KVNotInitialized(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	// Don't call InitTerminatedInstanceBucket
+
+	err = jsm.WriteTerminatedInstance("i-test", &vm.VM{})
+	assert.Error(t, err)
+
+	_, err = jsm.LoadTerminatedInstance("i-test")
+	assert.Error(t, err)
+
+	err = jsm.DeleteTerminatedInstance("i-test")
+	assert.Error(t, err)
+
+	_, err = jsm.ListTerminatedInstances()
+	assert.Error(t, err)
+}
+
+// --- Terminated KV bucket recovery tests ---
+
+// deleteTerminatedInstanceBucket deletes the underlying JetStream stream for the terminated-instances KV bucket.
+func deleteTerminatedInstanceBucket(t *testing.T, nc *nats.Conn) {
+	t.Helper()
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+	err = js.DeleteStream("KV_" + TerminatedInstanceBucket)
+	require.NoError(t, err)
+}
+
+func TestJetStreamManager_WriteTerminatedInstance_RecoverAfterStreamLost(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	deleteTerminatedInstanceBucket(t, nc)
+
+	testVM := &vm.VM{ID: "i-term-recover", Status: vm.StateTerminated, InstanceType: "t3.micro"}
+	err = jsm.WriteTerminatedInstance(testVM.ID, testVM)
+	require.NoError(t, err, "WriteTerminatedInstance should recover after stream loss")
+
+	loaded, err := jsm.LoadTerminatedInstance(testVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+	assert.Equal(t, "i-term-recover", loaded.ID)
+
+	_ = jsm.DeleteTerminatedInstance(testVM.ID)
+}
+
+func TestJetStreamManager_LoadTerminatedInstance_RecoverAfterStreamLost(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	deleteTerminatedInstanceBucket(t, nc)
+
+	loaded, err := jsm.LoadTerminatedInstance("i-nonexistent")
+	require.NoError(t, err, "LoadTerminatedInstance should recover after stream loss")
+	assert.Nil(t, loaded)
+}
+
+func TestJetStreamManager_DeleteTerminatedInstance_RecoverAfterStreamLost(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	deleteTerminatedInstanceBucket(t, nc)
+
+	err = jsm.DeleteTerminatedInstance("i-any")
+	require.NoError(t, err, "DeleteTerminatedInstance should recover after stream loss")
+}
+
+func TestJetStreamManager_ListTerminatedInstances_RecoverAfterStreamLost(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	deleteTerminatedInstanceBucket(t, nc)
+
+	instances, err := jsm.ListTerminatedInstances()
+	require.NoError(t, err, "ListTerminatedInstances should recover after stream loss")
+	assert.Empty(t, instances)
+}

hive/daemon/resource.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/types"
 )
 
 // errInsufficientCapacity is returned by allocateForLaunch when MinCount
@@ -38,7 +38,7 @@ func canAllocateCount(availVCPU, allocVCPU int, availMem, allocMem float64,
 
 // resourceStatsForType computes the InstanceTypeCap for a single instance type
 // given the remaining host resources. Pure function — no locks or side effects.
-func resourceStatsForType(remainVCPU int, remainMem float64, it *ec2.InstanceTypeInfo) config.InstanceTypeCap {
+func resourceStatsForType(remainVCPU int, remainMem float64, it *ec2.InstanceTypeInfo) types.InstanceTypeCap {
 	vCPUs := instanceTypeVCPUs(it)
 	memGB := float64(instanceTypeMemoryMiB(it)) / 1024.0
 
@@ -54,7 +54,7 @@ func resourceStatsForType(remainVCPU int, remainMem float64, it *ec2.InstanceTyp
 		name = *it.InstanceType
 	}
 
-	return config.InstanceTypeCap{
+	return types.InstanceTypeCap{
 		Name:      name,
 		VCPU:      int(vCPUs),
 		MemoryGB:  memGB,

hive/daemon/shutdown.go

@@ -53,6 +53,7 @@ func (d *Daemon) handleShutdownGate(msg *nats.Msg) {
 	var req ShutdownRequest
 	if err := json.Unmarshal(msg.Data, &req); err != nil {
 		slog.Error("handleShutdownGate: failed to unmarshal request", "error", err)
+		d.respondShutdownACK(msg, ShutdownACK{Node: d.node, Phase: "gate", Error: err.Error()})
 		return
 	}
 
@@ -105,6 +106,7 @@ func (d *Daemon) handleShutdownDrain(msg *nats.Msg) {
 	var req ShutdownRequest
 	if err := json.Unmarshal(msg.Data, &req); err != nil {
 		slog.Error("handleShutdownDrain: failed to unmarshal request", "error", err)
+		d.respondShutdownACK(msg, ShutdownACK{Node: d.node, Phase: "drain", Error: err.Error()})
 		return
 	}
 
@@ -162,6 +164,7 @@ func (d *Daemon) handleShutdownStorage(msg *nats.Msg) {
 	var req ShutdownRequest
 	if err := json.Unmarshal(msg.Data, &req); err != nil {
 		slog.Error("handleShutdownStorage: failed to unmarshal request", "error", err)
+		d.respondShutdownACK(msg, ShutdownACK{Node: d.node, Phase: "storage", Error: err.Error()})
 		return
 	}
 
@@ -196,6 +199,7 @@ func (d *Daemon) handleShutdownPersist(msg *nats.Msg) {
 	var req ShutdownRequest
 	if err := json.Unmarshal(msg.Data, &req); err != nil {
 		slog.Error("handleShutdownPersist: failed to unmarshal request", "error", err)
+		d.respondShutdownACK(msg, ShutdownACK{Node: d.node, Phase: "persist", Error: err.Error()})
 		return
 	}
 
@@ -226,6 +230,7 @@ func (d *Daemon) handleShutdownInfra(msg *nats.Msg) {
 	var req ShutdownRequest
 	if err := json.Unmarshal(msg.Data, &req); err != nil {
 		slog.Error("handleShutdownInfra: failed to unmarshal request", "error", err)
+		d.respondShutdownACK(msg, ShutdownACK{Node: d.node, Phase: "infra", Error: err.Error()})
 		return
 	}
 
@@ -290,9 +295,10 @@ func (d *Daemon) publishShutdownProgress(phase string, total, remaining int) {
 	}
 	data, err := json.Marshal(progress)
 	if err != nil {
+		slog.Error("Failed to marshal shutdown progress", "error", err)
 		return
 	}
 	if err := d.natsConn.Publish("hive.cluster.shutdown.progress", data); err != nil {
-		slog.Debug("Failed to publish shutdown progress", "error", err)
+		slog.Warn("Failed to publish shutdown progress", "error", err)
 	}
 }

hive/daemon/state_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/hive/hive/config"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
@@ -62,6 +62,7 @@ func createDaemonWithJetStream(t *testing.T) *Daemon {
 	require.NoError(t, err)
 	require.NoError(t, daemon.jsManager.InitKVBucket())
 	require.NoError(t, daemon.jsManager.InitClusterStateBucket())
+	require.NoError(t, daemon.jsManager.InitTerminatedInstanceBucket())
 
 	return daemon
 }
@@ -642,15 +643,20 @@ func TestRestoreInstances_ShuttingDownFinalizedToTerminated(t *testing.T) {
 	daemon.Instances.VMS = make(map[string]*vm.VM)
 	simulateCleanRestore(t, daemon)
 
-	instance, ok := daemon.Instances.VMS["i-restore-shut"]
-	require.True(t, ok, "instance should be loaded from state")
-	assert.Equal(t, vm.StateTerminated, instance.Status,
+	// shutting-down→terminated should be migrated to terminated KV
+	_, ok := daemon.Instances.VMS["i-restore-shut"]
+	assert.False(t, ok, "terminated instance should be removed from local map")
+
+	terminatedInst, err := daemon.jsManager.LoadTerminatedInstance("i-restore-shut")
+	require.NoError(t, err)
+	require.NotNil(t, terminatedInst, "terminated instance should exist in terminated KV")
+	assert.Equal(t, vm.StateTerminated, terminatedInst.Status,
 		"shutting-down instance should be finalized to terminated on recovery")
 }
 
-// TestRestoreInstances_TerminatedSkipped verifies that terminated instances
-// are loaded but not relaunched or modified.
-func TestRestoreInstances_TerminatedSkipped(t *testing.T) {
+// TestRestoreInstances_TerminatedRemoved verifies that terminated instances
+// are removed from per-node state on restore (no migration, just cleanup).
+func TestRestoreInstances_TerminatedRemoved(t *testing.T) {
 	daemon := createDaemonWithJetStream(t)
 
 	daemon.Instances.VMS["i-restore-term"] = &vm.VM{
@@ -662,10 +668,8 @@ func TestRestoreInstances_TerminatedSkipped(t *testing.T) {
 	daemon.Instances.VMS = make(map[string]*vm.VM)
 	simulateCleanRestore(t, daemon)
 
-	instance, ok := daemon.Instances.VMS["i-restore-term"]
-	require.True(t, ok, "terminated instance should still be loaded")
-	assert.Equal(t, vm.StateTerminated, instance.Status,
-		"terminated instance should remain terminated")
+	_, ok := daemon.Instances.VMS["i-restore-term"]
+	assert.False(t, ok, "terminated instance should be removed from local map")
 }
 
 // TestRestoreInstances_UserStoppedMigratedToSharedKV verifies that instances
@@ -676,7 +680,7 @@ func TestRestoreInstances_UserStoppedMigratedToSharedKV(t *testing.T) {
 	daemon.Instances.VMS["i-restore-userstop"] = &vm.VM{
 		ID:     "i-restore-userstop",
 		Status: vm.StateStopped,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			StopInstance: true,
 		},
 	}
@@ -770,7 +774,7 @@ func TestRestoreInstances_MixedStates(t *testing.T) {
 	}
 	daemon.Instances.VMS["i-mix-stopped"] = &vm.VM{
 		ID: "i-mix-stopped", Status: vm.StateStopped,
-		Attributes: qmp.Attributes{StopInstance: true},
+		Attributes: types.EC2CommandAttributes{StopInstance: true},
 	}
 	require.NoError(t, daemon.WriteState())
 
@@ -785,10 +789,17 @@ func TestRestoreInstances_MixedStates(t *testing.T) {
 	require.NotNil(t, stoppedFromStopping, "stopping→stopped should exist in shared KV")
 	assert.Equal(t, vm.StateStopped, stoppedFromStopping.Status)
 
-	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-mix-shutting"].Status,
-		"shutting-down should finalize to terminated")
-	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-mix-terminated"].Status,
-		"terminated should remain terminated")
+	// shutting-down should finalize to terminated and migrate to terminated KV
+	assert.Nil(t, daemon.Instances.VMS["i-mix-shutting"],
+		"shutting-down→terminated should be migrated to terminated KV")
+	terminatedFromShutting, err2 := daemon.jsManager.LoadTerminatedInstance("i-mix-shutting")
+	require.NoError(t, err2)
+	require.NotNil(t, terminatedFromShutting, "shutting-down→terminated should exist in terminated KV")
+	assert.Equal(t, vm.StateTerminated, terminatedFromShutting.Status)
+
+	// already-terminated should be removed from local map (no migration, just cleanup)
+	assert.Nil(t, daemon.Instances.VMS["i-mix-terminated"],
+		"terminated should be removed from local map")
 
 	assert.Nil(t, daemon.Instances.VMS["i-mix-stopped"],
 		"user-stopped should be migrated to shared KV")
@@ -824,10 +835,16 @@ func TestRestoreInstances_StatePersistsAfterRecovery(t *testing.T) {
 	require.NotNil(t, stoppedInst)
 	assert.Equal(t, vm.StateStopped, stoppedInst.Status)
 
-	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-persist-shut"].Status)
+	// shutting-down→terminated should be migrated to terminated KV
+	assert.Nil(t, daemon.Instances.VMS["i-persist-shut"],
+		"terminated instance should be removed from local map")
+	terminatedInst, err := daemon.jsManager.LoadTerminatedInstance("i-persist-shut")
+	require.NoError(t, err)
+	require.NotNil(t, terminatedInst)
+	assert.Equal(t, vm.StateTerminated, terminatedInst.Status)
 
 	// Second restart: stopped instance is in shared KV (not per-node state),
-	// so local map won't have it. Terminated instance persists in per-node state.
+	// terminated instance is in terminated KV (not per-node state).
 	daemon.Instances.VMS = make(map[string]*vm.VM)
 	simulateCleanRestore(t, daemon)
 
@@ -837,8 +854,49 @@ func TestRestoreInstances_StatePersistsAfterRecovery(t *testing.T) {
 	require.NotNil(t, stoppedInst2, "stopped instance should persist in shared KV through second restart")
 	assert.Equal(t, vm.StateStopped, stoppedInst2.Status)
 
-	assert.Equal(t, vm.StateTerminated, daemon.Instances.VMS["i-persist-shut"].Status,
-		"terminated state should persist through second restart")
+	// Terminated instance should still be in terminated KV (within TTL window)
+	terminatedInst2, err := daemon.jsManager.LoadTerminatedInstance("i-persist-shut")
+	require.NoError(t, err)
+	require.NotNil(t, terminatedInst2, "terminated instance should persist in terminated KV through second restart")
+	assert.Equal(t, vm.StateTerminated, terminatedInst2.Status)
+}
+
+// TestMigrateInstanceToKV verifies the unified migration helper works for both stopped and terminated paths.
+func TestMigrateInstanceToKV(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	// Test stopped migration
+	stoppedVM := &vm.VM{ID: "i-migrate-stop", Status: vm.StateStopped}
+	daemon.Instances.VMS[stoppedVM.ID] = stoppedVM
+	assert.True(t, daemon.migrateStoppedToSharedKV(stoppedVM))
+	assert.Nil(t, daemon.Instances.VMS[stoppedVM.ID], "should be removed from local map")
+	loaded, err := daemon.jsManager.LoadStoppedInstance(stoppedVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, loaded)
+	assert.Equal(t, daemon.node, loaded.LastNode)
+
+	// Test terminated migration
+	terminatedVM := &vm.VM{ID: "i-migrate-term", Status: vm.StateTerminated}
+	daemon.Instances.VMS[terminatedVM.ID] = terminatedVM
+	assert.True(t, daemon.migrateTerminatedToKV(terminatedVM))
+	assert.Nil(t, daemon.Instances.VMS[terminatedVM.ID], "should be removed from local map")
+	loadedTerm, err := daemon.jsManager.LoadTerminatedInstance(terminatedVM.ID)
+	require.NoError(t, err)
+	require.NotNil(t, loadedTerm)
+	assert.Equal(t, daemon.node, loadedTerm.LastNode)
+}
+
+// TestMigrateInstanceToKV_NoJetStream verifies graceful failure when JetStream is nil.
+func TestMigrateInstanceToKV_NoJetStream(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+	daemon.jsManager = nil
+
+	vm1 := &vm.VM{ID: "i-no-js", Status: vm.StateStopped}
+	daemon.Instances.VMS[vm1.ID] = vm1
+	assert.False(t, daemon.migrateStoppedToSharedKV(vm1))
+	assert.False(t, daemon.migrateTerminatedToKV(vm1))
+	// Instance should still be in local map since migration failed
+	assert.NotNil(t, daemon.Instances.VMS[vm1.ID])
 }
 
 // TestStatePersistence_RoundTrip verifies that all instance fields survive
@@ -850,7 +908,7 @@ func TestStatePersistence_RoundTrip(t *testing.T) {
 		ID:           "i-roundtrip",
 		Status:       vm.StateRunning,
 		InstanceType: "t3.micro",
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			StopInstance:      false,
 			TerminateInstance: false,
 		},
@@ -881,7 +939,7 @@ func TestRestoreInstances_StoppedInstanceMigratedToSharedKV(t *testing.T) {
 		ID:           instanceID,
 		InstanceType: "t3.micro",
 		Status:       vm.StateStopped,
-		Attributes:   qmp.Attributes{StopInstance: true},
+		Attributes:   types.EC2CommandAttributes{StopInstance: true},
 	}
 	require.NoError(t, daemon.WriteState())
 

hive/formation/formation.go

@@ -56,6 +56,10 @@ type SharedCredentials struct {
 	NatsToken   string `json:"nats_token"`
 	ClusterName string `json:"cluster_name"`
 	Region      string `json:"region"`
+
+	// Admin credentials (generated by init node, shared to join nodes)
+	AdminAccessKey string `json:"admin_access_key,omitempty"`
+	AdminSecretKey string `json:"admin_secret_key,omitempty"`
 }
 
 // FormationServer is a lightweight HTTP server that coordinates cluster formation.

hive/gateway/auth_test.go

@@ -93,8 +93,8 @@ func (m *mockIAMService) GetUserPolicies(_, _ string) ([]handlers_iam.PolicyDocu
 func (m *mockIAMService) DecryptSecret(ciphertext string) (string, error) {
 	return handlers_iam.DecryptSecret(ciphertext, m.masterKey)
 }
-func (m *mockIAMService) SeedRootUser(_ *handlers_iam.BootstrapData) error { return nil }
-func (m *mockIAMService) IsEmpty() (bool, error)                           { return true, nil }
+func (m *mockIAMService) SeedBootstrap(_ *handlers_iam.BootstrapData) error { return nil }
+func (m *mockIAMService) IsEmpty() (bool, error)                            { return true, nil }
 func (m *mockIAMService) CreateAccount(_ string) (*handlers_iam.Account, error) {
 	return nil, nil
 }

hive/gateway/ec2.go

@@ -65,6 +65,9 @@ var ec2Actions = map[string]EC2Handler{
 	"StopInstances": ec2Handler(func(input *ec2.StopInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_instance.StopInstances(input, gw.NATSConn, accountID)
 	}),
+	"RebootInstances": ec2Handler(func(input *ec2.RebootInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_instance.RebootInstances(input, gw.NATSConn, accountID)
+	}),
 	"TerminateInstances": ec2Handler(func(input *ec2.TerminateInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_instance.TerminateInstances(input, gw.NATSConn, accountID)
 	}),
@@ -231,7 +234,11 @@ var ec2LocalActions = map[string]bool{
 }
 
 func (gw *GatewayConfig) EC2_Request(w http.ResponseWriter, r *http.Request) error {
-	body, _ := io.ReadAll(r.Body)
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		slog.Error("Failed to read EC2 request body", "error", err)
+		return errors.New(awserrors.ErrorInternalError)
+	}
 	queryArgs := ParseAWSQueryArgs(string(body))
 
 	action := queryArgs["Action"]

hive/gateway/ec2/instance/DescribeInstances.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log/slog"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -105,27 +106,22 @@ func DescribeInstances(input *ec2.DescribeInstancesInput, natsConn *nats.Conn, e
 		}
 	}
 
-	// Query stopped instances from shared KV (via queue group — only one daemon handles it)
-	stoppedReqMsg := nats.NewMsg("ec2.DescribeStoppedInstances")
-	stoppedReqMsg.Data = jsonData
-	stoppedReqMsg.Header.Set(utils.AccountIDHeader, accountID)
-	stoppedMsg, err := natsConn.RequestMsg(stoppedReqMsg, 3*time.Second)
-	if err != nil {
-		slog.Warn("DescribeInstances: Failed to query stopped instances (may not be available)", "err", err)
-	} else {
-		responseError, parseErr := utils.ValidateErrorPayload(stoppedMsg.Data)
-		if parseErr != nil {
-			slog.Warn("DescribeInstances: Stopped instances query returned error", "code", responseError.Code)
-		} else {
-			var stoppedOutput ec2.DescribeInstancesOutput
-			if err := json.Unmarshal(stoppedMsg.Data, &stoppedOutput); err != nil {
-				slog.Error("DescribeInstances: Failed to unmarshal stopped instances response", "err", err)
-			} else if stoppedOutput.Reservations != nil {
-				allReservations = append(allReservations, stoppedOutput.Reservations...)
-				slog.Info("DescribeInstances: Collected stopped instance reservations", "count", len(stoppedOutput.Reservations))
+	// Query stopped and terminated instances in parallel (both use queue groups — single responder each)
+	var kvMu sync.Mutex
+	var kvWg sync.WaitGroup
+	for _, topic := range []string{"ec2.DescribeStoppedInstances", "ec2.DescribeTerminatedInstances"} {
+		kvWg.Add(1)
+		go func(topic string) {
+			defer kvWg.Done()
+			reservations := queryInstanceBucket(natsConn, topic, jsonData, accountID)
+			if len(reservations) > 0 {
+				kvMu.Lock()
+				allReservations = append(allReservations, reservations...)
+				kvMu.Unlock()
 			}
-		}
+		}(topic)
 	}
+	kvWg.Wait()
 
 	// Build final aggregated response
 	output := &ec2.DescribeInstancesOutput{
@@ -135,3 +131,28 @@ func DescribeInstances(input *ec2.DescribeInstancesInput, natsConn *nats.Conn, e
 	slog.Info("DescribeInstances: Aggregated response", "total_reservations", len(allReservations))
 	return output, nil
 }
+
+// queryInstanceBucket sends a NATS request to a describe topic and returns the reservations.
+func queryInstanceBucket(natsConn *nats.Conn, topic string, jsonData []byte, accountID string) []*ec2.Reservation {
+	reqMsg := nats.NewMsg(topic)
+	reqMsg.Data = jsonData
+	reqMsg.Header.Set(utils.AccountIDHeader, accountID)
+	msg, err := natsConn.RequestMsg(reqMsg, 3*time.Second)
+	if err != nil {
+		slog.Warn("DescribeInstances: Failed to query instance bucket", "topic", topic, "err", err)
+		return nil
+	}
+	if responseError, parseErr := utils.ValidateErrorPayload(msg.Data); parseErr != nil {
+		slog.Warn("DescribeInstances: Instance bucket query returned error", "topic", topic, "code", responseError.Code)
+		return nil
+	}
+	var output ec2.DescribeInstancesOutput
+	if err := json.Unmarshal(msg.Data, &output); err != nil {
+		slog.Error("DescribeInstances: Failed to unmarshal instance bucket response", "topic", topic, "err", err)
+		return nil
+	}
+	if len(output.Reservations) > 0 {
+		slog.Info("DescribeInstances: Collected reservations from bucket", "topic", topic, "count", len(output.Reservations))
+	}
+	return output.Reservations
+}

hive/gateway/ec2/instance/RebootInstances.go

@@ -0,0 +1,79 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	"github.com/mulgadc/hive/hive/types"
+	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats.go"
+)
+
+// RebootInstances sends reboot commands to specified instances via NATS.
+// Unlike stop+start, reboot keeps the instance running and sends a QMP system_reset.
+// Returns an empty response on success (AWS returns no state-change data).
+func RebootInstances(input *ec2.RebootInstancesInput, natsConn *nats.Conn, accountID string) (*ec2.RebootInstancesOutput, error) {
+	if len(input.InstanceIds) == 0 {
+		return nil, fmt.Errorf("no instance IDs provided")
+	}
+
+	slog.Info("RebootInstances: Processing request", "instance_count", len(input.InstanceIds))
+
+	for _, instanceIDPtr := range input.InstanceIds {
+		if instanceIDPtr == nil {
+			continue
+		}
+		instanceID := *instanceIDPtr
+
+		command := types.EC2InstanceCommand{
+			ID: instanceID,
+			Attributes: types.EC2CommandAttributes{
+				RebootInstance: true,
+			},
+		}
+
+		jsonData, err := json.Marshal(command)
+		if err != nil {
+			slog.Error("RebootInstances: Failed to marshal command", "instance_id", instanceID, "err", err)
+			continue
+		}
+
+		subject := fmt.Sprintf("ec2.cmd.%s", instanceID)
+		reqMsg := nats.NewMsg(subject)
+		reqMsg.Data = jsonData
+		reqMsg.Header.Set(utils.AccountIDHeader, accountID)
+		msg, err := natsConn.RequestMsg(reqMsg, 5*time.Second)
+		if err != nil {
+			slog.Error("RebootInstances: Failed to send command", "instance_id", instanceID, "err", err)
+
+			// NATS timeout means no daemon has a subscription for this instance.
+			// Check if the instance exists in the stopped-instances KV bucket —
+			// if so, return IncorrectInstanceState instead of NotFound.
+			describeInput := &ec2.DescribeInstancesInput{
+				InstanceIds: []*string{&instanceID},
+			}
+			describeData, _ := json.Marshal(describeInput)
+			if reservations := queryInstanceBucket(natsConn, "ec2.DescribeStoppedInstances", describeData, accountID); len(reservations) > 0 {
+				return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+			}
+
+			return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+		}
+
+		// Check if the daemon returned an error response (e.g. ownership check failure)
+		if responseError, parseErr := utils.ValidateErrorPayload(msg.Data); parseErr != nil {
+			slog.Error("RebootInstances: Daemon returned error", "instance_id", instanceID, "code", *responseError.Code)
+			return nil, errors.New(*responseError.Code)
+		}
+
+		slog.Info("RebootInstances: Command sent successfully", "instance_id", instanceID)
+	}
+
+	slog.Info("RebootInstances: Completed", "total_instances", len(input.InstanceIds))
+	return &ec2.RebootInstancesOutput{}, nil
+}

hive/gateway/ec2/instance/RebootInstances_test.go

@@ -0,0 +1,130 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/hive/hive/awserrors"
+	"github.com/mulgadc/hive/hive/types"
+	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRebootInstances_Success(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-0123456789abcdef0"
+
+	nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
+		var cmd types.EC2InstanceCommand
+		err := json.Unmarshal(msg.Data, &cmd)
+		require.NoError(t, err)
+
+		assert.Equal(t, instanceID, cmd.ID)
+		assert.True(t, cmd.Attributes.RebootInstance)
+
+		msg.Respond([]byte(`{}`))
+	})
+
+	input := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	output, err := RebootInstances(input, nc, "123456789012")
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+}
+
+func TestRebootInstances_MultipleInstances(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	ids := []string{"i-001", "i-002"}
+
+	for _, id := range ids {
+		nc.Subscribe("ec2.cmd."+id, func(msg *nats.Msg) {
+			msg.Respond([]byte(`{}`))
+		})
+	}
+
+	input := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{aws.String(ids[0]), aws.String(ids[1])},
+	}
+
+	output, err := RebootInstances(input, nc, "123456789012")
+
+	require.NoError(t, err)
+	require.NotNil(t, output)
+}
+
+func TestRebootInstances_EmptyInstanceIds(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	input := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{},
+	}
+
+	_, err := RebootInstances(input, nc, "123456789012")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "no instance IDs provided")
+}
+
+func TestRebootInstances_NATSRequestFails(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	input := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{aws.String("i-nosubscriber")},
+	}
+
+	_, err := RebootInstances(input, nc, "123456789012")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+}
+
+func TestRebootInstances_StoppedInstance(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-stopped"
+
+	// No ec2.cmd subscription — simulates a stopped instance with no daemon listener.
+	// Subscribe to the stopped-instances describe topic to return the instance.
+	nc.Subscribe("ec2.DescribeStoppedInstances", func(msg *nats.Msg) {
+		resp := ec2.DescribeInstancesOutput{
+			Reservations: []*ec2.Reservation{
+				{Instances: []*ec2.Instance{{InstanceId: aws.String(instanceID)}}},
+			},
+		}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+
+	input := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	_, err := RebootInstances(input, nc, "123456789012")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorIncorrectInstanceState, err.Error())
+}
+
+func TestRebootInstances_DaemonError(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	instanceID := "i-error"
+
+	nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
+		msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorIncorrectInstanceState))
+	})
+
+	input := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	_, err := RebootInstances(input, nc, "123456789012")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorIncorrectInstanceState, err.Error())
+}

hive/gateway/ec2/instance/RunInstances.go

@@ -30,6 +30,10 @@ func ValidateRunInstancesInput(input *ec2.RunInstancesInput) (err error) {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
+	if input.MaxCount == nil {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+
 	if *input.MaxCount == 0 {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}

hive/gateway/ec2/instance/StopInstances.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -31,15 +31,9 @@ func StopInstances(input *ec2.StopInstancesInput, natsConn *nats.Conn, accountID
 		}
 		instanceID := *instanceIDPtr
 
-		// Build the QMP command to stop the instance
-		// Note: system_powerdown with stop_instance=true prevents auto-restart on daemon boot
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			QMPCommand: qmp.QMPCommand{
-				Execute:   "system_powerdown",
-				Arguments: map[string]any{},
-			},
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				StopInstance:      true, // Don't auto-restart on daemon boot
 				TerminateInstance: false,
 			},

hive/gateway/ec2/instance/StopInstances_test.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -18,12 +18,11 @@ func TestStopInstances_Success(t *testing.T) {
 	instanceID := "i-0123456789abcdef0"
 
 	nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		var cmd qmp.Command
+		var cmd types.EC2InstanceCommand
 		err := json.Unmarshal(msg.Data, &cmd)
 		require.NoError(t, err)
 
 		assert.Equal(t, instanceID, cmd.ID)
-		assert.Equal(t, "system_powerdown", cmd.QMPCommand.Execute)
 		assert.True(t, cmd.Attributes.StopInstance)
 		assert.False(t, cmd.Attributes.TerminateInstance)
 

hive/gateway/ec2/instance/TerminateInstances.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -36,15 +36,9 @@ func TerminateInstances(input *ec2.TerminateInstancesInput, natsConn *nats.Conn,
 		}
 		instanceID := *instanceIDPtr
 
-		// Build the QMP command to terminate the instance
-		// Note: stop_instance=true prevents restart on daemon/node restart
-		command := qmp.Command{
+		command := types.EC2InstanceCommand{
 			ID: instanceID,
-			QMPCommand: qmp.QMPCommand{
-				Execute:   "system_powerdown",
-				Arguments: map[string]any{},
-			},
-			Attributes: qmp.Attributes{
+			Attributes: types.EC2CommandAttributes{
 				StopInstance:      true, // Prevent restart on daemon/node restart
 				TerminateInstance: true,
 			},
@@ -57,18 +51,36 @@ func TerminateInstances(input *ec2.TerminateInstancesInput, natsConn *nats.Conn,
 			continue
 		}
 
-		// Send NATS request to the specific instance topic with account ID header
+		// Send NATS request to the specific instance topic with account ID header.
+		// Retry briefly on ErrNoResponders — after a cluster restart the
+		// per-instance NATS subscription may not have propagated to all
+		// servers yet.
 		subject := fmt.Sprintf("ec2.cmd.%s", instanceID)
-		reqMsg := nats.NewMsg(subject)
-		reqMsg.Data = jsonData
-		reqMsg.Header.Set(utils.AccountIDHeader, accountID)
-		msg, err := natsConn.RequestMsg(reqMsg, 5*time.Second)
+		var msg *nats.Msg
+		for attempt := range 3 {
+			reqMsg := nats.NewMsg(subject)
+			reqMsg.Data = jsonData
+			reqMsg.Header.Set(utils.AccountIDHeader, accountID)
+			msg, err = natsConn.RequestMsg(reqMsg, 5*time.Second)
+			if err == nil || !errors.Is(err, nats.ErrNoResponders) {
+				break
+			}
+			if attempt < 2 {
+				slog.Debug("TerminateInstances: No responder on per-instance topic, retrying",
+					"instance_id", instanceID, "attempt", attempt+1)
+				time.Sleep(time.Duration(attempt+1) * time.Second)
+			}
+		}
 		if err != nil {
 			// If no daemon owns this instance, try the ec2.terminate topic for stopped instances
 			if errors.Is(err, nats.ErrNoResponders) {
 				slog.Info("TerminateInstances: No responder on per-instance topic, trying ec2.terminate", "instance_id", instanceID)
 
-				terminateReq, _ := json.Marshal(terminateStoppedInstanceRequest{InstanceID: instanceID})
+				terminateReq, err := json.Marshal(terminateStoppedInstanceRequest{InstanceID: instanceID})
+				if err != nil {
+					slog.Error("TerminateInstances: Failed to marshal terminate request", "instance_id", instanceID, "err", err)
+					continue
+				}
 				terminateReqMsg := nats.NewMsg("ec2.terminate")
 				terminateReqMsg.Data = terminateReq
 				terminateReqMsg.Header.Set(utils.AccountIDHeader, accountID)
@@ -80,11 +92,17 @@ func TerminateInstances(input *ec2.TerminateInstancesInput, natsConn *nats.Conn,
 						continue
 					}
 				}
+
+				// Check if instance is already terminated (idempotent, matches AWS behavior)
+				if isAlreadyTerminated(natsConn, instanceID, accountID) {
+					slog.Info("TerminateInstances: Instance already terminated", "instance_id", instanceID)
+					stateChanges = append(stateChanges, newStateChange(instanceID, 48, "terminated", 48, "terminated"))
+					continue
+				}
 			}
 
 			slog.Error("TerminateInstances: Failed to send command", "instance_id", instanceID, "err", err)
-			stateChanges = append(stateChanges, newStateChange(instanceID, 16, "running", 16, "running"))
-			continue
+			return nil, fmt.Errorf("failed to terminate instance %s: %w", instanceID, err)
 		}
 
 		// Check if the daemon returned an error response (e.g. ownership check failure)
@@ -105,3 +123,36 @@ func TerminateInstances(input *ec2.TerminateInstancesInput, natsConn *nats.Conn,
 	slog.Info("TerminateInstances: Completed", "total_instances", len(stateChanges))
 	return output, nil
 }
+
+// isAlreadyTerminated checks if an instance exists in the terminated KV bucket.
+func isAlreadyTerminated(natsConn *nats.Conn, instanceID, accountID string) bool {
+	describeInput := &ec2.DescribeInstancesInput{
+		InstanceIds: []*string{&instanceID},
+	}
+	reqData, err := json.Marshal(describeInput)
+	if err != nil {
+		slog.Warn("isAlreadyTerminated: failed to marshal request", "instanceId", instanceID, "err", err)
+		return false
+	}
+	reqMsg := nats.NewMsg("ec2.DescribeTerminatedInstances")
+	reqMsg.Data = reqData
+	reqMsg.Header.Set(utils.AccountIDHeader, accountID)
+	msg, err := natsConn.RequestMsg(reqMsg, 3*time.Second)
+	if err != nil {
+		slog.Warn("isAlreadyTerminated: failed to query terminated instances", "instanceId", instanceID, "err", err)
+		return false
+	}
+	var output ec2.DescribeInstancesOutput
+	if unmarshalErr := json.Unmarshal(msg.Data, &output); unmarshalErr != nil {
+		slog.Warn("isAlreadyTerminated: failed to unmarshal response", "instanceId", instanceID, "err", unmarshalErr)
+		return false
+	}
+	for _, res := range output.Reservations {
+		for _, inst := range res.Instances {
+			if inst.InstanceId != nil && *inst.InstanceId == instanceID {
+				return true
+			}
+		}
+	}
+	return false
+}

hive/gateway/ec2/instance/TerminateInstances_test.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -18,12 +18,11 @@ func TestTerminateInstances_Success(t *testing.T) {
 	instanceID := "i-0123456789abcdef0"
 
 	nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		var cmd qmp.Command
+		var cmd types.EC2InstanceCommand
 		err := json.Unmarshal(msg.Data, &cmd)
 		require.NoError(t, err)
 
 		assert.Equal(t, instanceID, cmd.ID)
-		assert.Equal(t, "system_powerdown", cmd.QMPCommand.Execute)
 		assert.True(t, cmd.Attributes.StopInstance)
 		assert.True(t, cmd.Attributes.TerminateInstance)
 
@@ -115,18 +114,11 @@ func TestTerminateInstances_NATSRequestFails(t *testing.T) {
 		InstanceIds: []*string{aws.String(instanceID)},
 	}
 
-	output, err := TerminateInstances(input, nc, "123456789012")
-
-	require.NoError(t, err)
-	require.Len(t, output.TerminatingInstances, 1)
-
-	// On NATS failure, state should reflect "still running"
-	sc := output.TerminatingInstances[0]
-	assert.Equal(t, instanceID, *sc.InstanceId)
-	assert.Equal(t, int64(16), *sc.CurrentState.Code)
-	assert.Equal(t, "running", *sc.CurrentState.Name)
-	assert.Equal(t, int64(16), *sc.PreviousState.Code)
-	assert.Equal(t, "running", *sc.PreviousState.Name)
+	// When no subscriber exists and all fallback paths fail, an error is
+	// returned so the caller knows the terminate did not succeed.
+	_, err := TerminateInstances(input, nc, "123456789012")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), instanceID)
 }
 
 func TestTerminateInstances_MixedSuccessAndFailure(t *testing.T) {
@@ -143,25 +135,19 @@ func TestTerminateInstances_MixedSuccessAndFailure(t *testing.T) {
 		InstanceIds: []*string{aws.String(goodID), aws.String(badID)},
 	}
 
-	output, err := TerminateInstances(input, nc, "123456789012")
-
-	require.NoError(t, err)
-	require.Len(t, output.TerminatingInstances, 2)
-
-	// First: success → shutting-down
-	assert.Equal(t, goodID, *output.TerminatingInstances[0].InstanceId)
-	assert.Equal(t, "shutting-down", *output.TerminatingInstances[0].CurrentState.Name)
-
-	// Second: failure → still running
-	assert.Equal(t, badID, *output.TerminatingInstances[1].InstanceId)
-	assert.Equal(t, "running", *output.TerminatingInstances[1].CurrentState.Name)
+	// When a running instance cannot be reached (ErrNoResponders exhausted after
+	// retries), TerminateInstances returns an error rather than silently
+	// reporting the instance as "running".
+	_, err := TerminateInstances(input, nc, "123456789012")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), badID)
 }
 
 func TestTerminateInstances_VerifiesQMPAttributes(t *testing.T) {
 	_, nc := startTestNATSServer(t)
 
 	instanceID := "i-verify"
-	var receivedCmd qmp.Command
+	var receivedCmd types.EC2InstanceCommand
 
 	nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
 		json.Unmarshal(msg.Data, &receivedCmd)

hive/gateway/ec2/volume/AttachVolume.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/hive/hive/awserrors"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -48,12 +48,12 @@ func AttachVolume(input *ec2.AttachVolumeInput, natsConn *nats.Conn, accountID s
 		device = *input.Device
 	}
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			AttachVolume: true,
 		},
-		AttachVolumeData: &qmp.AttachVolumeData{
+		AttachVolumeData: &types.AttachVolumeData{
 			VolumeID: volumeID,
 			Device:   device,
 		},

hive/gateway/ec2/volume/DetachVolume.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/hive/hive/awserrors"
 	handlers_ec2_volume "github.com/mulgadc/hive/hive/handlers/ec2/volume"
-	"github.com/mulgadc/hive/hive/qmp"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -66,12 +66,12 @@ func DetachVolume(input *ec2.DetachVolumeInput, natsConn *nats.Conn, accountID s
 		device = *input.Device
 	}
 
-	command := qmp.Command{
+	command := types.EC2InstanceCommand{
 		ID: instanceID,
-		Attributes: qmp.Attributes{
+		Attributes: types.EC2CommandAttributes{
 			DetachVolume: true,
 		},
-		DetachVolumeData: &qmp.DetachVolumeData{
+		DetachVolumeData: &types.DetachVolumeData{
 			VolumeID: volumeID,
 			Device:   device,
 			Force:    input.Force != nil && *input.Force,

hive/gateway/gateway.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -18,6 +19,7 @@ import (
 	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/mulgadc/hive/hive/gateway/policy"
 	handlers_iam "github.com/mulgadc/hive/hive/handlers/iam"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -105,10 +107,55 @@ func (gw *GatewayConfig) SetupRoutes() http.Handler {
 	return r
 }
 
-// corsMiddleware adds CORS headers for browser requests from the dev UI.
+// corsAllowedOrigins builds the set of allowed CORS origins from localhost and
+// all local non-loopback IPs on the hive-ui port (default 3000). This allows
+// the UI to be accessed from any local address, not just localhost.
+func corsAllowedOrigins() map[string]struct{} {
+	origins := map[string]struct{}{
+		"https://localhost:3000": {},
+	}
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return origins
+	}
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		addrs, err := iface.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil || ip.IsLoopback() || ip.To4() == nil {
+				continue
+			}
+			origins[fmt.Sprintf("https://%s:3000", ip.String())] = struct{}{}
+		}
+	}
+	return origins
+}
+
+// corsMiddleware adds CORS headers for browser requests from the hive-ui.
 func corsMiddleware(next http.Handler) http.Handler {
+	allowed := corsAllowedOrigins()
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Access-Control-Allow-Origin", "https://localhost:3000")
+		origin := r.Header.Get("Origin")
+		if origin == "" {
+			origin = "https://localhost:3000"
+		}
+		if _, ok := allowed[origin]; ok {
+			w.Header().Set("Access-Control-Allow-Origin", origin)
+		} else {
+			w.Header().Set("Access-Control-Allow-Origin", "https://localhost:3000")
+		}
 		w.Header().Set("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,HEAD,OPTIONS")
 		w.Header().Set("Access-Control-Allow-Headers", "*")
 		w.Header().Set("Access-Control-Allow-Credentials", "true")
@@ -168,6 +215,14 @@ func (gw *GatewayConfig) GetService(r *http.Request) (string, error) {
 	return svc, nil
 }
 
+// isNATSTransient reports whether err represents a transient NATS/JetStream
+// failure that may resolve after cluster leader election completes.
+func isNATSTransient(err error) bool {
+	return err != nil && (errors.Is(err, nats.ErrNoResponders) ||
+		errors.Is(err, nats.ErrTimeout) ||
+		errors.Is(err, nats.ErrNoStreamResponse))
+}
+
 // checkPolicy evaluates IAM policies for the current request. Returns nil
 // if access is allowed, or an ErrorAccessDenied error if denied.
 // Root users bypass evaluation entirely. If the IAM service is unavailable,
@@ -203,7 +258,20 @@ func (gw *GatewayConfig) checkPolicy(r *http.Request, service, action string) er
 	// Resolve the IAM action string (e.g. "ec2:RunInstances")
 	iamAction := policy.IAMAction(service, action)
 
-	policies, err := gw.IAMService.GetUserPolicies(accountID, identity)
+	// Retry on transient NATS errors (e.g. during node failure / leader election).
+	var policies []handlers_iam.PolicyDocument
+	var err error
+	for attempt := range 3 {
+		policies, err = gw.IAMService.GetUserPolicies(accountID, identity)
+		if err == nil || !isNATSTransient(err) {
+			break
+		}
+		if attempt < 2 {
+			slog.Debug("checkPolicy: transient NATS error, retrying",
+				"user", identity, "attempt", attempt+1, "err", err)
+			time.Sleep(time.Duration(attempt+1) * 200 * time.Millisecond)
+		}
+	}
 	if err != nil {
 		slog.Error("checkPolicy: failed to get user policies", "user", identity, "err", err)
 		return errors.New(awserrors.ErrorInternalError)
@@ -347,11 +415,6 @@ func ParseArgsToStruct(input *any, args map[string]string) (err error) {
 
 }
 
-// NodeDiscoverResponse is the response from a node discovery request
-type NodeDiscoverResponse struct {
-	Node string `json:"node"`
-}
-
 // DiscoverActiveNodes discovers the number of active hive daemon nodes in the cluster
 // by publishing a discovery request and counting unique responses.
 // Returns the number of active nodes (minimum 1 if fallback is needed).
@@ -399,7 +462,7 @@ func (gw *GatewayConfig) DiscoverActiveNodes() int {
 			break
 		}
 
-		var response NodeDiscoverResponse
+		var response types.NodeDiscoverResponse
 		if err := json.Unmarshal(msg.Data, &response); err != nil {
 			slog.Debug("DiscoverActiveNodes: Failed to unmarshal response", "err", err)
 			continue

hive/gateway/gateway_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/mulgadc/hive/hive/awserrors"
 	handlers_iam "github.com/mulgadc/hive/hive/handlers/iam"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -312,7 +314,7 @@ func TestDiscoverActiveNodes_WithResponders(t *testing.T) {
 	for _, nodeName := range []string{"node-1", "node-2"} {
 		name := nodeName
 		_, err := nc.Subscribe("hive.nodes.discover", func(msg *nats.Msg) {
-			resp := NodeDiscoverResponse{Node: name}
+			resp := types.NodeDiscoverResponse{Node: name}
 			data, _ := json.Marshal(resp)
 			msg.Respond(data)
 		})
@@ -352,7 +354,7 @@ func TestDiscoverActiveNodes_DuplicateNodes(t *testing.T) {
 
 	for range 2 {
 		_, err := nc.Subscribe("hive.nodes.discover", func(msg *nats.Msg) {
-			resp := NodeDiscoverResponse{Node: "same-node"}
+			resp := types.NodeDiscoverResponse{Node: "same-node"}
 			data, _ := json.Marshal(resp)
 			msg.Respond(data)
 		})
@@ -824,6 +826,80 @@ func TestCheckPolicy_MissingAccountID(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInternalError, err.Error())
 }
 
+func TestCheckPolicy_NATSTransientRetriesAllAttempts(t *testing.T) {
+	calls := 0
+	mock := &policyMockIAMService{
+		getUserPoliciesFn: func(_, _ string) ([]handlers_iam.PolicyDocument, error) {
+			calls++
+			return nil, fmt.Errorf("get user: %w", nats.ErrNoResponders)
+		},
+	}
+	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
+	req := httptest.NewRequest("POST", "/", nil)
+	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
+	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
+	req = req.WithContext(ctx)
+
+	err := gw.checkPolicy(req, "ec2", "DescribeInstances")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInternalError, err.Error())
+	assert.Equal(t, 3, calls, "should have retried all 3 attempts")
+}
+
+func TestCheckPolicy_NATSTransientRetriesThenSucceeds(t *testing.T) {
+	calls := 0
+	mock := &policyMockIAMService{
+		getUserPoliciesFn: func(_, _ string) ([]handlers_iam.PolicyDocument, error) {
+			calls++
+			if calls < 3 {
+				return nil, fmt.Errorf("get user: %w", nats.ErrNoResponders)
+			}
+			return []handlers_iam.PolicyDocument{
+				{
+					Version: "2012-10-17",
+					Statement: []handlers_iam.Statement{
+						{Effect: "Allow", Action: handlers_iam.StringOrArr{"ec2:*"}, Resource: handlers_iam.StringOrArr{"*"}},
+					},
+				},
+			}, nil
+		},
+	}
+	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
+	req := httptest.NewRequest("POST", "/", nil)
+	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
+	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
+	req = req.WithContext(ctx)
+
+	err := gw.checkPolicy(req, "ec2", "DescribeInstances")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, calls, "should have retried until success")
+}
+
+func TestCheckPolicy_NonTransientErrorStillFails(t *testing.T) {
+	mock := &policyMockIAMService{
+		getUserPoliciesFn: func(_, _ string) ([]handlers_iam.PolicyDocument, error) {
+			return nil, errors.New("database corruption")
+		},
+	}
+	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
+	req := httptest.NewRequest("POST", "/", nil)
+	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
+	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
+	req = req.WithContext(ctx)
+
+	err := gw.checkPolicy(req, "ec2", "DescribeInstances")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInternalError, err.Error())
+}
+
+func TestIsNATSTransient(t *testing.T) {
+	assert.False(t, isNATSTransient(nil))
+	assert.True(t, isNATSTransient(nats.ErrNoResponders))
+	assert.True(t, isNATSTransient(nats.ErrTimeout))
+	assert.True(t, isNATSTransient(fmt.Errorf("get user: %w", nats.ErrNoResponders)))
+	assert.False(t, isNATSTransient(errors.New("some other error")))
+}
+
 func TestCorsMiddleware_GETRequest(t *testing.T) {
 	nextCalled := false
 	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -859,6 +935,43 @@ func TestCorsMiddleware_OPTIONSRequest(t *testing.T) {
 	assert.Equal(t, "https://localhost:3000", w.Header().Get("Access-Control-Allow-Origin"))
 }
 
+func TestCorsMiddleware_LocalIPOrigin(t *testing.T) {
+	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+	})
+
+	handler := corsMiddleware(next)
+
+	// Request from a local IP should be reflected
+	allowed := corsAllowedOrigins()
+	for origin := range allowed {
+		if origin == "https://localhost:3000" {
+			continue
+		}
+		req := httptest.NewRequest("GET", "/", nil)
+		req.Header.Set("Origin", origin)
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, req)
+		assert.Equal(t, origin, w.Header().Get("Access-Control-Allow-Origin"))
+		return // test one IP is enough
+	}
+}
+
+func TestCorsMiddleware_UnknownOrigin(t *testing.T) {
+	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+	})
+
+	handler := corsMiddleware(next)
+	req := httptest.NewRequest("GET", "/", nil)
+	req.Header.Set("Origin", "https://evil.example.com")
+	w := httptest.NewRecorder()
+	handler.ServeHTTP(w, req)
+
+	// Unknown origins fall back to localhost
+	assert.Equal(t, "https://localhost:3000", w.Header().Get("Access-Control-Allow-Origin"))
+}
+
 func TestSlogRequestLogger_CallsNext(t *testing.T) {
 	nextCalled := false
 	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -890,7 +1003,7 @@ func TestSlogRequestLogger_CapturesStatusCode(t *testing.T) {
 func TestEC2ActionMapCompleteness(t *testing.T) {
 	expectedActions := []string{
 		"DescribeInstances", "RunInstances", "StartInstances", "StopInstances",
-		"TerminateInstances", "DescribeInstanceTypes", "GetConsoleOutput",
+		"TerminateInstances", "RebootInstances", "DescribeInstanceTypes", "GetConsoleOutput",
 		"ModifyInstanceAttribute",
 		"CreateKeyPair", "DeleteKeyPair", "DescribeKeyPairs", "ImportKeyPair",
 		"DescribeImages", "CreateImage",

hive/gateway/iam.go

@@ -94,7 +94,11 @@ var iamActions = map[string]IAMHandler{
 }
 
 func (gw *GatewayConfig) IAM_Request(w http.ResponseWriter, r *http.Request) error {
-	body, _ := io.ReadAll(r.Body)
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		slog.Error("Failed to read IAM request body", "error", err)
+		return errors.New(awserrors.ErrorInternalError)
+	}
 	queryArgs := ParseAWSQueryArgs(string(body))
 
 	action := queryArgs["Action"]

hive/gateway/iam/handler_test.go

@@ -87,9 +87,9 @@ func (s *stubIAMService) LookupAccessKey(_ string) (*handlers_iam.AccessKey, err
 	return nil, nil
 }
 
-func (s *stubIAMService) DecryptSecret(_ string) (string, error)           { return "", nil }
-func (s *stubIAMService) SeedRootUser(_ *handlers_iam.BootstrapData) error { return nil }
-func (s *stubIAMService) IsEmpty() (bool, error)                           { return true, nil }
+func (s *stubIAMService) DecryptSecret(_ string) (string, error)            { return "", nil }
+func (s *stubIAMService) SeedBootstrap(_ *handlers_iam.BootstrapData) error { return nil }
+func (s *stubIAMService) IsEmpty() (bool, error)                            { return true, nil }
 
 func (s *stubIAMService) CreateAccount(_ string) (*handlers_iam.Account, error) {
 	return nil, nil

hive/gateway/iam_request_test.go

@@ -124,9 +124,9 @@ func (m *flexMockIAMService) LookupAccessKey(_ string) (*handlers_iam.AccessKey,
 	return nil, errors.New("not implemented")
 }
 
-func (m *flexMockIAMService) DecryptSecret(_ string) (string, error)           { return "", nil }
-func (m *flexMockIAMService) SeedRootUser(_ *handlers_iam.BootstrapData) error { return nil }
-func (m *flexMockIAMService) IsEmpty() (bool, error)                           { return true, nil }
+func (m *flexMockIAMService) DecryptSecret(_ string) (string, error)            { return "", nil }
+func (m *flexMockIAMService) SeedBootstrap(_ *handlers_iam.BootstrapData) error { return nil }
+func (m *flexMockIAMService) IsEmpty() (bool, error)                            { return true, nil }
 
 func (m *flexMockIAMService) CreateAccount(_ string) (*handlers_iam.Account, error) {
 	return nil, nil

hive/handlers/ec2/igw/service_impl.go

@@ -11,9 +11,9 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/hive/hive/awserrors"
-	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/config"
 	handlers_ec2_vpc "github.com/mulgadc/hive/hive/handlers/ec2/vpc"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -341,4 +341,3 @@ func (s *IGWServiceImpl) recordToEC2(record *IGWRecord) *ec2.InternetGateway {
 
 	return igw
 }
-

hive/handlers/ec2/image/service_impl.go

@@ -17,6 +17,7 @@ import (
 	"github.com/mulgadc/hive/hive/config"
 	handlers_ec2_snapshot "github.com/mulgadc/hive/hive/handlers/ec2/snapshot"
 	"github.com/mulgadc/hive/hive/objectstore"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/viperblock/viperblock"
 	vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3"
@@ -398,7 +399,7 @@ func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID string) er
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	snapReq := config.EBSSnapshotRequest{Volume: volumeID, SnapshotID: snapshotID}
+	snapReq := types.EBSSnapshotRequest{Volume: volumeID, SnapshotID: snapshotID}
 	snapData, err := json.Marshal(snapReq)
 	if err != nil {
 		slog.Error("snapshotRunningVolume: failed to marshal snapshot request", "volumeId", volumeID, "err", err)
@@ -411,7 +412,7 @@ func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID string) er
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	var snapResp config.EBSSnapshotResponse
+	var snapResp types.EBSSnapshotResponse
 	if err := json.Unmarshal(msg.Data, &snapResp); err != nil {
 		slog.Error("snapshotRunningVolume: failed to unmarshal response", "err", err)
 		return errors.New(awserrors.ErrorServerInternal)

hive/handlers/ec2/instance/service_impl.go

@@ -18,6 +18,7 @@ import (
 	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/objectstore"
+	hivetypes "github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/hive/hive/vm"
 	"github.com/mulgadc/viperblock/types"
@@ -63,11 +64,9 @@ instance-id: {{.InstanceID}}
 local-hostname: {{.Hostname}}
 `
 
-// cloudInitNetworkConfig enables DHCP on all NICs. Without this, Ubuntu cloud
-// images only configure the first NIC and the guest may have no IPv4 connectivity.
-// This is essential for DEV_NETWORKING dual-NIC setups (TAP + hostfwd) and
-// also ensures OVN DHCP works reliably on the TAP interface.
-const cloudInitNetworkConfig = `network:
+// cloudInitNetworkConfigWildcard enables DHCP on all NICs via wildcard match.
+// Used when there's no dual-NIC setup (non-VPC or VPC without DEV_NETWORKING).
+const cloudInitNetworkConfigWildcard = `network:
   version: 2
   ethernets:
     allnics:
@@ -77,6 +76,38 @@ const cloudInitNetworkConfig = `network:
       dhcp-identifier: mac
 `
 
+// generateNetworkConfig produces the cloud-init network-config for the instance.
+//
+// When both eniMAC and devMAC are provided (VPC + DEV_NETWORKING), it generates
+// per-interface config that suppresses the default route on the dev/hostfwd NIC.
+// Without this, both NICs get DHCP default routes at equal metric, causing
+// nondeterministic routing — SSH via hostfwd times out and VPC traffic leaks.
+//
+// The dev NIC still gets an IP via DHCP (needed for hostfwd port forwarding)
+// but dhcp4-overrides prevents it from installing routes or DNS.
+func generateNetworkConfig(eniMAC, devMAC string) string {
+	if eniMAC == "" || devMAC == "" {
+		return cloudInitNetworkConfigWildcard
+	}
+	return fmt.Sprintf(`network:
+  version: 2
+  ethernets:
+    vpc0:
+      match:
+        macaddress: "%s"
+      dhcp4: true
+      dhcp-identifier: mac
+    dev0:
+      match:
+        macaddress: "%s"
+      dhcp4: true
+      dhcp-identifier: mac
+      dhcp4-overrides:
+        use-routes: false
+        use-dns: false
+`, eniMAC, devMAC)
+}
+
 type CloudInitData struct {
 	Username            string
 	SSHKey              string
@@ -324,7 +355,7 @@ func (s *InstanceServiceImpl) prepareRootVolume(input *ec2.RunInstancesInput, im
 
 	// Append root volume to instance
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, config.EBSRequest{
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, hivetypes.EBSRequest{
 		Name:                imageId,
 		Boot:                true,
 		DeleteOnTermination: deleteOnTermination,
@@ -466,7 +497,7 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe
 	}
 
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, config.EBSRequest{
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, hivetypes.EBSRequest{
 		Name: efiVb.VolumeName,
 		Boot: false,
 		EFI:  true,
@@ -526,7 +557,7 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu
 	}
 
 	// Create the cloud-init ISO
-	err = s.createCloudInitISO(input, instance.ID, instance.AccountID, cloudInitVb)
+	err = s.createCloudInitISO(input, instance, cloudInitVb)
 	if err != nil {
 		return err
 	}
@@ -542,7 +573,7 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu
 	}
 
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, config.EBSRequest{
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, hivetypes.EBSRequest{
 		Name:      cloudInitVolumeName,
 		Boot:      false,
 		CloudInit: true,
@@ -553,7 +584,7 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu
 }
 
 // createCloudInitISO generates the cloud-init ISO image
-func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, instanceId string, accountID string, cloudInitVb *viperblock.VB) error {
+func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, instance *vm.VM, cloudInitVb *viperblock.VB) error {
 	// Create ISO writer
 	writer, err := iso9660.NewWriter()
 	if err != nil {
@@ -563,7 +594,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 	defer writer.Cleanup()
 
 	// Generate instance metadata
-	hostname := generateHostname(instanceId)
+	hostname := generateHostname(instance.ID)
 
 	// Retrieve SSH pubkey from S3
 	keyName := ""
@@ -571,7 +602,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 		keyName = *input.KeyName
 	}
 
-	keyPath := fmt.Sprintf("keys/%s/%s", accountID, keyName)
+	keyPath := fmt.Sprintf("keys/%s/%s", instance.AccountID, keyName)
 	result, err := s.objectStore.GetObject(&awss3.GetObjectInput{
 		Bucket: aws.String(s.config.Predastore.Bucket),
 		Key:    aws.String(keyPath),
@@ -614,7 +645,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 
 	// Add meta-data
 	metaData := CloudInitMetaData{
-		InstanceID: instanceId,
+		InstanceID: instance.ID,
 		Hostname:   hostname,
 	}
 
@@ -632,8 +663,10 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Add network-config (enables DHCP on all NICs)
-	err = writer.AddFile(strings.NewReader(cloudInitNetworkConfig), "network-config")
+	// Add network-config: per-interface when VPC+dev (suppresses dev default route),
+	// wildcard DHCP otherwise.
+	networkConfig := generateNetworkConfig(instance.ENIMac, instance.DevMAC)
+	err = writer.AddFile(strings.NewReader(networkConfig), "network-config")
 	if err != nil {
 		slog.Error("failed to add network-config file", "err", err)
 		return errors.New(awserrors.ErrorServerInternal)

hive/handlers/ec2/instance/service_impl_test.go

@@ -394,10 +394,48 @@ func TestMockInstanceService(t *testing.T) {
 	assert.Equal(t, "subnet-abc123", *inst.SubnetId)
 }
 
-func TestCloudInitNetworkConfig(t *testing.T) {
-	assert.Contains(t, cloudInitNetworkConfig, "version: 2")
-	assert.Contains(t, cloudInitNetworkConfig, "dhcp4: true")
-	assert.Contains(t, cloudInitNetworkConfig, "dhcp-identifier: mac")
+func TestCloudInitNetworkConfigWildcard(t *testing.T) {
+	// No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING)
+	cfg := generateNetworkConfig("", "")
+	assert.Contains(t, cfg, "version: 2")
+	assert.Contains(t, cfg, "dhcp4: true")
+	assert.Contains(t, cfg, "dhcp-identifier: mac")
+	assert.Contains(t, cfg, `name: "en*"`, "wildcard should match all en* interfaces")
+	assert.NotContains(t, cfg, "use-routes")
+}
+
+func TestCloudInitNetworkConfigDualNIC(t *testing.T) {
+	eniMAC := "02:00:00:61:ef:c2"
+	devMAC := "02:de:00:60:83:0d"
+
+	cfg := generateNetworkConfig(eniMAC, devMAC)
+
+	// Both MACs present in config
+	assert.Contains(t, cfg, eniMAC)
+	assert.Contains(t, cfg, devMAC)
+
+	// VPC NIC gets normal DHCP (with default route)
+	assert.Contains(t, cfg, "vpc0:")
+	assert.Contains(t, cfg, "dev0:")
+
+	// Dev NIC has route/DNS suppressed
+	assert.Contains(t, cfg, "use-routes: false")
+	assert.Contains(t, cfg, "use-dns: false")
+
+	// No wildcard match — per-interface only
+	assert.NotContains(t, cfg, `name: "en*"`)
+}
+
+func TestCloudInitNetworkConfigPartialMAC(t *testing.T) {
+	// Only ENI MAC (VPC without dev) → wildcard
+	cfg := generateNetworkConfig("02:00:00:61:ef:c2", "")
+	assert.Contains(t, cfg, `name: "en*"`)
+	assert.NotContains(t, cfg, "use-routes")
+
+	// Only dev MAC (shouldn't happen, but defensive) → wildcard
+	cfg = generateNetworkConfig("", "02:de:00:60:83:0d")
+	assert.Contains(t, cfg, `name: "en*"`)
+	assert.NotContains(t, cfg, "use-routes")
 }
 
 func TestCloudInitVolumeNamePerInstance(t *testing.T) {

hive/handlers/ec2/snapshot/service_impl.go

@@ -17,6 +17,7 @@ import (
 	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/objectstore"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/nats-io/nats.go"
@@ -217,7 +218,7 @@ func (s *SnapshotServiceImpl) CreateSnapshot(input *ec2.CreateSnapshotInput, acc
 	// This sends a NATS message to the EBS daemon that owns the volume, which
 	// calls vb.CreateSnapshot() on the live viperblock instance.
 	if s.natsConn != nil {
-		snapReq := config.EBSSnapshotRequest{Volume: volumeID, SnapshotID: snapshotID}
+		snapReq := types.EBSSnapshotRequest{Volume: volumeID, SnapshotID: snapshotID}
 		snapData, err := json.Marshal(snapReq)
 		if err != nil {
 			slog.Error("CreateSnapshot: failed to marshal ebs.snapshot request", "volumeId", volumeID, "err", err)
@@ -232,7 +233,7 @@ func (s *SnapshotServiceImpl) CreateSnapshot(input *ec2.CreateSnapshotInput, acc
 			slog.Error("CreateSnapshot: ebs.snapshot NATS request failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		} else {
-			var snapResp config.EBSSnapshotResponse
+			var snapResp types.EBSSnapshotResponse
 			if err := json.Unmarshal(msg.Data, &snapResp); err != nil {
 				slog.Error("CreateSnapshot: failed to unmarshal ebs.snapshot response", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
 				return nil, errors.New(awserrors.ErrorServerInternal)

hive/handlers/ec2/volume/service_impl.go

@@ -17,6 +17,7 @@ import (
 	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/objectstore"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/viperblock/viperblock"
 	s3backend "github.com/mulgadc/viperblock/viperblock/backends/s3"
@@ -776,7 +777,7 @@ func (s *VolumeServiceImpl) DeleteVolume(input *ec2.DeleteVolumeInput, accountID
 
 	// Notify viperblockd to stop nbdkit/WAL syncer (best-effort)
 	if s.natsConn != nil {
-		deleteReq := config.EBSDeleteRequest{Volume: volumeID}
+		deleteReq := types.EBSDeleteRequest{Volume: volumeID}
 		deleteData, err := json.Marshal(deleteReq)
 		if err != nil {
 			slog.Error("DeleteVolume failed to marshal ebs.delete request", "volumeId", volumeID, "err", err)
@@ -785,7 +786,7 @@ func (s *VolumeServiceImpl) DeleteVolume(input *ec2.DeleteVolumeInput, accountID
 			if err != nil {
 				slog.Warn("ebs.delete notification failed (volume may not be mounted)", "volumeId", volumeID, "err", err)
 			} else {
-				var deleteResp config.EBSDeleteResponse
+				var deleteResp types.EBSDeleteResponse
 				if json.Unmarshal(msg.Data, &deleteResp) == nil && deleteResp.Error != "" {
 					slog.Error("ebs.delete returned error", "volumeId", volumeID, "err", deleteResp.Error)
 					return nil, errors.New(awserrors.ErrorServerInternal)

hive/handlers/ec2/volume/service_impl_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/mulgadc/hive/hive/awserrors"
 	"github.com/mulgadc/hive/hive/config"
 	"github.com/mulgadc/hive/hive/objectstore"
+	"github.com/mulgadc/hive/hive/types"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
@@ -1357,7 +1358,7 @@ func TestDeleteVolume_WithNATSNotification(t *testing.T) {
 
 	// Subscribe to ebs.delete and reply with success
 	sub, err := nc.Subscribe("ebs.delete", func(msg *nats.Msg) {
-		resp := config.EBSDeleteResponse{Volume: volumeID, Success: true}
+		resp := types.EBSDeleteResponse{Volume: volumeID, Success: true}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})
@@ -1496,7 +1497,7 @@ func TestDeleteVolume_NATSErrorResponse(t *testing.T) {
 
 	// Subscribe and respond with an error
 	sub, err := nc.Subscribe("ebs.delete", func(msg *nats.Msg) {
-		resp := config.EBSDeleteResponse{Volume: volumeID, Error: "volume still mounted"}
+		resp := types.EBSDeleteResponse{Volume: volumeID, Error: "volume still mounted"}
 		data, _ := json.Marshal(resp)
 		msg.Respond(data)
 	})

hive/handlers/ec2/vpc/service_impl.go

@@ -294,32 +294,6 @@ func (s *VPCServiceImpl) DescribeVpcs(input *ec2.DescribeVpcsInput, accountID st
 		vpcs = append(vpcs, s.vpcRecordToEC2(&record, accountID))
 	}
 
-	// Include global default VPCs (created at startup) for non-root accounts.
-	// This matches AWS behavior where all accounts have a default VPC.
-	if accountID != utils.GlobalAccountID {
-		globalPrefix := utils.GlobalAccountID + "."
-		for _, key := range keys {
-			if !strings.HasPrefix(key, globalPrefix) {
-				continue
-			}
-			entry, err := s.vpcKV.Get(key)
-			if err != nil {
-				continue
-			}
-			var record VPCRecord
-			if err := json.Unmarshal(entry.Value(), &record); err != nil {
-				continue
-			}
-			if !record.IsDefault {
-				continue
-			}
-			if len(vpcIDs) > 0 && !vpcIDs[record.VpcId] {
-				continue
-			}
-			vpcs = append(vpcs, s.vpcRecordToEC2(&record, accountID))
-		}
-	}
-
 	// If specific VPC IDs were requested but not found, return error
 	if len(vpcIDs) > 0 {
 		found := make(map[string]bool)

hive/handlers/ec2/vpc/service_impl_test.go

@@ -1,6 +1,7 @@
 package handlers_ec2_vpc
 
 import (
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -873,6 +874,96 @@ func TestDetachENI_NotFound(t *testing.T) {
 	assert.ErrorContains(t, err, "InvalidNetworkInterfaceID.NotFound")
 }
 
+// --- Per-account isolation tests ---
+
+func TestEnsureDefaultVPC_PerAccountIsolation(t *testing.T) {
+	svc := setupTestVPCService(t)
+	accountA := "111111111111"
+	accountB := "222222222222"
+
+	require.NoError(t, svc.EnsureDefaultVPC(accountA))
+	require.NoError(t, svc.EnsureDefaultVPC(accountB))
+
+	// Each account should see only their own default VPC
+	descA, err := svc.DescribeVpcs(&ec2.DescribeVpcsInput{}, accountA)
+	require.NoError(t, err)
+	require.Len(t, descA.Vpcs, 1)
+	assert.Equal(t, accountA, *descA.Vpcs[0].OwnerId)
+
+	descB, err := svc.DescribeVpcs(&ec2.DescribeVpcsInput{}, accountB)
+	require.NoError(t, err)
+	require.Len(t, descB.Vpcs, 1)
+	assert.Equal(t, accountB, *descB.Vpcs[0].OwnerId)
+
+	// VPC IDs should be different
+	assert.NotEqual(t, *descA.Vpcs[0].VpcId, *descB.Vpcs[0].VpcId)
+}
+
+func TestEnsureDefaultVPC_IndependentVNIs(t *testing.T) {
+	svc, nc := setupTestVPCServiceWithNC(t)
+	accountA := "111111111111"
+	accountB := "222222222222"
+
+	// Capture VNIs from vpc.create events
+	vniCh := make(chan int64, 2)
+	sub, err := nc.Subscribe("vpc.create", func(msg *nats.Msg) {
+		var evt struct {
+			VNI int64 `json:"vni"`
+		}
+		if err := json.Unmarshal(msg.Data, &evt); err == nil {
+			vniCh <- evt.VNI
+		}
+	})
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	require.NoError(t, svc.EnsureDefaultVPC(accountA))
+	require.NoError(t, svc.EnsureDefaultVPC(accountB))
+
+	var vnis []int64
+	for range 2 {
+		select {
+		case vni := <-vniCh:
+			vnis = append(vnis, vni)
+		case <-time.After(2 * time.Second):
+			t.Fatal("timed out waiting for vpc.create event")
+		}
+	}
+	assert.NotEqual(t, vnis[0], vnis[1], "each account should get a unique VNI")
+}
+
+func TestDescribeVpcs_NoGlobalSharing(t *testing.T) {
+	svc := setupTestVPCService(t)
+	globalAccount := "000000000000"
+	otherAccount := "111111111111"
+
+	// Create default VPC for global account only
+	require.NoError(t, svc.EnsureDefaultVPC(globalAccount))
+
+	// Other account should NOT see the global default VPC
+	desc, err := svc.DescribeVpcs(&ec2.DescribeVpcsInput{}, otherAccount)
+	require.NoError(t, err)
+	assert.Empty(t, desc.Vpcs)
+}
+
+func TestGetDefaultSubnet_PerAccount(t *testing.T) {
+	svc := setupTestVPCService(t)
+	accountA := "111111111111"
+	accountB := "222222222222"
+
+	require.NoError(t, svc.EnsureDefaultVPC(accountA))
+	require.NoError(t, svc.EnsureDefaultVPC(accountB))
+
+	subA, err := svc.GetDefaultSubnet(accountA)
+	require.NoError(t, err)
+	subB, err := svc.GetDefaultSubnet(accountB)
+	require.NoError(t, err)
+
+	assert.NotEqual(t, subA.SubnetId, subB.SubnetId)
+	assert.Equal(t, "172.31.0.0/20", subA.CidrBlock)
+	assert.Equal(t, "172.31.0.0/20", subB.CidrBlock)
+}
+
 // --- EnsureDefaultVPC event test ---
 
 func TestEnsureDefaultVPC_PublishesEvents(t *testing.T) {

hive/handlers/iam/account_test.go

@@ -1,11 +1,15 @@
 package handlers_iam
 
 import (
+	"encoding/json"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/mulgadc/hive/hive/utils"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -218,7 +222,7 @@ func TestAccessKeyReturnsAccountID(t *testing.T) {
 }
 
 // ============================================================================
-// SeedRootUser Account-Scoped Tests
+// SeedBootstrap Account-Scoped Tests
 // ============================================================================
 
 // ============================================================================
@@ -340,16 +344,16 @@ func TestCrossAccount_LookupAccessKeyReturnsCorrectAccount(t *testing.T) {
 }
 
 // ============================================================================
-// SeedRootUser Account-Scoped Tests
+// SeedBootstrap Account-Scoped Tests
 // ============================================================================
 
-func TestSeedRootUser_AccountScoped(t *testing.T) {
+func TestSeedBootstrap_AccountScoped(t *testing.T) {
 	svc := setupTestIAMService(t)
 
 	encryptedSecret, err := EncryptSecret("root-secret", svc.masterKey)
 	require.NoError(t, err)
 
-	err = svc.SeedRootUser(&BootstrapData{
+	err = svc.SeedBootstrap(&BootstrapData{
 		AccessKeyID:     "AKIAROOTEXAMPLE12345",
 		EncryptedSecret: encryptedSecret,
 		AccountID:       utils.GlobalAccountID,
@@ -374,6 +378,59 @@ func TestSeedRootUser_AccountScoped(t *testing.T) {
 	account, err := svc.GetAccount(utils.GlobalAccountID)
 	require.NoError(t, err)
 	assert.Equal(t, utils.GlobalAccountID, account.AccountID)
-	assert.Equal(t, "Global", account.AccountName)
+	assert.Equal(t, "system", account.AccountName)
 	assert.Equal(t, "ACTIVE", account.Status)
 }
+
+// ============================================================================
+// Account Creation Event Tests
+// ============================================================================
+
+func TestCreateAccount_PublishesEvent(t *testing.T) {
+	opts := &server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  t.TempDir(),
+		NoLog:     true,
+		NoSigs:    true,
+	}
+	ns, err := server.NewServer(opts)
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	masterKey, err := GenerateMasterKey()
+	require.NoError(t, err)
+
+	svc, err := NewIAMServiceImpl(nc, masterKey, 1)
+	require.NoError(t, err)
+
+	eventCh := make(chan *nats.Msg, 1)
+	sub, err := nc.Subscribe("iam.account.created", func(msg *nats.Msg) {
+		eventCh <- msg
+	})
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	acc, err := svc.CreateAccount("Event Test")
+	require.NoError(t, err)
+
+	select {
+	case msg := <-eventCh:
+		var evt struct {
+			AccountID   string `json:"account_id"`
+			AccountName string `json:"account_name"`
+		}
+		require.NoError(t, json.Unmarshal(msg.Data, &evt))
+		assert.Equal(t, acc.AccountID, evt.AccountID)
+		assert.Equal(t, "Event Test", evt.AccountName)
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for iam.account.created event")
+	}
+}

hive/handlers/iam/crypto_test.go

@@ -270,6 +270,84 @@ func TestSaveBootstrapData_RoundTrip(t *testing.T) {
 	}
 }
 
+func TestBootstrapData_ExtendedRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "bootstrap.json")
+
+	data := &BootstrapData{
+		AccessKeyID:     "AKIASYSTEM1234567890",
+		EncryptedSecret: "c3lzdGVtLXNlY3JldA==",
+		AccountID:       "000000000000",
+		Admin: &AdminBootstrapData{
+			AccountID:       "000000000001",
+			AccountName:     "hive",
+			UserName:        "admin",
+			AccessKeyID:     "AKIAADMIN12345678901",
+			EncryptedSecret: "YWRtaW4tc2VjcmV0",
+		},
+	}
+
+	if err := SaveBootstrapData(path, data); err != nil {
+		t.Fatalf("SaveBootstrapData() error: %v", err)
+	}
+
+	loaded, err := LoadBootstrapData(path)
+	if err != nil {
+		t.Fatalf("LoadBootstrapData() error: %v", err)
+	}
+
+	// System fields
+	if loaded.AccessKeyID != data.AccessKeyID {
+		t.Fatalf("AccessKeyID: expected %q, got %q", data.AccessKeyID, loaded.AccessKeyID)
+	}
+	if loaded.AccountID != data.AccountID {
+		t.Fatalf("AccountID: expected %q, got %q", data.AccountID, loaded.AccountID)
+	}
+
+	// Admin fields
+	if loaded.Admin == nil {
+		t.Fatal("Admin field should not be nil")
+	}
+	if loaded.Admin.AccountID != "000000000001" {
+		t.Fatalf("Admin.AccountID: expected %q, got %q", "000000000001", loaded.Admin.AccountID)
+	}
+	if loaded.Admin.AccountName != "hive" {
+		t.Fatalf("Admin.AccountName: expected %q, got %q", "hive", loaded.Admin.AccountName)
+	}
+	if loaded.Admin.UserName != "admin" {
+		t.Fatalf("Admin.UserName: expected %q, got %q", "admin", loaded.Admin.UserName)
+	}
+	if loaded.Admin.AccessKeyID != "AKIAADMIN12345678901" {
+		t.Fatalf("Admin.AccessKeyID: expected %q, got %q", "AKIAADMIN12345678901", loaded.Admin.AccessKeyID)
+	}
+	if loaded.Admin.EncryptedSecret != "YWRtaW4tc2VjcmV0" {
+		t.Fatalf("Admin.EncryptedSecret mismatch")
+	}
+}
+
+func TestBootstrapData_NoAdmin_BackwardCompat(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "bootstrap.json")
+
+	// Write bootstrap data without admin field (old format)
+	oldJSON := `{"access_key_id":"AKIAOLD123","encrypted_secret":"b2xk","account_id":"000000000000"}`
+	if err := os.WriteFile(path, []byte(oldJSON), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	loaded, err := LoadBootstrapData(path)
+	if err != nil {
+		t.Fatalf("LoadBootstrapData() error: %v", err)
+	}
+
+	if loaded.AccessKeyID != "AKIAOLD123" {
+		t.Fatalf("AccessKeyID: expected %q, got %q", "AKIAOLD123", loaded.AccessKeyID)
+	}
+	if loaded.Admin != nil {
+		t.Fatal("Admin field should be nil for old-format bootstrap data")
+	}
+}
+
 func TestSaveBootstrapData_InvalidPath(t *testing.T) {
 	data := &BootstrapData{
 		AccessKeyID:     "AKIAIOSFODNN7EXAMPLE",

hive/handlers/iam/service.go

@@ -36,7 +36,7 @@ type IAMService interface {
 	// Auth (internal — used by SigV4 middleware and bootstrap, not exposed via gateway)
 	LookupAccessKey(accessKeyID string) (*AccessKey, error)
 	DecryptSecret(ciphertext string) (string, error)
-	SeedRootUser(data *BootstrapData) error
+	SeedBootstrap(data *BootstrapData) error
 	IsEmpty() (bool, error)
 
 	// Account operations

hive/handlers/iam/service_impl.go

@@ -33,6 +33,7 @@ const (
 // IAMServiceImpl implements IAM operations using NATS JetStream KV.
 type IAMServiceImpl struct {
 	js                   nats.JetStreamContext
+	natsConn             *nats.Conn
 	usersBucket          nats.KeyValue
 	accessKeysBucket     nats.KeyValue
 	policiesBucket       nats.KeyValue
@@ -43,37 +44,42 @@ type IAMServiceImpl struct {
 }
 
 // NewIAMServiceImpl creates a new IAM service backed by NATS JetStream KV.
-func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte) (*IAMServiceImpl, error) {
+// clusterSize sets the JetStream replication factor for KV buckets. For
+// multi-node clusters this must match the number of NATS servers so that
+// buckets survive node failures. Pass 1 for single-node or test setups.
+func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) {
 	if len(masterKey) != 32 {
 		return nil, fmt.Errorf("master key must be 32 bytes, got %d", len(masterKey))
 	}
 
+	replicas := max(clusterSize, 1)
+
 	js, err := natsConn.JetStream()
 	if err != nil {
 		return nil, fmt.Errorf("get JetStream context: %w", err)
 	}
 
-	usersBucket, err := getOrCreateBucket(js, KVBucketUsers, 10)
+	usersBucket, err := getOrCreateBucket(js, KVBucketUsers, 10, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init users bucket: %w", err)
 	}
 
-	accessKeysBucket, err := getOrCreateBucket(js, KVBucketAccessKeys, 5)
+	accessKeysBucket, err := getOrCreateBucket(js, KVBucketAccessKeys, 5, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init access keys bucket: %w", err)
 	}
 
-	policiesBucket, err := getOrCreateBucket(js, KVBucketPolicies, 10)
+	policiesBucket, err := getOrCreateBucket(js, KVBucketPolicies, 10, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init policies bucket: %w", err)
 	}
 
-	accountsBucket, err := getOrCreateBucket(js, KVBucketAccounts, 5)
+	accountsBucket, err := getOrCreateBucket(js, KVBucketAccounts, 5, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init accounts bucket: %w", err)
 	}
 
-	accountCounterBucket, err := getOrCreateBucket(js, KVBucketAccountCounter, 5)
+	accountCounterBucket, err := getOrCreateBucket(js, KVBucketAccountCounter, 5, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init account counter bucket: %w", err)
 	}
@@ -87,10 +93,12 @@ func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte) (*IAMServiceImpl,
 		"users_bucket", KVBucketUsers,
 		"access_keys_bucket", KVBucketAccessKeys,
 		"policies_bucket", KVBucketPolicies,
-		"accounts_bucket", KVBucketAccounts)
+		"accounts_bucket", KVBucketAccounts,
+		"replicas", replicas)
 
 	return &IAMServiceImpl{
 		js:                   js,
+		natsConn:             natsConn,
 		usersBucket:          usersBucket,
 		accessKeysBucket:     accessKeysBucket,
 		policiesBucket:       policiesBucket,
@@ -101,10 +109,11 @@ func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte) (*IAMServiceImpl,
 	}, nil
 }
 
-func getOrCreateBucket(js nats.JetStreamContext, name string, history uint8) (nats.KeyValue, error) {
+func getOrCreateBucket(js nats.JetStreamContext, name string, history uint8, replicas int) (nats.KeyValue, error) {
 	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{
-		Bucket:  name,
-		History: history,
+		Bucket:   name,
+		History:  history,
+		Replicas: replicas,
 	})
 	if err != nil {
 		kv, err = js.KeyValue(name)
@@ -133,10 +142,15 @@ func (s *IAMServiceImpl) CreateUser(accountID string, input *iam.CreateUserInput
 		}
 	}
 
+	userID, err := generateIAMID("AIDA")
+	if err != nil {
+		return nil, fmt.Errorf("generate user ID: %w", err)
+	}
+
 	kvKey := accountID + "." + userName
 	user := User{
 		UserName:         userName,
-		UserID:           generateIAMID("AIDA"),
+		UserID:           userID,
 		AccountID:        accountID,
 		ARN:              fmt.Sprintf("arn:aws:iam::%s:user%s%s", accountID, path, userName),
 		Path:             path,
@@ -298,8 +312,14 @@ func (s *IAMServiceImpl) CreateAccessKey(accountID string, input *iam.CreateAcce
 		return nil, errors.New(awserrors.ErrorIAMLimitExceeded)
 	}
 
-	accessKeyID := generateAccessKeyID()
-	secretAccessKey := admin.GenerateAWSSecretKey()
+	accessKeyID, err := generateAccessKeyID()
+	if err != nil {
+		return nil, fmt.Errorf("generate access key ID: %w", err)
+	}
+	secretAccessKey, err := admin.GenerateAWSSecretKey()
+	if err != nil {
+		return nil, fmt.Errorf("generate secret key: %w", err)
+	}
 
 	encryptedSecret, err := EncryptSecret(secretAccessKey, s.masterKey)
 	if err != nil {
@@ -504,24 +524,27 @@ func (s *IAMServiceImpl) DecryptSecret(ciphertext string) (string, error) {
 	return s.decrypter.Decrypt(ciphertext)
 }
 
-// SeedRootUser consumes bootstrap data to create the root IAM user and access
-// key in NATS KV. Uses conditional create (put-if-not-exists) for multi-node
-// race safety — the first node to call this wins; others skip silently.
-// Also creates the global account record (000000000000) if it doesn't exist.
-func (s *IAMServiceImpl) SeedRootUser(data *BootstrapData) error {
-	// Ensure the global account record exists
-	globalAccount := Account{
+// SeedBootstrap consumes bootstrap data to create the system root IAM user and
+// (optionally) the default admin account in NATS KV. Uses conditional create
+// (put-if-not-exists) for multi-node race safety — the first node wins; others
+// skip silently. Also creates the system account record (000000000000).
+// If data.Admin is non-nil, it also seeds the admin account, user, access key,
+// AdministratorAccess policy, and sets the account counter so subsequent
+// CreateAccount calls start at 000000000002.
+func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error {
+	// --- Seed system account (000000000000) ---
+	systemAccount := Account{
 		AccountID:   utils.GlobalAccountID,
-		AccountName: "Global",
+		AccountName: "system",
 		Status:      AccountStatusActive,
 		CreatedAt:   time.Now().UTC().Format(time.RFC3339),
 	}
-	accountData, err := json.Marshal(globalAccount)
+	accountData, err := json.Marshal(systemAccount)
 	if err != nil {
-		return fmt.Errorf("marshal global account: %w", err)
+		return fmt.Errorf("marshal system account: %w", err)
 	}
 	if _, err := s.accountsBucket.Create(utils.GlobalAccountID, accountData); err != nil && !errors.Is(err, nats.ErrKeyExists) {
-		return fmt.Errorf("seed global account: %w", err)
+		return fmt.Errorf("seed system account: %w", err)
 	}
 
 	kvKey := utils.GlobalAccountID + ".root"
@@ -546,9 +569,7 @@ func (s *IAMServiceImpl) SeedRootUser(data *BootstrapData) error {
 	_, err = s.usersBucket.Create(kvKey, userData)
 	if errors.Is(err, nats.ErrKeyExists) {
 		slog.Info("Root user already seeded by another node, skipping")
-		return nil
-	}
-	if err != nil {
+	} else if err != nil {
 		return fmt.Errorf("seed root user: %w", err)
 	}
 
@@ -567,16 +588,120 @@ func (s *IAMServiceImpl) SeedRootUser(data *BootstrapData) error {
 		return fmt.Errorf("marshal root access key: %w", err)
 	}
 
-	_, err = s.accessKeysBucket.Create(data.AccessKeyID, akData)
-	if errors.Is(err, nats.ErrKeyExists) {
+	if _, err = s.accessKeysBucket.Create(data.AccessKeyID, akData); errors.Is(err, nats.ErrKeyExists) {
 		slog.Info("Root access key already seeded by another node, skipping")
-		return nil
-	}
-	if err != nil {
+	} else if err != nil {
 		return fmt.Errorf("seed root access key: %w", err)
+	} else {
+		slog.Info("System root user seeded", "accountID", utils.GlobalAccountID, "accessKeyID", data.AccessKeyID)
 	}
 
-	slog.Info("Root IAM user seeded", "accountID", utils.GlobalAccountID, "accessKeyID", data.AccessKeyID)
+	// --- Seed admin account (000000000001) if present ---
+	if data.Admin != nil {
+		if err := s.seedAdminAccount(data.Admin); err != nil {
+			return fmt.Errorf("seed admin account: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// seedAdminAccount creates the default admin account, user, access key,
+// AdministratorAccess policy, and sets the account counter to 2.
+func (s *IAMServiceImpl) seedAdminAccount(admin *AdminBootstrapData) error {
+	now := time.Now().UTC().Format(time.RFC3339)
+
+	// Create admin account record
+	account := Account{
+		AccountID:   admin.AccountID,
+		AccountName: admin.AccountName,
+		Status:      AccountStatusActive,
+		CreatedAt:   now,
+	}
+	accountData, err := json.Marshal(account)
+	if err != nil {
+		return fmt.Errorf("marshal admin account: %w", err)
+	}
+	if _, err := s.accountsBucket.Create(admin.AccountID, accountData); err != nil && !errors.Is(err, nats.ErrKeyExists) {
+		return fmt.Errorf("store admin account: %w", err)
+	}
+
+	// Set account counter to 2 so next CreateAccount gets 000000000002
+	if _, err := s.accountCounterBucket.Create("next_id", []byte("2")); err != nil && !errors.Is(err, nats.ErrKeyExists) {
+		return fmt.Errorf("seed account counter: %w", err)
+	}
+
+	// Create AdministratorAccess policy
+	policyARN := fmt.Sprintf("arn:aws:iam::%s:policy/AdministratorAccess", admin.AccountID)
+	policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`
+	policyID, err := generateIAMID("ANPA")
+	if err != nil {
+		return fmt.Errorf("generate policy ID: %w", err)
+	}
+	policy := Policy{
+		PolicyName:     "AdministratorAccess",
+		PolicyID:       policyID,
+		ARN:            policyARN,
+		Path:           "/",
+		Description:    "Full administrator access",
+		PolicyDocument: policyDoc,
+		CreatedAt:      now,
+		DefaultVersion: "v1",
+		Tags:           []Tag{},
+	}
+	policyData, err := json.Marshal(policy)
+	if err != nil {
+		return fmt.Errorf("marshal admin policy: %w", err)
+	}
+	policyKVKey := admin.AccountID + ".AdministratorAccess"
+	if _, err := s.policiesBucket.Create(policyKVKey, policyData); err != nil && !errors.Is(err, nats.ErrKeyExists) {
+		return fmt.Errorf("store admin policy: %w", err)
+	}
+
+	// Create admin user (with policy already attached)
+	adminUserID, err := generateIAMID("AIDA")
+	if err != nil {
+		return fmt.Errorf("generate admin user ID: %w", err)
+	}
+	kvKey := admin.AccountID + "." + admin.UserName
+	adminUser := User{
+		UserName:         admin.UserName,
+		UserID:           adminUserID,
+		AccountID:        admin.AccountID,
+		ARN:              fmt.Sprintf("arn:aws:iam::%s:user/%s", admin.AccountID, admin.UserName),
+		Path:             "/",
+		CreatedAt:        now,
+		AccessKeys:       []string{admin.AccessKeyID},
+		Tags:             []Tag{},
+		AttachedPolicies: []string{policyARN},
+	}
+
+	userData, err := json.Marshal(adminUser)
+	if err != nil {
+		return fmt.Errorf("marshal admin user: %w", err)
+	}
+	if _, err := s.usersBucket.Create(kvKey, userData); err != nil && !errors.Is(err, nats.ErrKeyExists) {
+		return fmt.Errorf("store admin user: %w", err)
+	}
+
+	// Create admin access key
+	ak := AccessKey{
+		AccessKeyID:     admin.AccessKeyID,
+		SecretAccessKey: admin.EncryptedSecret,
+		UserName:        admin.UserName,
+		AccountID:       admin.AccountID,
+		Status:          AccessKeyStatusActive,
+		CreatedAt:       now,
+	}
+	akData, err := json.Marshal(ak)
+	if err != nil {
+		return fmt.Errorf("marshal admin access key: %w", err)
+	}
+	if _, err := s.accessKeysBucket.Create(admin.AccessKeyID, akData); err != nil && !errors.Is(err, nats.ErrKeyExists) {
+		return fmt.Errorf("store admin access key: %w", err)
+	}
+
+	slog.Info("Admin account seeded", "accountID", admin.AccountID, "userName", admin.UserName, "accessKeyID", admin.AccessKeyID)
 	return nil
 }
 
@@ -656,6 +781,19 @@ func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) {
 	}
 
 	slog.Info("Account created", "accountID", accountID, "name", name)
+
+	if s.natsConn != nil {
+		evt, err := json.Marshal(struct {
+			AccountID   string `json:"account_id"`
+			AccountName string `json:"account_name"`
+		}{AccountID: accountID, AccountName: name})
+		if err != nil {
+			slog.Error("Failed to marshal account creation event", "accountID", accountID, "error", err)
+		} else if err := s.natsConn.Publish("iam.account.created", evt); err != nil {
+			slog.Error("Failed to publish account creation event", "accountID", accountID, "error", err)
+		}
+	}
+
 	return &account, nil
 }
 
@@ -730,9 +868,13 @@ func (s *IAMServiceImpl) CreatePolicy(accountID string, input *iam.CreatePolicyI
 		return nil, errors.New(awserrors.ErrorIAMInvalidInput)
 	}
 
+	newPolicyID, err := generateIAMID("ANPA")
+	if err != nil {
+		return nil, fmt.Errorf("generate policy ID: %w", err)
+	}
 	policy := Policy{
 		PolicyName:     policyName,
-		PolicyID:       generateIAMID("ANPA"),
+		PolicyID:       newPolicyID,
 		ARN:            fmt.Sprintf("arn:aws:iam::%s:policy%s%s", accountID, path, policyName),
 		Path:           path,
 		Description:    aws.StringValue(input.Description),
@@ -1167,20 +1309,20 @@ func (s *IAMServiceImpl) getUser(accountID, userName string) (*User, error) {
 	return &user, nil
 }
 
-func generateIAMID(prefix string) string {
+func generateIAMID(prefix string) (string, error) {
 	b := make([]byte, 16)
 	if _, err := rand.Read(b); err != nil {
-		panic("crypto/rand failed: " + err.Error())
+		return "", fmt.Errorf("crypto/rand failure: %w", err)
 	}
-	return prefix + strings.ToUpper(hex.EncodeToString(b))[:17]
+	return prefix + strings.ToUpper(hex.EncodeToString(b))[:17], nil
 }
 
-func generateAccessKeyID() string {
+func generateAccessKeyID() (string, error) {
 	b := make([]byte, 10)
 	if _, err := rand.Read(b); err != nil {
-		panic("crypto/rand failed: " + err.Error())
+		return "", fmt.Errorf("crypto/rand failure: %w", err)
 	}
-	return "AKIA" + strings.ToUpper(hex.EncodeToString(b))
+	return "AKIA" + strings.ToUpper(hex.EncodeToString(b)), nil
 }
 
 func parseCreatedAt(raw string) time.Time {

hive/handlers/iam/service_impl_test.go

@@ -47,7 +47,7 @@ func setupTestIAMService(t *testing.T) *IAMServiceImpl {
 	masterKey, err := GenerateMasterKey()
 	require.NoError(t, err)
 
-	svc, err := NewIAMServiceImpl(nc, masterKey)
+	svc, err := NewIAMServiceImpl(nc, masterKey, 1)
 	require.NoError(t, err)
 	return svc
 }
@@ -661,16 +661,16 @@ func TestLookupAccessKey_InactiveKey(t *testing.T) {
 }
 
 // ============================================================================
-// SeedRootUser Tests
+// SeedBootstrap Tests
 // ============================================================================
 
-func TestSeedRootUser(t *testing.T) {
+func TestSeedBootstrap(t *testing.T) {
 	svc := setupTestIAMService(t)
 
 	encryptedSecret, err := EncryptSecret("test-secret-key", svc.masterKey)
 	require.NoError(t, err)
 
-	err = svc.SeedRootUser(&BootstrapData{
+	err = svc.SeedBootstrap(&BootstrapData{
 		AccessKeyID:     "AKIAEXAMPLE123456789",
 		EncryptedSecret: encryptedSecret,
 		AccountID:       utils.GlobalAccountID,
@@ -702,11 +702,11 @@ func TestSeedRootUser(t *testing.T) {
 	account, err := svc.GetAccount(utils.GlobalAccountID)
 	require.NoError(t, err)
 	assert.Equal(t, utils.GlobalAccountID, account.AccountID)
-	assert.Equal(t, "Global", account.AccountName)
+	assert.Equal(t, "system", account.AccountName)
 	assert.Equal(t, "ACTIVE", account.Status)
 }
 
-func TestSeedRootUser_Idempotent(t *testing.T) {
+func TestSeedBootstrap_Idempotent(t *testing.T) {
 	svc := setupTestIAMService(t)
 
 	encryptedSecret, _ := EncryptSecret("test-secret", svc.masterKey)
@@ -717,11 +717,11 @@ func TestSeedRootUser_Idempotent(t *testing.T) {
 	}
 
 	// First call seeds
-	err := svc.SeedRootUser(data)
+	err := svc.SeedBootstrap(data)
 	require.NoError(t, err)
 
 	// Second call should succeed (no-op, idempotent)
-	err = svc.SeedRootUser(data)
+	err = svc.SeedBootstrap(data)
 	require.NoError(t, err)
 
 	// Root user should still exist with original data
@@ -732,6 +732,100 @@ func TestSeedRootUser_Idempotent(t *testing.T) {
 	assert.Equal(t, "root", *out.User.UserName)
 }
 
+func TestSeedBootstrap_WithAdmin(t *testing.T) {
+	svc := setupTestIAMService(t)
+
+	systemSecret, err := EncryptSecret("system-secret", svc.masterKey)
+	require.NoError(t, err)
+	adminSecret, err := EncryptSecret("admin-secret", svc.masterKey)
+	require.NoError(t, err)
+
+	err = svc.SeedBootstrap(&BootstrapData{
+		AccessKeyID:     "AKIASYSTEM1234567890",
+		EncryptedSecret: systemSecret,
+		AccountID:       utils.GlobalAccountID,
+		Admin: &AdminBootstrapData{
+			AccountID:       "000000000001",
+			AccountName:     "hive",
+			UserName:        "admin",
+			AccessKeyID:     "AKIAADMIN12345678901",
+			EncryptedSecret: adminSecret,
+		},
+	})
+	require.NoError(t, err)
+
+	// Verify system root user exists
+	out, err := svc.GetUser(utils.GlobalAccountID, &iam.GetUserInput{
+		UserName: aws.String("root"),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "root", *out.User.UserName)
+
+	// Verify admin account created
+	account, err := svc.GetAccount("000000000001")
+	require.NoError(t, err)
+	assert.Equal(t, "000000000001", account.AccountID)
+	assert.Equal(t, "hive", account.AccountName)
+	assert.Equal(t, "ACTIVE", account.Status)
+
+	// Verify admin user exists
+	adminOut, err := svc.GetUser("000000000001", &iam.GetUserInput{
+		UserName: aws.String("admin"),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "admin", *adminOut.User.UserName)
+	assert.Contains(t, *adminOut.User.Arn, "000000000001")
+
+	// Verify admin access key
+	ak, err := svc.LookupAccessKey("AKIAADMIN12345678901")
+	require.NoError(t, err)
+	assert.Equal(t, "admin", ak.UserName)
+	assert.Equal(t, "000000000001", ak.AccountID)
+
+	decrypted, err := DecryptSecret(ak.SecretAccessKey, svc.masterKey)
+	require.NoError(t, err)
+	assert.Equal(t, "admin-secret", decrypted)
+
+	// Verify AdministratorAccess policy was created and attached
+	policies, err := svc.GetUserPolicies("000000000001", "admin")
+	require.NoError(t, err)
+	require.Len(t, policies, 1)
+	assert.Equal(t, "Allow", string(policies[0].Statement[0].Effect))
+	assert.Equal(t, StringOrArr{"*"}, policies[0].Statement[0].Action)
+	assert.Equal(t, StringOrArr{"*"}, policies[0].Statement[0].Resource)
+
+	// Verify account counter set to 2 (next CreateAccount gets 000000000002)
+	nextAccount, err := svc.CreateAccount("test-org")
+	require.NoError(t, err)
+	assert.Equal(t, "000000000002", nextAccount.AccountID)
+}
+
+func TestSeedBootstrap_AdminNil_BackwardCompat(t *testing.T) {
+	svc := setupTestIAMService(t)
+
+	encryptedSecret, err := EncryptSecret("test-secret", svc.masterKey)
+	require.NoError(t, err)
+
+	// No Admin field — backward compatibility with old bootstrap.json
+	err = svc.SeedBootstrap(&BootstrapData{
+		AccessKeyID:     "AKIAEXAMPLE123456789",
+		EncryptedSecret: encryptedSecret,
+		AccountID:       utils.GlobalAccountID,
+	})
+	require.NoError(t, err)
+
+	// Root user should exist
+	out, err := svc.GetUser(utils.GlobalAccountID, &iam.GetUserInput{
+		UserName: aws.String("root"),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "root", *out.User.UserName)
+
+	// Admin account should NOT exist
+	_, err = svc.GetAccount("000000000001")
+	assert.Error(t, err)
+}
+
 // ============================================================================
 // IsEmpty Tests
 // ============================================================================
@@ -758,34 +852,41 @@ func TestIsEmpty_False(t *testing.T) {
 // ============================================================================
 
 func TestGenerateIAMID(t *testing.T) {
-	id := generateIAMID("AIDA")
+	id, err := generateIAMID("AIDA")
+	assert.NoError(t, err)
 	assert.Equal(t, "AIDA", id[:4])
 	assert.True(t, len(id) == 21) // AIDA + 17 hex chars
 
 	// Two IDs should differ
-	id2 := generateIAMID("AIDA")
+	id2, err := generateIAMID("AIDA")
+	assert.NoError(t, err)
 	assert.NotEqual(t, id, id2)
 
 	// Policy ID prefix
-	pid := generateIAMID("ANPA")
+	pid, err := generateIAMID("ANPA")
+	assert.NoError(t, err)
 	assert.Equal(t, "ANPA", pid[:4])
 	assert.Len(t, pid, 21)
 }
 
 func TestGenerateAccessKeyID(t *testing.T) {
-	id := generateAccessKeyID()
+	id, err := generateAccessKeyID()
+	assert.NoError(t, err)
 	assert.Equal(t, "AKIA", id[:4])
 	assert.True(t, len(id) == 24) // AKIA + 20 hex chars
 
-	id2 := generateAccessKeyID()
+	id2, err := generateAccessKeyID()
+	assert.NoError(t, err)
 	assert.NotEqual(t, id, id2)
 }
 
 func TestGenerateSecretAccessKey(t *testing.T) {
-	secret := admin.GenerateAWSSecretKey()
+	secret, err := admin.GenerateAWSSecretKey()
+	assert.NoError(t, err)
 	assert.Len(t, secret, 40)
 
-	secret2 := admin.GenerateAWSSecretKey()
+	secret2, err := admin.GenerateAWSSecretKey()
+	assert.NoError(t, err)
 	assert.NotEqual(t, secret, secret2)
 }
 
@@ -1647,11 +1748,13 @@ func TestValidatePolicyDocument_ResourceARNFormat(t *testing.T) {
 // ============================================================================
 
 func TestGeneratePolicyID(t *testing.T) {
-	id := generateIAMID("ANPA")
+	id, err := generateIAMID("ANPA")
+	assert.NoError(t, err)
 	assert.Equal(t, "ANPA", id[:4])
 	assert.Len(t, id, 21) // ANPA + 17 hex chars
 
-	id2 := generateIAMID("ANPA")
+	id2, err := generateIAMID("ANPA")
+	assert.NoError(t, err)
 	assert.NotEqual(t, id, id2)
 }
 
@@ -1779,7 +1882,8 @@ func TestParseCreatedAt(t *testing.T) {
 
 func TestGenerateIAMID_AllUpperHex(t *testing.T) {
 	for range 20 {
-		id := generateIAMID("AIDA")
+		id, err := generateIAMID("AIDA")
+		assert.NoError(t, err)
 		suffix := id[4:]
 		for _, c := range suffix {
 			assert.True(t, (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F'),
@@ -1790,7 +1894,8 @@ func TestGenerateIAMID_AllUpperHex(t *testing.T) {
 
 func TestGenerateAccessKeyID_AllUpperHex(t *testing.T) {
 	for range 20 {
-		id := generateAccessKeyID()
+		id, err := generateAccessKeyID()
+		assert.NoError(t, err)
 		suffix := id[4:]
 		for _, c := range suffix {
 			assert.True(t, (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F'),

hive/handlers/iam/types.go

@@ -103,9 +103,20 @@ func (s StringOrArr) MarshalJSON() ([]byte, error) {
 // BootstrapData is the on-disk JSON file consumed on first gateway start
 // to seed the root IAM user into NATS KV.
 type BootstrapData struct {
+	AccessKeyID     string              `json:"access_key_id"`
+	EncryptedSecret string              `json:"encrypted_secret"`
+	AccountID       string              `json:"account_id"`
+	Admin           *AdminBootstrapData `json:"admin,omitempty"`
+}
+
+// AdminBootstrapData holds the default admin account credentials seeded
+// alongside the system account during first boot.
+type AdminBootstrapData struct {
+	AccountID       string `json:"account_id"`
+	AccountName     string `json:"account_name"`
+	UserName        string `json:"user_name"`
 	AccessKeyID     string `json:"access_key_id"`
 	EncryptedSecret string `json:"encrypted_secret"`
-	AccountID       string `json:"account_id"`
 }
 
 // Account represents a Hive account. Accounts namespace IAM users,

hive/qmp/qmp.go

@@ -2,8 +2,10 @@ package qmp
 
 import (
 	"encoding/json"
+	"fmt"
 	"net"
 	"sync"
+	"time"
 )
 
 type unmarshalTarget any
@@ -20,34 +22,6 @@ var CommandResponseTypes = map[string]unmarshalTarget{
 	"query-status": &Status{},
 }
 
-type Command struct {
-	ID               string            `json:"id"`
-	QMPCommand       QMPCommand        `json:"command"`
-	Attributes       Attributes        `json:"attributes"`
-	AttachVolumeData *AttachVolumeData `json:"attach_volume_data,omitempty"`
-	DetachVolumeData *DetachVolumeData `json:"detach_volume_data,omitempty"`
-}
-
-type Attributes struct {
-	StopInstance      bool `json:"stop_instance"`
-	TerminateInstance bool `json:"delete_instance"`
-	StartInstance     bool `json:"start_instance"`
-	AttachVolume      bool `json:"attach_volume"`
-	DetachVolume      bool `json:"detach_volume"`
-}
-
-type AttachVolumeData struct {
-	VolumeID string `json:"volume_id"`
-	Device   string `json:"device,omitempty"`
-}
-
-type DetachVolumeData struct {
-	VolumeID string `json:"volume_id"`
-	Device   string `json:"device,omitempty"`
-	Force    bool   `json:"force,omitempty"`
-}
-
-
 type Status struct {
 	Status     string `json:"status"`
 	Singlestep bool   `json:"singlestep"`
@@ -145,7 +119,11 @@ func NewQMPClient(path string) (*QMPClient, error) {
 		return nil, err
 	}
 
-	//conn.SetReadDeadline(time.Now().Add(30 * time.Second))
+	// Set deadline for greeting — a hung QEMU would block forever without this
+	if err := conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
+		_ = conn.Close()
+		return nil, fmt.Errorf("set read deadline: %w", err)
+	}
 
 	client := &QMPClient{
 		Conn:    conn,
@@ -157,7 +135,13 @@ func NewQMPClient(path string) (*QMPClient, error) {
 	var greeting QMPGreeting
 	if err := client.Decoder.Decode(&greeting); err != nil {
 		_ = conn.Close()
-		return nil, err
+		return nil, fmt.Errorf("waiting for QMP greeting: %w", err)
+	}
+
+	// Clear deadline for normal operations
+	if err := conn.SetReadDeadline(time.Time{}); err != nil {
+		_ = conn.Close()
+		return nil, fmt.Errorf("clear read deadline: %w", err)
 	}
 
 	// enable capabilities

hive/services/awsgw/awsgw.go

@@ -24,8 +24,12 @@ type Service struct {
 }
 
 func New(cfg any) (svc *Service, err error) {
+	c, ok := cfg.(*config.ClusterConfig)
+	if !ok {
+		return nil, fmt.Errorf("invalid config type for awsgw service")
+	}
 	svc = &Service{
-		Config: cfg.(*config.ClusterConfig),
+		Config: c,
 	}
 	return svc, nil
 }
@@ -85,22 +89,28 @@ func launchService(config *config.ClusterConfig) error {
 	// Initialize IAM service with NATS KV backend (required for auth).
 	// On multi-node clusters, JetStream KV requires cluster quorum which may
 	// not be available yet if nodes start concurrently. Retry with backoff.
-	iamService, err := initIAMService(natsConn, masterKey)
+	iamService, err := initIAMService(natsConn, masterKey, len(config.Nodes))
 	if err != nil {
 		return fmt.Errorf("initialize IAM service: %w", err)
 	}
 
-	// First boot: consume bootstrap.json → seed root user into NATS KV → delete file
+	// First boot: consume bootstrap.json → seed IAM users into NATS KV → delete file
 	bootstrapPath := filepath.Join(nodeConfig.BaseDir, "config", "bootstrap.json")
-	if data, err := handlers_iam.LoadBootstrapData(bootstrapPath); err == nil {
-		slog.Info("Bootstrap file found, seeding root IAM user")
-		if err := iamService.SeedRootUser(data); err != nil {
-			return fmt.Errorf("seed root user from bootstrap.json: %w", err)
+	data, err := handlers_iam.LoadBootstrapData(bootstrapPath)
+	switch {
+	case err == nil:
+		slog.Info("Bootstrap file found, seeding IAM users")
+		if err := iamService.SeedBootstrap(data); err != nil {
+			return fmt.Errorf("seed bootstrap from bootstrap.json: %w", err)
 		}
 		if err := os.Remove(bootstrapPath); err != nil {
 			slog.Warn("Failed to delete bootstrap file", "path", bootstrapPath, "err", err)
 		}
 		slog.Info("Bootstrap complete, bootstrap.json deleted")
+	case os.IsNotExist(err):
+		// No bootstrap file — normal after first boot
+	default:
+		return fmt.Errorf("load bootstrap from %s: %w", bootstrapPath, err)
 	}
 
 	// Create gateway with NATS connection
@@ -172,7 +182,7 @@ func connectNATS(host, token string) (*nats.Conn, error) {
 // initIAMService initializes the IAM service with retry/backoff. On multi-node
 // clusters, JetStream requires NATS cluster quorum before KV buckets can be
 // created. This retries for up to 5 minutes to allow late-joining nodes.
-func initIAMService(natsConn *nats.Conn, masterKey []byte) (*handlers_iam.IAMServiceImpl, error) {
+func initIAMService(natsConn *nats.Conn, masterKey []byte, clusterSize int) (*handlers_iam.IAMServiceImpl, error) {
 	const maxWait = 5 * time.Minute
 	retryDelay := 500 * time.Millisecond
 	start := time.Now()
@@ -180,7 +190,7 @@ func initIAMService(natsConn *nats.Conn, masterKey []byte) (*handlers_iam.IAMSer
 
 	for {
 		attempt++
-		svc, err := handlers_iam.NewIAMServiceImpl(natsConn, masterKey)
+		svc, err := handlers_iam.NewIAMServiceImpl(natsConn, masterKey, clusterSize)
 		if err == nil {
 			if attempt > 1 {
 				slog.Info("IAM service initialized after retry", "attempts", attempt, "elapsed", time.Since(start).Round(time.Second))

hive/services/hive/hive.go

@@ -1,6 +1,7 @@
 package hive
 
 import (
+	"fmt"
 	"log/slog"
 	"os"
 
@@ -17,8 +18,12 @@ type Service struct {
 }
 
 func New(cfg any) (svc *Service, err error) {
+	c, ok := cfg.(*config.ClusterConfig)
+	if !ok {
+		return nil, fmt.Errorf("invalid config type for hive service")
+	}
 	svc = &Service{
-		Config: cfg.(*config.ClusterConfig),
+		Config: c,
 	}
 	return svc, nil
 }

hive/services/hiveui/frontend/.gitignore

@@ -2,3 +2,4 @@ node_modules
 .tanstack
 *.tsbuildinfo
 .pnpm-store
+coverage

hive/services/hiveui/frontend/.npmrc

@@ -0,0 +1,2 @@
+minimum-release-time=3d
+exclude-packages-from-minimum-release-time=vite,@vitejs/*

hive/services/hiveui/frontend/.nvmrc

@@ -1 +1 @@
-24.12.0
+24.14.0