From 2351d4ff810b984ea7d33b041a1eb87e9e567fca Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Mon, 16 Mar 2026 22:47:08 +1100 Subject: [PATCH] New doc templates --- docs/creating-accounts/README.md | 61 +++++ docs/creating-an-environment/README.md | 119 +++++++++ docs/hive-admin-cli/README.md | 97 ++++++++ docs/hybrid-sync/README.md | 74 ++++++ docs/installing-hive-airgapped/README.md | 132 ++++++++++ docs/installing-hive-multi-node/README.md | 285 ++++++++++++++++++++++ docs/installing-hive/README.md | 198 +++++++++++++++ docs/launching-instances/README.md | 97 ++++++++ docs/moving-aws-workload/README.md | 102 ++++++++ docs/source-install/README.md | 194 +++++++++++++++ docs/vpc-networking/README.md | 97 ++++++++ 11 files changed, 1456 insertions(+) create mode 100644 docs/creating-accounts/README.md create mode 100644 docs/creating-an-environment/README.md create mode 100644 docs/hive-admin-cli/README.md create mode 100644 docs/hybrid-sync/README.md create mode 100644 docs/installing-hive-airgapped/README.md create mode 100644 docs/installing-hive-multi-node/README.md create mode 100644 docs/installing-hive/README.md create mode 100644 docs/launching-instances/README.md create mode 100644 docs/moving-aws-workload/README.md create mode 100644 docs/source-install/README.md create mode 100644 docs/vpc-networking/README.md diff --git a/docs/creating-accounts/README.md b/docs/creating-accounts/README.md new file mode 100644 index 000000000..9bbf45ceb --- /dev/null +++ b/docs/creating-accounts/README.md @@ -0,0 +1,61 @@ +--- +title: "Creating Accounts" +description: "Create and manage isolated user accounts with their own resources." +category: "Administration" +tags: + - accounts + - iam + - admin +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" +--- + +# Creating Accounts + +> Create and manage isolated user accounts with their own resources. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +Hive supports multi-tenant account isolation. Each account gets its own IAM credentials, AWS CLI profile, and isolated resource namespace. + +## Instructions + +## Create Account + +```bash +./bin/hive admin account create --name myteam +export AWS_PROFILE=hive-myteam +``` + +## Verify + +```bash +aws sts get-caller-identity +``` + +## Troubleshooting + +## Credentials not working + +Verify the AWS CLI configuration files exist and contain the correct profile: + +```bash +cat ~/.aws/config +cat ~/.aws/credentials +``` + +Ensure the `AWS_PROFILE` environment variable matches the profile name: + +```bash +echo $AWS_PROFILE +export AWS_PROFILE=hive-myteam +``` diff --git a/docs/creating-an-environment/README.md b/docs/creating-an-environment/README.md new file mode 100644 index 000000000..dda257f69 --- /dev/null +++ b/docs/creating-an-environment/README.md @@ -0,0 +1,119 @@ +--- +title: "Creating an Environment" +description: "End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances." +category: "Environments" +tags: + - environment + - vpc + - ec2 + - ssh +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "Debian Cloud Images" + url: "https://cloud.debian.org/images/cloud/bookworm/latest/" +--- + +# Creating an Environment + +> End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +This guide walks through the complete workflow for creating a production-ready Hive environment. + +**Steps:** +1. Create a user account +2. Import or create SSH keys +3. Create a VPC and subnet +4. Select and import an AMI +5. Launch EC2 instances +6. Connect via SSH +7. Monitor instance health + +## Instructions + +## 1. Import SSH Key + +```bash +aws ec2 import-key-pair --key-name "hive-key" --public-key-material fileb://~/.ssh/id_rsa.pub +``` + +## 2. Create VPC + +```bash +aws ec2 create-vpc --cidr-block 10.200.0.0/16 +export HIVE_VPC="vpc-XXX" + +aws ec2 create-subnet --vpc-id $HIVE_VPC --cidr-block 10.200.1.0/24 +export HIVE_SUBNET="subnet-XXX" +``` + +## 3. Import AMI + +```bash +./bin/hive admin images list +./bin/hive admin images import --name debian-12-x86_64 +export HIVE_AMI="ami-XXX" +``` + +## 4. Launch Instance + +```bash +aws ec2 run-instances \ + --image-id $HIVE_AMI \ + --instance-type t3.small \ + --key-name hive-key \ + --subnet-id $HIVE_SUBNET \ + --count 1 +``` + +## 5. Connect via SSH + +```bash +ps auxw | grep $INSTANCE_ID +# Look for: hostfwd=tcp:127.0.0.1:-:22 + +ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p +``` + +## Troubleshooting + +## Instance stuck in pending + +Check the Hive daemon and QEMU logs for boot errors: + +```bash +ls ~/hive/logs/ +cat ~/hive/logs/daemon.log +``` + +Verify the AMI exists and architecture matches your host: + +```bash +aws ec2 describe-images --image-ids $HIVE_AMI +``` + +## SSH connection refused + +cloud-init takes 30-60 seconds to configure networking and SSH after boot. Wait and retry. + +Verify the SSH port forwarding is active: + +```bash +ps auxw | grep hostfwd +``` + +Ensure you're using the correct key and port: + +```bash +ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p +``` diff --git a/docs/hive-admin-cli/README.md b/docs/hive-admin-cli/README.md new file mode 100644 index 000000000..ca6dd263a --- /dev/null +++ b/docs/hive-admin-cli/README.md @@ -0,0 +1,97 @@ +--- +title: "Hive Admin CLI" +description: "Complete reference for the Hive administration CLI. Manage accounts, nodes, VMs, and services." +category: "Administration" +tags: + - cli + - admin + - reference +badge: cli +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" +--- + +# Hive Admin CLI + +> Complete reference for the Hive administration CLI. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +The `hive` binary is the central administration tool for managing your Hive infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control. + +**Binary location:** `~/Development/mulga/hive/bin/hive` + +All services in the Hive platform are managed through this single binary. + +## Instructions + +## Account Management + +```bash +./bin/hive admin account create --name myteam +export AWS_PROFILE=hive-myteam +``` + +## Node Management + +```bash +./bin/hive get nodes +``` + +``` +NAME STATUS IP REGION AZ UPTIME VMs +node1 Ready 127.0.0.1 ap-southeast-2 ap-southeast-2a 2m 0 +node2 Ready 127.0.0.2 ap-southeast-2 ap-southeast-2a 2m 0 +node3 Ready 127.0.0.3 ap-southeast-2 ap-southeast-2a 2m 0 +``` + +## Monitor Resources + +```bash +./bin/hive top nodes +``` + +## Image Management + +```bash +./bin/hive admin images list +./bin/hive admin images import --name debian-12-arm64 +``` + +## Cluster Shutdown + +```bash +./bin/hive admin cluster shutdown +``` + +## Troubleshooting + +## Permission denied running hive + +The binary may not be executable. Fix permissions: + +```bash +chmod +x ./bin/hive +``` + +If you get permission errors during operations, ensure you're running with appropriate privileges. Some OVN and networking commands require `sudo`. + +## Services fail to start + +Check the daemon logs for specific errors: + +```bash +ls ~/hive/logs/ +cat ~/hive/logs/daemon.log +``` + +Common causes include port conflicts, missing OVN configuration, or untrusted CA certificates. diff --git a/docs/hybrid-sync/README.md b/docs/hybrid-sync/README.md new file mode 100644 index 000000000..282306be3 --- /dev/null +++ b/docs/hybrid-sync/README.md @@ -0,0 +1,74 @@ +--- +title: "Hybrid Sync" +description: "Synchronize data between Hive and AWS when network connectivity is available." +category: "Migration" +tags: + - hybrid + - sync + - s3 +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" +--- + +# Hybrid Sync + +> Synchronize data between Hive and AWS when network connectivity is available. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +Hive's hybrid mode enables bidirectional data synchronization between local infrastructure and AWS cloud services. Ideal for intermittent connectivity environments. + +## Instructions + +## Push Local to Cloud + +```bash +aws s3 sync s3://local-bucket/ s3://cloud-bucket/ \ + --source-region hive --region us-east-1 +``` + +## Pull Cloud to Local + +```bash +aws s3 sync s3://cloud-bucket/ s3://local-bucket/ \ + --source-region us-east-1 --region hive +``` + +## EBS Volume Backup + +```bash +rsync -avz /data/ user@cloud-server:/backup/hive-data/ +``` + +## Troubleshooting + +## Sync fails mid-transfer + +S3 sync is idempotent — re-run the same command to resume where it left off: + +```bash +aws s3 sync s3://local-bucket/ s3://cloud-bucket/ \ + --source-region hive --region us-east-1 +``` + +Only changed or missing files will be transferred on subsequent runs. + +## Permissions errors during sync + +Verify your AWS credentials are configured for both the source and destination: + +```bash +aws sts get-caller-identity --profile hive +aws sts get-caller-identity --profile default +``` + +Ensure the IAM user has `s3:GetObject`, `s3:PutObject`, and `s3:ListBucket` permissions on both buckets. diff --git a/docs/installing-hive-airgapped/README.md b/docs/installing-hive-airgapped/README.md new file mode 100644 index 000000000..c27a2c6dc --- /dev/null +++ b/docs/installing-hive-airgapped/README.md @@ -0,0 +1,132 @@ +--- +title: "Air-Gapped Installation" +description: "Deploy Hive in environments without internet connectivity. Covers offline package caching, USB deployment, and package verification." +category: "Getting Started" +tags: + - install + - air-gapped + - offline +badge: advanced +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "apt-cacher-ng" + url: "https://wiki.debian.org/AptCacherNg" +--- + +# Air-Gapped Installation + +> Deploy Hive in environments without internet connectivity. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +In air-gapped or disconnected environments, Hive can be deployed without internet access. This guide covers preparing offline packages on a connected machine, creating USB deployment media, and installing on the target server with package verification. + +**Deployment methods:** +- **apt-cacher-ng** — Local APT cache for Debian/Ubuntu packages +- **Go module cache** — Pre-downloaded Go dependencies +- **USB deployment** — Portable installation media with all dependencies +- **GPG verification** — Cryptographic package integrity checks + +## Instructions + +## Step 1. Prepare packages (on a connected machine) + +Set up a local APT cache to collect all required packages: + +```bash +sudo apt install apt-cacher-ng +sudo systemctl enable --now apt-cacher-ng +``` + +Pre-download all Hive dependencies: + +```bash +sudo apt install --download-only nbdkit nbdkit-plugin-dev pkg-config \ + qemu-system qemu-utils qemu-kvm libvirt-daemon-system \ + libvirt-clients libvirt-dev make gcc unzip xz-utils file \ + ovn-central ovn-host openvswitch-switch +``` + +## Step 2. Create USB deployment media + +```bash +mkdir -p /media/hive-deploy/{apt-packages,go-cache,hive-source} +cp /var/cache/apt/archives/*.deb /media/hive-deploy/apt-packages/ +cp -r ~/go/pkg/mod/cache/ /media/hive-deploy/go-cache/ +cp -r ~/Development/mulga/hive /media/hive-deploy/hive-source/ +``` + +## Step 3. Verify package integrity + +Sign and verify packages before transferring to the target: + +```bash +gpg --import /media/hive-deploy/mulga-signing-key.asc +gpg --verify hive-v1.0.tar.gz.sig hive-v1.0.tar.gz +sha256sum -c /media/hive-deploy/checksums.sha256 +``` + +## Step 4. Install on the target server + +Mount the USB and install: + +```bash +sudo mount /dev/sdb1 /mnt/usb +sudo dpkg -i /mnt/usb/apt-packages/*.deb +cp -r /mnt/usb/hive-source ~/Development/mulga/hive +cd ~/Development/mulga/hive && make build +``` + +Then follow the [Source Install](/docs/source-install) guide from Step 3 (Setup OVN) onwards. + +## Troubleshooting + +## Missing dependencies after dpkg install + +Some packages may have unresolved dependencies. Fix with: + +```bash +sudo dpkg -i /mnt/usb/apt-packages/*.deb +sudo apt-get install -f +``` + +The `-f` flag tells apt to fix broken dependencies using what's available locally. + +## GPG verification fails + +The signing key may not match or the download was corrupted. Re-import the key and verify: + +```bash +gpg --import /media/hive-deploy/mulga-signing-key.asc +gpg --verify hive-v1.0.tar.gz.sig hive-v1.0.tar.gz +``` + +If verification still fails, re-download the package on the connected machine and compare checksums: + +```bash +sha256sum hive-v1.0.tar.gz +``` + +## Go module cache errors + +The Go module cache may have been incompletely copied. On the connected machine, re-export the full cache: + +```bash +cp -r ~/go/pkg/mod/cache/ /media/hive-deploy/go-cache/ +``` + +On the target, restore it: + +```bash +mkdir -p ~/go/pkg/mod/ +cp -r /mnt/usb/go-cache ~/go/pkg/mod/cache +``` diff --git a/docs/installing-hive-multi-node/README.md b/docs/installing-hive-multi-node/README.md new file mode 100644 index 000000000..89672487f --- /dev/null +++ b/docs/installing-hive-multi-node/README.md @@ -0,0 +1,285 @@ +--- +title: "Multi-Node Installation" +description: "Deploy Hive across multiple servers to create an availability zone with high availability, data durability, and fault tolerance." +category: "Getting Started" +tags: + - install + - multi node + - cluster +badge: availability-zone +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "Predastore (S3)" + url: "https://github.com/mulgadc/predastore" + - title: "Viperblock (EBS)" + url: "https://github.com/mulgadc/viperblock" +--- + +# Multi-Node Installation + +> Deploy Hive across multiple servers to create an availability zone. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +A Hive cluster operates as a fully distributed infrastructure region — similar to an AWS Region — where multiple nodes provide high availability, data durability, and fault tolerance. + +**Services distributed across nodes:** +- **NATS** — Clustered message bus for request routing and JetStream replication +- **Predastore** — Raft consensus with Reed-Solomon erasure coding (RS 2+1) +- **Viperblock** — Block storage co-located with compute nodes +- **Hive Daemon** — Independent AWS API request serving per node + +Cluster formation is automatic. When initializing with `--nodes 3`, the init node starts a formation server and waits for peers to join. Once all nodes register, each receives the full cluster topology — credentials, CA certificates, NATS routes, Predastore peer lists — no manual configuration synchronization required. + +**Network requirements:** +- Minimum 1 NIC per server +- 2 NICs recommended for production (management + overlay) +- UDP port 6081 open between hosts (Geneve tunnels) + +## Instructions + +## Step 1. Install Hive on each server + +Run the binary installer on **every server** in the cluster: + +```bash +curl https://install.mulgadc.com/ | bash +``` + +## Step 2. Set node IP variables + +On **each server**, export the IPs for all nodes (replace with your actual IPs): + +```bash +export HIVE_NODE1=192.168.1.10 +export HIVE_NODE2=192.168.1.11 +export HIVE_NODE3=192.168.1.12 +``` + +To find your server's IP: + +```bash +ip addr show | grep "inet " +``` + +## Step 3. Setup OVN networking + +OVN must be configured on every server before forming the cluster. Server 1 runs OVN central (NB and SB databases) and must be set up first. + +**Server 1 — OVN central + compute (run first):** + +```bash +hive setup-ovn --management --encap-ip=$HIVE_NODE1 +``` + +Verify OVN central is ready before proceeding: + +```bash +sudo ovn-sbctl show +``` + +**Server 2 — Compute node (after server 1 is ready):** + +```bash +hive setup-ovn --ovn-remote=tcp:$HIVE_NODE1:6642 --encap-ip=$HIVE_NODE2 +``` + +**Server 3 — Compute node (after server 1 is ready):** + +```bash +hive setup-ovn --ovn-remote=tcp:$HIVE_NODE1:6642 --encap-ip=$HIVE_NODE3 +``` + +Verify all chassis have registered (from server 1): + +```bash +sudo ovn-sbctl show +``` + +You should see 3 chassis entries, one per server, each with a Geneve encap IP. + +## Step 4. Form the cluster + +The formation process requires running init and join commands concurrently. The init node blocks until all expected nodes have joined. + +**Server 1 — Initialize the cluster:** + +```bash +hive admin init \ + --node node1 \ + --nodes 3 \ + --bind $HIVE_NODE1 \ + --cluster-bind $HIVE_NODE1 \ + --port 4432 \ + --hive-dir ~/hive/ \ + --config-dir ~/hive/config/ \ + --region us-east-1 \ + --az us-east-1a +``` + +**Server 2 — Join the cluster (while init is running):** + +```bash +hive admin join \ + --node node2 \ + --bind $HIVE_NODE2 \ + --cluster-bind $HIVE_NODE2 \ + --host $HIVE_NODE1:4432 \ + --data-dir ~/hive/ \ + --config-dir ~/hive/config/ \ + --region us-east-1 \ + --az us-east-1a +``` + +**Server 3 — Join the cluster (while init is running):** + +```bash +hive admin join \ + --node node3 \ + --bind $HIVE_NODE3 \ + --cluster-bind $HIVE_NODE3 \ + --host $HIVE_NODE1:4432 \ + --data-dir ~/hive/ \ + --config-dir ~/hive/config/ \ + --region us-east-1 \ + --az us-east-1a +``` + +All three processes will exit with a cluster summary once formation is complete. + +## Step 5. Start services and verify + +Start services on **all servers**: + +```bash +hive start +``` + +From any node, verify the cluster: + +```bash +hive get nodes +``` + +``` +NAME STATUS IP REGION AZ UPTIME VMs +node1 Ready 192.168.1.10 us-east-1 us-east-1a 2m 0 +node2 Ready 192.168.1.11 us-east-1 us-east-1a 2m 0 +node3 Ready 192.168.1.12 us-east-1 us-east-1a 2m 0 +``` + +## Step 6. Trust the CA certificate + +On **each server**, trust the CA generated during init: + +```bash +sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates +export AWS_PROFILE=hive +``` + +## Step 7. Launch instances across the cluster + +Connect to any node's AWS Gateway — all nodes serve the same cluster state: + +```bash +aws ec2 run-instances \ + --image-id $HIVE_AMI \ + --instance-type t3.small \ + --key-name hive-key \ + --subnet-id $HIVE_SUBNET \ + --count 3 +``` + +Hive distributes instances across nodes based on available capacity. Check which node each instance landed on: + +```bash +hive get vms +``` + +## Shutdown + +A graceful shutdown coordinates all nodes — drains VMs, flushes storage, and stops services: + +```bash +hive admin cluster shutdown +``` + +## Troubleshooting + +## Nodes not joining the cluster + +The init command must still be running when you execute join on other servers. If init has already exited, re-run it. + +Check that all servers can reach the init node on the formation port: + +```bash +curl -s http://$HIVE_NODE1:4432/health +``` + +## OVN chassis not registering + +Compute nodes need to reach OVN central on server 1. Verify from server 1: + +```bash +sudo ovn-sbctl show +``` + +If chassis are missing, check firewall rules for port 6642: + +```bash +sudo ss -tlnp | grep 6642 +``` + +Re-run the OVN setup on the affected compute node. + +## Cross-host VMs cannot communicate + +This indicates a Geneve tunnel issue. Verify tunnels are established: + +```bash +sudo ovs-vsctl show | grep -i geneve +``` + +Ensure UDP port 6081 is open between all hosts: + +```bash +sudo ss -ulnp | grep 6081 +``` + +If tunnels are missing, verify `--encap-ip` was set correctly during OVN setup. + +## Health check fails on a node + +Check if the daemon is running and listening on the correct IP: + +```bash +curl -s http://$HIVE_NODE1:4432/health +``` + +Review the daemon logs for errors: + +```bash +cat ~/hive/logs/daemon.log +``` + +Verify `--bind` was set to the correct IP during cluster formation. + +## NATS cluster routing issues + +Check NATS logs for connection errors between nodes: + +```bash +grep -i "route\|cluster" ~/hive/logs/nats.log +``` + +Ensure `--cluster-bind` IPs are reachable between all servers. diff --git a/docs/installing-hive/README.md b/docs/installing-hive/README.md new file mode 100644 index 000000000..c65116e5a --- /dev/null +++ b/docs/installing-hive/README.md @@ -0,0 +1,198 @@ +--- +title: "Installing Hive" +description: "Install Hive on a single server using the binary installer. Get a working region with EC2, VPC, EBS, and S3 in minutes." +category: "Getting Started" +tags: + - install + - single node + - quickstart +badge: quickstart +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "Predastore (S3)" + url: "https://github.com/mulgadc/predastore" + - title: "Viperblock (EBS)" + url: "https://github.com/mulgadc/viperblock" +--- + +# Installing Hive + +> Install Hive on a single server using the binary installer. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +Hive is an open-source infrastructure platform that brings core AWS services — EC2, VPC, EBS, and S3 — to bare-metal, edge, and on-prem environments. + +This guide installs Hive on a single server using the binary installer. For multi-server clusters, see [Multi-Node Installation](/docs/installing-hive-multi-node). To build from source, see [Source Install](/docs/source-install). + +**Supported Operating Systems:** +- Ubuntu 22.04 / 24.04 / 25.10 +- Debian 12.13 + +**What gets installed:** +- Hive daemon and CLI +- QEMU/KVM for virtual machine management +- OVN/Open vSwitch for VPC networking +- Predastore (S3-compatible object storage) +- Viperblock (EBS-compatible block storage) +- AWS CLI v2 + +## Instructions + +## Step 1. Install Hive + +```bash +curl https://install.mulgadc.com/ | bash +``` + +The installer downloads the Hive binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI). + +## Step 2. Initialize a region + +Create your first region and availability zone: + +```bash +hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1 +``` + +## Step 3. Trust the CA certificate + +Hive generates a local CA during initialization. Add it to your system trust store: + +```bash +sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates +``` + +## Step 4. Start services + +```bash +hive start +``` + +Set the AWS profile to use the default admin account: + +```bash +export AWS_PROFILE=hive +``` + +## Step 5. Import an AMI + +List available images and import one matching your architecture: + +```bash +hive admin images list +hive admin images import --name debian-12-arm64 +``` + +## Step 6. Create a VPC and launch an instance + +```bash +aws ec2 create-vpc --cidr-block 10.200.0.0/16 +export HIVE_VPC="vpc-XXX" + +aws ec2 create-subnet --vpc-id $HIVE_VPC --cidr-block 10.200.1.0/24 +export HIVE_SUBNET="subnet-XXX" + +aws ec2 import-key-pair --key-name "hive-key" --public-key-material fileb://~/.ssh/id_rsa.pub + +aws ec2 run-instances \ + --image-id $HIVE_AMI \ + --instance-type t3.small \ + --key-name hive-key \ + --subnet-id $HIVE_SUBNET \ + --count 1 +``` + +## Step 7. Connect via SSH + +Find the forwarded SSH port and connect: + +```bash +ps auxw | grep hostfwd +# Look for: hostfwd=tcp:127.0.0.1:-:22 + +ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p +``` + +## Troubleshooting + +## hive command not found + +The binary is not in your PATH. Run the installer again or add the install directory to your PATH: + +```bash +export PATH=$PATH:~/hive/bin/ +``` + +Add this to your `~/.bashrc` or `~/.zshrc` for persistence. + +## OVN services not starting + +Check if the OVN controller is active: + +```bash +sudo systemctl is-active ovn-controller +sudo ovn-sbctl show +``` + +If inactive, re-run the OVN setup that the installer performs. Check system logs for details: + +```bash +journalctl -u ovn-controller --no-pager -n 20 +``` + +## CA certificate not trusted + +AWS CLI will reject HTTPS connections if the Hive CA is not trusted. Re-add the certificate: + +```bash +sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates +``` + +Verify it was added: + +```bash +ls -la /usr/local/share/ca-certificates/hive-ca.crt +``` + +## Instance stuck in pending + +Check the Hive daemon logs for errors: + +```bash +ls ~/hive/logs/ +cat ~/hive/logs/daemon.log +``` + +Verify the AMI was imported successfully: + +```bash +aws ec2 describe-images +``` + +## SSH connection refused + +cloud-init takes 30-60 seconds to configure the instance after boot. Wait and retry. + +Verify the SSH port forwarding is active: + +```bash +ps auxw | grep hostfwd +``` + +If no `hostfwd` entry appears, the instance may not have started correctly. Check instance state: + +```bash +aws ec2 describe-instances --instance-ids $INSTANCE_ID +``` diff --git a/docs/launching-instances/README.md b/docs/launching-instances/README.md new file mode 100644 index 000000000..25d3e9526 --- /dev/null +++ b/docs/launching-instances/README.md @@ -0,0 +1,97 @@ +--- +title: "Launching Instances" +description: "Launch, manage, and connect to EC2-compatible virtual machines on Hive." +category: "Environments" +tags: + - ec2 + - instances + - vm +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "AWS EC2 CLI" + url: "https://docs.aws.amazon.com/cli/latest/reference/ec2/" +--- + +# Launching Instances + +> Launch, manage, and connect to EC2-compatible virtual machines on Hive. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +Hive provides EC2-compatible VM management built on QEMU/KVM. Instances support cloud-init, SSH key injection, VPC networking, and standard AWS lifecycle operations. + +**Supported operations:** +- `run-instances` — Launch new VMs +- `describe-instances` — Query state +- `stop-instances` / `start-instances` — Lifecycle +- `terminate-instances` — Permanent removal + +## Instructions + +## Launch + +```bash +aws ec2 run-instances \ + --image-id $HIVE_AMI \ + --instance-type t3.small \ + --key-name hive-key \ + --subnet-id $HIVE_SUBNET \ + --count 1 + +export INSTANCE_ID="i-XXX" +``` + +## Manage + +```bash +aws ec2 describe-instances --instance-ids $INSTANCE_ID +aws ec2 stop-instances --instance-ids $INSTANCE_ID +aws ec2 start-instances --instance-ids $INSTANCE_ID +aws ec2 terminate-instances --instance-ids $INSTANCE_ID +``` + +## SSH (Development) + +```bash +ps auxw | grep $INSTANCE_ID +ssh -i ~/.ssh/hive-key ec2-user@127.0.0.1 -p +``` + +## Troubleshooting + +## Instance fails to boot + +Check QEMU logs for the instance and verify the AMI architecture matches your host: + +```bash +cat ~/hive/logs/daemon.log +aws ec2 describe-images --image-ids $HIVE_AMI +``` + +If the AMI is for a different architecture (e.g. arm64 on an x86_64 host), import the correct image: + +```bash +./bin/hive admin images list +./bin/hive admin images import --name debian-12-x86_64 +``` + +## Cannot SSH into instance + +cloud-init needs time to configure the instance after boot. Wait 30-60 seconds and retry. + +Verify the SSH key was specified correctly when launching: + +```bash +aws ec2 describe-instances --instance-ids $INSTANCE_ID +``` + +Check the `KeyName` field matches the key you're using to connect. diff --git a/docs/moving-aws-workload/README.md b/docs/moving-aws-workload/README.md new file mode 100644 index 000000000..ccfd72adc --- /dev/null +++ b/docs/moving-aws-workload/README.md @@ -0,0 +1,102 @@ +--- +title: "Moving an AWS Workload to Mulga" +description: "Migrate existing AWS workloads to Hive using compatible APIs, SDKs, and Terraform." +category: "Migration" +tags: + - migration + - aws + - terraform +badge: new +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "AWS CLI Reference" + url: "https://docs.aws.amazon.com/cli/latest/reference/" + - title: "Terraform AWS Provider" + url: "https://registry.terraform.io/providers/hashicorp/aws/latest" +--- + +# Moving an AWS Workload to Mulga + +> Migrate existing AWS workloads to Hive using compatible APIs, SDKs, and Terraform. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +Hive provides drop-in compatibility with AWS APIs, making it possible to migrate existing workloads with minimal changes. + +**Supported Services:** VPC, EC2, EBS, S3, IAM + +**Compatible Tools:** AWS CLI, AWS SDKs, Terraform, any S3-compatible client + +## Instructions + +## Configure AWS CLI + +```bash +export AWS_PROFILE=hive +aws ec2 describe-instances +aws s3 ls +``` + +## Terraform + +```hcl +provider "aws" { + region = "ap-southeast-2" + access_key = "your-hive-access-key" + secret_key = "your-hive-secret-key" + + endpoints { + ec2 = "https://localhost:9999" + s3 = "https://localhost:8443" + } + + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true +} +``` + +## Hybrid S3 Sync + +```bash +aws s3 sync s3://local-bucket/ s3://cloud-bucket/ --source-region hive --region us-east-1 +``` + +## Troubleshooting + +## Terraform provider errors + +Ensure all three skip flags are set in your provider configuration: + +```hcl +skip_credentials_validation = true +skip_metadata_api_check = true +skip_requesting_account_id = true +``` + +Without these, Terraform will try to validate credentials and metadata against real AWS endpoints. + +## S3 signature errors + +Hive uses AWS Signature V4. Ensure your AWS CLI is version 2.0 or higher: + +```bash +aws --version +``` + +If using an older version, upgrade: + +```bash +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install --update +``` diff --git a/docs/source-install/README.md b/docs/source-install/README.md new file mode 100644 index 000000000..8b09da751 --- /dev/null +++ b/docs/source-install/README.md @@ -0,0 +1,194 @@ +--- +title: "Source Install" +description: "Build Hive from source for development, custom builds, or contributing. Includes dependency setup, compilation, and OVN configuration." +category: "Getting Started" +tags: + - install + - source + - development +badge: source +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "Predastore (S3)" + url: "https://github.com/mulgadc/predastore" + - title: "Viperblock (EBS)" + url: "https://github.com/mulgadc/viperblock" + - title: "Go Downloads" + url: "https://go.dev/dl/" + - title: "AWS CLI Install" + url: "https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" +--- + +# Source Install + +> Build Hive from source for development, custom builds, or contributing. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +This guide covers building Hive from source code. Use this if you want to contribute to Hive, build custom versions, or inspect the codebase. For production deployments, the [binary installer](/docs/installing-hive) is recommended. + +**Supported Operating Systems:** +- Ubuntu 22.04 / 24.04 / 25.10 +- Debian 12.13 + +**Requirements:** +- Go 1.26.1+ +- GCC, make, pkg-config +- QEMU/KVM +- OVN/Open vSwitch +- AWS CLI v2 + +## Instructions + +## Step 1. Install system dependencies + +### Quick Install + +Bootstrap all dependencies in one step: + +```bash +sudo make -C hive quickinstall +``` + +Ensure Go is in your PATH: + +```bash +export PATH=$PATH:/usr/local/go/bin/ +``` + +### Manual Install + +Alternatively, install packages individually: + +```bash +sudo add-apt-repository universe +sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file ovn-central ovn-host openvswitch-switch +``` + +Install Go 1.26.1+ from [https://go.dev/dl/](https://go.dev/dl/) and verify: + +```bash +go version +``` + +Install AWS CLI v2: + +```bash +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install +aws --version +``` + +## Step 2. Clone and build + +```bash +mkdir -p ~/Development/mulga/ +cd ~/Development/mulga/ +git clone https://github.com/mulgadc/hive.git +cd hive +./scripts/clone-deps.sh # Clone viperblock + predastore +./scripts/dev-setup.sh # Setup complete dev environment +``` + +Confirm `./bin/hive` exists and is executable. + +## Step 3. Setup OVN + +For a single-node dev environment: + +```bash +./scripts/setup-ovn.sh --management +``` + +This creates the `br-int` integration bridge, starts `ovn-controller`, configures Geneve tunnel endpoints, and enables IP forwarding. + +## Step 4. Initialize + +Create a region and availability zone: + +```bash +./bin/hive admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1 +``` + +## Step 5. Trust the CA certificate + +```bash +sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates +``` + +## Step 6. Start services + +```bash +./scripts/start-dev.sh +export AWS_PROFILE=hive +``` + +## Troubleshooting + +## Go not found in PATH + +After installing Go, it may not be in your current shell session. Add it: + +```bash +export PATH=$PATH:/usr/local/go/bin/ +``` + +Add this to your `~/.bashrc` or `~/.zshrc` for persistence. Verify with: + +```bash +go version +``` + +## ./bin/hive missing after build + +The build did not complete successfully. Re-run the setup: + +```bash +./scripts/dev-setup.sh +``` + +Check the output for compilation errors. Common causes are missing Go version or missing system packages. + +## OVN services not starting + +Verify the setup script was run with `--management`: + +```bash +./scripts/setup-ovn.sh --management +``` + +Check OVN controller status: + +```bash +sudo systemctl is-active ovn-controller +sudo ovn-sbctl show +``` + +## CA certificate not trusted + +AWS CLI will reject HTTPS connections to Hive services. Re-add the certificate: + +```bash +sudo cp ~/hive/config/ca.pem /usr/local/share/ca-certificates/hive-ca.crt +sudo update-ca-certificates +``` + +## clone-deps.sh fails + +Ensure you have access to the GitHub repositories. If using SSH and it fails, try HTTPS: + +```bash +git config --global url."https://github.com/".insteadOf "git@github.com:" +./scripts/clone-deps.sh +``` diff --git a/docs/vpc-networking/README.md b/docs/vpc-networking/README.md new file mode 100644 index 000000000..a93f74c16 --- /dev/null +++ b/docs/vpc-networking/README.md @@ -0,0 +1,97 @@ +--- +title: "VPC Networking" +description: "Create and manage Virtual Private Clouds with subnets, DHCP, and Geneve overlay networking." +category: "Environments" +tags: + - vpc + - networking + - ovn +resources: + - title: "Hive Repository" + url: "https://github.com/mulgadc/hive" + - title: "OVN Architecture" + url: "https://www.ovn.org/en/" +--- + +# VPC Networking + +> Create and manage Virtual Private Clouds with subnets, DHCP, and Geneve overlay networking. + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +Every EC2 instance runs inside a VPC with an isolated virtual network. Hive uses OVN (Open Virtual Network) to provide the networking layer. + +**How it works:** +- VPC becomes an OVN logical router +- Subnet becomes an OVN logical switch with DHCP +- Each instance gets an OVN port with automatic IP assignment +- Cross-host traffic uses Geneve tunnels (UDP 6081) + +## Instructions + +## Create VPC + +```bash +aws ec2 create-vpc --cidr-block 10.200.0.0/16 +export HIVE_VPC="vpc-XXX" +``` + +## Create Subnet + +```bash +aws ec2 create-subnet --vpc-id $HIVE_VPC --cidr-block 10.200.1.0/24 +export HIVE_SUBNET="subnet-XXX" +``` + +## Verify + +```bash +aws ec2 describe-vpcs +aws ec2 describe-subnets +sudo ovn-nbctl lr-list +sudo ovn-nbctl ls-list +``` + +## Troubleshooting + +## VPC creation fails + +Ensure OVN services are running and the vpcd daemon is active: + +```bash +sudo systemctl is-active ovn-controller +``` + +Check the vpcd logs for errors: + +```bash +cat ~/hive/logs/vpcd.log +``` + +## Instances cannot reach each other + +This typically means Geneve tunnels are not established between hosts. Verify tunnel configuration: + +```bash +sudo ovs-vsctl show | grep -i geneve +``` + +Ensure UDP port 6081 is open between all hosts: + +```bash +sudo ss -ulnp | grep 6081 +``` + +From inside a VM, check that the private IP was assigned via DHCP: + +```bash +ip addr show +```