From cf12a3272e2ccc5181a0709a3a9060ef1d141bb2 Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:42:43 +1000 Subject: [PATCH] IAC for Proxmox / AWS terraform examples --- .gitignore | 8 + INSTALL.md | 24 ++- scripts/iac/README.md | 8 + scripts/iac/aws/README.md | 206 +++++++++++++++++++++++ scripts/iac/aws/main.tf | 224 +++++++++++++++++++++++++ scripts/iac/proxmox/.env.example | 53 ++++++ scripts/iac/proxmox/README.md | 246 ++++++++++++++++++++++++++++ scripts/iac/proxmox/cloud-config.tf | 40 +++++ scripts/iac/proxmox/files.tf | 17 ++ scripts/iac/proxmox/main.tf | 32 ++++ scripts/iac/proxmox/variables.tf | 60 +++++++ scripts/iac/proxmox/vms.tf | 71 ++++++++ 12 files changed, 980 insertions(+), 9 deletions(-) create mode 100644 scripts/iac/README.md create mode 100644 scripts/iac/aws/README.md create mode 100644 scripts/iac/aws/main.tf create mode 100644 scripts/iac/proxmox/.env.example create mode 100644 scripts/iac/proxmox/README.md create mode 100644 scripts/iac/proxmox/cloud-config.tf create mode 100644 scripts/iac/proxmox/files.tf create mode 100644 scripts/iac/proxmox/main.tf create mode 100644 scripts/iac/proxmox/variables.tf create mode 100644 scripts/iac/proxmox/vms.tf diff --git a/.gitignore b/.gitignore index 1aaa2746e..261af9103 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,11 @@ +# Terraform/OpenTofu +.terraform/ +.terraform.lock.hcl +*.tfstate +*.tfstate.* +*.tfvars +!example.tfvars + # Go workspace go.work go.work.sum diff --git a/INSTALL.md b/INSTALL.md index b08516d69..e1d31efe9 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -45,7 +45,7 @@ sudo add-apt-repository universe sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file ``` -Ensure the Go toolkit is installed for version 1.26.0 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/). +Ensure the Go toolkit is installed for version 1.25.7 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/). Confirm Go is correctly installed, and set in your $PATH. @@ -262,22 +262,28 @@ aws ec2 describe-instance-types { "InstanceTypes": [ { - "InstanceType": "m8a.small", + "InstanceType": "t3.small", "CurrentGeneration": true, - "SupportedRootDeviceTypes": ["ebs"], - "SupportedVirtualizationTypes": ["hvm"], + "SupportedRootDeviceTypes": [ + "ebs" + ], + "SupportedVirtualizationTypes": [ + "hvm" + ], "Hypervisor": "kvm", "ProcessorInfo": { - "SupportedArchitectures": ["x86_64"] + "SupportedArchitectures": [ + "x86_64" + ] }, "VCpuInfo": { - "DefaultVCpus": 2 + "DefaultVCpus": 2 }, "MemoryInfo": { - "SizeInMiB": 1024 + "SizeInMiB": 2048 }, "BurstablePerformanceSupported": false - } + }, ] } ``` @@ -285,7 +291,7 @@ aws ec2 describe-instance-types Export the instance-type: ```sh -export HIVE_INSTANCE="m8a.small" +export HIVE_INSTANCE="t3.small" ``` ### Run Instance diff --git a/scripts/iac/README.md b/scripts/iac/README.md new file mode 100644 index 000000000..277aeefd2 --- /dev/null +++ b/scripts/iac/README.md @@ -0,0 +1,8 @@ +# IAC + +Collection of OpenTofu/Terraform templates for creating Hive instances. + +| Template | Description | +|---|---| +| [proxmox/](proxmox/) | Provisions a 3-node Hive cluster on Proxmox VE bare-metal hypervisors | +| [aws/](aws/) | Provisions infrastructure using the Terraform AWS provider, with MulgaOS AWS-compatible API as the local endpoint | diff --git a/scripts/iac/aws/README.md b/scripts/iac/aws/README.md new file mode 100644 index 000000000..f88bd9e72 --- /dev/null +++ b/scripts/iac/aws/README.md @@ -0,0 +1,206 @@ +# MulgaOS AWS Infrastructure as Code + +This directory contains OpenTofu/Terraform configuration files for deploying infrastructure on MulgaOS using AWS-compatible APIs. + +## Prerequisites + +1. **OpenTofu or Terraform installed** + ```bash + # Install OpenTofu (recommended) + # See: https://opentofu.org/docs/intro/install/ + + # Or use Terraform + # See: https://developer.hashicorp.com/terraform/install + ``` + +2. **MulgaOS services running** + - `hive daemon` - Core daemon with NATS subscriptions + - `awsd` - AWS API gateway (default: http://localhost:8080) + - `predastore` - S3-compatible storage + - `viperblock` - EBS-compatible block storage + +3. **AWS credentials configured** + ```bash + # Option 1: Environment variables + export AWS_ACCESS_KEY_ID=your_access_key + export AWS_SECRET_ACCESS_KEY=your_secret_key + + # Option 2: AWS CLI profile + aws configure --profile hive + export AWS_PROFILE=hive + ``` + +4. **AMI available** + Before running, ensure you have a Debian 12 AMI imported: + ```bash + # Import an AMI using the hive CLI or aws CLI + aws --endpoint-url http://localhost:8080 ec2 describe-images + ``` + +## Configuration + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `TF_VAR_mulgaos_endpoint` | `http://localhost:8080` | MulgaOS API endpoint | +| `TF_VAR_region` | `ap-southeast-2` | AWS region (for compatibility) | +| `AWS_ACCESS_KEY_ID` | - | MulgaOS access key | +| `AWS_SECRET_ACCESS_KEY` | - | MulgaOS secret key | + +### terraform.tfvars (optional) + +Create a `terraform.tfvars` file for custom settings: + +```hcl +mulgaos_endpoint = "http://192.168.1.100:8080" +region = "us-west-2" +``` + +## Usage + +### Initialize + +```bash +cd iac/aws +tofu init +# or: terraform init +``` + +### Validate + +```bash +tofu validate +# or: terraform validate +``` + +### Plan + +```bash +tofu plan +# or: terraform plan +``` + +### Apply + +```bash +tofu apply +# or: terraform apply +``` + +### Destroy + +```bash +tofu destroy +# or: terraform destroy +``` + +## What Gets Created + +The `main.tf` creates: + +1. **VPC** (`10.1.1.0/16`) - Virtual Private Cloud +2. **Internet Gateway** - For public internet access +3. **Public Subnet** (`10.1.2.0/24`) - With auto-assign public IP +4. **Route Table** - Routes `0.0.0.0/0` through IGW +5. **Security Group** - Allows SSH (22), HTTP (80), HTTPS (443) inbound +6. **Placement Group** - Spread strategy for HA +7. **3 EC2 Instances** - t3.small Debian 12 VMs +8. **SSH Key Pair** - ED25519 key saved to `hive-test.pem` + +## SSH Access + +After `tofu apply`, SSH into instances: + +```bash +# Using the generated key +ssh -i hive-test.pem admin@ + +# For local development with QEMU port forwarding +# Find the forwarded port from QEMU args (-netdev user,id=net0,hostfwd=tcp:127.0.0.1:PORT-:22) +ssh -p PORT -i hive-test.pem admin@127.0.0.1 +``` + +## Troubleshooting + +### "No valid credential sources found" + +Ensure AWS credentials are set: +```bash +export AWS_ACCESS_KEY_ID=your_key +export AWS_SECRET_ACCESS_KEY=your_secret +``` + +### "Error: No AMI found" + +Import a Debian 12 AMI first, or modify `main.tf` to use a different AMI filter. + +### Connection refused + +Ensure MulgaOS services are running: +```bash +# Check if awsd gateway is running +curl http://localhost:8080/health + +# Check daemon status +hive daemon status +``` + +### "InvalidAction" errors + +Some AWS operations may not be implemented yet. Check `TODO.md` for the current implementation status. + +## Customization + +### Using a different AMI + +Modify the `data "aws_ami"` block in `main.tf`: + +```hcl +data "aws_ami" "custom" { + most_recent = true + owners = ["self"] # Your imported AMIs + + filter { + name = "name" + values = ["my-custom-ami-*"] + } +} +``` + +### Changing instance count + +Modify the `count` parameter: + +```hcl +resource "aws_instance" "vm" { + count = 5 # Change from 3 to 5 + # ... +} +``` + +### Different instance type + +```hcl +resource "aws_instance" "vm" { + instance_type = "t3.medium" # Change from t3.small + # ... +} +``` + +## Files + +| File | Description | +|------|-------------| +| `main.tf` | Main Terraform/OpenTofu configuration | +| `hive-test.pem` | Generated SSH private key (after apply) | +| `.terraform/` | Provider plugins (after init) | +| `terraform.tfstate` | State file (after apply) | +| `terraform.tfvars` | Custom variables (optional) | + +## Related Documentation + +- [MulgaOS DEV.md](../../DEV.md) - Development guide +- [MulgaOS TODO.md](../../TODO.md) - Implementation status +- [OpenTofu Documentation](https://opentofu.org/docs/) +- [AWS Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) diff --git a/scripts/iac/aws/main.tf b/scripts/iac/aws/main.tf new file mode 100644 index 000000000..6b1a921a5 --- /dev/null +++ b/scripts/iac/aws/main.tf @@ -0,0 +1,224 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0" + } + } +} + +variable "region" { + type = string + default = "ap-southeast-2" # Sydney, change if you want +} + +# MulgaOS AWS-compatible endpoints +# Set these environment variables or use terraform.tfvars: +# export AWS_ACCESS_KEY_ID=your_access_key +# export AWS_SECRET_ACCESS_KEY=your_secret_key +# export TF_VAR_mulgaos_endpoint=http://localhost:8080 +variable "mulgaos_endpoint" { + type = string + default = "http://localhost:8080" + description = "MulgaOS API endpoint (awsd gateway)" +} + +provider "aws" { + region = var.region + + # MulgaOS custom endpoints + endpoints { + ec2 = var.mulgaos_endpoint + s3 = var.mulgaos_endpoint + iam = var.mulgaos_endpoint + sts = var.mulgaos_endpoint + } + + # Skip AWS credential validation (MulgaOS handles auth) + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + + # For local development, skip region validation + skip_region_validation = true +} + +data "aws_availability_zones" "available" { + state = "available" +} + +# Debian 12 AMI (official Debian account) +# For MulgaOS, you may need to import a local AMI first +data "aws_ami" "debian12" { + most_recent = true + owners = ["136693071363"] # Debian official + + filter { + name = "name" + values = ["debian-12-amd64-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } +} + +# Keypair: hive-test +resource "tls_private_key" "hive_test" { + algorithm = "ED25519" +} + +resource "aws_key_pair" "hive_test" { + key_name = "hive-test" + public_key = tls_private_key.hive_test.public_key_openssh +} + +# Save private key locally for SSH +resource "local_file" "hive_test_pem" { + filename = "${path.module}/hive-test.pem" + content = tls_private_key.hive_test.private_key_openssh + file_permission = "0600" +} + +# 1: VPC 10.1.1.0/16 +resource "aws_vpc" "main" { + cidr_block = "10.1.1.0/16" + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + Name = "hive-test-vpc" + } +} + +# Internet access for a public subnet +resource "aws_internet_gateway" "igw" { + vpc_id = aws_vpc.main.id + + tags = { + Name = "hive-test-igw" + } +} + +# 2: Subnet 10.1.2.0/24 (public) +resource "aws_subnet" "public" { + vpc_id = aws_vpc.main.id + cidr_block = "10.1.2.0/24" + availability_zone = data.aws_availability_zones.available.names[0] + map_public_ip_on_launch = true + + tags = { + Name = "hive-test-public-subnet" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.igw.id + } + + tags = { + Name = "hive-test-public-rt" + } +} + +resource "aws_route_table_association" "public" { + subnet_id = aws_subnet.public.id + route_table_id = aws_route_table.public.id +} + +# 3: Security group +resource "aws_security_group" "web_ssh" { + name = "hive-test-sg" + description = "Allow inbound 22,80,443; allow all outbound" + vpc_id = aws_vpc.main.id + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTP" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTPS" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + description = "All outbound" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "hive-test-sg" + } +} + +# 4: Ensure 3 servers on different underlying hardware (spread placement group) +resource "aws_placement_group" "spread" { + name = "hive-test-spread" + strategy = "spread" +} + +resource "aws_instance" "vm" { + count = 3 + ami = data.aws_ami.debian12.id + instance_type = "t3.small" + + subnet_id = aws_subnet.public.id + vpc_security_group_ids = [aws_security_group.web_ssh.id] + key_name = aws_key_pair.hive_test.key_name + + associate_public_ip_address = true + placement_group = aws_placement_group.spread.name + + tags = { + Name = "hive-test-vm-${count.index + 1}" + } +} + +output "instance_public_ips" { + value = aws_instance.vm[*].public_ip +} + +output "ssh_commands" { + value = [ + for ip in aws_instance.vm[*].public_ip : + "ssh -i hive-test.pem admin@${ip}" + ] +} diff --git a/scripts/iac/proxmox/.env.example b/scripts/iac/proxmox/.env.example new file mode 100644 index 000000000..1c62b7eb5 --- /dev/null +++ b/scripts/iac/proxmox/.env.example @@ -0,0 +1,53 @@ +# Proxmox IaC Environment Configuration +# Copy this file to .env and fill in your values: +# cp .env.example .env +# +# Then source it before running tofu: +# source .env && tofu plan + +# --- Proxmox API --- + +# API token for Proxmox VE (read by the provider directly) +export PROXMOX_VE_API_TOKEN="terraform@pve!provider=YOUR_TOKEN_HERE" + +# Proxmox VE API endpoint URL +export TF_VAR_proxmox_endpoint="https://pve-host.example.com:8006/" + +# --- SSH Access --- + +# SSH username on Proxmox hosts (used by provider for file uploads) +export TF_VAR_proxmox_ssh_username="terraform" + +# Path to SSH private key for Proxmox host access +export TF_VAR_ssh_private_key_path="~/.ssh/proxmox-host-tf" + +# Path to SSH public key injected into VMs via cloud-init (for tf-user) +export TF_VAR_ssh_public_key_path="~/.ssh/cloud-init.pub" + +# --- Cluster Nodes (exactly 3 required) --- +# Each node object: +# name - Proxmox node name (as shown in the Proxmox UI) +# address - SSH-reachable hostname or IP of the Proxmox host +# bridge - Network bridge for VM interfaces (e.g. vmbr0, vmbr3) +# datastore_id - Proxmox storage ID for VM disks (e.g. local, local-lvm) + +export TF_VAR_nodes='[ + { + "name": "node1", + "address": "node1.example.com", + "bridge": "vmbr0", + "datastore_id": "local" + }, + { + "name": "node2", + "address": "node2.example.com", + "bridge": "vmbr0", + "datastore_id": "local" + }, + { + "name": "node3", + "address": "node3.example.com", + "bridge": "vmbr0", + "datastore_id": "local" + } +]' diff --git a/scripts/iac/proxmox/README.md b/scripts/iac/proxmox/README.md new file mode 100644 index 000000000..c29ed1674 --- /dev/null +++ b/scripts/iac/proxmox/README.md @@ -0,0 +1,246 @@ +# Proxmox IaC + +OpenTofu/Terraform templates for provisioning a 3-node Hive development cluster on Proxmox VE. + +## Prerequisites + +- [OpenTofu](https://opentofu.org/) (or Terraform >= 1.6.0) +- 3 Proxmox VE nodes with: + - A Terraform API token (`Datacenter > Permissions > API Tokens`) + - SSH access for a `terraform` user on each node + - Debian 12 cloud image uploaded to `local:iso/` on each node +- SSH keypair for cloud-init VM access +- A hardcoded Debian ISO is used, which needs to be pre-installed on each Proxmox host ("debian-12-genericcloud-amd64-20240211-1654.img") https://cdimage.debian.org/images/cloud/bookworm/20240211-1654/debian-12-genericcloud-amd64-20240211-1654.qcow2 + +## Setup + +1. Copy the example environment file and fill in your values: + +```sh +cp .env.example .env +# Edit .env with your Proxmox endpoint, SSH keys, and node configuration +``` + +2. Set your Proxmox API token separately (do not store in `.env`): + +```sh +export PROXMOX_VE_API_TOKEN="terraform@pve!provider=YOUR_TOKEN_SECRET" +``` + +3. Initialize and deploy: + +```sh +source .env +tofu init +tofu plan +tofu apply +``` + +### Example `.env` file + +```sh +# --- Proxmox API --- + +# API token - set manually via: export PROXMOX_VE_API_TOKEN="terraform@pve!provider=..." +# Format: @!= + +# Proxmox VE API endpoint (the URL you use to access the web UI) +export TF_VAR_proxmox_endpoint="https://pve1.lab.example.com:8006/" + +# --- SSH Access --- + +# SSH username on the Proxmox hosts (used by the provider for file uploads via SCP) +export TF_VAR_proxmox_ssh_username="terraform" + +# Path to SSH private key that authenticates as the above user on each Proxmox host +export TF_VAR_ssh_private_key_path="~/.ssh/proxmox-tf" + +# Path to SSH public key injected into VMs via cloud-init (the tf-user account) +export TF_VAR_ssh_public_key_path="~/.ssh/proxmox-tf-cloudinit.pub" + +# --- Cluster Nodes (exactly 3 required) --- +# +# Each node object: +# name - Proxmox node name exactly as shown in the Proxmox UI sidebar +# address - SSH-reachable hostname or IP of the Proxmox host +# bridge - Linux bridge for VM network interfaces (check Proxmox UI > Node > Network) +# datastore_id - Proxmox storage pool for VM disks (check Proxmox UI > Node > Storage) + +export TF_VAR_nodes='[ + { + "name": "pve1", + "address": "pve1.lab.example.com", + "bridge": "vmbr0", + "datastore_id": "local-lvm" + }, + { + "name": "pve2", + "address": "pve2.lab.example.com", + "bridge": "vmbr0", + "datastore_id": "local-lvm" + }, + { + "name": "pve3", + "address": "pve3.lab.example.com", + "bridge": "vmbr0", + "datastore_id": "local-lvm" + } +]' +``` + +### Environment variable reference + +| Variable | Required | Description | +|---|---|---| +| `PROXMOX_VE_API_TOKEN` | Yes | Proxmox API token (read by provider directly) | +| `TF_VAR_proxmox_endpoint` | Yes | Proxmox VE API URL (e.g. `https://host:8006/`) | +| `TF_VAR_proxmox_ssh_username` | No | SSH user on Proxmox hosts (default: `terraform`) | +| `TF_VAR_ssh_private_key_path` | Yes | SSH private key for Proxmox host access | +| `TF_VAR_ssh_public_key_path` | Yes | SSH public key injected into VMs via cloud-init | +| `TF_VAR_nodes` | Yes | JSON array of exactly 3 node objects | + +Each node object in `TF_VAR_nodes`: + +| Field | Example | Description | +|---|---|---| +| `name` | `pve1` | Proxmox node name (as shown in UI sidebar) | +| `address` | `pve1.lab.example.com` | SSH-reachable hostname or IP | +| `bridge` | `vmbr0` | Network bridge for VM interfaces | +| `datastore_id` | `local-lvm` | Storage pool for VM disks | + +## Deploy + +```sh +source .env +tofu init +tofu plan # validates config - fails if required env vars are missing +tofu apply +``` + +Example output: + +```sh +Plan: 6 to add, 0 to change, 0 to destroy. + +Apply complete! Resources: 6 added, 0 changed, 0 destroyed. + +Outputs: + +iac_mulgaos_ips = { + "node1" = { + "data" = "10.1.3.169" + "management" = "10.1.3.170" + } + "node2" = { + "data" = "10.1.2.21" + "management" = "10.1.3.171" + } + "node3" = { + "data" = "10.1.2.22" + "management" = "10.1.3.172" + } +} +``` + +## Destroy + +```sh +source .env +tofu destroy +``` + +## SSH access + +Connect to provisioned VMs using the cloud-init public key: + +```sh +ssh -i ~/.ssh/your-cloud-init-key tf-user@ +``` + +## Multi-node cluster setup + +After VMs are provisioned, initialize the Hive cluster manually. Consider moving to Ansible for automation. + +Note, modify the NODE IP addresses with the output from the terraform deployment above. + +### Node 1 (init) + +Management traffic over first network adapter, with Open vSwitch (Geneve) traffic on the 2nd for VPC traffic. + +```sh +cd ~/Development/mulga/hive +make + +export HIVE_REGION="ap-southeast-2" +export HIVE_AZ="ap-southeast-2a" +export NODE1_MGMT_IP="10.1.3.170" +export NODE2_MGMT_IP="10.1.3.171" +export NODE3_MGMT_IP="10.1.3.172" + +./bin/hive admin init \ + --node node1 \ + --bind $NODE1_MGMT_IP \ + --cluster-bind $NODE1_MGMT_IP \ + --cluster-routes $NODE1_MGMT_IP:4248 \ + --predastore-nodes $NODE1_MGMT_IP,$NODE2_MGMT_IP,$NODE3_MGMT_IP \ + --port 4432 \ + --hive-dir ~/hive/ \ + --config-dir ~/hive/config/ \ + --region $HIVE_REGION \ + --az $HIVE_AZ +``` + +### Node 2 (join) + +```sh +cd ~/Development/mulga/hive +make + +export HIVE_REGION="ap-southeast-2" +export HIVE_AZ="ap-southeast-2" +export NODE1_MGMT_IP="10.1.3.170" +export NODE2_MGMT_IP="10.1.3.171" +export NODE3_MGMT_IP="10.1.3.172" + +./bin/hive admin join \ + --node node2 \ + --bind $NODE2_MGMT_IP \ + --cluster-bind $NODE2_MGMT_IP \ + --cluster-routes $NODE1_MGMT_IP:4248 \ + --host $NODE1_MGMT_IP:4432 \ + --data-dir ~/hive/ \ + --config-dir ~/hive/config/ \ + --region $HIVE_REGION \ + --az $HIVE_AZ +``` + +### Node 3 (join) + +```sh +cd ~/Development/mulga/hive +make + +export HIVE_REGION="ap-southeast-2" +export HIVE_AZ="ap-southeast-2" +export NODE1_MGMT_IP="10.1.3.170" +export NODE2_MGMT_IP="10.1.3.171" +export NODE3_MGMT_IP="10.1.3.172" + +./bin/hive admin join \ + --node node3 \ + --bind $NODE3_MGMT_IP \ + --cluster-bind $NODE3_MGMT_IP \ + --cluster-routes $NODE1_MGMT_IP:4248 \ + --host $NODE1_MGMT_IP:4432 \ + --data-dir ~/hive/ \ + --config-dir ~/hive/config/ \ + --region $HIVE_REGION \ + --az $HIVE_AZ +``` + +## Known issues + +- `~/hive/config/hive.toml` - Does not add node1, node2, node3 from config automatically +- `~/hive/config/predastore/predastore.toml` - Uses previous static local node config, needs to use the IPs for each node in the cluster +- On multi-node deployments, NATS on the primary can timeout waiting for other nodes to start, causing a race condition where NATS fails and all dependent services fail +- When adding 3 nodes, `nats.conf` is not updated with the 3rd node. Each node's cluster name is hardcoded to `C1` instead of per-node names diff --git a/scripts/iac/proxmox/cloud-config.tf b/scripts/iac/proxmox/cloud-config.tf new file mode 100644 index 000000000..2f3ea5636 --- /dev/null +++ b/scripts/iac/proxmox/cloud-config.tf @@ -0,0 +1,40 @@ +data "local_file" "ssh_public_key" { + filename = pathexpand(var.ssh_public_key_path) +} + +resource "proxmox_virtual_environment_file" "iac_mulgaos_config" { + for_each = local.nodes + + content_type = "snippets" + datastore_id = "local" + node_name = each.key + + source_raw { + data = <> /home/tf-user/.bashrc" + - "export PATH=$PATH:/usr/local/go/bin && cd /home/tf-user/Development/mulga/hive && ./scripts/clone-deps.sh && ./scripts/dev-setup.sh" + - chown -R tf-user:tf-user /home/tf-user/Development/ + - echo "done" > /tmp/vendor-cloud-init-done +EOF + + file_name = "iac-mulgaos-vendor-cloud-config.yaml" + } +} diff --git a/scripts/iac/proxmox/files.tf b/scripts/iac/proxmox/files.tf new file mode 100644 index 000000000..63d34d178 --- /dev/null +++ b/scripts/iac/proxmox/files.tf @@ -0,0 +1,17 @@ +# Files already exist, skip + +#resource "proxmox_virtual_environment_download_file" "debian_cloud_image_neon" { +# content_type = "iso" +# datastore_id = "local" +# node_name = "neon" +# url = "https://cdimage.debian.org/images/cloud/bookworm/20240211-1654/debian-12-genericcloud-amd64-20240211-1654.qcow2" +# file_name = "debian-12-genericcloud-amd64-20240211-1654.img" +#} + +#resource "proxmox_virtual_environment_download_file" "debian_cloud_image_radon" { +# content_type = "iso" +# datastore_id = "local" +# node_name = "radon" +# url = "https://cdimage.debian.org/images/cloud/bookworm/20240211-1654/debian-12-genericcloud-amd64-20240211-1654.qcow2" +# file_name = "debian-12-genericcloud-amd64-20240211-1654.img" +#} \ No newline at end of file diff --git a/scripts/iac/proxmox/main.tf b/scripts/iac/proxmox/main.tf new file mode 100644 index 000000000..abac03044 --- /dev/null +++ b/scripts/iac/proxmox/main.tf @@ -0,0 +1,32 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "0.52.0" + } + } +} + +locals { + nodes = { for idx, node in var.nodes : node.name => merge(node, { index = idx + 1 }) } +} + +# API token is read from PROXMOX_VE_API_TOKEN env var automatically +provider "proxmox" { + endpoint = var.proxmox_endpoint + insecure = true + + ssh { + agent = false + username = var.proxmox_ssh_username + private_key = file(pathexpand(var.ssh_private_key_path)) + + dynamic "node" { + for_each = local.nodes + content { + name = node.key + address = node.value.address + } + } + } +} diff --git a/scripts/iac/proxmox/variables.tf b/scripts/iac/proxmox/variables.tf new file mode 100644 index 000000000..098931848 --- /dev/null +++ b/scripts/iac/proxmox/variables.tf @@ -0,0 +1,60 @@ +variable "proxmox_endpoint" { + type = string + description = "Proxmox VE API endpoint URL (e.g. https://pve.example.com:8006/)" + + validation { + condition = can(regex("^https?://", var.proxmox_endpoint)) + error_message = "proxmox_endpoint must be a valid URL starting with http:// or https://" + } +} + +variable "proxmox_ssh_username" { + type = string + description = "SSH username for Proxmox host access" + default = "terraform" +} + +variable "ssh_private_key_path" { + type = string + description = "Path to SSH private key for Proxmox host access" + + validation { + condition = length(var.ssh_private_key_path) > 0 + error_message = "ssh_private_key_path must not be empty." + } +} + +variable "ssh_public_key_path" { + type = string + description = "Path to SSH public key injected into VMs via cloud-init" + + validation { + condition = length(var.ssh_public_key_path) > 0 + error_message = "ssh_public_key_path must not be empty." + } +} + +variable "nodes" { + type = list(object({ + name = string + address = string + bridge = string + datastore_id = string + })) + description = "Exactly 3 Proxmox nodes for the Hive cluster" + + validation { + condition = length(var.nodes) == 3 + error_message = "Exactly 3 nodes must be defined for the Hive cluster." + } + + validation { + condition = alltrue([for n in var.nodes : length(n.name) > 0 && length(n.address) > 0]) + error_message = "Each node must have a non-empty name and address." + } + + validation { + condition = length(distinct([for n in var.nodes : n.name])) == length(var.nodes) + error_message = "Each node must have a unique name." + } +} diff --git a/scripts/iac/proxmox/vms.tf b/scripts/iac/proxmox/vms.tf new file mode 100644 index 000000000..41b16ceb0 --- /dev/null +++ b/scripts/iac/proxmox/vms.tf @@ -0,0 +1,71 @@ +resource "proxmox_virtual_environment_vm" "iac_mulgaos" { + for_each = local.nodes + + name = "iac-dev-mulgaos-${each.value.index}" + description = "Managed by Terraform" + tags = ["terraform", "mulgaos"] + node_name = each.key + + initialization { + datastore_id = "local" + vendor_data_file_id = proxmox_virtual_environment_file.iac_mulgaos_config[each.key].id + + user_account { + username = "tf-user" + keys = [trimspace(data.local_file.ssh_public_key.content)] + } + + # Management + ip_config { + ipv4 { + address = "dhcp" + } + } + + # Data/VM + ip_config { + ipv4 { + address = "dhcp" + } + } + } + + cpu { + cores = 4 + type = "host" + } + + memory { + dedicated = 8192 + } + + agent { + enabled = true + } + + network_device { + bridge = each.value.bridge + } + + network_device { + bridge = each.value.bridge + } + + disk { + file_id = "local:iso/debian-12-genericcloud-amd64-20240211-1654.img" + datastore_id = each.value.datastore_id + interface = "virtio0" + iothread = true + discard = "on" + size = 32 + } +} + +output "iac_mulgaos_ips" { + value = { + for name, vm in proxmox_virtual_environment_vm.iac_mulgaos : name => { + management = vm.ipv4_addresses[1][0] + data = vm.ipv4_addresses[2][0] + } + } +}