multi-server tweaks, ec2 health, IAC templates, misc cleanup

2026-10-15 17:11:30 +00:00 · 2026-02-17 09:10:13 +10:00
parent 962d85edf2
commit bb533dfbb6
32 changed files with 3497 additions and 104 deletions
@@ -6,3 +6,4 @@ Collection of OpenTofu/Terraform templates for creating Hive instances.
 |---|---|
 | [proxmox/](proxmox/) | Provisions a 3-node Hive cluster on Proxmox VE bare-metal hypervisors |
 | [aws/](aws/) | Provisions infrastructure using the Terraform AWS provider, with MulgaOS AWS-compatible API as the local endpoint |
+| [proxmox-template/](proxmox-template/) | Builds a Proxmox VM template with all Hive dependencies pre-installed (QEMU, Go, repos, modules) |
@@ -0,0 +1,60 @@
+# Proxmox Template Builder — Environment Configuration
+#
+# Copy to .env and fill in your values:
+#   cp .env.example .env
+#
+# Then source it before running tofu:
+#   source .env && tofu plan
+
+# --- Proxmox API ---
+
+# API token for Proxmox VE (read by the provider directly)
+export PROXMOX_VE_API_TOKEN="terraform@pve!provider=YOUR_TOKEN_HERE"
+
+# Proxmox VE API endpoint URL
+export TF_VAR_proxmox_endpoint="https://pve-host.example.com:8006/"
+
+# --- Proxmox Node ---
+# Build the template on ONE node. After creation, use vzdump to copy to others.
+#
+#   name    - Proxmox node name (as shown in the Proxmox UI)
+#   address - SSH-reachable hostname or IP of the Proxmox host
+
+export TF_VAR_proxmox_node_name="node1"
+export TF_VAR_proxmox_node_address="node1.example.com"
+
+# --- SSH Access ---
+
+# SSH username on Proxmox hosts (used by provider for file uploads + qm commands)
+export TF_VAR_proxmox_ssh_username="terraform"
+
+# Path to SSH private key for Proxmox host access
+export TF_VAR_ssh_private_key_path="~/.ssh/proxmox-host-tf"
+
+# Path to SSH private key for VM access (must be the pair of the public key below)
+export TF_VAR_vm_ssh_private_key_path="~/.ssh/cloud-init"
+
+# Path to SSH public key injected into VMs via cloud-init (for tf-user)
+export TF_VAR_ssh_public_key_path="~/.ssh/cloud-init.pub"
+
+# --- Template ---
+
+# VM ID for the template (pick a high number to avoid conflicts)
+export TF_VAR_template_vmid=9000
+
+# Short tag for the base image (used in Proxmox template tags)
+export TF_VAR_base_image_tag="debian-12"
+
+# Storage and network for the build node
+export TF_VAR_datastore_id="local-lvm"
+export TF_VAR_bridge="vmbr0"
+
+# --- Optional ---
+
+# Base image — change this when testing a newer Debian/Ubuntu version:
+# export TF_VAR_base_image="local:iso/debian-13-genericcloud-amd64.img"
+# export TF_VAR_base_image_tag="debian-13"
+# export TF_VAR_template_vmid=9010
+
+# Clone a specific branch instead of main:
+# export TF_VAR_git_branch="feature/my-branch"
@@ -0,0 +1,6 @@
+.env
+.terraform/
+.terraform.lock.hcl
+*.tfstate
+*.tfstate.backup
+*.tfplan
@@ -0,0 +1,124 @@
+# Hive Node Template Builder
+
+Builds a Proxmox VM template with all Hive dependencies pre-installed. Cloning from this template reduces cluster spin-up from ~15 minutes to ~1-2 minutes (`git pull` + `make build`).
+
+## What's in the template
+
+| Component | Details |
+|---|---|
+| System packages | QEMU, nbdkit, libvirt, gcc, jq, curl, etc. (`make install-system`) |
+| Go | 1.26.0 (`make install-go`) |
+| AWS CLI | v2 (`make install-aws`) |
+| Repositories | hive, viperblock, predastore (with `go.work` configured) |
+| Module cache | `go mod download` completed for all repos |
+| Cloud images | Ubuntu 24.04 for nested VMs (`~/images/ubuntu-24.04.img`) |
+| Tuning | sysctl (rmem_max/wmem_max), kvm group, ufw disabled |
+| Generalized | Clean machine-id, SSH host keys, cloud-init — each clone boots fresh |
+
+## Quick Start
+
+```bash
+cd scripts/iac/proxmox-template
+
+# 1. Configure
+cp .env.example .env
+# Edit .env — set your Proxmox endpoint, API token, node, and SSH key paths.
+
+# 2. Set API token (not stored in .env for security)
+export PROXMOX_VE_API_TOKEN="terraform@pve!provider=YOUR_TOKEN"
+
+# 3. Build
+source .env
+tofu init
+tofu plan
+tofu apply
+```
+
+Build takes ~10-15 minutes (apt packages, Go, cloud image download). When done, you'll have a template on the build node at the configured VMID (default 9000).
+
+## Distribute to Other Nodes
+
+After `tofu apply`, the output prints ready-to-run commands. Example for a 3-node cluster:
+
+```bash
+# Export from build node
+ssh -i ~/.ssh/proxmox-host-tf terraform@<BUILD_NODE> \
+  'vzdump 9000 --dumpdir /tmp --mode stop --compress zstd'
+
+# Copy to other nodes
+scp -3 -i ~/.ssh/proxmox-host-tf \
+  terraform@<BUILD_NODE>:/tmp/vzdump-qemu-9000-*.vma.zst \
+  terraform@<TARGET_NODE>:/tmp/
+
+# Restore on each target node (use a unique VMID and the node's datastore)
+ssh -i ~/.ssh/proxmox-host-tf terraform@<TARGET_NODE> \
+  'qmrestore /tmp/vzdump-qemu-9000-*.vma.zst <NEW_VMID> --storage <DATASTORE> && qm template <NEW_VMID>'
+```
+
+After distributing, each node has its own copy of the template with a unique VMID.
+
+## Using the Template
+
+Update `scripts/iac/proxmox/vms.tf` to clone from the template instead of booting from a raw cloud image. Or clone manually from the Proxmox UI.
+
+On each cloned VM after boot:
+
+```bash
+# Pull latest code
+cd ~/Development/mulga/hive && git pull
+cd ~/Development/mulga/viperblock && git pull
+cd ~/Development/mulga/predastore && git pull
+
+# Build (~30s with cached modules)
+cd ~/Development/mulga/hive && export PATH=/usr/local/go/bin:$PATH && make build
+```
+
+## Testing a Different Base Image
+
+Upload the new image to Proxmox, then build a separate template:
+
+```bash
+# Debian 13 example
+export TF_VAR_base_image="local:iso/debian-13-genericcloud-amd64.img"
+export TF_VAR_base_image_tag="debian-13"
+export TF_VAR_template_vmid=9010   # Keep the debian-12 template at 9000
+
+source .env
+tofu apply
+```
+
+Templates are tagged in the Proxmox UI with the base image name (`debian-12`, `debian-13`, etc.) so you can tell them apart.
+
+## Rebuilding
+
+When dependencies change (new Go version, new system packages):
+
+```bash
+source .env
+tofu destroy     # Remove the old template
+tofu apply       # Build fresh
+# Then re-distribute with vzdump
+```
+
+## SSH Key Setup
+
+This template uses **two separate key pairs** (matching the `scripts/iac/proxmox/` setup):
+
+| Key | Used for | Variable |
+|---|---|---|
+| Proxmox host key | SSH to Proxmox hosts (provider, `qm template` command) | `ssh_private_key_path` |
+| VM / cloud-init key | SSH to VMs (cloud-init injection, provisioning) | `vm_ssh_private_key_path` + `ssh_public_key_path` |
+
+If you use the same key for both, set all three variables to the same key pair.
+
+## Files
+
+| File | Description |
+|------|-------------|
+| `main.tf` | Proxmox provider configuration |
+| `variables.tf` | All configurable variables with defaults |
+| `template-builder.tf` | VM creation, provisioning, template conversion |
+| `outputs.tf` | Template info + vzdump/restore commands |
+| `scripts/provision.sh` | Install deps, clone repos, cache modules, generalize |
+| `.env.example` | Environment variable template |
+| `.gitignore` | Ignores .env, state files, provider plugins |
@@ -0,0 +1,27 @@
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "0.52.0"
+    }
+  }
+}
+
+# API token is read from PROXMOX_VE_API_TOKEN env var automatically
+provider "proxmox" {
+  endpoint = var.proxmox_endpoint
+  insecure = true
+
+  ssh {
+    agent       = false
+    username    = var.proxmox_ssh_username
+    private_key = file(pathexpand(var.ssh_private_key_path))
+
+    node {
+      name    = var.proxmox_node_name
+      address = var.proxmox_node_address
+    }
+  }
+}
@@ -0,0 +1,43 @@
+output "template_vmid" {
+  description = "VM ID of the created template"
+  value       = proxmox_virtual_environment_vm.builder.vm_id
+}
+
+output "template_node" {
+  description = "Proxmox node where the template lives"
+  value       = var.proxmox_node_name
+}
+
+output "base_image_tag" {
+  description = "Base image tag for this template"
+  value       = var.base_image_tag
+}
+
+# Ready-to-run command: export the template for distribution to other Proxmox nodes
+output "vzdump_export_command" {
+  description = "Run this to export the template (vzdump) for copying to other nodes"
+  value       = <<-EOT
+    ssh -i ${var.ssh_private_key_path} ${var.proxmox_ssh_username}@${var.proxmox_node_address} \
+      'vzdump ${var.template_vmid} --dumpdir /tmp --mode stop --compress zstd'
+  EOT
+}
+
+# After exporting, copy to each target node and restore
+output "distribute_commands" {
+  description = "Copy and restore the template on other Proxmox nodes"
+  value       = <<-EOT
+    # 1. Export from build node (${var.proxmox_node_name}):
+    ssh -i ${var.ssh_private_key_path} ${var.proxmox_ssh_username}@${var.proxmox_node_address} \
+      'vzdump ${var.template_vmid} --dumpdir /tmp --mode stop --compress zstd'
+
+    # 2. Copy to each target node:
+    scp -3 \
+      -i ${var.ssh_private_key_path} \
+      ${var.proxmox_ssh_username}@${var.proxmox_node_address}:/tmp/vzdump-qemu-${var.template_vmid}-*.vma.zst \
+      ${var.proxmox_ssh_username}@<TARGET_NODE_ADDRESS>:/tmp/
+
+    # 3. On each target node — restore and convert to template:
+    ssh -i ${var.ssh_private_key_path} ${var.proxmox_ssh_username}@<TARGET_NODE_ADDRESS> \
+      'qmrestore /tmp/vzdump-qemu-${var.template_vmid}-*.vma.zst <NEW_VMID> --storage <DATASTORE> && qm template <NEW_VMID>'
+  EOT
+}
@@ -0,0 +1,176 @@
+#!/bin/bash
+# provision.sh — Installs all Hive dependencies, clones repos, caches Go modules,
+# and generalizes the VM for use as a Proxmox template.
+#
+# Runs as tf-user via Terraform remote-exec. Uses sudo for system operations.
+# Environment: GIT_BRANCH (default: main)
+
+set -euo pipefail
+
+GIT_BRANCH="${GIT_BRANCH:-main}"
+WORK_DIR="$HOME/Development/mulga"
+
+echo "========================================"
+echo "  Hive Template Provisioning"
+echo "  Branch: ${GIT_BRANCH}"
+echo "  Arch:   $(uname -m)"
+echo "========================================"
+
+# ── 1. Wait for cloud-init ──────────────────────────────────────────────────
+echo ""
+echo "=== [1/8] Waiting for cloud-init to complete ==="
+cloud-init status --wait
+
+# ── 2. Clone repositories ───────────────────────────────────────────────────
+echo ""
+echo "=== [2/8] Cloning repositories ==="
+mkdir -p "$WORK_DIR"
+cd "$WORK_DIR"
+
+for repo in hive viperblock predastore; do
+    if [ ! -d "$repo" ]; then
+        echo "  Cloning $repo (branch: $GIT_BRANCH)..."
+        git clone -b "$GIT_BRANCH" "https://github.com/mulgadc/${repo}.git"
+    else
+        echo "  $repo already exists, skipping"
+    fi
+done
+
+# ── 3. Install system dependencies via Makefile ─────────────────────────────
+echo ""
+echo "=== [3/8] Installing system dependencies ==="
+cd "$WORK_DIR/hive"
+sudo make quickinstall
+
+# Ensure Go is in PATH for the rest of the script
+export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"
+echo "  Go version: $(go version)"
+echo "  AWS CLI:    $(aws --version)"
+
+# ── 4. Set up go.work ───────────────────────────────────────────────────────
+echo ""
+echo "=== [4/8] Creating go.work ==="
+
+# Detect Go version from go.mod
+GO_VERSION=$(grep '^go ' "$WORK_DIR/hive/go.mod" | awk '{print $2}')
+echo "  Using Go version from go.mod: ${GO_VERSION}"
+
+cat > "$WORK_DIR/hive/go.work" << GOWORK
+go ${GO_VERSION}
+
+use (
+    .
+    ../viperblock
+    ../viperblock/nbd
+    ../viperblock/nbd/libguestfs.org/nbdkit
+    ../predastore
+)
+GOWORK
+
+echo "  Created $WORK_DIR/hive/go.work"
+
+# ── 5. Download Go modules ──────────────────────────────────────────────────
+echo ""
+echo "=== [5/8] Downloading Go modules ==="
+cd "$WORK_DIR/hive"       && go mod download && echo "  hive: done"
+cd "$WORK_DIR/viperblock"  && go mod download && echo "  viperblock: done"
+cd "$WORK_DIR/predastore"  && go mod download && echo "  predastore: done"
+
+# Sub-modules
+if [ -f "$WORK_DIR/viperblock/nbd/go.mod" ]; then
+    cd "$WORK_DIR/viperblock/nbd" && go mod download && echo "  viperblock/nbd: done"
+fi
+if [ -f "$WORK_DIR/viperblock/nbd/libguestfs.org/nbdkit/go.mod" ]; then
+    cd "$WORK_DIR/viperblock/nbd/libguestfs.org/nbdkit" && go mod download && echo "  viperblock/nbd/nbdkit: done"
+fi
+
+# ── 6. Pre-download cloud images for nested VMs ─────────────────────────────
+echo ""
+echo "=== [6/8] Pre-downloading cloud images ==="
+mkdir -p "$HOME/images"
+
+ARCH=$(dpkg --print-architecture)
+IMAGE_URL=""
+if [ "$ARCH" = "amd64" ]; then
+    IMAGE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+elif [ "$ARCH" = "arm64" ]; then
+    IMAGE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img"
+fi
+
+if [ -n "$IMAGE_URL" ]; then
+    echo "  Downloading Ubuntu 24.04 cloud image ($ARCH)..."
+    curl -L -o "$HOME/images/ubuntu-24.04.img" "$IMAGE_URL"
+    echo "  Saved to ~/images/ubuntu-24.04.img"
+else
+    echo "  Unknown arch ($ARCH), skipping cloud image download"
+fi
+
+# ── 7. System tuning ────────────────────────────────────────────────────────
+echo ""
+echo "=== [7/8] Applying system tuning ==="
+
+# Persistent sysctl for NATS / high-throughput networking
+sudo tee /etc/sysctl.d/99-hive.conf > /dev/null << 'SYSCTL'
+net.core.rmem_max=4194304
+net.core.wmem_max=4194304
+SYSCTL
+echo "  sysctl: net.core.rmem_max/wmem_max = 4MB"
+
+# Add user to kvm group for QEMU access
+sudo usermod -aG kvm "$(whoami)"
+echo "  Added $(whoami) to kvm group"
+
+# Disable ufw if present
+if command -v ufw >/dev/null 2>&1; then
+    sudo ufw disable 2>/dev/null || true
+    echo "  ufw disabled"
+fi
+
+# ── 8. Generalize for templating ────────────────────────────────────────────
+echo ""
+echo "=== [8/8] Cleaning up for template ==="
+
+# Apt caches
+sudo apt-get clean
+sudo rm -rf /var/lib/apt/lists/*
+echo "  Cleared apt cache"
+
+# Logs
+sudo find /var/log -type f -name "*.log" -exec truncate -s 0 {} \; 2>/dev/null || true
+sudo rm -rf /var/log/journal/*
+echo "  Truncated logs"
+
+# Temp files
+sudo rm -rf /tmp/* /var/tmp/*
+
+# Machine ID — systemd regenerates on next boot.
+# This is critical: without this, all clones get the same DHCP client ID → IP conflicts.
+sudo truncate -s 0 /etc/machine-id
+sudo rm -f /var/lib/dbus/machine-id
+echo "  Cleared machine-id (will regenerate on boot)"
+
+# SSH host keys — regenerated on boot by cloud-init or systemd
+sudo rm -f /etc/ssh/ssh_host_*
+echo "  Removed SSH host keys (will regenerate on boot)"
+
+# Cloud-init state — forces cloud-init to run again on next boot.
+# This ensures each clone gets its own hostname, SSH keys, user config, etc.
+sudo cloud-init clean
+echo "  Cleaned cloud-init state (will re-run on boot)"
+
+# Shell history
+history -c 2>/dev/null || true
+cat /dev/null > "$HOME/.bash_history" 2>/dev/null || true
+
+# Provisioning script itself
+rm -f /tmp/provision.sh
+
+# Sync filesystem
+sync
+
+echo ""
+echo "========================================"
+echo "  Template provisioning complete!"
+echo "  VM is ready to be shut down and"
+echo "  converted to a Proxmox template."
+echo "========================================"
@@ -0,0 +1,130 @@
+data "local_file" "ssh_public_key" {
+  filename = pathexpand(var.ssh_public_key_path)
+}
+
+# Minimal cloud-init: just enough to bootstrap SSH + guest agent.
+# Heavy provisioning is done via remote-exec after boot.
+resource "proxmox_virtual_environment_file" "cloud_config" {
+  content_type = "snippets"
+  datastore_id = "local"
+  node_name    = var.proxmox_node_name
+
+  source_raw {
+    # NOTE: #cloud-config MUST start at column 0 — cloud-init uses it as a magic header.
+    data = <<EOF
+#cloud-config
+package_update: true
+packages:
+  - qemu-guest-agent
+  - git
+  - make
+  - curl
+  - sudo
+  - ca-certificates
+runcmd:
+  - systemctl enable qemu-guest-agent
+  - systemctl start qemu-guest-agent
+EOF
+
+    file_name = "hive-template-cloud-config.yaml"
+  }
+}
+
+resource "proxmox_virtual_environment_vm" "builder" {
+  name      = "${var.template_name}-builder"
+  node_name = var.proxmox_node_name
+  vm_id     = var.template_vmid
+
+  tags = sort(["terraform", "hive-template", var.base_image_tag])
+
+  initialization {
+    datastore_id        = "local"
+    vendor_data_file_id = proxmox_virtual_environment_file.cloud_config.id
+
+    user_account {
+      username = var.vm_ssh_user
+      keys     = [trimspace(data.local_file.ssh_public_key.content)]
+    }
+
+    ip_config {
+      ipv4 {
+        address = "dhcp"
+      }
+    }
+  }
+
+  cpu {
+    cores = var.cpu_cores
+    type  = "host"
+  }
+
+  memory {
+    dedicated = var.memory_mb
+  }
+
+  agent {
+    enabled = true
+  }
+
+  network_device {
+    bridge = var.bridge
+  }
+
+  disk {
+    file_id      = var.base_image
+    datastore_id = var.datastore_id
+    interface    = "virtio0"
+    iothread     = true
+    discard      = "on"
+    size         = var.disk_size_gb
+  }
+
+  # SSH connection to the VM for provisioning.
+  # Uses the VM key (vm_ssh_private_key_path), NOT the Proxmox host key.
+  # Waits for guest agent to report an IP, then connects.
+  connection {
+    type        = "ssh"
+    user        = var.vm_ssh_user
+    private_key = file(pathexpand(var.vm_ssh_private_key_path))
+    host        = element([for addr in flatten(self.ipv4_addresses) : addr if !startswith(addr, "127.")], 0)
+    timeout     = "10m"
+  }
+
+  # Copy provisioning script to VM
+  provisioner "file" {
+    source      = "${path.module}/scripts/provision.sh"
+    destination = "/tmp/provision.sh"
+  }
+
+  # Run provisioning (installs deps, clones repos, caches modules, cleans up)
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /tmp/provision.sh",
+      "GIT_BRANCH='${var.git_branch}' /tmp/provision.sh",
+    ]
+  }
+}
+
+# After provisioning completes, shut down the VM and convert to a Proxmox template.
+# This runs on the Proxmox host via SSH (not on the VM — it's being shut down).
+resource "null_resource" "convert_template" {
+  depends_on = [proxmox_virtual_environment_vm.builder]
+
+  triggers = {
+    builder_id = proxmox_virtual_environment_vm.builder.vm_id
+  }
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      ssh -i '${pathexpand(var.ssh_private_key_path)}' \
+        -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        '${var.proxmox_ssh_username}@${var.proxmox_node_address}' \
+        "sudo qm shutdown ${proxmox_virtual_environment_vm.builder.vm_id} --timeout 60 && \
+         echo 'Waiting for VM to stop...' && \
+         while sudo qm status ${proxmox_virtual_environment_vm.builder.vm_id} | grep -q running; do sleep 2; done && \
+         echo 'Converting to template...' && \
+         sudo qm template ${proxmox_virtual_environment_vm.builder.vm_id} && \
+         echo 'Template ${var.template_vmid} created successfully'"
+    EOT
+  }
+}
@@ -0,0 +1,117 @@
+# --- Proxmox Connection ---
+
+variable "proxmox_endpoint" {
+  type        = string
+  description = "Proxmox VE API endpoint URL (e.g. https://pve.example.com:8006/)"
+}
+
+variable "proxmox_node_name" {
+  type        = string
+  description = "Proxmox node name to build the template on (as shown in the Proxmox UI)"
+}
+
+variable "proxmox_node_address" {
+  type        = string
+  description = "SSH-reachable hostname or IP of the Proxmox node"
+}
+
+variable "proxmox_ssh_username" {
+  type        = string
+  description = "SSH username for Proxmox host access"
+  default     = "terraform"
+}
+
+# --- SSH Keys ---
+# Proxmox host and VM typically use different key pairs.
+# - ssh_private_key_path  → authenticates to the Proxmox host (provider + qm commands)
+# - vm_ssh_private_key_path → authenticates to the VM for provisioning (remote-exec)
+# - ssh_public_key_path → injected into the VM via cloud-init
+
+variable "ssh_private_key_path" {
+  type        = string
+  description = "Path to SSH private key for Proxmox host access"
+}
+
+variable "vm_ssh_private_key_path" {
+  type        = string
+  description = "Path to SSH private key for VM access (must match ssh_public_key_path)"
+}
+
+variable "ssh_public_key_path" {
+  type        = string
+  description = "Path to SSH public key injected into the VM via cloud-init"
+}
+
+# --- Base Image ---
+
+variable "base_image" {
+  type        = string
+  description = "Proxmox image file ID for the base OS (e.g. local:iso/debian-12-genericcloud-amd64-20240211-1654.img)"
+  default     = "local:iso/debian-12-genericcloud-amd64-20240211-1654.img"
+}
+
+variable "base_image_tag" {
+  type        = string
+  description = "Short tag identifying the base image, used in template tags for tracking"
+  default     = "debian-12"
+}
+
+# --- Template Settings ---
+
+variable "template_name" {
+  type        = string
+  description = "Name for the resulting Proxmox template"
+  default     = "hive-node-template"
+}
+
+variable "template_vmid" {
+  type        = number
+  description = "VM ID for the template (pick a high number to avoid conflicts, e.g. 9000)"
+  default     = 9000
+}
+
+variable "vm_ssh_user" {
+  type        = string
+  description = "Username created in the VM via cloud-init"
+  default     = "tf-user"
+}
+
+# --- VM Resources ---
+
+variable "cpu_cores" {
+  type        = number
+  description = "CPU cores for the builder VM (more = faster builds)"
+  default     = 4
+}
+
+variable "memory_mb" {
+  type        = number
+  description = "Memory in MB for the builder VM"
+  default     = 8192
+}
+
+variable "disk_size_gb" {
+  type        = number
+  description = "Disk size in GB (needs room for OS + Go + modules + cloud images)"
+  default     = 32
+}
+
+variable "datastore_id" {
+  type        = string
+  description = "Proxmox storage for the VM disk"
+  default     = "local-lvm"
+}
+
+variable "bridge" {
+  type        = string
+  description = "Network bridge for the VM"
+  default     = "vmbr0"
+}
+
+# --- Git ---
+
+variable "git_branch" {
+  type        = string
+  description = "Git branch to clone for hive, viperblock, and predastore"
+  default     = "main"
+}
@@ -29,10 +29,14 @@ export TF_VAR_ssh_public_key_path="~/.ssh/cloud-init.pub"
 # You can define 1 or more nodes — VMs are distributed across them.
 #
 # Each node object:
-#   name         - Proxmox node name (as shown in the Proxmox UI)
-#   address      - SSH-reachable hostname or IP of the Proxmox host
-#   bridge       - Network bridge for VM interfaces (e.g. vmbr0, vmbr3)
-#   datastore_id - Proxmox storage ID for VM disks (e.g. local, local-lvm)
+#   name          - Proxmox node name (as shown in the Proxmox UI)
+#   address       - SSH-reachable hostname or IP of the Proxmox host
+#   bridge        - Network bridge for VM interfaces (e.g. vmbr0, vmbr3)
+#   datastore_id  - Proxmox storage ID for VM disks (e.g. local, local-lvm)
+#   template_vmid - (optional) VMID of a pre-built template on this node.
+#                   Build with scripts/iac/proxmox-template/, then distribute
+#                   with vzdump. When set, VMs clone from the template (fast).
+#                   When omitted, VMs boot from os_image (slower).

 export TF_VAR_nodes='[
  {
@@ -54,3 +58,11 @@ export TF_VAR_nodes='[
    "datastore_id": "local"
  }
 ]'
+
+# Example with templates (VMIDs may differ per node after vzdump restore):
+#
+# export TF_VAR_nodes='[
+#   { "name": "node1", "address": "node1.example.com", "bridge": "vmbr0", "datastore_id": "local", "template_vmid": 9999 },
+#   { "name": "node2", "address": "node2.example.com", "bridge": "vmbr0", "datastore_id": "local-lvm", "template_vmid": 9997 },
+#   { "name": "node3", "address": "node3.example.com", "bridge": "vmbr0", "datastore_id": "ssd", "template_vmid": 9998 }
+# ]'
@@ -2,6 +2,38 @@ data "local_file" "ssh_public_key" {
  filename = pathexpand(var.ssh_public_key_path)
 }

+# When cloning from a template, packages are pre-installed — skip apt update/upgrade.
+locals {
+  cloud_config_template = <<-EOF
+    #cloud-config
+    runcmd:
+      - timedatectl set-timezone America/Los_Angeles
+      - systemctl enable qemu-guest-agent
+      - systemctl start qemu-guest-agent
+      - echo "done" > /tmp/vendor-cloud-init-done
+  EOF
+
+  cloud_config_image = <<-EOF
+    #cloud-config
+    package_update: true
+    package_upgrade: true
+    packages:
+      - qemu-guest-agent
+      - net-tools
+      - git
+      - make
+    runcmd:
+      - timedatectl set-timezone America/Los_Angeles
+      - systemctl enable qemu-guest-agent
+      - systemctl start qemu-guest-agent
+      - ufw disable
+      - sysctl -w net.core.rmem_max=4194304
+      - sysctl -w net.core.wmem_max=4194304
+      - usermod -aG kvm tf-user
+      - echo "done" > /tmp/vendor-cloud-init-done
+  EOF
+}
+
 resource "proxmox_virtual_environment_file" "cloud_config" {
  count = var.node_count

@@ -10,26 +42,7 @@ resource "proxmox_virtual_environment_file" "cloud_config" {
  node_name    = var.nodes[count.index % length(var.nodes)].name

  source_raw {
-    data = <<EOF
-#cloud-config
-package_update: true
-package_upgrade: true
-packages:
-  - qemu-guest-agent
-  - net-tools
-  - git
-  - make
-runcmd:
-  - timedatectl set-timezone America/Los_Angeles
-  - systemctl enable qemu-guest-agent
-  - systemctl start qemu-guest-agent
-  - ufw disable
-  - sysctl -w net.core.rmem_max=4194304
-  - sysctl -w net.core.wmem_max=4194304
-  - usermod -aG kvm tf-user
-  - echo "done" > /tmp/vendor-cloud-init-done
-EOF
-
+    data      = var.nodes[count.index % length(var.nodes)].template_vmid != null ? local.cloud_config_template : local.cloud_config_image
    file_name = "${var.cluster_name}-${count.index + 1}-cloud-config.yaml"
  }
 }
@@ -36,12 +36,13 @@ variable "ssh_public_key_path" {

 variable "nodes" {
  type = list(object({
-    name         = string
-    address      = string
-    bridge       = string
-    datastore_id = string
+    name          = string
+    address       = string
+    bridge        = string
+    datastore_id  = string
+    template_vmid = optional(number)
  }))
-  description = "Proxmox hosts for VM placement (VMs are distributed round-robin)"
+  description = "Proxmox hosts for VM placement (VMs are distributed round-robin). Set template_vmid to clone from a pre-built template instead of booting from os_image."

  validation {
    condition     = length(var.nodes) > 0
@@ -100,6 +101,6 @@ variable "disk_size_gb" {

 variable "os_image" {
  type        = string
-  description = "Proxmox image file ID for VM boot disk"
+  description = "Proxmox image file ID for VM boot disk (used when node has no template_vmid)"
  default     = "local:iso/debian-12-genericcloud-amd64-20240211-1654.img"
 }
@@ -51,12 +51,24 @@ resource "proxmox_virtual_environment_vm" "hive_node" {
    bridge = var.nodes[count.index % length(var.nodes)].bridge
  }

-  disk {
-    file_id      = var.os_image
-    datastore_id = var.nodes[count.index % length(var.nodes)].datastore_id
-    interface    = "virtio0"
-    iothread     = true
-    discard      = "on"
-    size         = var.disk_size_gb
+  # Clone from template if the target node has template_vmid set, otherwise boot from cloud image.
+  dynamic "clone" {
+    for_each = var.nodes[count.index % length(var.nodes)].template_vmid != null ? [1] : []
+    content {
+      vm_id = var.nodes[count.index % length(var.nodes)].template_vmid
+      full  = true
+    }
+  }
+
+  dynamic "disk" {
+    for_each = var.nodes[count.index % length(var.nodes)].template_vmid == null ? [1] : []
+    content {
+      file_id      = var.os_image
+      datastore_id = var.nodes[count.index % length(var.nodes)].datastore_id
+      interface    = "virtio0"
+      iothread     = true
+      discard      = "on"
+      size         = var.disk_size_gb
+    }
  }
 }
@@ -126,6 +126,22 @@ start_service_foreground() {
    $command
 }

+# Function to set OOM score for a service (Linux only)
+# Protects infrastructure services from OOM killer (-500 = less likely to be killed)
+set_oom_score() {
+    local name="$1"
+    local score="$2"
+    local pid_file="$LOGS_DIR/${name}.pid"
+    if [ -f "$pid_file" ]; then
+        local pid=$(cat "$pid_file")
+        if [ -d "/proc/$pid" ]; then
+            echo "$score" > "/proc/$pid/oom_score_adj" 2>/dev/null && \
+                echo "  OOM score for $name (PID $pid): $score" || \
+                echo "  Warning: Could not set OOM score for $name"
+        fi
+    fi
+}
+
 # Function to check if service is responsive
 check_service() {
    local name="$1"
@@ -188,6 +204,7 @@ if has_service "nats"; then

    NATS_CMD="./bin/hive service nats start"
    start_service "nats" "$NATS_CMD"
+    set_oom_score "nats" "-500"
 else
    echo "1️⃣  Skipping NATS (not a local service)"
 fi
@@ -231,6 +248,7 @@ if has_service "predastore"; then

    PREDASTORE_CMD="./bin/hive service predastore start"
    start_service "predastore" "$PREDASTORE_CMD"
+    set_oom_score "predastore" "-500"
    check_service "Predastore" "$HIVE_PREDASTORE_HOST" "$HIVE_PREDASTORE_PORT"
 else
    echo "2️⃣  Skipping Predastore (not a local service)"
@@ -266,6 +284,7 @@ if has_service "viperblock"; then

    VIPERBLOCK_CMD="./bin/hive service viperblock start"
    start_service "viperblock" "$VIPERBLOCK_CMD"
+    set_oom_score "viperblock" "-500"
 else
    echo "3️⃣  Skipping Viperblock (not a local service)"
 fi
@@ -281,6 +300,7 @@ if has_service "daemon"; then

    HIVE_CMD="./bin/hive service hive start"
    start_service "hive" "$HIVE_CMD"
+    set_oom_score "hive" "-500"
 else
    echo "4️⃣. Skipping Hive Gateway (not a local service)"
 fi
@@ -300,6 +320,7 @@ if has_service "awsgw"; then

    AWSGW_CMD="./bin/hive service awsgw start"
    start_service "awsgw" "$AWSGW_CMD"
+    set_oom_score "awsgw" "-500"
 else
    echo "5️⃣. Skipping AWS Gateway (not a local service)"
 fi
@@ -312,6 +333,7 @@ if [ "${UI}" != "false" ] && has_service "ui"; then

    HIVEUI_CMD="./bin/hive service hive-ui start"
    start_service "hive-ui" "$HIVEUI_CMD"
+    set_oom_score "hive-ui" "-500"
 else
    echo ""
    echo "6️⃣. Skipping Hive UI"
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Workload performance testing (memory, disk, cpu, network)
+
+sudo apt update
+sudo apt install -y make gcc git
+
+wget https://go.dev/dl/go1.26.0.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.26.0.linux-amd64.tar.gz
+
+echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc
+source $HOME/.bashrc
+
+git clone https://github.com/dgraph-io/badger.git
+cd badger
+
+# disable jemalloc
+sed -i 's/-tags=jemalloc/-tags=nojemalloc/g' test.sh
+
+# run 4 tests, otherwise OOM on <8GB RAM
+sed -i 's/-parallel 16/-parallel 4/g; s/-parallel=16/-parallel=4/g' test.sh
+
+# run tests
+./test.sh