From 962d85edf2a79215ffebd2802609b598a35032fd Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Sat, 14 Feb 2026 19:27:13 +1000 Subject: [PATCH] Improved IAC automation testing --- .gitignore | 1 + .../improvements/iac-testing-hive.md | 341 ++++++++++++++++++ scripts/iac/configure-cluster.sh | 195 ++++++++++ scripts/iac/hive-test.sh | 223 ++++++++++++ scripts/iac/proxmox/.env.example | 5 +- scripts/iac/proxmox/README.md | 193 ++++------ scripts/iac/proxmox/cloud-config.tf | 13 +- scripts/iac/proxmox/main.tf | 8 +- scripts/iac/proxmox/outputs.tf | 14 + scripts/iac/proxmox/variables.tf | 51 ++- scripts/iac/proxmox/vms.tf | 35 +- scripts/iac/test-cluster.sh | 71 ++++ 12 files changed, 985 insertions(+), 165 deletions(-) create mode 100644 docs/development/improvements/iac-testing-hive.md create mode 100755 scripts/iac/configure-cluster.sh create mode 100755 scripts/iac/hive-test.sh create mode 100644 scripts/iac/proxmox/outputs.tf create mode 100755 scripts/iac/test-cluster.sh diff --git a/.gitignore b/.gitignore index bf5d475e1..fa7d5bba6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.tfstate.* *.tfvars !example.tfvars +scripts/iac/proxmox/clusters/ # Go workspace go.work diff --git a/docs/development/improvements/iac-testing-hive.md b/docs/development/improvements/iac-testing-hive.md new file mode 100644 index 000000000..119b3a0a8 --- /dev/null +++ b/docs/development/improvements/iac-testing-hive.md @@ -0,0 +1,341 @@ +# Reusable IaC Test Environments for Hive + +## Summary + +Make the Proxmox IaC in `scripts/iac/proxmox/` reusable and parameterized so that multiple isolated Hive clusters can be provisioned, configured, tested, and destroyed independently. Introduce a four-phase lifecycle (Provision, Configure, Test, Destroy) with a single wrapper script, inspired by Kubernetes kubetest2, QEMU avocado, and SUSE Rancher patterns. + +**Status: Planned** + +## Context / Problem Statement + +The current Proxmox IaC works well for its original purpose — provisioning the 3 static dev servers (10.1.3.170-172). But it has several limitations that prevent reuse: + +### What exists today + +| File | Purpose | +|---|---| +| `scripts/iac/proxmox/main.tf` | Provider config (`bpg/proxmox` v0.52.0), locals for node map | +| `scripts/iac/proxmox/vms.tf` | VM resources, hardcoded 4 vCPU / 8GB RAM / 32GB disk, output `iac_mulgaos_ips` | +| `scripts/iac/proxmox/variables.tf` | Input vars: `proxmox_endpoint`, `ssh_*_key_path`, `nodes` (exactly 3) | +| `scripts/iac/proxmox/cloud-config.tf` | Cloud-init: installs packages, clones hive, runs `quickinstall` + `dev-setup.sh` | +| `scripts/iac/proxmox/README.md` | Manual cluster setup instructions (init node 1, join nodes 2-3) | + +### Current gaps + +1. **Hardcoded to 3 nodes** — `variables.tf` validates `length(var.nodes) == 3`, can't run a 1-node smoke test or a 5-node stress test +2. **Single state file** — `terraform.tfstate` lives in the working directory, no way to have parallel environments without manually copying the directory (which is what `proxmox.orig/` is doing today) +3. **Hardcoded VM names** — `iac-dev-mulgaos-${index}` means two environments would collide on VM names +4. **Hardcoded specs** — 4 vCPU, 8GB RAM, 32GB disk baked into `vms.tf`, no way to test with different sizes +5. **Cloud-init does too much** — clones hive from GitHub, runs `quickinstall`, runs `dev-setup.sh`. This couples provisioning to a specific hive version and makes iteration slow (full cloud-init reruns on every `tofu apply`) +6. **No automated post-provisioning** — after `tofu apply`, the README has manual SSH commands for `hive admin init` on node 1, `hive admin join` on nodes 2-3, then manual `hive start` on each +7. **No test orchestration** — no way to automatically run E2E tests against a provisioned cluster + +## Design Principles + +Drawing from how other infrastructure projects handle automated test environments: + +### Kubernetes kubetest2 + +Pluggable deployer interface with lifecycle phases: `up -> test -> down`. Deployers are swappable (kind, GKE, EKS) but the lifecycle contract is the same. The test harness is decoupled from infrastructure creation — tests don't know or care whether they're running against kind or GKE. + +**Takeaway**: Separate the lifecycle wrapper from the infrastructure backend. Today we have Proxmox; tomorrow we might add AWS or libvirt. The `up/test/down` contract stays the same. + +### QEMU avocado + +Parameterized test matrices — vary CPU count, memory, disk type, architecture across test runs. Test cases are decoupled from environment parameters. A single test suite runs against many different configurations. + +**Takeaway**: Make node count, CPU, memory, and disk size variables, not constants. Enable test matrix runs like "3-node/4-core vs 5-node/8-core". + +### SUSE Rancher + +Two-layer IaC — Layer 1 provisions VMs (bare OS), Layer 2 deploys software (Kubernetes/Rancher). Clean separation means you can re-deploy software without re-provisioning VMs, and vice versa. + +**Takeaway**: Cloud-init should only set up the OS. Hive installation and cluster formation belong in a separate configure phase that can be re-run independently. + +### Terraform best practices + +Separate state per environment (workspaces are for identical configs in different accounts, not divergent environments). Variables for everything that changes. Outputs from one phase become inputs to the next. + +**Takeaway**: Use `-state=clusters/{name}/terraform.tfstate` for state isolation. Write structured outputs (node IPs, SSH config) to `clusters/{name}/inventory.json` for the configure phase to consume. + +## Proposed Changes + +### Four-phase lifecycle + +``` +Provision (tofu) -> Configure (shell) -> Test (shell) -> Destroy (tofu) + | | | | + v v v v + VMs exist Hive running Tests pass VMs gone + IPs known Cluster formed Results saved State cleaned +``` + +### Phase 1: Provision + +Terraform/OpenTofu creates VMs with a minimal OS. No hive-specific setup. + +**Changes to `scripts/iac/proxmox/variables.tf`:** + +```hcl +variable "cluster_name" { + type = string + description = "Unique name for this cluster (used in VM names, state path, tags)" + + validation { + condition = can(regex("^[a-z0-9-]+$", var.cluster_name)) + error_message = "cluster_name must contain only lowercase letters, numbers, and hyphens." + } +} + +variable "node_count" { + type = number + description = "Number of nodes in the cluster (1-10)" + default = 3 + + validation { + condition = var.node_count >= 1 && var.node_count <= 10 + error_message = "node_count must be between 1 and 10." + } +} + +variable "cpu_cores" { + type = number + description = "vCPU cores per VM" + default = 4 +} + +variable "memory_mb" { + type = number + description = "Memory in MB per VM" + default = 8192 +} + +variable "disk_size_gb" { + type = number + description = "Root disk size in GB per VM" + default = 32 +} + +variable "os_image" { + type = string + description = "Cloud image file ID on Proxmox (local:iso/...)" + default = "local:iso/debian-12-genericcloud-amd64-20240211-1654.img" +} +``` + +The existing `nodes` variable (list of Proxmox host objects) stays — it defines *where* VMs land. The new `node_count` controls *how many* VMs are created. VMs are distributed across Proxmox hosts round-robin. The `length(var.nodes) == 3` validation is removed; any number of Proxmox hosts is valid. + +**Changes to `scripts/iac/proxmox/vms.tf`:** + +```hcl +resource "proxmox_virtual_environment_vm" "hive_node" { + count = var.node_count + + name = "${var.cluster_name}-${count.index + 1}" + description = "Managed by Terraform - cluster: ${var.cluster_name}" + tags = ["terraform", "hive", var.cluster_name] + node_name = var.nodes[count.index % length(var.nodes)].name + + cpu { + cores = var.cpu_cores + type = "host" + } + + memory { + dedicated = var.memory_mb + } + + disk { + file_id = var.os_image + datastore_id = var.nodes[count.index % length(var.nodes)].datastore_id + interface = "virtio0" + iothread = true + discard = "on" + size = var.disk_size_gb + } + + # ... initialization, network_device, agent blocks unchanged +} +``` + +**Changes to `scripts/iac/proxmox/cloud-config.tf`:** + +Slim down to OS essentials only. Remove hive clone, `quickinstall`, `clone-deps.sh`, `dev-setup.sh`: + +```yaml +#cloud-config +package_update: true +package_upgrade: true +packages: + - qemu-guest-agent + - net-tools + - git + - make +runcmd: + - timedatectl set-timezone America/Los_Angeles + - systemctl enable qemu-guest-agent + - systemctl start qemu-guest-agent + - ufw disable + - sysctl -w net.core.rmem_max=4194304 + - sysctl -w net.core.wmem_max=4194304 + - usermod -aG kvm tf-user + - echo "done" > /tmp/vendor-cloud-init-done +``` + +Hive installation moves to Phase 2 (Configure), where it can be iterated without re-provisioning VMs. + +**New `scripts/iac/proxmox/outputs.tf`:** + +Structured inventory output for the configure phase: + +```hcl +output "inventory" { + description = "Cluster inventory for configure phase" + value = { + cluster_name = var.cluster_name + nodes = [ + for i, vm in proxmox_virtual_environment_vm.hive_node : { + name = "${var.cluster_name}-${i + 1}" + index = i + 1 + management = vm.ipv4_addresses[1][0] + data = vm.ipv4_addresses[2][0] + } + ] + ssh_user = "tf-user" + ssh_key_path = var.ssh_private_key_path + } +} +``` + +**State isolation:** + +Each cluster gets its own state file via `-state` flag: + +```bash +tofu apply -state="clusters/${CLUSTER_NAME}/terraform.tfstate" +``` + +The `clusters/` directory is gitignored. Each cluster's state, inventory, and logs live in `clusters/{name}/`. + +### Phase 2: Configure + +A shell script that reads the Terraform outputs and sets up Hive on the provisioned VMs. + +**`scripts/iac/configure-cluster.sh `:** + +1. Read inventory from `tofu output -json -state=clusters/{name}/terraform.tfstate` +2. Wait for cloud-init to complete on all nodes (poll `/tmp/vendor-cloud-init-done`) +3. On each node (parallel): clone/pull hive repo, run `make quickinstall`, `clone-deps.sh`, `dev-setup.sh`, `make build` +4. On node 1: `hive admin init` with management IPs from inventory +5. On remaining nodes (parallel): `hive admin join` pointing at node 1 +6. On all nodes: `hive start` +7. Wait for health checks: poll `http://:4432/health` on all nodes + +This replaces the manual SSH commands currently in `scripts/iac/proxmox/README.md`. The `hive admin init` and `hive admin join` commands are generated from inventory data rather than hardcoded IPs. + +### Phase 3: Test + +**`scripts/iac/test-cluster.sh `:** + +1. Read inventory to get node IPs +2. Set `HIVE_ENDPOINT=https://:9999` +3. Run E2E tests from `tests/e2e/` against the real cluster +4. Save results to `clusters/{name}/test-results/` + +This reuses the existing E2E test logic, currently Docker-only (`tests/e2e/run-e2e.sh`, `tests/e2e/run-multinode-e2e.sh`), pointed at real cluster IPs instead of localhost containers. + +### Phase 4: Destroy + +```bash +tofu destroy -state="clusters/${CLUSTER_NAME}/terraform.tfstate" +rm -rf "clusters/${CLUSTER_NAME}/" +``` + +### Lifecycle wrapper + +**`scripts/iac/hive-test.sh`** — single entry point: + +```bash +#!/bin/bash +# Usage: hive-test.sh [options] +# +# Commands: +# up Provision VMs (Phase 1) +# configure Install and start Hive (Phase 2) +# test Run E2E tests (Phase 3) +# down Destroy VMs and clean up (Phase 4) +# full Run all phases: up -> configure -> test -> down +# status Show cluster status (node IPs, health) +# +# Options (passed through to tofu apply): +# --node-count=N Number of nodes (default: 3) +# --cpu-cores=N vCPU per node (default: 4) +# --memory-mb=N Memory per node in MB (default: 8192) +# --disk-size-gb=N Disk per node in GB (default: 32) + +# Examples: +# hive-test.sh full my-test-cluster +# hive-test.sh up stress-test --node-count=5 --cpu-cores=8 +# hive-test.sh down stress-test +``` + +The `full` command mirrors kubetest2's `--up --test --down` pattern: provision, configure, test, then tear down regardless of test outcome (with exit code preserved). + +## Files to Modify + +| File | Change | +|---|---| +| `scripts/iac/proxmox/variables.tf` | Remove `length == 3` validation, add `cluster_name`, `node_count`, `cpu_cores`, `memory_mb`, `disk_size_gb`, `os_image` | +| `scripts/iac/proxmox/vms.tf` | Use `count` instead of `for_each` on `local.nodes`, parameterize VM name/specs, round-robin placement | +| `scripts/iac/proxmox/cloud-config.tf` | Remove hive clone/quickinstall/dev-setup, keep OS essentials only | +| `scripts/iac/proxmox/main.tf` | No changes (provider config stays the same) | +| `scripts/iac/proxmox/outputs.tf` | **New** — structured inventory output (IPs, SSH config, cluster name) | +| `scripts/iac/configure-cluster.sh` | **New** — Phase 2: install hive, form cluster, start services | +| `scripts/iac/test-cluster.sh` | **New** — Phase 3: run E2E tests against real cluster | +| `scripts/iac/hive-test.sh` | **New** — lifecycle wrapper (up/configure/test/down/full) | +| `scripts/iac/proxmox/.gitignore` | Add `clusters/` directory | + +## Multi-Environment Usage Examples + +```bash +# Source Proxmox credentials (unchanged from today) +source scripts/iac/proxmox/.env + +# Environment 1: standard 3-node dev cluster +./scripts/iac/hive-test.sh up hive-dev1 + +# Environment 2: 5-node stress test with larger VMs +./scripts/iac/hive-test.sh up hive-stress --node-count=5 --cpu-cores=8 --memory-mb=16384 + +# Configure and test environment 1 +./scripts/iac/hive-test.sh configure hive-dev1 +./scripts/iac/hive-test.sh test hive-dev1 + +# Full lifecycle (CI use case): provision, configure, test, destroy +./scripts/iac/hive-test.sh full hive-ci-$(date +%s) + +# Check status of any cluster +./scripts/iac/hive-test.sh status hive-dev1 + +# Tear down when done +./scripts/iac/hive-test.sh down hive-stress +./scripts/iac/hive-test.sh down hive-dev1 +``` + +## Testing + +- Provision a 1-node cluster, verify VM comes up with correct name and specs +- Provision a 3-node cluster (matches current setup), verify parity with existing behavior +- Run configure phase, verify all nodes have hive running and cluster formed +- Run test phase, verify E2E tests execute against real cluster +- Run destroy phase, verify VMs are removed and state is cleaned +- Provision two clusters simultaneously, verify no naming or state conflicts +- Test the `full` lifecycle end-to-end + +## Future Work + +- **CI integration**: GitHub Actions workflow that provisions real Proxmox VMs for E2E tests on PR, with automatic teardown +- **AWS provider parity**: Same lifecycle wrapper for a future `scripts/iac/aws/` backend, keeping the `up/configure/test/down` contract +- **Test matrices**: Parameterized runs varying node count and instance sizes across a matrix (like avocado), e.g. `{1,3,5} nodes x {4,8} cores` +- **Ansible migration**: Replace `configure-cluster.sh` with Ansible playbooks when shell scripting complexity grows (inventory.json maps cleanly to Ansible inventory) +- **Re-configure without re-provision**: Since cloud-init no longer installs hive, `configure` can be re-run to deploy a new hive build without destroying/recreating VMs diff --git a/scripts/iac/configure-cluster.sh b/scripts/iac/configure-cluster.sh new file mode 100755 index 000000000..988cb5b7e --- /dev/null +++ b/scripts/iac/configure-cluster.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Configure a provisioned Hive cluster: clone repo, build, form cluster, start services. +# Usage: configure-cluster.sh + +CLUSTER_NAME="${1:?Usage: configure-cluster.sh }" +STATE_FILE="${2:?Usage: configure-cluster.sh }" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOFU_DIR="$SCRIPT_DIR/proxmox" + +SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10" + +REGION="ap-southeast-2" +AZ="ap-southeast-2a" + +inventory=$(tofu -chdir="$TOFU_DIR" output -json -state="$STATE_FILE" inventory) +NODE_COUNT=$(echo "$inventory" | jq -r '.node_count') +SSH_KEY=$(eval echo "$(echo "$inventory" | jq -r '.ssh_key_path')") +SSH_USER=$(echo "$inventory" | jq -r '.ssh_user') + +# Build arrays of node IPs +declare -a MGMT_IPS +for i in $(seq 0 $((NODE_COUNT - 1))); do + MGMT_IPS+=("$(echo "$inventory" | jq -r ".nodes[$i].management")") +done + +remote() { + local ip="$1" + shift + ssh -i "$SSH_KEY" $SSH_OPTS "$SSH_USER@$ip" "$@" +} + +remote_bg() { + local ip="$1" + shift + ssh -i "$SSH_KEY" $SSH_OPTS "$SSH_USER@$ip" "$@" & +} + +echo "==> Cluster '$CLUSTER_NAME': $NODE_COUNT nodes" +for i in $(seq 0 $((NODE_COUNT - 1))); do + echo " node$((i + 1)): ${MGMT_IPS[$i]}" +done + +# --- Step 1: Wait for cloud-init --- +echo "" +echo "==> Waiting for cloud-init to complete on all nodes..." +for i in $(seq 0 $((NODE_COUNT - 1))); do + ip="${MGMT_IPS[$i]}" + printf " node%d (%s): " "$((i + 1))" "$ip" + attempts=0 + while ! remote "$ip" "test -f /tmp/vendor-cloud-init-done" 2>/dev/null; do + attempts=$((attempts + 1)) + if [ $attempts -ge 60 ]; then + echo "TIMEOUT" + echo "Error: cloud-init did not complete on $ip after 5 minutes" + exit 1 + fi + sleep 5 + done + echo "done" +done + +# --- Step 2: Clone repo and build on all nodes --- +echo "" +echo "==> Cloning hive repo and building on all nodes..." +PIDS=() +for i in $(seq 0 $((NODE_COUNT - 1))); do + ip="${MGMT_IPS[$i]}" + ( + echo " node$((i + 1)) ($ip): cloning..." + remote "$ip" "mkdir -p ~/Development/mulga && cd ~/Development/mulga && git clone https://github.com/mulgadc/hive.git && export PATH=\$PATH:/usr/local/go/bin && sudo make -C hive quickinstall" + echo " node$((i + 1)) ($ip): clone-deps + dev-setup..." + remote "$ip" "export PATH=\$PATH:/usr/local/go/bin && cd ~/Development/mulga/hive && ./scripts/clone-deps.sh && ./scripts/dev-setup.sh" + echo " node$((i + 1)) ($ip): build complete" + ) & + PIDS+=($!) +done + +for pid in "${PIDS[@]}"; do + wait "$pid" +done +echo " All nodes built successfully" + +# --- Step 3: Form cluster --- +echo "" +echo "==> Forming cluster..." + +# Build comma-separated IP lists +ALL_MGMT_IPS=$(IFS=,; echo "${MGMT_IPS[*]}") +NODE1_IP="${MGMT_IPS[0]}" + +# Node 1: init +echo " Starting init on node1 ($NODE1_IP)..." +remote_bg "$NODE1_IP" "export PATH=\$PATH:/usr/local/go/bin && cd ~/Development/mulga/hive && ./bin/hive admin init \ + --node node1 \ + --bind $NODE1_IP \ + --cluster-bind $NODE1_IP \ + --cluster-routes $NODE1_IP:4248 \ + --predastore-nodes $ALL_MGMT_IPS \ + --port 4432 \ + --nodes $NODE_COUNT \ + --region $REGION \ + --az $AZ \ + --hive-dir ~/hive/ \ + --config-dir ~/hive/config/" +INIT_PID=$! + +# Wait for formation server to be ready +echo " Waiting for formation server on $NODE1_IP..." +attempts=0 +while ! curl -s --connect-timeout 3 "http://$NODE1_IP:4432/formation/health" >/dev/null 2>&1; do + attempts=$((attempts + 1)) + if [ $attempts -ge 60 ]; then + echo "Error: Formation server did not start on $NODE1_IP after 5 minutes" + kill $INIT_PID 2>/dev/null || true + exit 1 + fi + sleep 5 +done +echo " Formation server ready" + +# Nodes 2..N: join +if [ "$NODE_COUNT" -gt 1 ]; then + JOIN_PIDS=() + for i in $(seq 1 $((NODE_COUNT - 1))); do + ip="${MGMT_IPS[$i]}" + node_num=$((i + 1)) + echo " Starting join on node$node_num ($ip)..." + remote_bg "$ip" "export PATH=\$PATH:/usr/local/go/bin && cd ~/Development/mulga/hive && ./bin/hive admin join \ + --node node$node_num \ + --bind $ip \ + --cluster-bind $ip \ + --cluster-routes $NODE1_IP:4248 \ + --host $NODE1_IP:4432 \ + --data-dir ~/hive/ \ + --config-dir ~/hive/config/ \ + --region $REGION \ + --az $AZ" + JOIN_PIDS+=($!) + done + + for pid in "${JOIN_PIDS[@]}"; do + wait "$pid" + done +fi + +# Wait for init process to complete +wait $INIT_PID +echo " Cluster formation complete" + +# --- Step 4: Start services --- +echo "" +echo "==> Starting services on all nodes..." +PIDS=() +for i in $(seq 0 $((NODE_COUNT - 1))); do + ip="${MGMT_IPS[$i]}" + ( + echo " node$((i + 1)) ($ip): starting services..." + remote "$ip" "export PATH=\$PATH:/usr/local/go/bin && cd ~/Development/mulga/hive && HIVE_SKIP_BUILD=true ./scripts/start-dev.sh" + echo " node$((i + 1)) ($ip): services started" + ) & + PIDS+=($!) +done + +for pid in "${PIDS[@]}"; do + wait "$pid" +done + +# --- Step 5: Health check --- +echo "" +echo "==> Waiting for all nodes to become healthy..." +for i in $(seq 0 $((NODE_COUNT - 1))); do + ip="${MGMT_IPS[$i]}" + printf " node%d (%s): " "$((i + 1))" "$ip" + attempts=0 + while true; do + health=$(curl -s --connect-timeout 3 "http://$ip:4432/health" 2>/dev/null || echo "") + if [ -n "$health" ]; then + echo "healthy" + break + fi + attempts=$((attempts + 1)) + if [ $attempts -ge 30 ]; then + echo "TIMEOUT" + echo "Warning: node$((i + 1)) did not become healthy after 150 seconds" + break + fi + sleep 5 + done +done + +echo "" +echo "==> Cluster '$CLUSTER_NAME' configured successfully" diff --git a/scripts/iac/hive-test.sh b/scripts/iac/hive-test.sh new file mode 100755 index 000000000..a9b8d7caf --- /dev/null +++ b/scripts/iac/hive-test.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOFU_DIR="$SCRIPT_DIR/proxmox" +CLUSTERS_DIR="$TOFU_DIR/clusters" + +usage() { + cat <<'USAGE' +Usage: hive-test.sh [options] + +Commands: + up Provision VMs with OpenTofu + configure Clone repo, build, form cluster, start services + test Run smoke tests against the cluster + down Destroy VMs and clean up state + full up → configure → test → down (preserves test exit code) + status Show cluster health and node IPs + ssh SSH to node N (1-based) + +Options: + --node-count=N Number of VMs (default: 3) + --memory-mb=N Memory per VM in MB (default: 16384) + --cpu-cores=N CPU cores per VM (default: 4) + --disk-size-gb=N Disk size per VM in GB (default: 32) + +Examples: + ./scripts/iac/hive-test.sh up hive-test1 + ./scripts/iac/hive-test.sh configure hive-test1 + ./scripts/iac/hive-test.sh test hive-test1 + ./scripts/iac/hive-test.sh status hive-test1 + ./scripts/iac/hive-test.sh ssh hive-test1 2 + ./scripts/iac/hive-test.sh down hive-test1 + ./scripts/iac/hive-test.sh full hive-test1 --node-count=3 +USAGE + exit 1 +} + +[ $# -lt 2 ] && usage + +COMMAND="$1" +CLUSTER_NAME="$2" +shift 2 + +# Parse options +NODE_COUNT=3 +MEMORY_MB=16384 +CPU_CORES=4 +DISK_SIZE_GB=32 +SSH_NODE="" + +for arg in "$@"; do + case "$arg" in + --node-count=*) NODE_COUNT="${arg#*=}" ;; + --memory-mb=*) MEMORY_MB="${arg#*=}" ;; + --cpu-cores=*) CPU_CORES="${arg#*=}" ;; + --disk-size-gb=*) DISK_SIZE_GB="${arg#*=}" ;; + [0-9]*) SSH_NODE="$arg" ;; + *) echo "Unknown option: $arg"; usage ;; + esac +done + +STATE_DIR="$CLUSTERS_DIR/$CLUSTER_NAME" +STATE_FILE="$STATE_DIR/terraform.tfstate" + +tofu_cmd() { + tofu -chdir="$TOFU_DIR" "$@" +} + +get_inventory() { + tofu_cmd output -json -state="$STATE_FILE" inventory 2>/dev/null +} + +get_node_ip() { + local index=$1 + get_inventory | jq -r ".nodes[$((index - 1))].management" +} + +get_ssh_key() { + get_inventory | jq -r ".ssh_key_path" +} + +get_node_count() { + get_inventory | jq -r ".node_count" +} + +cmd_up() { + echo "==> Provisioning cluster '$CLUSTER_NAME' ($NODE_COUNT nodes, ${MEMORY_MB}MB RAM, ${CPU_CORES} cores)" + mkdir -p "$STATE_DIR" + + tofu_cmd init + + tofu_cmd apply \ + -state="$STATE_FILE" \ + -var="cluster_name=$CLUSTER_NAME" \ + -var="node_count=$NODE_COUNT" \ + -var="memory_mb=$MEMORY_MB" \ + -var="cpu_cores=$CPU_CORES" \ + -var="disk_size_gb=$DISK_SIZE_GB" \ + -auto-approve + + echo "" + echo "==> Cluster '$CLUSTER_NAME' provisioned" + echo "" + get_inventory | jq '.nodes[] | {name, management, data}' +} + +cmd_configure() { + echo "==> Configuring cluster '$CLUSTER_NAME'" + if [ ! -f "$STATE_FILE" ]; then + echo "Error: No state file found at $STATE_FILE. Run 'up' first." + exit 1 + fi + + "$SCRIPT_DIR/configure-cluster.sh" "$CLUSTER_NAME" "$STATE_FILE" +} + +cmd_test() { + echo "==> Testing cluster '$CLUSTER_NAME'" + if [ ! -f "$STATE_FILE" ]; then + echo "Error: No state file found at $STATE_FILE. Run 'up' first." + exit 1 + fi + + "$SCRIPT_DIR/test-cluster.sh" "$CLUSTER_NAME" "$STATE_FILE" +} + +cmd_down() { + echo "==> Destroying cluster '$CLUSTER_NAME'" + if [ ! -f "$STATE_FILE" ]; then + echo "Error: No state file found at $STATE_FILE." + exit 1 + fi + + tofu_cmd destroy \ + -state="$STATE_FILE" \ + -var="cluster_name=$CLUSTER_NAME" \ + -var="node_count=$NODE_COUNT" \ + -var="memory_mb=$MEMORY_MB" \ + -var="cpu_cores=$CPU_CORES" \ + -var="disk_size_gb=$DISK_SIZE_GB" \ + -auto-approve + + rm -rf "$STATE_DIR" + echo "==> Cluster '$CLUSTER_NAME' destroyed" +} + +cmd_full() { + cmd_up + + cmd_configure + + local test_exit=0 + cmd_test || test_exit=$? + + cmd_down + + exit $test_exit +} + +cmd_status() { + if [ ! -f "$STATE_FILE" ]; then + echo "Error: No state file found at $STATE_FILE." + exit 1 + fi + + local inventory + inventory=$(get_inventory) + local count + count=$(echo "$inventory" | jq -r '.node_count') + local ssh_key + ssh_key=$(echo "$inventory" | jq -r '.ssh_key_path') + + echo "==> Cluster '$CLUSTER_NAME' status ($count nodes)" + echo "" + + for i in $(seq 1 "$count"); do + local ip + ip=$(echo "$inventory" | jq -r ".nodes[$((i - 1))].management") + local data_ip + data_ip=$(echo "$inventory" | jq -r ".nodes[$((i - 1))].data") + local name + name=$(echo "$inventory" | jq -r ".nodes[$((i - 1))].name") + + printf " %-20s mgmt=%-16s data=%-16s " "$name" "$ip" "$data_ip" + + local health + if health=$(curl -s --connect-timeout 3 "http://$ip:4432/health" 2>/dev/null); then + echo "health=$health" + else + echo "health=unreachable" + fi + done +} + +cmd_ssh() { + if [ -z "$SSH_NODE" ]; then + echo "Usage: hive-test.sh ssh " + exit 1 + fi + + local ip + ip=$(get_node_ip "$SSH_NODE") + local ssh_key + ssh_key=$(get_ssh_key) + + echo "==> SSH to $CLUSTER_NAME node $SSH_NODE ($ip)" + ssh -i "$(eval echo "$ssh_key")" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "tf-user@$ip" +} + +case "$COMMAND" in + up) cmd_up ;; + configure) cmd_configure ;; + test) cmd_test ;; + down) cmd_down ;; + full) cmd_full ;; + status) cmd_status ;; + ssh) cmd_ssh ;; + *) echo "Unknown command: $COMMAND"; usage ;; +esac diff --git a/scripts/iac/proxmox/.env.example b/scripts/iac/proxmox/.env.example index 1c62b7eb5..8cfce20d9 100644 --- a/scripts/iac/proxmox/.env.example +++ b/scripts/iac/proxmox/.env.example @@ -24,7 +24,10 @@ export TF_VAR_ssh_private_key_path="~/.ssh/proxmox-host-tf" # Path to SSH public key injected into VMs via cloud-init (for tf-user) export TF_VAR_ssh_public_key_path="~/.ssh/cloud-init.pub" -# --- Cluster Nodes (exactly 3 required) --- +# --- Proxmox Nodes --- +# Physical Proxmox hosts where VMs are placed (round-robin). +# You can define 1 or more nodes — VMs are distributed across them. +# # Each node object: # name - Proxmox node name (as shown in the Proxmox UI) # address - SSH-reachable hostname or IP of the Proxmox host diff --git a/scripts/iac/proxmox/README.md b/scripts/iac/proxmox/README.md index c29ed1674..d4aa7b597 100644 --- a/scripts/iac/proxmox/README.md +++ b/scripts/iac/proxmox/README.md @@ -1,39 +1,87 @@ # Proxmox IaC -OpenTofu/Terraform templates for provisioning a 3-node Hive development cluster on Proxmox VE. +OpenTofu/Terraform templates for provisioning Hive development clusters on Proxmox VE. ## Prerequisites - [OpenTofu](https://opentofu.org/) (or Terraform >= 1.6.0) -- 3 Proxmox VE nodes with: +- 1 or more Proxmox VE nodes with: - A Terraform API token (`Datacenter > Permissions > API Tokens`) - SSH access for a `terraform` user on each node - Debian 12 cloud image uploaded to `local:iso/` on each node - SSH keypair for cloud-init VM access - A hardcoded Debian ISO is used, which needs to be pre-installed on each Proxmox host ("debian-12-genericcloud-amd64-20240211-1654.img") https://cdimage.debian.org/images/cloud/bookworm/20240211-1654/debian-12-genericcloud-amd64-20240211-1654.qcow2 -## Setup +## Quick Start (hive-test.sh) -1. Copy the example environment file and fill in your values: +The `hive-test.sh` wrapper script manages the full lifecycle: provision, configure, test, and destroy. + +```sh +# Source environment +source scripts/iac/proxmox/.env +source ~/prox # Proxmox API token + +# Provision a 3-node cluster +./scripts/iac/hive-test.sh up my-cluster + +# Configure: clone repo, build, form cluster, start services +./scripts/iac/hive-test.sh configure my-cluster + +# Run smoke tests +./scripts/iac/hive-test.sh test my-cluster + +# Check cluster health +./scripts/iac/hive-test.sh status my-cluster + +# SSH to a specific node +./scripts/iac/hive-test.sh ssh my-cluster 2 + +# Destroy when done +./scripts/iac/hive-test.sh down my-cluster + +# Or run the full lifecycle (up → configure → test → down) +./scripts/iac/hive-test.sh full my-cluster +``` + +### Options + +``` +--node-count=N Number of VMs (default: 3) +--memory-mb=N Memory per VM in MB (default: 16384) +--cpu-cores=N CPU cores per VM (default: 4) +--disk-size-gb=N Disk size per VM in GB (default: 32) +``` + +Example with custom sizing: + +```sh +./scripts/iac/hive-test.sh up my-cluster --node-count=5 --memory-mb=8192 --cpu-cores=2 +``` + +Per-cluster state is stored in `scripts/iac/proxmox/clusters//terraform.tfstate`, allowing multiple clusters to coexist. + +## Manual Setup + +### 1. Configure environment ```sh cp .env.example .env # Edit .env with your Proxmox endpoint, SSH keys, and node configuration ``` -2. Set your Proxmox API token separately (do not store in `.env`): +### 2. Set API token ```sh export PROXMOX_VE_API_TOKEN="terraform@pve!provider=YOUR_TOKEN_SECRET" ``` -3. Initialize and deploy: +### 3. Deploy ```sh source .env tofu init -tofu plan -tofu apply +tofu plan -var="cluster_name=dev1" +tofu apply -var="cluster_name=dev1" ``` ### Example `.env` file @@ -58,8 +106,9 @@ export TF_VAR_ssh_private_key_path="~/.ssh/proxmox-tf" # Path to SSH public key injected into VMs via cloud-init (the tf-user account) export TF_VAR_ssh_public_key_path="~/.ssh/proxmox-tf-cloudinit.pub" -# --- Cluster Nodes (exactly 3 required) --- +# --- Proxmox Nodes --- # +# Physical Proxmox hosts where VMs are placed (round-robin). # Each node object: # name - Proxmox node name exactly as shown in the Proxmox UI sidebar # address - SSH-reachable hostname or IP of the Proxmox host @@ -97,7 +146,7 @@ export TF_VAR_nodes='[ | `TF_VAR_proxmox_ssh_username` | No | SSH user on Proxmox hosts (default: `terraform`) | | `TF_VAR_ssh_private_key_path` | Yes | SSH private key for Proxmox host access | | `TF_VAR_ssh_public_key_path` | Yes | SSH public key injected into VMs via cloud-init | -| `TF_VAR_nodes` | Yes | JSON array of exactly 3 node objects | +| `TF_VAR_nodes` | Yes | JSON array of Proxmox node objects | Each node object in `TF_VAR_nodes`: @@ -108,45 +157,22 @@ Each node object in `TF_VAR_nodes`: | `bridge` | `vmbr0` | Network bridge for VM interfaces | | `datastore_id` | `local-lvm` | Storage pool for VM disks | -## Deploy +### Cluster variables (passed via `-var` or `TF_VAR_`) -```sh -source .env -tofu init -tofu plan # validates config - fails if required env vars are missing -tofu apply -``` - -Example output: - -```sh -Plan: 6 to add, 0 to change, 0 to destroy. - -Apply complete! Resources: 6 added, 0 changed, 0 destroyed. - -Outputs: - -iac_mulgaos_ips = { - "node1" = { - "data" = "10.1.3.169" - "management" = "10.1.3.170" - } - "node2" = { - "data" = "10.1.2.21" - "management" = "10.1.3.171" - } - "node3" = { - "data" = "10.1.2.22" - "management" = "10.1.3.172" - } -} -``` +| Variable | Default | Description | +|---|---|---| +| `cluster_name` | (required) | Cluster name — used in VM names and tags | +| `node_count` | `3` | Number of VMs to create (1-10) | +| `cpu_cores` | `4` | CPU cores per VM | +| `memory_mb` | `16384` | Memory per VM in MB | +| `disk_size_gb` | `32` | Disk size per VM in GB | +| `os_image` | `local:iso/debian-12-...` | Proxmox image for boot disk | ## Destroy ```sh source .env -tofu destroy +tofu destroy -var="cluster_name=dev1" ``` ## SSH access @@ -157,87 +183,6 @@ Connect to provisioned VMs using the cloud-init public key: ssh -i ~/.ssh/your-cloud-init-key tf-user@ ``` -## Multi-node cluster setup - -After VMs are provisioned, initialize the Hive cluster manually. Consider moving to Ansible for automation. - -Note, modify the NODE IP addresses with the output from the terraform deployment above. - -### Node 1 (init) - -Management traffic over first network adapter, with Open vSwitch (Geneve) traffic on the 2nd for VPC traffic. - -```sh -cd ~/Development/mulga/hive -make - -export HIVE_REGION="ap-southeast-2" -export HIVE_AZ="ap-southeast-2a" -export NODE1_MGMT_IP="10.1.3.170" -export NODE2_MGMT_IP="10.1.3.171" -export NODE3_MGMT_IP="10.1.3.172" - -./bin/hive admin init \ - --node node1 \ - --bind $NODE1_MGMT_IP \ - --cluster-bind $NODE1_MGMT_IP \ - --cluster-routes $NODE1_MGMT_IP:4248 \ - --predastore-nodes $NODE1_MGMT_IP,$NODE2_MGMT_IP,$NODE3_MGMT_IP \ - --port 4432 \ - --hive-dir ~/hive/ \ - --config-dir ~/hive/config/ \ - --region $HIVE_REGION \ - --az $HIVE_AZ -``` - -### Node 2 (join) - -```sh -cd ~/Development/mulga/hive -make - -export HIVE_REGION="ap-southeast-2" -export HIVE_AZ="ap-southeast-2" -export NODE1_MGMT_IP="10.1.3.170" -export NODE2_MGMT_IP="10.1.3.171" -export NODE3_MGMT_IP="10.1.3.172" - -./bin/hive admin join \ - --node node2 \ - --bind $NODE2_MGMT_IP \ - --cluster-bind $NODE2_MGMT_IP \ - --cluster-routes $NODE1_MGMT_IP:4248 \ - --host $NODE1_MGMT_IP:4432 \ - --data-dir ~/hive/ \ - --config-dir ~/hive/config/ \ - --region $HIVE_REGION \ - --az $HIVE_AZ -``` - -### Node 3 (join) - -```sh -cd ~/Development/mulga/hive -make - -export HIVE_REGION="ap-southeast-2" -export HIVE_AZ="ap-southeast-2" -export NODE1_MGMT_IP="10.1.3.170" -export NODE2_MGMT_IP="10.1.3.171" -export NODE3_MGMT_IP="10.1.3.172" - -./bin/hive admin join \ - --node node3 \ - --bind $NODE3_MGMT_IP \ - --cluster-bind $NODE3_MGMT_IP \ - --cluster-routes $NODE1_MGMT_IP:4248 \ - --host $NODE1_MGMT_IP:4432 \ - --data-dir ~/hive/ \ - --config-dir ~/hive/config/ \ - --region $HIVE_REGION \ - --az $HIVE_AZ -``` - ## Known issues - `~/hive/config/hive.toml` - Does not add node1, node2, node3 from config automatically diff --git a/scripts/iac/proxmox/cloud-config.tf b/scripts/iac/proxmox/cloud-config.tf index 2f3ea5636..e28d2d765 100644 --- a/scripts/iac/proxmox/cloud-config.tf +++ b/scripts/iac/proxmox/cloud-config.tf @@ -2,12 +2,12 @@ data "local_file" "ssh_public_key" { filename = pathexpand(var.ssh_public_key_path) } -resource "proxmox_virtual_environment_file" "iac_mulgaos_config" { - for_each = local.nodes +resource "proxmox_virtual_environment_file" "cloud_config" { + count = var.node_count content_type = "snippets" datastore_id = "local" - node_name = each.key + node_name = var.nodes[count.index % length(var.nodes)].name source_raw { data = <> /home/tf-user/.bashrc" - - "export PATH=$PATH:/usr/local/go/bin && cd /home/tf-user/Development/mulga/hive && ./scripts/clone-deps.sh && ./scripts/dev-setup.sh" - - chown -R tf-user:tf-user /home/tf-user/Development/ - echo "done" > /tmp/vendor-cloud-init-done EOF - file_name = "iac-mulgaos-vendor-cloud-config.yaml" + file_name = "${var.cluster_name}-${count.index + 1}-cloud-config.yaml" } } diff --git a/scripts/iac/proxmox/main.tf b/scripts/iac/proxmox/main.tf index abac03044..4a5f81440 100644 --- a/scripts/iac/proxmox/main.tf +++ b/scripts/iac/proxmox/main.tf @@ -7,10 +7,6 @@ terraform { } } -locals { - nodes = { for idx, node in var.nodes : node.name => merge(node, { index = idx + 1 }) } -} - # API token is read from PROXMOX_VE_API_TOKEN env var automatically provider "proxmox" { endpoint = var.proxmox_endpoint @@ -22,9 +18,9 @@ provider "proxmox" { private_key = file(pathexpand(var.ssh_private_key_path)) dynamic "node" { - for_each = local.nodes + for_each = var.nodes content { - name = node.key + name = node.value.name address = node.value.address } } diff --git a/scripts/iac/proxmox/outputs.tf b/scripts/iac/proxmox/outputs.tf new file mode 100644 index 000000000..8eebac06f --- /dev/null +++ b/scripts/iac/proxmox/outputs.tf @@ -0,0 +1,14 @@ +output "inventory" { + value = { + cluster_name = var.cluster_name + node_count = var.node_count + nodes = [for i, vm in proxmox_virtual_environment_vm.hive_node : { + name = "${var.cluster_name}-${i + 1}" + index = i + 1 + management = vm.ipv4_addresses[1][0] + data = vm.ipv4_addresses[2][0] + }] + ssh_user = "tf-user" + ssh_key_path = trimsuffix(var.ssh_public_key_path, ".pub") + } +} diff --git a/scripts/iac/proxmox/variables.tf b/scripts/iac/proxmox/variables.tf index 098931848..a1d0027ee 100644 --- a/scripts/iac/proxmox/variables.tf +++ b/scripts/iac/proxmox/variables.tf @@ -41,11 +41,11 @@ variable "nodes" { bridge = string datastore_id = string })) - description = "Exactly 3 Proxmox nodes for the Hive cluster" + description = "Proxmox hosts for VM placement (VMs are distributed round-robin)" validation { - condition = length(var.nodes) == 3 - error_message = "Exactly 3 nodes must be defined for the Hive cluster." + condition = length(var.nodes) > 0 + error_message = "At least one Proxmox node must be defined." } validation { @@ -58,3 +58,48 @@ variable "nodes" { error_message = "Each node must have a unique name." } } + +variable "cluster_name" { + type = string + description = "Name for the Hive cluster (used in VM names and tags)" + + validation { + condition = can(regex("^[a-z0-9-]+$", var.cluster_name)) + error_message = "cluster_name must contain only lowercase letters, numbers, and hyphens." + } +} + +variable "node_count" { + type = number + description = "Number of Hive VMs to create" + default = 3 + + validation { + condition = var.node_count >= 1 && var.node_count <= 10 + error_message = "node_count must be between 1 and 10." + } +} + +variable "cpu_cores" { + type = number + description = "Number of CPU cores per VM" + default = 4 +} + +variable "memory_mb" { + type = number + description = "Memory in MB per VM" + default = 16384 +} + +variable "disk_size_gb" { + type = number + description = "Disk size in GB per VM" + default = 32 +} + +variable "os_image" { + type = string + description = "Proxmox image file ID for VM boot disk" + default = "local:iso/debian-12-genericcloud-amd64-20240211-1654.img" +} diff --git a/scripts/iac/proxmox/vms.tf b/scripts/iac/proxmox/vms.tf index 41b16ceb0..708be2d7d 100644 --- a/scripts/iac/proxmox/vms.tf +++ b/scripts/iac/proxmox/vms.tf @@ -1,14 +1,14 @@ -resource "proxmox_virtual_environment_vm" "iac_mulgaos" { - for_each = local.nodes +resource "proxmox_virtual_environment_vm" "hive_node" { + count = var.node_count - name = "iac-dev-mulgaos-${each.value.index}" + name = "${var.cluster_name}-${count.index + 1}" description = "Managed by Terraform" - tags = ["terraform", "mulgaos"] - node_name = each.key + tags = ["terraform", "hive", var.cluster_name] + node_name = var.nodes[count.index % length(var.nodes)].name initialization { datastore_id = "local" - vendor_data_file_id = proxmox_virtual_environment_file.iac_mulgaos_config[each.key].id + vendor_data_file_id = proxmox_virtual_environment_file.cloud_config[count.index].id user_account { username = "tf-user" @@ -31,12 +31,12 @@ resource "proxmox_virtual_environment_vm" "iac_mulgaos" { } cpu { - cores = 4 + cores = var.cpu_cores type = "host" } memory { - dedicated = 8192 + dedicated = var.memory_mb } agent { @@ -44,28 +44,19 @@ resource "proxmox_virtual_environment_vm" "iac_mulgaos" { } network_device { - bridge = each.value.bridge + bridge = var.nodes[count.index % length(var.nodes)].bridge } network_device { - bridge = each.value.bridge + bridge = var.nodes[count.index % length(var.nodes)].bridge } disk { - file_id = "local:iso/debian-12-genericcloud-amd64-20240211-1654.img" - datastore_id = each.value.datastore_id + file_id = var.os_image + datastore_id = var.nodes[count.index % length(var.nodes)].datastore_id interface = "virtio0" iothread = true discard = "on" - size = 32 - } -} - -output "iac_mulgaos_ips" { - value = { - for name, vm in proxmox_virtual_environment_vm.iac_mulgaos : name => { - management = vm.ipv4_addresses[1][0] - data = vm.ipv4_addresses[2][0] - } + size = var.disk_size_gb } } diff --git a/scripts/iac/test-cluster.sh b/scripts/iac/test-cluster.sh new file mode 100755 index 000000000..c93e13b4d --- /dev/null +++ b/scripts/iac/test-cluster.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run smoke tests against a configured Hive cluster. +# Usage: test-cluster.sh + +CLUSTER_NAME="${1:?Usage: test-cluster.sh }" +STATE_FILE="${2:?Usage: test-cluster.sh }" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOFU_DIR="$SCRIPT_DIR/proxmox" + +inventory=$(tofu -chdir="$TOFU_DIR" output -json -state="$STATE_FILE" inventory) +NODE1_IP=$(echo "$inventory" | jq -r '.nodes[0].management') + +AWS_ENDPOINT="https://$NODE1_IP:9999" +PASS=0 +FAIL=0 + +run_test() { + local name="$1" + shift + printf " %-40s " "$name" + if output=$("$@" 2>&1); then + echo "PASS" + PASS=$((PASS + 1)) + else + echo "FAIL" + echo " $output" | head -5 + FAIL=$((FAIL + 1)) + fi +} + +echo "==> Smoke tests for cluster '$CLUSTER_NAME'" +echo " Endpoint: $AWS_ENDPOINT" +echo "" + +# Wait for gateway to be reachable +echo "==> Waiting for AWS gateway..." +attempts=0 +while ! curl -sk --connect-timeout 3 "$AWS_ENDPOINT" >/dev/null 2>&1; do + attempts=$((attempts + 1)) + if [ $attempts -ge 30 ]; then + echo "Error: Gateway not reachable at $AWS_ENDPOINT after 150 seconds" + exit 1 + fi + sleep 5 +done +echo " Gateway reachable" +echo "" + +echo "==> Running tests..." + +run_test "describe-regions" \ + aws --endpoint-url "$AWS_ENDPOINT" --no-verify-ssl --region ap-southeast-2 \ + ec2 describe-regions + +run_test "describe-instance-types" \ + aws --endpoint-url "$AWS_ENDPOINT" --no-verify-ssl --region ap-southeast-2 \ + ec2 describe-instance-types --query 'InstanceTypes[0].InstanceType' + +run_test "describe-instances" \ + aws --endpoint-url "$AWS_ENDPOINT" --no-verify-ssl --region ap-southeast-2 \ + ec2 describe-instances + +echo "" +echo "==> Results: $PASS passed, $FAIL failed" + +if [ $FAIL -gt 0 ]; then + exit 1 +fi