mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
IAC for Proxmox / AWS terraform examples
This commit is contained in:
Ben Duncan
@@ -1,3 +1,11 @@
|
||||
# Terraform/OpenTofu
|
||||
.terraform/
|
||||
.terraform.lock.hcl
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
*.tfvars
|
||||
!example.tfvars
|
||||
|
||||
# Go workspace
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
+15
-9
@@ -45,7 +45,7 @@ sudo add-apt-repository universe
|
||||
sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file
|
||||
```
|
||||
|
||||
Ensure the Go toolkit is installed for version 1.26.0 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
|
||||
Ensure the Go toolkit is installed for version 1.25.7 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
|
||||
|
||||
Confirm Go is correctly installed, and set in your $PATH.
|
||||
|
||||
@@ -262,22 +262,28 @@ aws ec2 describe-instance-types
|
||||
{
|
||||
"InstanceTypes": [
|
||||
{
|
||||
"InstanceType": "m8a.small",
|
||||
"InstanceType": "t3.small",
|
||||
"CurrentGeneration": true,
|
||||
"SupportedRootDeviceTypes": ["ebs"],
|
||||
"SupportedVirtualizationTypes": ["hvm"],
|
||||
"SupportedRootDeviceTypes": [
|
||||
"ebs"
|
||||
],
|
||||
"SupportedVirtualizationTypes": [
|
||||
"hvm"
|
||||
],
|
||||
"Hypervisor": "kvm",
|
||||
"ProcessorInfo": {
|
||||
"SupportedArchitectures": ["x86_64"]
|
||||
"SupportedArchitectures": [
|
||||
"x86_64"
|
||||
]
|
||||
},
|
||||
"VCpuInfo": {
|
||||
"DefaultVCpus": 2
|
||||
"DefaultVCpus": 2
|
||||
},
|
||||
"MemoryInfo": {
|
||||
"SizeInMiB": 1024
|
||||
"SizeInMiB": 2048
|
||||
},
|
||||
"BurstablePerformanceSupported": false
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -285,7 +291,7 @@ aws ec2 describe-instance-types
|
||||
Export the instance-type:
|
||||
|
||||
```sh
|
||||
export HIVE_INSTANCE="m8a.small"
|
||||
export HIVE_INSTANCE="t3.small"
|
||||
```
|
||||
|
||||
### Run Instance
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# IAC
|
||||
|
||||
Collection of OpenTofu/Terraform templates for creating Hive instances.
|
||||
|
||||
| Template | Description |
|
||||
|---|---|
|
||||
| [proxmox/](proxmox/) | Provisions a 3-node Hive cluster on Proxmox VE bare-metal hypervisors |
|
||||
| [aws/](aws/) | Provisions infrastructure using the Terraform AWS provider, with MulgaOS AWS-compatible API as the local endpoint |
|
||||
@@ -0,0 +1,206 @@
|
||||
# MulgaOS AWS Infrastructure as Code
|
||||
|
||||
This directory contains OpenTofu/Terraform configuration files for deploying infrastructure on MulgaOS using AWS-compatible APIs.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **OpenTofu or Terraform installed**
|
||||
```bash
|
||||
# Install OpenTofu (recommended)
|
||||
# See: https://opentofu.org/docs/intro/install/
|
||||
|
||||
# Or use Terraform
|
||||
# See: https://developer.hashicorp.com/terraform/install
|
||||
```
|
||||
|
||||
2. **MulgaOS services running**
|
||||
- `hive daemon` - Core daemon with NATS subscriptions
|
||||
- `awsd` - AWS API gateway (default: http://localhost:8080)
|
||||
- `predastore` - S3-compatible storage
|
||||
- `viperblock` - EBS-compatible block storage
|
||||
|
||||
3. **AWS credentials configured**
|
||||
```bash
|
||||
# Option 1: Environment variables
|
||||
export AWS_ACCESS_KEY_ID=your_access_key
|
||||
export AWS_SECRET_ACCESS_KEY=your_secret_key
|
||||
|
||||
# Option 2: AWS CLI profile
|
||||
aws configure --profile hive
|
||||
export AWS_PROFILE=hive
|
||||
```
|
||||
|
||||
4. **AMI available**
|
||||
Before running, ensure you have a Debian 12 AMI imported:
|
||||
```bash
|
||||
# Import an AMI using the hive CLI or aws CLI
|
||||
aws --endpoint-url http://localhost:8080 ec2 describe-images
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TF_VAR_mulgaos_endpoint` | `http://localhost:8080` | MulgaOS API endpoint |
|
||||
| `TF_VAR_region` | `ap-southeast-2` | AWS region (for compatibility) |
|
||||
| `AWS_ACCESS_KEY_ID` | - | MulgaOS access key |
|
||||
| `AWS_SECRET_ACCESS_KEY` | - | MulgaOS secret key |
|
||||
|
||||
### terraform.tfvars (optional)
|
||||
|
||||
Create a `terraform.tfvars` file for custom settings:
|
||||
|
||||
```hcl
|
||||
mulgaos_endpoint = "http://192.168.1.100:8080"
|
||||
region = "us-west-2"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Initialize
|
||||
|
||||
```bash
|
||||
cd iac/aws
|
||||
tofu init
|
||||
# or: terraform init
|
||||
```
|
||||
|
||||
### Validate
|
||||
|
||||
```bash
|
||||
tofu validate
|
||||
# or: terraform validate
|
||||
```
|
||||
|
||||
### Plan
|
||||
|
||||
```bash
|
||||
tofu plan
|
||||
# or: terraform plan
|
||||
```
|
||||
|
||||
### Apply
|
||||
|
||||
```bash
|
||||
tofu apply
|
||||
# or: terraform apply
|
||||
```
|
||||
|
||||
### Destroy
|
||||
|
||||
```bash
|
||||
tofu destroy
|
||||
# or: terraform destroy
|
||||
```
|
||||
|
||||
## What Gets Created
|
||||
|
||||
The `main.tf` creates:
|
||||
|
||||
1. **VPC** (`10.1.1.0/16`) - Virtual Private Cloud
|
||||
2. **Internet Gateway** - For public internet access
|
||||
3. **Public Subnet** (`10.1.2.0/24`) - With auto-assign public IP
|
||||
4. **Route Table** - Routes `0.0.0.0/0` through IGW
|
||||
5. **Security Group** - Allows SSH (22), HTTP (80), HTTPS (443) inbound
|
||||
6. **Placement Group** - Spread strategy for HA
|
||||
7. **3 EC2 Instances** - t3.small Debian 12 VMs
|
||||
8. **SSH Key Pair** - ED25519 key saved to `hive-test.pem`
|
||||
|
||||
## SSH Access
|
||||
|
||||
After `tofu apply`, SSH into instances:
|
||||
|
||||
```bash
|
||||
# Using the generated key
|
||||
ssh -i hive-test.pem admin@<public_ip>
|
||||
|
||||
# For local development with QEMU port forwarding
|
||||
# Find the forwarded port from QEMU args (-netdev user,id=net0,hostfwd=tcp:127.0.0.1:PORT-:22)
|
||||
ssh -p PORT -i hive-test.pem admin@127.0.0.1
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "No valid credential sources found"
|
||||
|
||||
Ensure AWS credentials are set:
|
||||
```bash
|
||||
export AWS_ACCESS_KEY_ID=your_key
|
||||
export AWS_SECRET_ACCESS_KEY=your_secret
|
||||
```
|
||||
|
||||
### "Error: No AMI found"
|
||||
|
||||
Import a Debian 12 AMI first, or modify `main.tf` to use a different AMI filter.
|
||||
|
||||
### Connection refused
|
||||
|
||||
Ensure MulgaOS services are running:
|
||||
```bash
|
||||
# Check if awsd gateway is running
|
||||
curl http://localhost:8080/health
|
||||
|
||||
# Check daemon status
|
||||
hive daemon status
|
||||
```
|
||||
|
||||
### "InvalidAction" errors
|
||||
|
||||
Some AWS operations may not be implemented yet. Check `TODO.md` for the current implementation status.
|
||||
|
||||
## Customization
|
||||
|
||||
### Using a different AMI
|
||||
|
||||
Modify the `data "aws_ami"` block in `main.tf`:
|
||||
|
||||
```hcl
|
||||
data "aws_ami" "custom" {
|
||||
most_recent = true
|
||||
owners = ["self"] # Your imported AMIs
|
||||
|
||||
filter {
|
||||
name = "name"
|
||||
values = ["my-custom-ami-*"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Changing instance count
|
||||
|
||||
Modify the `count` parameter:
|
||||
|
||||
```hcl
|
||||
resource "aws_instance" "vm" {
|
||||
count = 5 # Change from 3 to 5
|
||||
# ...
|
||||
}
|
||||
```
|
||||
|
||||
### Different instance type
|
||||
|
||||
```hcl
|
||||
resource "aws_instance" "vm" {
|
||||
instance_type = "t3.medium" # Change from t3.small
|
||||
# ...
|
||||
}
|
||||
```
|
||||
|
||||
## Files
|
||||
|
||||
| File | Description |
|
||||
|------|-------------|
|
||||
| `main.tf` | Main Terraform/OpenTofu configuration |
|
||||
| `hive-test.pem` | Generated SSH private key (after apply) |
|
||||
| `.terraform/` | Provider plugins (after init) |
|
||||
| `terraform.tfstate` | State file (after apply) |
|
||||
| `terraform.tfvars` | Custom variables (optional) |
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [MulgaOS DEV.md](../../DEV.md) - Development guide
|
||||
- [MulgaOS TODO.md](../../TODO.md) - Implementation status
|
||||
- [OpenTofu Documentation](https://opentofu.org/docs/)
|
||||
- [AWS Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs)
|
||||
@@ -0,0 +1,224 @@
|
||||
terraform {
|
||||
required_version = ">= 1.6.0"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 5.0"
|
||||
}
|
||||
tls = {
|
||||
source = "hashicorp/tls"
|
||||
version = ">= 4.0"
|
||||
}
|
||||
local = {
|
||||
source = "hashicorp/local"
|
||||
version = ">= 2.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
variable "region" {
|
||||
type = string
|
||||
default = "ap-southeast-2" # Sydney, change if you want
|
||||
}
|
||||
|
||||
# MulgaOS AWS-compatible endpoints
|
||||
# Set these environment variables or use terraform.tfvars:
|
||||
# export AWS_ACCESS_KEY_ID=your_access_key
|
||||
# export AWS_SECRET_ACCESS_KEY=your_secret_key
|
||||
# export TF_VAR_mulgaos_endpoint=http://localhost:8080
|
||||
variable "mulgaos_endpoint" {
|
||||
type = string
|
||||
default = "http://localhost:8080"
|
||||
description = "MulgaOS API endpoint (awsd gateway)"
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = var.region
|
||||
|
||||
# MulgaOS custom endpoints
|
||||
endpoints {
|
||||
ec2 = var.mulgaos_endpoint
|
||||
s3 = var.mulgaos_endpoint
|
||||
iam = var.mulgaos_endpoint
|
||||
sts = var.mulgaos_endpoint
|
||||
}
|
||||
|
||||
# Skip AWS credential validation (MulgaOS handles auth)
|
||||
skip_credentials_validation = true
|
||||
skip_metadata_api_check = true
|
||||
skip_requesting_account_id = true
|
||||
|
||||
# For local development, skip region validation
|
||||
skip_region_validation = true
|
||||
}
|
||||
|
||||
data "aws_availability_zones" "available" {
|
||||
state = "available"
|
||||
}
|
||||
|
||||
# Debian 12 AMI (official Debian account)
|
||||
# For MulgaOS, you may need to import a local AMI first
|
||||
data "aws_ami" "debian12" {
|
||||
most_recent = true
|
||||
owners = ["136693071363"] # Debian official
|
||||
|
||||
filter {
|
||||
name = "name"
|
||||
values = ["debian-12-amd64-*"]
|
||||
}
|
||||
|
||||
filter {
|
||||
name = "virtualization-type"
|
||||
values = ["hvm"]
|
||||
}
|
||||
|
||||
filter {
|
||||
name = "root-device-type"
|
||||
values = ["ebs"]
|
||||
}
|
||||
}
|
||||
|
||||
# Keypair: hive-test
|
||||
resource "tls_private_key" "hive_test" {
|
||||
algorithm = "ED25519"
|
||||
}
|
||||
|
||||
resource "aws_key_pair" "hive_test" {
|
||||
key_name = "hive-test"
|
||||
public_key = tls_private_key.hive_test.public_key_openssh
|
||||
}
|
||||
|
||||
# Save private key locally for SSH
|
||||
resource "local_file" "hive_test_pem" {
|
||||
filename = "${path.module}/hive-test.pem"
|
||||
content = tls_private_key.hive_test.private_key_openssh
|
||||
file_permission = "0600"
|
||||
}
|
||||
|
||||
# 1: VPC 10.1.1.0/16
|
||||
resource "aws_vpc" "main" {
|
||||
cidr_block = "10.1.1.0/16"
|
||||
enable_dns_hostnames = true
|
||||
enable_dns_support = true
|
||||
|
||||
tags = {
|
||||
Name = "hive-test-vpc"
|
||||
}
|
||||
}
|
||||
|
||||
# Internet access for a public subnet
|
||||
resource "aws_internet_gateway" "igw" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
tags = {
|
||||
Name = "hive-test-igw"
|
||||
}
|
||||
}
|
||||
|
||||
# 2: Subnet 10.1.2.0/24 (public)
|
||||
resource "aws_subnet" "public" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
cidr_block = "10.1.2.0/24"
|
||||
availability_zone = data.aws_availability_zones.available.names[0]
|
||||
map_public_ip_on_launch = true
|
||||
|
||||
tags = {
|
||||
Name = "hive-test-public-subnet"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_route_table" "public" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
route {
|
||||
cidr_block = "0.0.0.0/0"
|
||||
gateway_id = aws_internet_gateway.igw.id
|
||||
}
|
||||
|
||||
tags = {
|
||||
Name = "hive-test-public-rt"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_route_table_association" "public" {
|
||||
subnet_id = aws_subnet.public.id
|
||||
route_table_id = aws_route_table.public.id
|
||||
}
|
||||
|
||||
# 3: Security group
|
||||
resource "aws_security_group" "web_ssh" {
|
||||
name = "hive-test-sg"
|
||||
description = "Allow inbound 22,80,443; allow all outbound"
|
||||
vpc_id = aws_vpc.main.id
|
||||
|
||||
ingress {
|
||||
description = "SSH"
|
||||
from_port = 22
|
||||
to_port = 22
|
||||
protocol = "tcp"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
ingress {
|
||||
description = "HTTP"
|
||||
from_port = 80
|
||||
to_port = 80
|
||||
protocol = "tcp"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
ingress {
|
||||
description = "HTTPS"
|
||||
from_port = 443
|
||||
to_port = 443
|
||||
protocol = "tcp"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
egress {
|
||||
description = "All outbound"
|
||||
from_port = 0
|
||||
to_port = 0
|
||||
protocol = "-1"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
tags = {
|
||||
Name = "hive-test-sg"
|
||||
}
|
||||
}
|
||||
|
||||
# 4: Ensure 3 servers on different underlying hardware (spread placement group)
|
||||
resource "aws_placement_group" "spread" {
|
||||
name = "hive-test-spread"
|
||||
strategy = "spread"
|
||||
}
|
||||
|
||||
resource "aws_instance" "vm" {
|
||||
count = 3
|
||||
ami = data.aws_ami.debian12.id
|
||||
instance_type = "t3.small"
|
||||
|
||||
subnet_id = aws_subnet.public.id
|
||||
vpc_security_group_ids = [aws_security_group.web_ssh.id]
|
||||
key_name = aws_key_pair.hive_test.key_name
|
||||
|
||||
associate_public_ip_address = true
|
||||
placement_group = aws_placement_group.spread.name
|
||||
|
||||
tags = {
|
||||
Name = "hive-test-vm-${count.index + 1}"
|
||||
}
|
||||
}
|
||||
|
||||
output "instance_public_ips" {
|
||||
value = aws_instance.vm[*].public_ip
|
||||
}
|
||||
|
||||
output "ssh_commands" {
|
||||
value = [
|
||||
for ip in aws_instance.vm[*].public_ip :
|
||||
"ssh -i hive-test.pem admin@${ip}"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
# Proxmox IaC Environment Configuration
|
||||
# Copy this file to .env and fill in your values:
|
||||
# cp .env.example .env
|
||||
#
|
||||
# Then source it before running tofu:
|
||||
# source .env && tofu plan
|
||||
|
||||
# --- Proxmox API ---
|
||||
|
||||
# API token for Proxmox VE (read by the provider directly)
|
||||
export PROXMOX_VE_API_TOKEN="terraform@pve!provider=YOUR_TOKEN_HERE"
|
||||
|
||||
# Proxmox VE API endpoint URL
|
||||
export TF_VAR_proxmox_endpoint="https://pve-host.example.com:8006/"
|
||||
|
||||
# --- SSH Access ---
|
||||
|
||||
# SSH username on Proxmox hosts (used by provider for file uploads)
|
||||
export TF_VAR_proxmox_ssh_username="terraform"
|
||||
|
||||
# Path to SSH private key for Proxmox host access
|
||||
export TF_VAR_ssh_private_key_path="~/.ssh/proxmox-host-tf"
|
||||
|
||||
# Path to SSH public key injected into VMs via cloud-init (for tf-user)
|
||||
export TF_VAR_ssh_public_key_path="~/.ssh/cloud-init.pub"
|
||||
|
||||
# --- Cluster Nodes (exactly 3 required) ---
|
||||
# Each node object:
|
||||
# name - Proxmox node name (as shown in the Proxmox UI)
|
||||
# address - SSH-reachable hostname or IP of the Proxmox host
|
||||
# bridge - Network bridge for VM interfaces (e.g. vmbr0, vmbr3)
|
||||
# datastore_id - Proxmox storage ID for VM disks (e.g. local, local-lvm)
|
||||
|
||||
export TF_VAR_nodes='[
|
||||
{
|
||||
"name": "node1",
|
||||
"address": "node1.example.com",
|
||||
"bridge": "vmbr0",
|
||||
"datastore_id": "local"
|
||||
},
|
||||
{
|
||||
"name": "node2",
|
||||
"address": "node2.example.com",
|
||||
"bridge": "vmbr0",
|
||||
"datastore_id": "local"
|
||||
},
|
||||
{
|
||||
"name": "node3",
|
||||
"address": "node3.example.com",
|
||||
"bridge": "vmbr0",
|
||||
"datastore_id": "local"
|
||||
}
|
||||
]'
|
||||
@@ -0,0 +1,246 @@
|
||||
# Proxmox IaC
|
||||
|
||||
OpenTofu/Terraform templates for provisioning a 3-node Hive development cluster on Proxmox VE.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [OpenTofu](https://opentofu.org/) (or Terraform >= 1.6.0)
|
||||
- 3 Proxmox VE nodes with:
|
||||
- A Terraform API token (`Datacenter > Permissions > API Tokens`)
|
||||
- SSH access for a `terraform` user on each node
|
||||
- Debian 12 cloud image uploaded to `local:iso/` on each node
|
||||
- SSH keypair for cloud-init VM access
|
||||
- A hardcoded Debian ISO is used, which needs to be pre-installed on each Proxmox host ("debian-12-genericcloud-amd64-20240211-1654.img") https://cdimage.debian.org/images/cloud/bookworm/20240211-1654/debian-12-genericcloud-amd64-20240211-1654.qcow2
|
||||
|
||||
## Setup
|
||||
|
||||
1. Copy the example environment file and fill in your values:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# Edit .env with your Proxmox endpoint, SSH keys, and node configuration
|
||||
```
|
||||
|
||||
2. Set your Proxmox API token separately (do not store in `.env`):
|
||||
|
||||
```sh
|
||||
export PROXMOX_VE_API_TOKEN="terraform@pve!provider=YOUR_TOKEN_SECRET"
|
||||
```
|
||||
|
||||
3. Initialize and deploy:
|
||||
|
||||
```sh
|
||||
source .env
|
||||
tofu init
|
||||
tofu plan
|
||||
tofu apply
|
||||
```
|
||||
|
||||
### Example `.env` file
|
||||
|
||||
```sh
|
||||
# --- Proxmox API ---
|
||||
|
||||
# API token - set manually via: export PROXMOX_VE_API_TOKEN="terraform@pve!provider=..."
|
||||
# Format: <user>@<realm>!<token-name>=<token-secret>
|
||||
|
||||
# Proxmox VE API endpoint (the URL you use to access the web UI)
|
||||
export TF_VAR_proxmox_endpoint="https://pve1.lab.example.com:8006/"
|
||||
|
||||
# --- SSH Access ---
|
||||
|
||||
# SSH username on the Proxmox hosts (used by the provider for file uploads via SCP)
|
||||
export TF_VAR_proxmox_ssh_username="terraform"
|
||||
|
||||
# Path to SSH private key that authenticates as the above user on each Proxmox host
|
||||
export TF_VAR_ssh_private_key_path="~/.ssh/proxmox-tf"
|
||||
|
||||
# Path to SSH public key injected into VMs via cloud-init (the tf-user account)
|
||||
export TF_VAR_ssh_public_key_path="~/.ssh/proxmox-tf-cloudinit.pub"
|
||||
|
||||
# --- Cluster Nodes (exactly 3 required) ---
|
||||
#
|
||||
# Each node object:
|
||||
# name - Proxmox node name exactly as shown in the Proxmox UI sidebar
|
||||
# address - SSH-reachable hostname or IP of the Proxmox host
|
||||
# bridge - Linux bridge for VM network interfaces (check Proxmox UI > Node > Network)
|
||||
# datastore_id - Proxmox storage pool for VM disks (check Proxmox UI > Node > Storage)
|
||||
|
||||
export TF_VAR_nodes='[
|
||||
{
|
||||
"name": "pve1",
|
||||
"address": "pve1.lab.example.com",
|
||||
"bridge": "vmbr0",
|
||||
"datastore_id": "local-lvm"
|
||||
},
|
||||
{
|
||||
"name": "pve2",
|
||||
"address": "pve2.lab.example.com",
|
||||
"bridge": "vmbr0",
|
||||
"datastore_id": "local-lvm"
|
||||
},
|
||||
{
|
||||
"name": "pve3",
|
||||
"address": "pve3.lab.example.com",
|
||||
"bridge": "vmbr0",
|
||||
"datastore_id": "local-lvm"
|
||||
}
|
||||
]'
|
||||
```
|
||||
|
||||
### Environment variable reference
|
||||
|
||||
| Variable | Required | Description |
|
||||
|---|---|---|
|
||||
| `PROXMOX_VE_API_TOKEN` | Yes | Proxmox API token (read by provider directly) |
|
||||
| `TF_VAR_proxmox_endpoint` | Yes | Proxmox VE API URL (e.g. `https://host:8006/`) |
|
||||
| `TF_VAR_proxmox_ssh_username` | No | SSH user on Proxmox hosts (default: `terraform`) |
|
||||
| `TF_VAR_ssh_private_key_path` | Yes | SSH private key for Proxmox host access |
|
||||
| `TF_VAR_ssh_public_key_path` | Yes | SSH public key injected into VMs via cloud-init |
|
||||
| `TF_VAR_nodes` | Yes | JSON array of exactly 3 node objects |
|
||||
|
||||
Each node object in `TF_VAR_nodes`:
|
||||
|
||||
| Field | Example | Description |
|
||||
|---|---|---|
|
||||
| `name` | `pve1` | Proxmox node name (as shown in UI sidebar) |
|
||||
| `address` | `pve1.lab.example.com` | SSH-reachable hostname or IP |
|
||||
| `bridge` | `vmbr0` | Network bridge for VM interfaces |
|
||||
| `datastore_id` | `local-lvm` | Storage pool for VM disks |
|
||||
|
||||
## Deploy
|
||||
|
||||
```sh
|
||||
source .env
|
||||
tofu init
|
||||
tofu plan # validates config - fails if required env vars are missing
|
||||
tofu apply
|
||||
```
|
||||
|
||||
Example output:
|
||||
|
||||
```sh
|
||||
Plan: 6 to add, 0 to change, 0 to destroy.
|
||||
|
||||
Apply complete! Resources: 6 added, 0 changed, 0 destroyed.
|
||||
|
||||
Outputs:
|
||||
|
||||
iac_mulgaos_ips = {
|
||||
"node1" = {
|
||||
"data" = "10.1.3.169"
|
||||
"management" = "10.1.3.170"
|
||||
}
|
||||
"node2" = {
|
||||
"data" = "10.1.2.21"
|
||||
"management" = "10.1.3.171"
|
||||
}
|
||||
"node3" = {
|
||||
"data" = "10.1.2.22"
|
||||
"management" = "10.1.3.172"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Destroy
|
||||
|
||||
```sh
|
||||
source .env
|
||||
tofu destroy
|
||||
```
|
||||
|
||||
## SSH access
|
||||
|
||||
Connect to provisioned VMs using the cloud-init public key:
|
||||
|
||||
```sh
|
||||
ssh -i ~/.ssh/your-cloud-init-key tf-user@<VM_IP>
|
||||
```
|
||||
|
||||
## Multi-node cluster setup
|
||||
|
||||
After VMs are provisioned, initialize the Hive cluster manually. Consider moving to Ansible for automation.
|
||||
|
||||
Note, modify the NODE IP addresses with the output from the terraform deployment above.
|
||||
|
||||
### Node 1 (init)
|
||||
|
||||
Management traffic over first network adapter, with Open vSwitch (Geneve) traffic on the 2nd for VPC traffic.
|
||||
|
||||
```sh
|
||||
cd ~/Development/mulga/hive
|
||||
make
|
||||
|
||||
export HIVE_REGION="ap-southeast-2"
|
||||
export HIVE_AZ="ap-southeast-2a"
|
||||
export NODE1_MGMT_IP="10.1.3.170"
|
||||
export NODE2_MGMT_IP="10.1.3.171"
|
||||
export NODE3_MGMT_IP="10.1.3.172"
|
||||
|
||||
./bin/hive admin init \
|
||||
--node node1 \
|
||||
--bind $NODE1_MGMT_IP \
|
||||
--cluster-bind $NODE1_MGMT_IP \
|
||||
--cluster-routes $NODE1_MGMT_IP:4248 \
|
||||
--predastore-nodes $NODE1_MGMT_IP,$NODE2_MGMT_IP,$NODE3_MGMT_IP \
|
||||
--port 4432 \
|
||||
--hive-dir ~/hive/ \
|
||||
--config-dir ~/hive/config/ \
|
||||
--region $HIVE_REGION \
|
||||
--az $HIVE_AZ
|
||||
```
|
||||
|
||||
### Node 2 (join)
|
||||
|
||||
```sh
|
||||
cd ~/Development/mulga/hive
|
||||
make
|
||||
|
||||
export HIVE_REGION="ap-southeast-2"
|
||||
export HIVE_AZ="ap-southeast-2"
|
||||
export NODE1_MGMT_IP="10.1.3.170"
|
||||
export NODE2_MGMT_IP="10.1.3.171"
|
||||
export NODE3_MGMT_IP="10.1.3.172"
|
||||
|
||||
./bin/hive admin join \
|
||||
--node node2 \
|
||||
--bind $NODE2_MGMT_IP \
|
||||
--cluster-bind $NODE2_MGMT_IP \
|
||||
--cluster-routes $NODE1_MGMT_IP:4248 \
|
||||
--host $NODE1_MGMT_IP:4432 \
|
||||
--data-dir ~/hive/ \
|
||||
--config-dir ~/hive/config/ \
|
||||
--region $HIVE_REGION \
|
||||
--az $HIVE_AZ
|
||||
```
|
||||
|
||||
### Node 3 (join)
|
||||
|
||||
```sh
|
||||
cd ~/Development/mulga/hive
|
||||
make
|
||||
|
||||
export HIVE_REGION="ap-southeast-2"
|
||||
export HIVE_AZ="ap-southeast-2"
|
||||
export NODE1_MGMT_IP="10.1.3.170"
|
||||
export NODE2_MGMT_IP="10.1.3.171"
|
||||
export NODE3_MGMT_IP="10.1.3.172"
|
||||
|
||||
./bin/hive admin join \
|
||||
--node node3 \
|
||||
--bind $NODE3_MGMT_IP \
|
||||
--cluster-bind $NODE3_MGMT_IP \
|
||||
--cluster-routes $NODE1_MGMT_IP:4248 \
|
||||
--host $NODE1_MGMT_IP:4432 \
|
||||
--data-dir ~/hive/ \
|
||||
--config-dir ~/hive/config/ \
|
||||
--region $HIVE_REGION \
|
||||
--az $HIVE_AZ
|
||||
```
|
||||
|
||||
## Known issues
|
||||
|
||||
- `~/hive/config/hive.toml` - Does not add node1, node2, node3 from config automatically
|
||||
- `~/hive/config/predastore/predastore.toml` - Uses previous static local node config, needs to use the IPs for each node in the cluster
|
||||
- On multi-node deployments, NATS on the primary can timeout waiting for other nodes to start, causing a race condition where NATS fails and all dependent services fail
|
||||
- When adding 3 nodes, `nats.conf` is not updated with the 3rd node. Each node's cluster name is hardcoded to `C1` instead of per-node names
|
||||
@@ -0,0 +1,40 @@
|
||||
data "local_file" "ssh_public_key" {
|
||||
filename = pathexpand(var.ssh_public_key_path)
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_file" "iac_mulgaos_config" {
|
||||
for_each = local.nodes
|
||||
|
||||
content_type = "snippets"
|
||||
datastore_id = "local"
|
||||
node_name = each.key
|
||||
|
||||
source_raw {
|
||||
data = <<EOF
|
||||
#cloud-config
|
||||
package_update: true
|
||||
package_upgrade: true
|
||||
packages:
|
||||
- qemu-guest-agent
|
||||
- net-tools
|
||||
- git
|
||||
- make
|
||||
runcmd:
|
||||
- timedatectl set-timezone America/Los_Angeles
|
||||
- systemctl enable qemu-guest-agent
|
||||
- systemctl start qemu-guest-agent
|
||||
- ufw disable
|
||||
- sysctl -w net.core.rmem_max=4194304
|
||||
- sysctl -w net.core.wmem_max=4194304
|
||||
- usermod -aG kvm tf-user
|
||||
- mkdir -p /home/tf-user/Development/mulga/
|
||||
- "cd /home/tf-user/Development/mulga && git clone https://github.com/mulgadc/hive.git && make -C hive quickinstall"
|
||||
- "echo 'export PATH=$PATH:/usr/local/go/bin/' >> /home/tf-user/.bashrc"
|
||||
- "export PATH=$PATH:/usr/local/go/bin && cd /home/tf-user/Development/mulga/hive && ./scripts/clone-deps.sh && ./scripts/dev-setup.sh"
|
||||
- chown -R tf-user:tf-user /home/tf-user/Development/
|
||||
- echo "done" > /tmp/vendor-cloud-init-done
|
||||
EOF
|
||||
|
||||
file_name = "iac-mulgaos-vendor-cloud-config.yaml"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
# Files already exist, skip
|
||||
|
||||
#resource "proxmox_virtual_environment_download_file" "debian_cloud_image_neon" {
|
||||
# content_type = "iso"
|
||||
# datastore_id = "local"
|
||||
# node_name = "neon"
|
||||
# url = "https://cdimage.debian.org/images/cloud/bookworm/20240211-1654/debian-12-genericcloud-amd64-20240211-1654.qcow2"
|
||||
# file_name = "debian-12-genericcloud-amd64-20240211-1654.img"
|
||||
#}
|
||||
|
||||
#resource "proxmox_virtual_environment_download_file" "debian_cloud_image_radon" {
|
||||
# content_type = "iso"
|
||||
# datastore_id = "local"
|
||||
# node_name = "radon"
|
||||
# url = "https://cdimage.debian.org/images/cloud/bookworm/20240211-1654/debian-12-genericcloud-amd64-20240211-1654.qcow2"
|
||||
# file_name = "debian-12-genericcloud-amd64-20240211-1654.img"
|
||||
#}
|
||||
@@ -0,0 +1,32 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
proxmox = {
|
||||
source = "bpg/proxmox"
|
||||
version = "0.52.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
locals {
|
||||
nodes = { for idx, node in var.nodes : node.name => merge(node, { index = idx + 1 }) }
|
||||
}
|
||||
|
||||
# API token is read from PROXMOX_VE_API_TOKEN env var automatically
|
||||
provider "proxmox" {
|
||||
endpoint = var.proxmox_endpoint
|
||||
insecure = true
|
||||
|
||||
ssh {
|
||||
agent = false
|
||||
username = var.proxmox_ssh_username
|
||||
private_key = file(pathexpand(var.ssh_private_key_path))
|
||||
|
||||
dynamic "node" {
|
||||
for_each = local.nodes
|
||||
content {
|
||||
name = node.key
|
||||
address = node.value.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
variable "proxmox_endpoint" {
|
||||
type = string
|
||||
description = "Proxmox VE API endpoint URL (e.g. https://pve.example.com:8006/)"
|
||||
|
||||
validation {
|
||||
condition = can(regex("^https?://", var.proxmox_endpoint))
|
||||
error_message = "proxmox_endpoint must be a valid URL starting with http:// or https://"
|
||||
}
|
||||
}
|
||||
|
||||
variable "proxmox_ssh_username" {
|
||||
type = string
|
||||
description = "SSH username for Proxmox host access"
|
||||
default = "terraform"
|
||||
}
|
||||
|
||||
variable "ssh_private_key_path" {
|
||||
type = string
|
||||
description = "Path to SSH private key for Proxmox host access"
|
||||
|
||||
validation {
|
||||
condition = length(var.ssh_private_key_path) > 0
|
||||
error_message = "ssh_private_key_path must not be empty."
|
||||
}
|
||||
}
|
||||
|
||||
variable "ssh_public_key_path" {
|
||||
type = string
|
||||
description = "Path to SSH public key injected into VMs via cloud-init"
|
||||
|
||||
validation {
|
||||
condition = length(var.ssh_public_key_path) > 0
|
||||
error_message = "ssh_public_key_path must not be empty."
|
||||
}
|
||||
}
|
||||
|
||||
variable "nodes" {
|
||||
type = list(object({
|
||||
name = string
|
||||
address = string
|
||||
bridge = string
|
||||
datastore_id = string
|
||||
}))
|
||||
description = "Exactly 3 Proxmox nodes for the Hive cluster"
|
||||
|
||||
validation {
|
||||
condition = length(var.nodes) == 3
|
||||
error_message = "Exactly 3 nodes must be defined for the Hive cluster."
|
||||
}
|
||||
|
||||
validation {
|
||||
condition = alltrue([for n in var.nodes : length(n.name) > 0 && length(n.address) > 0])
|
||||
error_message = "Each node must have a non-empty name and address."
|
||||
}
|
||||
|
||||
validation {
|
||||
condition = length(distinct([for n in var.nodes : n.name])) == length(var.nodes)
|
||||
error_message = "Each node must have a unique name."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
resource "proxmox_virtual_environment_vm" "iac_mulgaos" {
|
||||
for_each = local.nodes
|
||||
|
||||
name = "iac-dev-mulgaos-${each.value.index}"
|
||||
description = "Managed by Terraform"
|
||||
tags = ["terraform", "mulgaos"]
|
||||
node_name = each.key
|
||||
|
||||
initialization {
|
||||
datastore_id = "local"
|
||||
vendor_data_file_id = proxmox_virtual_environment_file.iac_mulgaos_config[each.key].id
|
||||
|
||||
user_account {
|
||||
username = "tf-user"
|
||||
keys = [trimspace(data.local_file.ssh_public_key.content)]
|
||||
}
|
||||
|
||||
# Management
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
|
||||
# Data/VM
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = 4
|
||||
type = "host"
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = 8192
|
||||
}
|
||||
|
||||
agent {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
network_device {
|
||||
bridge = each.value.bridge
|
||||
}
|
||||
|
||||
network_device {
|
||||
bridge = each.value.bridge
|
||||
}
|
||||
|
||||
disk {
|
||||
file_id = "local:iso/debian-12-genericcloud-amd64-20240211-1654.img"
|
||||
datastore_id = each.value.datastore_id
|
||||
interface = "virtio0"
|
||||
iothread = true
|
||||
discard = "on"
|
||||
size = 32
|
||||
}
|
||||
}
|
||||
|
||||
output "iac_mulgaos_ips" {
|
||||
value = {
|
||||
for name, vm in proxmox_virtual_environment_vm.iac_mulgaos : name => {
|
||||
management = vm.ipv4_addresses[1][0]
|
||||
data = vm.ipv4_addresses[2][0]
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user