diff --git a/tests/infrastructure/.gitignore b/tests/infrastructure/.gitignore new file mode 100644 index 0000000..01615b5 --- /dev/null +++ b/tests/infrastructure/.gitignore @@ -0,0 +1,6 @@ +*.tfstate* +.terraform/ +*.tfvars +!terraform.tfvars.example +.terraform.lock.hcl +.api-token diff --git a/tests/infrastructure/README.md b/tests/infrastructure/README.md new file mode 100644 index 0000000..4c8af90 --- /dev/null +++ b/tests/infrastructure/README.md @@ -0,0 +1,150 @@ +# Nested Proxmox VE Test Infrastructure + +This Terraform configuration provisions a throwaway nested Proxmox VE virtual machine on an existing Proxmox host. The nested instance is used as a target for PSProxmoxVE integration tests, providing a real PVE API to test against without risking production infrastructure. + +## Prerequisites + +- **Terraform** >= 1.5.0 +- **Proxmox VE ISO** downloaded from [proxmox.com/en/downloads](https://www.proxmox.com/en/downloads) +- **API token** on the existing Proxmox host with full administrator privileges (Datastore.Allocate, VM.Allocate, VM.Config.*, Sys.Modify, etc.) +- **curl** and **jq** installed on the machine running Terraform (used by provisioner scripts) +- **SSH agent** running with a key that can access the Proxmox host (used by the bpg/proxmox provider for file uploads) +- A **routable IP address** available for the nested PVE instance (see Network section below) +- **Nested virtualization** enabled on the Proxmox host (see Intel vs AMD notes below) + +## Quick Start + +1. Copy the example variables file and fill in your values: + + ```bash + cp terraform.tfvars.example terraform.tfvars + # Edit terraform.tfvars with your Proxmox host details, ISO path, and network config + ``` + +2. Initialize Terraform and download the provider: + + ```bash + terraform init + ``` + +3. Review the plan: + + ```bash + terraform plan + ``` + +4. Apply to create the nested PVE VM: + + ```bash + terraform apply + ``` + + This will: + - Upload the PVE ISO to the target node + - Generate an answer file for unattended installation + - Create and start the nested VM + - Wait for the PVE API to become responsive (up to 10 minutes) + - Create an API token (`root@pam!integration`) for integration tests + +5. After apply completes, retrieve the test connection details: + + ```bash + terraform output pve_test_url + terraform output -raw pve_test_api_token + ``` + +## How It Works + +### Answer File + +The `answer.toml.tftpl` template generates a TOML answer file that automates the Proxmox VE installer. It configures the root password, network settings (static IP), disk layout, and other installation parameters so that no manual interaction is required during installation. + +### Wait Script + +After the VM is created and booted from the ISO, the `scripts/wait-for-api.sh` script polls the nested PVE API endpoint every 10 seconds for up to 10 minutes. Installation typically takes 3-7 minutes depending on disk and CPU performance. The script exits successfully once the API returns a valid version response. + +### API Token Creation + +Once the API is responsive, a provisioner script authenticates to the nested PVE using the root password and creates an API token (`root@pam!integration`) with full privileges (privsep=0). The token value is saved to `.api-token` and exposed via the `pve_test_api_token` output for use in integration tests. + +## Cleanup + +To destroy the nested PVE VM and all associated resources: + +```bash +terraform destroy +``` + +This removes the VM, uploaded ISO, and answer file snippet from the Proxmox host. The `.api-token` file is also cleaned up locally. + +## Network Configuration + +The nested PVE instance requires a static IP address that is: + +- **Routable** from the machine running integration tests (CI runner or developer workstation) +- **Not in use** by any other device on the network +- On the **same subnet** as the network bridge (`vmbr0` by default) on the host + +The nested PVE will configure its own `vmbr0` bridge internally, but from the host's perspective it appears as a single VM with the assigned IP address. + +If your test environment uses VLANs or an isolated test network, adjust the `network_bridge` variable accordingly and ensure the CI runner has connectivity to that network. + +## Resource Requirements + +The nested PVE VM requires sufficient resources to run Proxmox VE and potentially host lightweight test VMs inside it: + +| Resource | Minimum | Default | Recommended | +|----------|---------|---------|-------------| +| CPU cores | 2 | 4 | 4+ | +| Memory | 4096 MB | 8192 MB | 8192+ MB | +| Disk | 32 GB | 64 GB | 64+ GB | + +Ensure the Proxmox host has enough free resources to accommodate these allocations. + +## Intel vs AMD Nested Virtualization + +Nested virtualization must be enabled on the host for the nested PVE to function as a hypervisor itself. The CPU type is set to `host` to pass through virtualization extensions. + +### Intel + +Nested virtualization is typically enabled by default on modern Intel CPUs. Verify with: + +```bash +cat /sys/module/kvm_intel/parameters/nested +``` + +If it shows `N`, enable it: + +```bash +echo "options kvm_intel nested=1" > /etc/modprobe.d/kvm-intel.conf +modprobe -r kvm_intel && modprobe kvm_intel +``` + +### AMD + +AMD nested virtualization support varies. Check with: + +```bash +cat /sys/module/kvm_amd/parameters/nested +``` + +If it shows `0`, enable it: + +```bash +echo "options kvm_amd nested=1" > /etc/modprobe.d/kvm-amd.conf +modprobe -r kvm_amd && modprobe kvm_amd +``` + +Note that AMD nested virtualization can be less stable than Intel in some configurations. If you encounter issues with nested VMs inside the nested PVE, the integration tests themselves (which test the PVE API, not nested VM creation) will still work -- only tests that attempt to create VMs inside the nested PVE would be affected. + +## Files + +| File | Purpose | +|------|---------| +| `main.tf` | Provider config, VM resource, provisioners | +| `variables.tf` | Input variable definitions with defaults | +| `outputs.tf` | Test connection details for integration tests | +| `answer.toml.tftpl` | Unattended PVE installer answer file template | +| `scripts/wait-for-api.sh` | Polls PVE API until responsive | +| `terraform.tfvars.example` | Example variable values | +| `.gitignore` | Excludes state, provider cache, secrets | diff --git a/tests/infrastructure/answer.toml.tftpl b/tests/infrastructure/answer.toml.tftpl new file mode 100644 index 0000000..7415025 --- /dev/null +++ b/tests/infrastructure/answer.toml.tftpl @@ -0,0 +1,18 @@ +[global] +keyboard = "en-us" +country = "us" +fqdn = "pve-test.local" +mailto = "test@test.local" +timezone = "UTC" +root_password = "${root_password}" + +[network] +source = "from-answer" +cidr = "${cidr}" +dns = "${dns}" +gateway = "${gateway}" +filter.ID_NET_NAME = "*" + +[disk-setup] +filesystem = "ext4" +disk_list = ["sda"] diff --git a/tests/infrastructure/main.tf b/tests/infrastructure/main.tf new file mode 100644 index 0000000..86b606f --- /dev/null +++ b/tests/infrastructure/main.tf @@ -0,0 +1,140 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + proxmox = { + source = "bpg/proxmox" + version = ">= 0.70.0" + } + } +} + +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + insecure = var.proxmox_insecure + + ssh { + agent = true + } +} + +# Upload PVE ISO to the target node +resource "proxmox_virtual_environment_file" "pve_iso" { + content_type = "iso" + datastore_id = var.iso_storage + node_name = var.target_node + + source_file { + path = var.iso_file + } +} + +# Answer file for unattended PVE installation +resource "proxmox_virtual_environment_file" "answer_file" { + content_type = "snippets" + datastore_id = var.answer_file_storage + node_name = var.target_node + + source_raw { + data = templatefile("${path.module}/answer.toml.tftpl", { + root_password = var.test_vm_password + cidr = "${var.test_vm_ip}/${var.test_vm_netmask_bits}" + dns = var.test_vm_dns + gateway = var.test_vm_gateway + }) + file_name = "pve-test-answer.toml" + } +} + +# Nested PVE VM +resource "proxmox_virtual_environment_vm" "nested_pve" { + name = var.vm_name + node_name = var.target_node + vm_id = var.vm_id + description = "Automated nested PVE test instance - safe to destroy" + tags = ["test", "nested-pve", "auto-managed"] + + machine = "q35" + bios = "ovmf" + + cpu { + type = "host" + cores = var.cores + sockets = 1 + } + + memory { + dedicated = var.memory + } + + efi_disk { + datastore_id = var.disk_storage + type = "4m" + } + + disk { + datastore_id = var.disk_storage + interface = "scsi0" + size = var.disk_size + file_format = "raw" + } + + cdrom { + enabled = true + file_id = proxmox_virtual_environment_file.pve_iso.id + interface = "ide2" + } + + network_device { + bridge = var.network_bridge + model = "virtio" + } + + boot_order = ["scsi0", "ide2"] + + operating_system { + type = "l26" + } + + on_boot = true + started = true + + lifecycle { + ignore_changes = [cdrom] + } +} + +# Wait for PVE API to become responsive after installation +resource "null_resource" "wait_for_api" { + depends_on = [proxmox_virtual_environment_vm.nested_pve] + + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-api.sh ${var.test_vm_ip} 8006 600" + } +} + +# Create a test API token on the nested PVE for integration tests +resource "null_resource" "create_api_token" { + depends_on = [null_resource.wait_for_api] + + provisioner "local-exec" { + command = <<-EOT + # Get a ticket first + TICKET_DATA=$(curl -sk -d "username=root@pam&password=${var.test_vm_password}" \ + "https://${var.test_vm_ip}:8006/api2/json/access/ticket") + TICKET=$(echo "$TICKET_DATA" | jq -r '.data.ticket') + CSRF=$(echo "$TICKET_DATA" | jq -r '.data.CSRFPreventionToken') + + # Create API token + TOKEN_DATA=$(curl -sk -X POST \ + -H "Cookie: PVEAuthCookie=$TICKET" \ + -H "CSRFPreventionToken: $CSRF" \ + -d "tokenid=integration" \ + -d "privsep=0" \ + "https://${var.test_vm_ip}:8006/api2/json/access/users/root@pam/token/integration") + + TOKEN_VALUE=$(echo "$TOKEN_DATA" | jq -r '.data.value') + echo "root@pam!integration=$TOKEN_VALUE" > ${path.module}/.api-token + EOT + } +} diff --git a/tests/infrastructure/outputs.tf b/tests/infrastructure/outputs.tf new file mode 100644 index 0000000..a972799 --- /dev/null +++ b/tests/infrastructure/outputs.tf @@ -0,0 +1,25 @@ +output "pve_test_host" { + value = var.test_vm_ip +} + +output "pve_test_port" { + value = 8006 +} + +output "pve_test_url" { + value = "https://${var.test_vm_ip}:8006" +} + +output "pve_test_vm_id" { + value = var.vm_id +} + +output "pve_test_node_name" { + value = "pve" + description = "Default node name inside a fresh PVE install" +} + +output "pve_test_api_token" { + value = fileexists("${path.module}/.api-token") ? trimspace(file("${path.module}/.api-token")) : "not-yet-created" + sensitive = true +} diff --git a/tests/infrastructure/scripts/wait-for-api.sh b/tests/infrastructure/scripts/wait-for-api.sh new file mode 100755 index 0000000..2855385 --- /dev/null +++ b/tests/infrastructure/scripts/wait-for-api.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOST="$1" +PORT="${2:-8006}" +MAX_WAIT="${3:-600}" +INTERVAL=10 + +echo "Waiting for PVE API at https://${HOST}:${PORT}..." +elapsed=0 +while [ $elapsed -lt $MAX_WAIT ]; do + if curl -sk --connect-timeout 5 "https://${HOST}:${PORT}/api2/json/version" 2>/dev/null | grep -q '"version"'; then + echo "PVE API is responsive after ${elapsed}s" + exit 0 + fi + echo " Not ready yet (${elapsed}s elapsed)..." + sleep $INTERVAL + elapsed=$((elapsed + INTERVAL)) +done + +echo "ERROR: PVE API not responsive after ${MAX_WAIT}s" +exit 1 diff --git a/tests/infrastructure/terraform.tfvars.example b/tests/infrastructure/terraform.tfvars.example new file mode 100644 index 0000000..f5d11f2 --- /dev/null +++ b/tests/infrastructure/terraform.tfvars.example @@ -0,0 +1,18 @@ +# Connection to your existing Proxmox host +proxmox_endpoint = "https://pve.example.com:8006" +proxmox_api_token = "root@pam!terraform=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" +target_node = "pve" + +# Path to PVE ISO (download from proxmox.com/en/downloads) +iso_file = "/path/to/proxmox-ve_9.1-1.iso" + +# Network config for nested PVE (must be routable from CI runner) +test_vm_ip = "192.168.1.200" +test_vm_gateway = "192.168.1.1" + +# Optional overrides +# cores = 4 +# memory = 8192 +# disk_size = 64 +# disk_storage = "local-lvm" +# network_bridge = "vmbr0" diff --git a/tests/infrastructure/variables.tf b/tests/infrastructure/variables.tf new file mode 100644 index 0000000..ec6cc40 --- /dev/null +++ b/tests/infrastructure/variables.tf @@ -0,0 +1,109 @@ +variable "proxmox_endpoint" { + description = "URL of the existing Proxmox VE API (e.g. https://pve.example.com:8006)" + type = string +} + +variable "proxmox_api_token" { + description = "API token for authenticating with the existing Proxmox host (user@realm!tokenid=secret)" + type = string + sensitive = true +} + +variable "proxmox_insecure" { + description = "Whether to skip TLS verification when connecting to the Proxmox API" + type = bool + default = true +} + +variable "target_node" { + description = "Name of the Proxmox node where the nested PVE VM will be created" + type = string +} + +variable "vm_id" { + description = "VMID to assign to the nested PVE virtual machine" + type = number + default = 99900 +} + +variable "vm_name" { + description = "Name for the nested PVE virtual machine" + type = string + default = "pve-test-nested" +} + +variable "cores" { + description = "Number of CPU cores to allocate to the nested PVE VM" + type = number + default = 4 +} + +variable "memory" { + description = "Amount of memory in MB to allocate to the nested PVE VM" + type = number + default = 8192 +} + +variable "disk_size" { + description = "Size of the primary disk in GB for the nested PVE VM" + type = number + default = 64 +} + +variable "disk_storage" { + description = "Proxmox storage pool for VM disks (must support raw format)" + type = string + default = "local-lvm" +} + +variable "iso_storage" { + description = "Proxmox storage pool for uploading the PVE ISO" + type = string + default = "local" +} + +variable "iso_file" { + description = "Local path to the Proxmox VE installation ISO file" + type = string +} + +variable "network_bridge" { + description = "Network bridge on the host to attach the nested PVE VM to" + type = string + default = "vmbr0" +} + +variable "test_vm_ip" { + description = "Static IP address to assign to the nested PVE instance" + type = string +} + +variable "test_vm_gateway" { + description = "Default gateway for the nested PVE instance" + type = string +} + +variable "test_vm_netmask_bits" { + description = "CIDR prefix length for the nested PVE network (e.g. 24 for /24)" + type = string + default = "24" +} + +variable "test_vm_dns" { + description = "DNS server for the nested PVE instance" + type = string + default = "1.1.1.1" +} + +variable "test_vm_password" { + description = "Root password for the nested PVE instance" + type = string + sensitive = true + default = "Testpass123!" +} + +variable "answer_file_storage" { + description = "Proxmox storage pool for the answer file snippet (must support snippets content type)" + type = string + default = "local" +}