diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 382ada9..b20f66a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -24,6 +24,9 @@ name: Integration Tests # node with the most free memory and refuses to run when # no node has enough headroom — the token's user needs # PVEAuditor on /nodes for the memory stats. +# STORAGE_VM_IP - Static CIDR address for the shared storage/answer-server +# VM on the CI VLAN (default: 172.16.60.60/24) +# STORAGE_VM_GATEWAY - Gateway + DNS for that VM (default: 172.16.60.1) # # Optional secrets (for skip_provision mode): # PVETEST_HOST - Hostname or IP of a pre-existing nested PVE @@ -122,11 +125,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} volumes: - /opt/pve-integration:/opt/pve-integration - # WARNING: The Docker socket is mounted to allow storage containers to be - # managed from within the job container. This grants root-equivalent access - # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted - # runners — never on shared runners. - - /var/run/docker.sock:/var/run/docker.sock steps: - uses: actions/checkout@v7 @@ -138,6 +136,8 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} + STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} + STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} @@ -201,11 +201,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} volumes: - /opt/pve-integration:/opt/pve-integration - # WARNING: The Docker socket is mounted to allow storage containers to be - # stopped from within the job container. This grants root-equivalent access - # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted - # runners — never on shared runners. - - /var/run/docker.sock:/var/run/docker.sock steps: - uses: actions/checkout@v7 @@ -217,6 +212,8 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} + STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} + STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} diff --git a/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 b/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 index 91901d4..6fd84a0 100644 --- a/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 +++ b/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 @@ -4,7 +4,7 @@ Pester 5 integration tests for shared storage backends (NFS, iSCSI). These tests require the multi-node integration test infrastructure with - Docker-based storage services (iSCSI target + NFS server). They are + the shared storage VM (iSCSI target + NFS server on the CI VLAN). They are SKIPPED when the storage env vars are not set. Required environment variables (in addition to base integration vars): diff --git a/tests/infrastructure/main.tf b/tests/infrastructure/main.tf index 675adab..0155296 100644 --- a/tests/infrastructure/main.tf +++ b/tests/infrastructure/main.tf @@ -3,11 +3,8 @@ terraform { required_providers { proxmox = { source = "bpg/proxmox" - version = ">= 0.70.0" - } - docker = { - source = "kreuzwerker/docker" - version = ">= 3.0.0" + # disk.import_from and content_type "import" need >= 0.79.0 + version = ">= 0.79.0" } } } @@ -18,11 +15,6 @@ provider "proxmox" { insecure = var.proxmox_insecure } -provider "docker" { - # Uses the Docker socket from the dev-infra container - # (mounted at /var/run/docker.sock) -} - resource "proxmox_virtual_environment_file" "auto_iso" { for_each = var.pve_isos content_type = "iso" @@ -91,9 +83,10 @@ resource "proxmox_virtual_environment_vm" "nested_pve" { started = true - # VMs must not boot until the HTTP answer server is running, - # otherwise the PVE auto-installer can't fetch its answer file. - depends_on = [docker_container.answer_server] + # VMs must not boot until the storage VM exists — it serves the HTTP answer + # files the PVE auto-installer fetches. run-integration.sh additionally + # configures the storage VM (phase one) before applying these resources. + depends_on = [proxmox_virtual_environment_vm.storage] lifecycle { ignore_changes = [started, cdrom] diff --git a/tests/infrastructure/outputs.tf b/tests/infrastructure/outputs.tf index 565f7bb..dea5cd4 100644 --- a/tests/infrastructure/outputs.tf +++ b/tests/infrastructure/outputs.tf @@ -8,17 +8,7 @@ output "pve_test_node_name" { description = "Default node name inside a fresh PVE install" } -output "storage_ip" { - description = "IP address where storage services are reachable" - value = var.docker_host_ip -} - -output "storage_iscsi_iqn" { - description = "iSCSI target IQN" - value = var.storage_iscsi_iqn -} - -output "storage_nfs_export" { - description = "NFS export path" - value = "${var.docker_host_ip}:/srv/nfs/shared" +output "storage_vm_ip" { + description = "Static address of the shared NFS/iSCSI storage VM" + value = var.storage_vm_ip } diff --git a/tests/infrastructure/scripts/setup-storage-server.sh b/tests/infrastructure/scripts/setup-storage-server.sh new file mode 100644 index 0000000..8e76149 --- /dev/null +++ b/tests/infrastructure/scripts/setup-storage-server.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Configures the shared services VM inside the CI sandbox VLAN: NFS export, +# iSCSI target, and the HTTP answer server the PVE auto-installers fetch +# from. Idempotent — safe to re-run against an already-configured VM. +# +# Usage: setup-storage-server.sh +# is the local directory holding default.toml and answers/. + +set -euo pipefail + +STORAGE_IP="${1:?Usage: setup-storage-server.sh }" +SSH_KEY="${2:?missing SSH private key path}" +ISCSI_IQN="${3:?missing iSCSI IQN}" +ANSWER_DIR="${4:?missing answer directory}" + +# Same pinned image the Terraform-managed runner container used previously. +ANSWER_SERVER_IMAGE="slothcroissant/proxmox-auto-installer-server@sha256:0f45d7bfe6e3cc76aa00fc578e40b80b9054e377db18a79122866fe5522bc7ed" + +SSH_OPTS="-i ${SSH_KEY} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" + +echo "=== Configuring storage services on ${STORAGE_IP} ===" + +# Cloud-init may still be running on first boot; wait for SSH to accept the +# key login it installs. The last attempt's stderr is kept so an auth failure +# is distinguishable from an unreachable host. +for i in $(seq 1 60); do + if ssh ${SSH_OPTS} -o ConnectTimeout=5 "ubuntu@${STORAGE_IP}" true 2>/tmp/storage-ssh-err; then + break + fi + if [[ $i -eq 60 ]]; then + echo "ERROR: storage VM at ${STORAGE_IP} not reachable over SSH" >&2 + cat /tmp/storage-ssh-err >&2 + exit 1 + fi + sleep 5 +done + +# sshd accepts logins before cloud-init's first-boot work (and its apt/dpkg +# locks) is finished; a degraded cloud-init result is fine for our purposes, +# and a hung cloud-init must fail fast, not eat the job timeout. +timeout 300 ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ + "sudo cloud-init status --wait" || true + +echo "Copying answer files..." +ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ + "sudo mkdir -p /opt/answer-server && sudo rm -rf /opt/answer-server/answers && sudo chown ubuntu /opt/answer-server" +scp ${SSH_OPTS} -r \ + "${ANSWER_DIR}/default.toml" "${ANSWER_DIR}/answers" \ + "ubuntu@${STORAGE_IP}:/opt/answer-server/" + +ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ + "sudo ISCSI_IQN='${ISCSI_IQN}' ANSWER_SERVER_IMAGE='${ANSWER_SERVER_IMAGE}' bash -s" <<'REMOTE' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +# cloud-init's first-boot apt activity can hold the dpkg lock briefly +for i in $(seq 1 30); do + apt-get update -qq >/dev/null 2>&1 && break + sleep 5 +done +apt-get install -y -qq nfs-kernel-server tgt docker.io >/dev/null + +# Ubuntu's nfs-kernel-server package does not create /etc/exports.d +mkdir -p /etc/exports.d /srv/nfs/shared /srv/iscsi +echo '/srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)' > /etc/exports.d/pvetest.exports +exportfs -ra + +if [ ! -f /srv/iscsi/lun0.img ]; then + truncate -s 10G /srv/iscsi/lun0.img +fi +cat > /etc/tgt/conf.d/pvetest.conf < + backing-store /srv/iscsi/lun0.img + +CONF + +systemctl enable --now nfs-kernel-server tgt >/dev/null || { + journalctl -u nfs-kernel-server -u tgt --no-pager -n 30 >&2 + exit 1 +} +systemctl restart tgt + +docker rm -f pvetest-answer-server >/dev/null 2>&1 || true +docker run -d --name pvetest-answer-server --restart unless-stopped \ + --network host \ + -v /opt/answer-server/answers:/app/answers \ + -v /opt/answer-server/default.toml:/app/default.toml \ + "${ANSWER_SERVER_IMAGE}" >/dev/null + +# Ordering requirement: the PVE installers fetch answers as soon as this +# script returns, so :8000 must accept connections before exit (any HTTP +# status counts as listening). +for i in $(seq 1 20); do + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 2 http://127.0.0.1:8000/ || true) + [ "$code" != "000" ] && break + if [ "$i" -eq 20 ]; then + echo "ERROR: answer server did not start listening on :8000" >&2 + docker logs pvetest-answer-server 2>&1 | tail -20 >&2 + exit 1 + fi + sleep 3 +done + +echo "Services ready: NFS /srv/nfs/shared, iSCSI ${ISCSI_IQN}, answer server :8000" +REMOTE + +echo "Storage server configuration complete." diff --git a/tests/infrastructure/storage-vm.tf b/tests/infrastructure/storage-vm.tf new file mode 100644 index 0000000..40c76c6 --- /dev/null +++ b/tests/infrastructure/storage-vm.tf @@ -0,0 +1,80 @@ +# ── Shared storage VM ─────────────────────────────────────────────── +# One small Ubuntu guest in the CI pool serves NFS + iSCSI to every nested +# PVE node. It lives on the same isolated VLAN as the nodes under test, so +# no firewall path out of the CI sandbox is needed. Package install and +# service configuration happen over SSH (setup-storage-server.sh) after boot. + +resource "proxmox_virtual_environment_file" "storage_cloud_image" { + content_type = "import" + datastore_id = var.iso_storage + node_name = var.target_node + overwrite = true + + source_file { + path = var.cloud_image_path + } +} + +resource "proxmox_virtual_environment_vm" "storage" { + name = "pvetest-storage" + node_name = var.target_node + vm_id = var.storage_vmid + pool_id = var.pool_id + + cpu { + type = "host" + cores = 2 + sockets = 1 + } + + memory { + dedicated = 2048 + } + + disk { + datastore_id = var.disk_storage + interface = "scsi0" + size = 32 + import_from = proxmox_virtual_environment_file.storage_cloud_image.id + } + + initialization { + datastore_id = var.disk_storage + + ip_config { + ipv4 { + address = var.storage_vm_ip + gateway = var.storage_vm_gateway + } + } + + dns { + servers = [var.storage_vm_gateway] + } + + user_account { + username = "ubuntu" + # Password is console-only: Ubuntu cloud images refuse SSH password auth + # and PVE's cloud-init never sets ssh_pwauth. SSH uses the key. + password = var.test_vm_password + keys = [trimspace(var.storage_vm_ssh_public_key)] + } + } + + network_device { + bridge = var.network_bridge + model = "virtio" + } + + operating_system { + type = "l26" + } + + # Resized Ubuntu cloud images kernel-panic on boot without a serial console + # (bpg/terraform-provider-proxmox documented issue). + serial_device { + device = "socket" + } + + started = true +} diff --git a/tests/infrastructure/variables.tf b/tests/infrastructure/variables.tf index 111ee50..4c18863 100644 --- a/tests/infrastructure/variables.tf +++ b/tests/infrastructure/variables.tf @@ -61,7 +61,7 @@ variable "disk_storage" { } variable "iso_storage" { - description = "Proxmox storage pool for uploading the ISO (must accept ISO content type)" + description = "Proxmox storage pool for uploads (must accept the iso AND import content types — import is not enabled by default on most storages)" type = string default = "local" } @@ -78,28 +78,40 @@ variable "test_vm_password" { sensitive = true } -variable "storage_iscsi_iqn" { - description = "iSCSI target IQN for the test storage" - type = string - default = "iqn.2024-01.local.test:storage" +variable "storage_vmid" { + description = "VMID for the shared storage VM (must be inside the CI pool's reserved range)" + type = number + default = 5080 } -variable "storage_iscsi_lun_size" { - description = "Size of the iSCSI LUN backing file" +variable "storage_vm_ip" { + description = "Static IPv4 address in CIDR form for the storage VM, outside the VLAN's DHCP range" type = string - default = "10G" + default = "172.16.60.60/24" + + validation { + condition = can(cidrnetmask(var.storage_vm_ip)) + error_message = "storage_vm_ip must be CIDR notation, e.g. 172.16.60.60/24." + } } -variable "docker_host_ip" { - description = "IP of the Docker host, used by PVE nodes to reach storage containers" +variable "storage_vm_ssh_public_key" { + description = "SSH public key granted to the storage VM's ubuntu user (cloud images refuse password SSH; required for provision, unused on destroy)" type = string + default = "" } -variable "answer_server_dir" { - description = "Host path to the answer server root directory (contains default.toml and answers/ subdirectory)" +variable "storage_vm_gateway" { + description = "Gateway (and DNS server) for the storage VM" type = string + default = "172.16.60.1" } +variable "cloud_image_path" { + description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)" + type = string + default = "" +} variable "pool_id" { description = "Resource pool the nested VMs are created in (a pool-scoped API token can only allocate here)"