diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 382ada9..b20f66a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -24,6 +24,9 @@ name: Integration Tests # node with the most free memory and refuses to run when # no node has enough headroom — the token's user needs # PVEAuditor on /nodes for the memory stats. +# STORAGE_VM_IP - Static CIDR address for the shared storage/answer-server +# VM on the CI VLAN (default: 172.16.60.60/24) +# STORAGE_VM_GATEWAY - Gateway + DNS for that VM (default: 172.16.60.1) # # Optional secrets (for skip_provision mode): # PVETEST_HOST - Hostname or IP of a pre-existing nested PVE @@ -122,11 +125,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} volumes: - /opt/pve-integration:/opt/pve-integration - # WARNING: The Docker socket is mounted to allow storage containers to be - # managed from within the job container. This grants root-equivalent access - # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted - # runners — never on shared runners. - - /var/run/docker.sock:/var/run/docker.sock steps: - uses: actions/checkout@v7 @@ -138,6 +136,8 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} + STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} + STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} @@ -201,11 +201,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} volumes: - /opt/pve-integration:/opt/pve-integration - # WARNING: The Docker socket is mounted to allow storage containers to be - # stopped from within the job container. This grants root-equivalent access - # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted - # runners — never on shared runners. - - /var/run/docker.sock:/var/run/docker.sock steps: - uses: actions/checkout@v7 @@ -217,6 +212,8 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} + STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} + STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} diff --git a/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 b/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 index 91901d4..6fd84a0 100644 --- a/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 +++ b/tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1 @@ -4,7 +4,7 @@ Pester 5 integration tests for shared storage backends (NFS, iSCSI). These tests require the multi-node integration test infrastructure with - Docker-based storage services (iSCSI target + NFS server). They are + the shared storage VM (iSCSI target + NFS server on the CI VLAN). They are SKIPPED when the storage env vars are not set. Required environment variables (in addition to base integration vars): diff --git a/tests/infrastructure/README.md b/tests/infrastructure/README.md index c2d82c9..cc4ab41 100644 --- a/tests/infrastructure/README.md +++ b/tests/infrastructure/README.md @@ -163,7 +163,9 @@ Note that AMD nested virtualization can be less stable than Intel in some config | `main.tf` | Provider config, VM resource, provisioners | | `variables.tf` | Input variable definitions with defaults | | `outputs.tf` | Test connection details for integration tests | +| `storage-vm.tf` | Shared NFS/iSCSI/answer-server VM on the CI VLAN | | `answer.toml.tftpl` | Unattended PVE installer answer file template | +| `scripts/setup-storage-server.sh` | Configures services on the storage VM over SSH | | `scripts/wait-for-api.sh` | Polls PVE API until responsive | | `terraform.tfvars.example` | Example variable values | | `.gitignore` | Excludes state, provider cache, secrets | diff --git a/tests/infrastructure/docker-compose.storage.yml b/tests/infrastructure/docker-compose.storage.yml deleted file mode 100644 index 40eb511..0000000 --- a/tests/infrastructure/docker-compose.storage.yml +++ /dev/null @@ -1,66 +0,0 @@ -## Shared storage services for integration testing. -## Provides iSCSI target and NFS server accessible from nested PVE nodes. -## -## These run on the GitHub runner (or any Docker host on the same network -## as the PVE nodes) using host networking so the PVE nodes can reach them. -## -## Usage: -## docker compose -f tests/infrastructure/docker-compose.storage.yml up -d -## docker compose -f tests/infrastructure/docker-compose.storage.yml down -v - -services: - iscsi-target: - image: ubuntu:24.04 - container_name: pvetest-iscsi - network_mode: host - privileged: true - volumes: - - iscsi-data:/srv/iscsi - environment: - ISCSI_IQN: ${ISCSI_IQN:-iqn.2024-01.local.test:storage} - ISCSI_LUN_SIZE: ${ISCSI_LUN_SIZE:-10G} - entrypoint: ["/bin/bash", "-c"] - command: - - | - set -e - apt-get update -qq && apt-get install -y -qq tgt >/dev/null 2>&1 - mkdir -p /srv/iscsi - if [ ! -f /srv/iscsi/lun0.img ]; then - truncate -s $$ISCSI_LUN_SIZE /srv/iscsi/lun0.img - fi - tgtd --foreground & - sleep 2 - # Create target if it does not already exist - if ! tgtadm --lld iscsi --op show --mode target | grep -q "Target 1: $$ISCSI_IQN"; then - tgtadm --lld iscsi --op new --mode target --tid 1 -T $$ISCSI_IQN - fi - # Create logical unit (LUN 1) if it does not already exist - if ! tgtadm --lld iscsi --op show --mode logicalunit --tid 1 2>/dev/null | grep -qE "LUN:[[:space:]]*1($|[^0-9])"; then - tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 --backing-store /srv/iscsi/lun0.img - fi - # Bind target to all initiators if not already bound - if ! tgtadm --lld iscsi --op show --mode target --tid 1 2>/dev/null | grep -q "Initiator-address: ALL"; then - tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL - fi - echo "iSCSI target ready: $$ISCSI_IQN (port 3260)" - wait - restart: unless-stopped - - nfs-server: - image: erichough/nfs-server:2.2.1 - container_name: pvetest-nfs - network_mode: host - privileged: true - volumes: - - nfs-data:/srv/nfs/shared - # NOTE: /lib/modules is required by the NFS kernel module loader. - # This mount is Linux-host-specific and will fail on macOS/Windows Docker Desktop. - # Integration tests using this compose file must run on a Linux host (e.g. GitHub-hosted runners or the dev container). - - /lib/modules:/lib/modules:ro - environment: - NFS_EXPORT_0: /srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash) - restart: unless-stopped - -volumes: - iscsi-data: - nfs-data: diff --git a/tests/infrastructure/main.tf b/tests/infrastructure/main.tf index 675adab..0155296 100644 --- a/tests/infrastructure/main.tf +++ b/tests/infrastructure/main.tf @@ -3,11 +3,8 @@ terraform { required_providers { proxmox = { source = "bpg/proxmox" - version = ">= 0.70.0" - } - docker = { - source = "kreuzwerker/docker" - version = ">= 3.0.0" + # disk.import_from and content_type "import" need >= 0.79.0 + version = ">= 0.79.0" } } } @@ -18,11 +15,6 @@ provider "proxmox" { insecure = var.proxmox_insecure } -provider "docker" { - # Uses the Docker socket from the dev-infra container - # (mounted at /var/run/docker.sock) -} - resource "proxmox_virtual_environment_file" "auto_iso" { for_each = var.pve_isos content_type = "iso" @@ -91,9 +83,10 @@ resource "proxmox_virtual_environment_vm" "nested_pve" { started = true - # VMs must not boot until the HTTP answer server is running, - # otherwise the PVE auto-installer can't fetch its answer file. - depends_on = [docker_container.answer_server] + # VMs must not boot until the storage VM exists — it serves the HTTP answer + # files the PVE auto-installer fetches. run-integration.sh additionally + # configures the storage VM (phase one) before applying these resources. + depends_on = [proxmox_virtual_environment_vm.storage] lifecycle { ignore_changes = [started, cdrom] diff --git a/tests/infrastructure/outputs.tf b/tests/infrastructure/outputs.tf index 565f7bb..dea5cd4 100644 --- a/tests/infrastructure/outputs.tf +++ b/tests/infrastructure/outputs.tf @@ -8,17 +8,7 @@ output "pve_test_node_name" { description = "Default node name inside a fresh PVE install" } -output "storage_ip" { - description = "IP address where storage services are reachable" - value = var.docker_host_ip -} - -output "storage_iscsi_iqn" { - description = "iSCSI target IQN" - value = var.storage_iscsi_iqn -} - -output "storage_nfs_export" { - description = "NFS export path" - value = "${var.docker_host_ip}:/srv/nfs/shared" +output "storage_vm_ip" { + description = "Static address of the shared NFS/iSCSI storage VM" + value = var.storage_vm_ip } diff --git a/tests/infrastructure/scripts/run-integration.sh b/tests/infrastructure/scripts/run-integration.sh index 6acaf61..aa238e5 100644 --- a/tests/infrastructure/scripts/run-integration.sh +++ b/tests/infrastructure/scripts/run-integration.sh @@ -4,11 +4,12 @@ # Single source of truth for the provision → test → cleanup lifecycle. # Called by both the GitHub Actions workflow and the local dev container. # -# Provisions two PVE nodes per version (a/b) for cluster testing, plus -# Docker containers on the runner host for iSCSI/NFS shared storage. +# Provisions two PVE nodes per version (a/b) for cluster testing, plus a +# small storage VM on the same isolated VLAN serving NFS, iSCSI, and the +# auto-install answer files. # # Usage: -# run-integration.sh provision [8|9|all] Provision nested PVE VMs + start storage containers +# run-integration.sh provision [8|9|all] Provision storage VM + nested PVE VMs # run-integration.sh test [8|9|all] [filter] Run integration tests (default: all, no filter) # run-integration.sh cleanup [8|9|all] Destroy provisioned VMs # run-integration.sh all [8|9|all] Full lifecycle: provision → test → cleanup @@ -42,6 +43,10 @@ # MODULE_ARTIFACT Path to built module DLLs (default: ./publish/netstandard2.0) # PVE_VERSIONS Space-separated versions to provision (default: "9 8") # STORAGE_ISCSI_IQN iSCSI IQN for storage target (default: iqn.2024-01.local.test:storage) +# STORAGE_VM_IP Static CIDR address for the storage VM on the CI VLAN, +# outside the DHCP range (default: 172.16.60.60/24) +# STORAGE_VM_GATEWAY Gateway + DNS for the storage VM (default: 172.16.60.1) +# STORAGE_VMID VMID for the storage VM (default: 5080) set -euo pipefail @@ -50,7 +55,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INFRA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" REPO_ROOT="$(cd "$INFRA_DIR/../.." && pwd)" -# ── Defaults ──────────────────────────────────────────────────────── +# ── Defaults ────────────────────────────────────────────────────── CACHE_DIR="${CACHE_DIR:-/opt/pve-integration}" # Always use a path under CACHE_DIR (shared mount) so files are visible # to sibling Docker containers. Do NOT use RUNNER_TEMP — it's container-local @@ -61,11 +66,18 @@ MODULE_ARTIFACT="${MODULE_ARTIFACT:-$REPO_ROOT/publish/netstandard2.0}" PVE_VERSIONS="${PVE_VERSIONS:-9 8}" SKIP_PROVISION="${SKIP_PROVISION:-false}" STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}" -STORAGE_COMPOSE="$INFRA_DIR/docker-compose.storage.yml" +STORAGE_VM_IP="${STORAGE_VM_IP:-172.16.60.60/24}" +STORAGE_VM_GATEWAY="${STORAGE_VM_GATEWAY:-172.16.60.1}" +STORAGE_VMID="${STORAGE_VMID:-5080}" +STORAGE_VM_HOST="${STORAGE_VM_IP%%/*}" +# Keypair for SSH to the storage VM (cloud images refuse password SSH) +STORAGE_VM_SSH_KEY="${STORAGE_VM_SSH_KEY:-$WORK_DIR/storage-vm-key}" +# Must match CLOUD_IMAGE_FILENAME in ensure-cloud-images.sh +CLOUD_IMAGE_NAME="noble-server-cloudimg-amd64.qcow2" # Store Terraform state on the shared mount so it persists across CI jobs TF_STATE_FILE="$WORK_DIR/terraform.tfstate" -# ── Node config ─────────────────────────────────────────────────── +# ── Node config ───────────────────────────────────────────────── # Each version gets two nodes: a (primary) and b (secondary). # ISOs are per-version; nodes within a version share the same base ISO. @@ -78,6 +90,14 @@ pve_iso() { esac } +# Generic auto-install ISO name; embeds the answer-server host so a cached +# ISO baked with an older answer URL is never reused. +pve_auto_iso() { + local base + base="$(pve_iso "$1")" + echo "${base%.iso}-auto-${STORAGE_VM_HOST//./-}.iso" +} + pve_vmid() { case "$1" in 9a) echo "${PVE9A_VMID:-5091}" ;; 9b) echo "${PVE9B_VMID:-5092}" ;; @@ -122,7 +142,7 @@ expand_nodes() { ALL_NODES="$(expand_nodes)" -# ── CI helpers ────────────────────────────────────────────────────── +# ── CI helpers ──────────────────────────────────────────────────── ci_mask() { [[ "${GITHUB_ACTIONS:-}" == "true" ]] && echo "::add-mask::$1" || true; } ci_error() { [[ "${GITHUB_ACTIONS:-}" == "true" ]] && echo "::error::$1" || echo "ERROR: $1" >&2; } @@ -157,7 +177,8 @@ pve_free_nodes() { # the parent hypervisor. resolve_target_node() { local count="$1" - local need=$(( count * ${PVE_VM_MEM_GB:-8} + ${PVE_MEM_HEADROOM_GB:-8} )) + # +2 GiB for the storage VM provisioned alongside the nested nodes + local need=$(( count * ${PVE_VM_MEM_GB:-8} + 2 + ${PVE_MEM_HEADROOM_GB:-8} )) local nodes nodes="$(pve_free_nodes || true)" @@ -186,7 +207,7 @@ resolve_target_node() { export PVE_TARGET_NODE } -# ── Subcommands ───────────────────────────────────────────────────── +# ── Subcommands ─────────────────────────────────────────────────── cmd_provision() { local requested="${1:-all}" @@ -203,7 +224,7 @@ cmd_provision() { log "Starting provisioning..." log " Versions: $provision_versions" log " Nodes:$provision_nodes" - log " Storage: Docker containers (iSCSI + NFS)" + log " Storage: dedicated VM at $STORAGE_VM_HOST (NFS + iSCSI + answer server)" require_env PVE_ENDPOINT require_env PVE_API_TOKEN require_env PVE_PASSWORD @@ -225,19 +246,6 @@ cmd_provision() { CLOUD_IMAGE_PATH=$(echo "$cloud_output" | grep "^CLOUD_IMAGE_PATH=" | cut -d= -f2) OVA_PATH=$(echo "$cloud_output" | grep "^OVA_PATH=" | cut -d= -f2) - # Discover Docker host IP early — needed for the HTTP auto-install ISO URL - # and for the docker_host_ip Terraform variable. - local storage_ip - storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}') - if [ -z "$storage_ip" ]; then - storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1) - fi - if [ -z "$storage_ip" ]; then - ci_error "Could not determine Docker host IP for storage services" - exit 1 - fi - log "Docker host IP: $storage_ip" - # Generate per-MAC answer files for the HTTP answer server. # Each node gets a file named by its MAC address so the server can # route the correct answer to each VM during auto-install. @@ -267,12 +275,12 @@ cmd_provision() { for v in $provision_versions; do local base_iso_name generic_iso base_iso_name="$(pve_iso "$v")" - generic_iso="$WORK_DIR/${base_iso_name%.iso}-http-auto.iso" + generic_iso="$WORK_DIR/$(pve_auto_iso "$v")" if [ ! -f "$generic_iso" ]; then log "Preparing HTTP auto-install ISO for PVE $v..." proxmox-auto-install-assistant prepare-iso \ --fetch-from http \ - --url "http://${storage_ip}:8000/answer" \ + --url "http://${STORAGE_VM_HOST}:8000/answer" \ --on-first-boot "$SCRIPT_DIR/first-boot.sh" \ --tmp "$WORK_DIR" \ --output "$generic_iso" \ @@ -297,9 +305,7 @@ cmd_provision() { # Build pve_isos map: version -> ISO path (one per version) local isos='{}' for v in $PVE_VERSIONS; do - local iso_name - iso_name="$(pve_iso "$v")" - local iso_path="$WORK_DIR/${iso_name%.iso}-http-auto.iso" + local iso_path="$WORK_DIR/$(pve_auto_iso "$v")" isos="$(jq --arg key "$v" --arg path "$iso_path" \ '. + {($key): $path}' <<<"$isos")" done @@ -325,6 +331,39 @@ cmd_provision() { jq -n --argjson pve_instances "$instances" --argjson pve_isos "$isos" \ '{pve_instances: $pve_instances, pve_isos: $pve_isos}' > "$tfvars" + # The nested PVE installers fetch their answer files from the storage VM, + # so it must be provisioned and configured before they boot: phase one + # creates and configures the storage VM, phase two everything else. + if [[ ! -f "$STORAGE_VM_SSH_KEY" ]]; then + ssh-keygen -t ed25519 -f "$STORAGE_VM_SSH_KEY" -N '' -C 'pvetest-storage' >/dev/null + fi + + tf_apply() { + # TMPDIR: use work dir to avoid filling the container's /tmp with + # multi-GB ISO uploads. + (cd "$INFRA_DIR" && \ + TMPDIR="$WORK_DIR" \ + TF_VAR_proxmox_endpoint="$PVE_ENDPOINT" \ + TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \ + TF_VAR_target_node="$PVE_TARGET_NODE" \ + TF_VAR_test_vm_password="$PVE_PASSWORD" \ + TF_VAR_cloud_image_path="$CLOUD_IMAGE_PATH" \ + TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \ + TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \ + TF_VAR_storage_vmid="$STORAGE_VMID" \ + TF_VAR_storage_vm_ssh_public_key="$(cat "${STORAGE_VM_SSH_KEY}.pub")" \ + terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" "$@") + } + + log "Running Terraform apply (storage VM)..." + tf_apply \ + -target=proxmox_virtual_environment_file.storage_cloud_image \ + -target=proxmox_virtual_environment_vm.storage + + log "Configuring storage VM at $STORAGE_VM_HOST..." + bash "$SCRIPT_DIR/setup-storage-server.sh" \ + "$STORAGE_VM_HOST" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server" + log "Running Terraform apply (PVE nodes)..." local tf_targets="" if [[ "$requested" != "all" ]]; then @@ -334,26 +373,10 @@ cmd_provision() { for node in $provision_nodes; do tf_targets="$tf_targets -target=proxmox_virtual_environment_vm.nested_pve[\"$node\"]" done - # Always include shared Docker storage and answer server resources - tf_targets="$tf_targets -target=docker_image.ubuntu" - tf_targets="$tf_targets -target=docker_container.iscsi_target" - tf_targets="$tf_targets -target=docker_container.nfs_server" - tf_targets="$tf_targets -target=docker_container.answer_server" - tf_targets="$tf_targets -target=docker_volume.iscsi_data" - tf_targets="$tf_targets -target=docker_volume.nfs_data" log "Terraform targets: $tf_targets" fi - # TMPDIR: use work dir to avoid filling the container's /tmp with multi-GB ISO uploads. - (cd "$INFRA_DIR" && \ - TMPDIR="$WORK_DIR" \ - TF_VAR_proxmox_endpoint="$PVE_ENDPOINT" \ - TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \ - TF_VAR_target_node="$PVE_TARGET_NODE" \ - TF_VAR_test_vm_password="$PVE_PASSWORD" \ - TF_VAR_docker_host_ip="$storage_ip" \ - TF_VAR_answer_server_dir="$WORK_DIR/answer-server" \ - terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets) + tf_apply $tf_targets # Wait for PVE instances to boot and discover IPs for node in $provision_nodes; do @@ -401,7 +424,7 @@ cmd_provision() { config=$(jq \ --arg cloud_image "${CLOUD_IMAGE_PATH:-}" \ --arg ova "${OVA_PATH:-}" \ - --arg storage_ip "$storage_ip" \ + --arg storage_ip "$STORAGE_VM_HOST" \ --arg storage_iqn "$STORAGE_ISCSI_IQN" \ '. + { storage: {ip: $storage_ip, iscsi_iqn: $storage_iqn, nfs_export: ($storage_ip + ":/srv/nfs/shared")}, @@ -480,7 +503,7 @@ cmd_test() { export PVETEST_OVA_PATH=$(jq -r '.ova_path' "$CONFIG_FILE") # Secondary node (b) export PVETEST_HOST_B=$(jq -r ".pve${v}.nodes.b.host" "$CONFIG_FILE") - # Storage services (Docker on runner) + # Storage services (dedicated VM on the CI VLAN) export PVETEST_STORAGE_VM_IP=$(jq -r '.storage.ip' "$CONFIG_FILE") export PVETEST_ISCSI_IQN=$(jq -r '.storage.iscsi_iqn' "$CONFIG_FILE") export PVETEST_NFS_EXPORT=$(jq -r '.storage.nfs_export' "$CONFIG_FILE") @@ -552,11 +575,19 @@ cmd_test() { } \$config.Filter.Tag = 'Integration' + \$config.Run.PassThru = \$true \$config.Output.Verbosity = 'Detailed' \$config.TestResult.Enabled = \$true \$config.TestResult.OutputFormat = 'NUnitXml' \$config.TestResult.OutputPath = \"TestResults/integration-results-pve\${PveVersion}.xml\" - Invoke-Pester -Configuration \$config + \$result = Invoke-Pester -Configuration \$config + if (-not \$result -or \$result.TotalCount -eq 0) { + Write-Error 'No integration tests were discovered or executed' + exit 1 + } + if (\$result.FailedCount -gt 0) { + exit 1 + } " || test_exit=$? if [[ $test_exit -ne 0 ]]; then @@ -581,13 +612,6 @@ cmd_cleanup() { fi require_env PVE_TARGET_NODE - # Discover Docker host IP for the docker_host_ip variable - local storage_ip - storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}') - if [ -z "$storage_ip" ]; then - storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1) - fi - # Build tfvars for all versions (Terraform needs the full variable map) local tfvars="$WORK_DIR/instances.tfvars.json" if [[ ! -f "$tfvars" ]]; then @@ -616,10 +640,6 @@ cmd_cleanup() { (cd "$INFRA_DIR" && terraform init -input=false 2>/dev/null) - # Ensure answer server dir exists (terraform destroy validates host_path mounts) - mkdir -p "$WORK_DIR/answer-server/answers" - touch "$WORK_DIR/answer-server/default.toml" - # Build -target flags when destroying a subset local tf_targets="" if [[ "$requested" != "all" ]]; then @@ -641,8 +661,9 @@ cmd_cleanup() { TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \ TF_VAR_target_node="$PVE_TARGET_NODE" \ TF_VAR_test_vm_password="${PVE_PASSWORD:-placeholder}" \ - TF_VAR_docker_host_ip="${storage_ip:-127.0.0.1}" \ - TF_VAR_answer_server_dir="${WORK_DIR}/answer-server" \ + TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \ + TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \ + TF_VAR_storage_vmid="$STORAGE_VMID" \ terraform destroy -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets) # Clean up work directory when destroying all @@ -665,6 +686,8 @@ cmd_force_cleanup() { log "Force cleanup — bypassing Terraform, using direct API calls..." + # preflight-cleanup.sh reads PVE_TARGET_NODE from the environment; in auto + # mode resolve it from the node provision picked. if [[ -z "${PVE_TARGET_NODE:-}" || "$PVE_TARGET_NODE" == "auto" ]]; then PVE_TARGET_NODE="$(cat "$TARGET_NODE_FILE" 2>/dev/null || true)" [[ -n "$PVE_TARGET_NODE" ]] && export PVE_TARGET_NODE @@ -674,14 +697,13 @@ cmd_force_cleanup() { # Track which versions we've already cleaned up ISOs for (generic ISOs are shared) local cleaned_iso_versions="" for node in $cleanup_nodes; do - local vm_id v iso_name iso_file + local vm_id v iso_file vm_id="$(pve_vmid "$node")" v="${node%[ab]}" - iso_name="$(pve_iso "$v")" # Only clean up the generic ISO once per version iso_file="" if [[ ! " $cleaned_iso_versions " =~ " $v " ]]; then - iso_file="${iso_name%.iso}-http-auto.iso" + iso_file="$(pve_auto_iso "$v")" cleaned_iso_versions="$cleaned_iso_versions $v" fi log "Force cleaning $node (VMID $vm_id)..." @@ -691,11 +713,13 @@ cmd_force_cleanup() { || true done - # Always stop Docker containers in force mode — leaving them causes - # Terraform to fail on next provision (container already exists). - log "Stopping storage and answer server containers..." - docker rm -f pvetest-iscsi pvetest-nfs pvetest-answer-server 2>/dev/null || true - docker volume rm pvetest-iscsi-data pvetest-nfs-data 2>/dev/null || true + # Unconditional: the state wipe below is unconditional too, and a storage + # VM surviving its state entry cannot be reclaimed by the next provision. + log "Force cleaning storage VM (VMID $STORAGE_VMID)..." + bash "$SCRIPT_DIR/preflight-cleanup.sh" \ + "${PVE_ENDPOINT:-}" "${PVE_API_TOKEN:-}" \ + "$STORAGE_VMID" "$CLOUD_IMAGE_NAME" "$INFRA_DIR" \ + || true # Remove Terraform state (both local and shared mount) so next provision starts clean. # Keep .terraform.lock.hcl (provider version lock) for reproducibility. @@ -704,8 +728,9 @@ cmd_force_cleanup() { rm -f "$TF_STATE_FILE" "${TF_STATE_FILE}.backup" rm -rf "$INFRA_DIR/.terraform" - # Remove work artifacts + # Remove work artifacts, including locally cached auto-install ISOs rm -f "$CONFIG_FILE" "$WORK_DIR"/instances.tfvars.json "$TARGET_NODE_FILE" + rm -f "$WORK_DIR"/*-auto-*.iso "$WORK_DIR"/*-http-auto.iso log "Force cleanup complete. Next provision will start from scratch." } @@ -759,7 +784,7 @@ cmd_all() { return $test_exit } -# ── Main ──────────────────────────────────────────────────────────── +# ── Main ─────────────────────────────────────────────────────────── main() { local cmd="${1:-}" shift || true @@ -775,7 +800,7 @@ main() { echo "Usage: $(basename "$0") {provision|test|cleanup|force-cleanup|taint|all} [8|9|all] [test-filter]" echo "" echo "Subcommands:" - echo " provision [8|9|all] Provision nested PVE VMs + storage containers" + echo " provision [8|9|all] Provision storage VM + nested PVE VMs" echo " test [8|9|all] [filter] Run integration tests (default: all versions, no filter)" echo " cleanup [8|9|all] Destroy resources via terraform destroy (default: all)" echo " force-cleanup [8|9|all] Bypass Terraform — destroy via API + wipe state (recovery)" diff --git a/tests/infrastructure/scripts/setup-storage-server.sh b/tests/infrastructure/scripts/setup-storage-server.sh new file mode 100644 index 0000000..8e76149 --- /dev/null +++ b/tests/infrastructure/scripts/setup-storage-server.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Configures the shared services VM inside the CI sandbox VLAN: NFS export, +# iSCSI target, and the HTTP answer server the PVE auto-installers fetch +# from. Idempotent — safe to re-run against an already-configured VM. +# +# Usage: setup-storage-server.sh +# is the local directory holding default.toml and answers/. + +set -euo pipefail + +STORAGE_IP="${1:?Usage: setup-storage-server.sh }" +SSH_KEY="${2:?missing SSH private key path}" +ISCSI_IQN="${3:?missing iSCSI IQN}" +ANSWER_DIR="${4:?missing answer directory}" + +# Same pinned image the Terraform-managed runner container used previously. +ANSWER_SERVER_IMAGE="slothcroissant/proxmox-auto-installer-server@sha256:0f45d7bfe6e3cc76aa00fc578e40b80b9054e377db18a79122866fe5522bc7ed" + +SSH_OPTS="-i ${SSH_KEY} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" + +echo "=== Configuring storage services on ${STORAGE_IP} ===" + +# Cloud-init may still be running on first boot; wait for SSH to accept the +# key login it installs. The last attempt's stderr is kept so an auth failure +# is distinguishable from an unreachable host. +for i in $(seq 1 60); do + if ssh ${SSH_OPTS} -o ConnectTimeout=5 "ubuntu@${STORAGE_IP}" true 2>/tmp/storage-ssh-err; then + break + fi + if [[ $i -eq 60 ]]; then + echo "ERROR: storage VM at ${STORAGE_IP} not reachable over SSH" >&2 + cat /tmp/storage-ssh-err >&2 + exit 1 + fi + sleep 5 +done + +# sshd accepts logins before cloud-init's first-boot work (and its apt/dpkg +# locks) is finished; a degraded cloud-init result is fine for our purposes, +# and a hung cloud-init must fail fast, not eat the job timeout. +timeout 300 ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ + "sudo cloud-init status --wait" || true + +echo "Copying answer files..." +ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ + "sudo mkdir -p /opt/answer-server && sudo rm -rf /opt/answer-server/answers && sudo chown ubuntu /opt/answer-server" +scp ${SSH_OPTS} -r \ + "${ANSWER_DIR}/default.toml" "${ANSWER_DIR}/answers" \ + "ubuntu@${STORAGE_IP}:/opt/answer-server/" + +ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ + "sudo ISCSI_IQN='${ISCSI_IQN}' ANSWER_SERVER_IMAGE='${ANSWER_SERVER_IMAGE}' bash -s" <<'REMOTE' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +# cloud-init's first-boot apt activity can hold the dpkg lock briefly +for i in $(seq 1 30); do + apt-get update -qq >/dev/null 2>&1 && break + sleep 5 +done +apt-get install -y -qq nfs-kernel-server tgt docker.io >/dev/null + +# Ubuntu's nfs-kernel-server package does not create /etc/exports.d +mkdir -p /etc/exports.d /srv/nfs/shared /srv/iscsi +echo '/srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)' > /etc/exports.d/pvetest.exports +exportfs -ra + +if [ ! -f /srv/iscsi/lun0.img ]; then + truncate -s 10G /srv/iscsi/lun0.img +fi +cat > /etc/tgt/conf.d/pvetest.conf < + backing-store /srv/iscsi/lun0.img + +CONF + +systemctl enable --now nfs-kernel-server tgt >/dev/null || { + journalctl -u nfs-kernel-server -u tgt --no-pager -n 30 >&2 + exit 1 +} +systemctl restart tgt + +docker rm -f pvetest-answer-server >/dev/null 2>&1 || true +docker run -d --name pvetest-answer-server --restart unless-stopped \ + --network host \ + -v /opt/answer-server/answers:/app/answers \ + -v /opt/answer-server/default.toml:/app/default.toml \ + "${ANSWER_SERVER_IMAGE}" >/dev/null + +# Ordering requirement: the PVE installers fetch answers as soon as this +# script returns, so :8000 must accept connections before exit (any HTTP +# status counts as listening). +for i in $(seq 1 20); do + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 2 http://127.0.0.1:8000/ || true) + [ "$code" != "000" ] && break + if [ "$i" -eq 20 ]; then + echo "ERROR: answer server did not start listening on :8000" >&2 + docker logs pvetest-answer-server 2>&1 | tail -20 >&2 + exit 1 + fi + sleep 3 +done + +echo "Services ready: NFS /srv/nfs/shared, iSCSI ${ISCSI_IQN}, answer server :8000" +REMOTE + +echo "Storage server configuration complete." diff --git a/tests/infrastructure/storage-vm.tf b/tests/infrastructure/storage-vm.tf new file mode 100644 index 0000000..40c76c6 --- /dev/null +++ b/tests/infrastructure/storage-vm.tf @@ -0,0 +1,80 @@ +# ── Shared storage VM ─────────────────────────────────────────────── +# One small Ubuntu guest in the CI pool serves NFS + iSCSI to every nested +# PVE node. It lives on the same isolated VLAN as the nodes under test, so +# no firewall path out of the CI sandbox is needed. Package install and +# service configuration happen over SSH (setup-storage-server.sh) after boot. + +resource "proxmox_virtual_environment_file" "storage_cloud_image" { + content_type = "import" + datastore_id = var.iso_storage + node_name = var.target_node + overwrite = true + + source_file { + path = var.cloud_image_path + } +} + +resource "proxmox_virtual_environment_vm" "storage" { + name = "pvetest-storage" + node_name = var.target_node + vm_id = var.storage_vmid + pool_id = var.pool_id + + cpu { + type = "host" + cores = 2 + sockets = 1 + } + + memory { + dedicated = 2048 + } + + disk { + datastore_id = var.disk_storage + interface = "scsi0" + size = 32 + import_from = proxmox_virtual_environment_file.storage_cloud_image.id + } + + initialization { + datastore_id = var.disk_storage + + ip_config { + ipv4 { + address = var.storage_vm_ip + gateway = var.storage_vm_gateway + } + } + + dns { + servers = [var.storage_vm_gateway] + } + + user_account { + username = "ubuntu" + # Password is console-only: Ubuntu cloud images refuse SSH password auth + # and PVE's cloud-init never sets ssh_pwauth. SSH uses the key. + password = var.test_vm_password + keys = [trimspace(var.storage_vm_ssh_public_key)] + } + } + + network_device { + bridge = var.network_bridge + model = "virtio" + } + + operating_system { + type = "l26" + } + + # Resized Ubuntu cloud images kernel-panic on boot without a serial console + # (bpg/terraform-provider-proxmox documented issue). + serial_device { + device = "socket" + } + + started = true +} diff --git a/tests/infrastructure/storage.tf b/tests/infrastructure/storage.tf deleted file mode 100644 index f217b0a..0000000 --- a/tests/infrastructure/storage.tf +++ /dev/null @@ -1,112 +0,0 @@ -# ── Answer server container ────────────────────────────────────────── - -resource "docker_container" "answer_server" { - name = "pvetest-answer-server" - # Pin to digest for reproducibility. Update by pulling latest and running: - # docker inspect slothcroissant/proxmox-auto-installer-server:latest --format '{{index .RepoDigests 0}}' - image = "slothcroissant/proxmox-auto-installer-server@sha256:0f45d7bfe6e3cc76aa00fc578e40b80b9054e377db18a79122866fe5522bc7ed" - restart = "unless-stopped" - must_run = true - start = true - - network_mode = "host" - - volumes { - host_path = "${var.answer_server_dir}/answers" - container_path = "/app/answers" - } - - volumes { - host_path = "${var.answer_server_dir}/default.toml" - container_path = "/app/default.toml" - } -} - -# ── Docker images & volumes ────────────────────────────────────────── - -resource "docker_image" "ubuntu" { - # Pin to digest for reproducibility. Update: - # docker pull ubuntu:24.04 && docker inspect ubuntu:24.04 --format '{{index .RepoDigests 0}}' - name = "ubuntu@sha256:186072bba1b2f436cbb91ef2567abca677337cfc786c86e107d25b7072feef0c" -} - -resource "docker_volume" "iscsi_data" { - name = "pvetest-iscsi-data" -} - -resource "docker_volume" "nfs_data" { - name = "pvetest-nfs-data" -} - -# ── iSCSI target container ────────────────────────────────────────── - -resource "docker_container" "iscsi_target" { - name = "pvetest-iscsi" - image = docker_image.ubuntu.image_id - privileged = true - restart = "unless-stopped" - - network_mode = "host" - - volumes { - volume_name = docker_volume.iscsi_data.name - container_path = "/srv/iscsi" - } - - env = [ - "ISCSI_IQN=${var.storage_iscsi_iqn}", - "ISCSI_LUN_SIZE=${var.storage_iscsi_lun_size}", - ] - - entrypoint = ["/bin/bash", "-c"] - command = [<<-EOT - set -e - apt-get update -qq && apt-get install -y -qq tgt >/dev/null 2>&1 - mkdir -p /srv/iscsi - if [ ! -f /srv/iscsi/lun0.img ]; then - truncate -s $${ISCSI_LUN_SIZE} /srv/iscsi/lun0.img - fi - tgtd --foreground & - sleep 2 - if ! tgtadm --lld iscsi --op show --mode target | grep -q "Target 1: $${ISCSI_IQN}"; then - tgtadm --lld iscsi --op new --mode target --tid 1 -T $${ISCSI_IQN} - fi - if ! tgtadm --lld iscsi --op show --mode logicalunit --tid 1 2>/dev/null | grep -qE "LUN:[[:space:]]*1($$|[^0-9])"; then - tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 --backing-store /srv/iscsi/lun0.img - fi - if ! tgtadm --lld iscsi --op show --mode target --tid 1 2>/dev/null | grep -q "Initiator-address: ALL"; then - tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL - fi - echo "iSCSI target ready: $${ISCSI_IQN} (port 3260)" - wait - EOT - ] -} - -# ── NFS server container ──────────────────────────────────────────── - -resource "docker_container" "nfs_server" { - name = "pvetest-nfs" - # Pin to digest for reproducibility. Update: - # docker pull erichough/nfs-server:2.2.1 && docker inspect erichough/nfs-server:2.2.1 --format '{{index .RepoDigests 0}}' - image = "erichough/nfs-server@sha256:1efd4ece380c5ba27479417585224ef857006daa46ab84560a28c1224bc71e9e" - privileged = true - restart = "unless-stopped" - - network_mode = "host" - - volumes { - volume_name = docker_volume.nfs_data.name - container_path = "/srv/nfs/shared" - } - - volumes { - host_path = "/lib/modules" - container_path = "/lib/modules" - read_only = true - } - - env = [ - "NFS_EXPORT_0=/srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)", - ] -} diff --git a/tests/infrastructure/variables.tf b/tests/infrastructure/variables.tf index 111ee50..4c18863 100644 --- a/tests/infrastructure/variables.tf +++ b/tests/infrastructure/variables.tf @@ -61,7 +61,7 @@ variable "disk_storage" { } variable "iso_storage" { - description = "Proxmox storage pool for uploading the ISO (must accept ISO content type)" + description = "Proxmox storage pool for uploads (must accept the iso AND import content types — import is not enabled by default on most storages)" type = string default = "local" } @@ -78,28 +78,40 @@ variable "test_vm_password" { sensitive = true } -variable "storage_iscsi_iqn" { - description = "iSCSI target IQN for the test storage" - type = string - default = "iqn.2024-01.local.test:storage" +variable "storage_vmid" { + description = "VMID for the shared storage VM (must be inside the CI pool's reserved range)" + type = number + default = 5080 } -variable "storage_iscsi_lun_size" { - description = "Size of the iSCSI LUN backing file" +variable "storage_vm_ip" { + description = "Static IPv4 address in CIDR form for the storage VM, outside the VLAN's DHCP range" type = string - default = "10G" + default = "172.16.60.60/24" + + validation { + condition = can(cidrnetmask(var.storage_vm_ip)) + error_message = "storage_vm_ip must be CIDR notation, e.g. 172.16.60.60/24." + } } -variable "docker_host_ip" { - description = "IP of the Docker host, used by PVE nodes to reach storage containers" +variable "storage_vm_ssh_public_key" { + description = "SSH public key granted to the storage VM's ubuntu user (cloud images refuse password SSH; required for provision, unused on destroy)" type = string + default = "" } -variable "answer_server_dir" { - description = "Host path to the answer server root directory (contains default.toml and answers/ subdirectory)" +variable "storage_vm_gateway" { + description = "Gateway (and DNS server) for the storage VM" type = string + default = "172.16.60.1" } +variable "cloud_image_path" { + description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)" + type = string + default = "" +} variable "pool_id" { description = "Resource pool the nested VMs are created in (a pool-scoped API token can only allocate here)"