Merge pull request #87 from GoodOlClint/ci/pester-exit-sandbox-storage

ci: storage and answer services move to a VM inside the CI sandbox

.github/workflows/integration-tests.yml

@@ -24,6 +24,9 @@ name: Integration Tests
 #                        node with the most free memory and refuses to run when
 #                        no node has enough headroom — the token's user needs
 #                        PVEAuditor on /nodes for the memory stats.
+#   STORAGE_VM_IP      - Static CIDR address for the shared storage/answer-server
+#                        VM on the CI VLAN (default: 172.16.60.60/24)
+#   STORAGE_VM_GATEWAY - Gateway + DNS for that VM (default: 172.16.60.1)
 #
 # Optional secrets (for skip_provision mode):
 #   PVETEST_HOST       - Hostname or IP of a pre-existing nested PVE
@@ -122,11 +125,6 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
       volumes:
         - /opt/pve-integration:/opt/pve-integration
-        # WARNING: The Docker socket is mounted to allow storage containers to be
-        # managed from within the job container. This grants root-equivalent access
-        # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted
-        # runners — never on shared runners.
-        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
       - uses: actions/checkout@v7
@@ -138,6 +136,8 @@ jobs:
           PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
           PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }}
           PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }}
+          STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }}
+          STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }}
           TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }}
           TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }}
           TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }}
@@ -201,11 +201,6 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
       volumes:
         - /opt/pve-integration:/opt/pve-integration
-        # WARNING: The Docker socket is mounted to allow storage containers to be
-        # stopped from within the job container. This grants root-equivalent access
-        # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted
-        # runners — never on shared runners.
-        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
       - uses: actions/checkout@v7
@@ -217,6 +212,8 @@ jobs:
           PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
           PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }}
           PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }}
+          STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }}
+          STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }}
           TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }}
           TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }}
           TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }}

tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1

@@ -4,7 +4,7 @@
     Pester 5 integration tests for shared storage backends (NFS, iSCSI).
 
     These tests require the multi-node integration test infrastructure with
-    Docker-based storage services (iSCSI target + NFS server). They are
+    the shared storage VM (iSCSI target + NFS server on the CI VLAN). They are
     SKIPPED when the storage env vars are not set.
 
     Required environment variables (in addition to base integration vars):

tests/infrastructure/README.md

@@ -163,7 +163,9 @@ Note that AMD nested virtualization can be less stable than Intel in some config
 | `main.tf` | Provider config, VM resource, provisioners |
 | `variables.tf` | Input variable definitions with defaults |
 | `outputs.tf` | Test connection details for integration tests |
+| `storage-vm.tf` | Shared NFS/iSCSI/answer-server VM on the CI VLAN |
 | `answer.toml.tftpl` | Unattended PVE installer answer file template |
+| `scripts/setup-storage-server.sh` | Configures services on the storage VM over SSH |
 | `scripts/wait-for-api.sh` | Polls PVE API until responsive |
 | `terraform.tfvars.example` | Example variable values |
 | `.gitignore` | Excludes state, provider cache, secrets |

tests/infrastructure/docker-compose.storage.yml

@@ -1,66 +0,0 @@
-## Shared storage services for integration testing.
-## Provides iSCSI target and NFS server accessible from nested PVE nodes.
-##
-## These run on the GitHub runner (or any Docker host on the same network
-## as the PVE nodes) using host networking so the PVE nodes can reach them.
-##
-## Usage:
-##   docker compose -f tests/infrastructure/docker-compose.storage.yml up -d
-##   docker compose -f tests/infrastructure/docker-compose.storage.yml down -v
-
-services:
-  iscsi-target:
-    image: ubuntu:24.04
-    container_name: pvetest-iscsi
-    network_mode: host
-    privileged: true
-    volumes:
-      - iscsi-data:/srv/iscsi
-    environment:
-      ISCSI_IQN: ${ISCSI_IQN:-iqn.2024-01.local.test:storage}
-      ISCSI_LUN_SIZE: ${ISCSI_LUN_SIZE:-10G}
-    entrypoint: ["/bin/bash", "-c"]
-    command:
-      - |
-        set -e
-        apt-get update -qq && apt-get install -y -qq tgt >/dev/null 2>&1
-        mkdir -p /srv/iscsi
-        if [ ! -f /srv/iscsi/lun0.img ]; then
-          truncate -s $$ISCSI_LUN_SIZE /srv/iscsi/lun0.img
-        fi
-        tgtd --foreground &
-        sleep 2
-        # Create target if it does not already exist
-        if ! tgtadm --lld iscsi --op show --mode target | grep -q "Target 1: $$ISCSI_IQN"; then
-          tgtadm --lld iscsi --op new --mode target --tid 1 -T $$ISCSI_IQN
-        fi
-        # Create logical unit (LUN 1) if it does not already exist
-        if ! tgtadm --lld iscsi --op show --mode logicalunit --tid 1 2>/dev/null | grep -qE "LUN:[[:space:]]*1($|[^0-9])"; then
-          tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 --backing-store /srv/iscsi/lun0.img
-        fi
-        # Bind target to all initiators if not already bound
-        if ! tgtadm --lld iscsi --op show --mode target --tid 1 2>/dev/null | grep -q "Initiator-address: ALL"; then
-          tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL
-        fi
-        echo "iSCSI target ready: $$ISCSI_IQN (port 3260)"
-        wait
-    restart: unless-stopped
-
-  nfs-server:
-    image: erichough/nfs-server:2.2.1
-    container_name: pvetest-nfs
-    network_mode: host
-    privileged: true
-    volumes:
-      - nfs-data:/srv/nfs/shared
-      # NOTE: /lib/modules is required by the NFS kernel module loader.
-      # This mount is Linux-host-specific and will fail on macOS/Windows Docker Desktop.
-      # Integration tests using this compose file must run on a Linux host (e.g. GitHub-hosted runners or the dev container).
-      - /lib/modules:/lib/modules:ro
-    environment:
-      NFS_EXPORT_0: /srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)
-    restart: unless-stopped
-
-volumes:
-  iscsi-data:
-  nfs-data:

tests/infrastructure/main.tf

@@ -3,11 +3,8 @@ terraform {
   required_providers {
     proxmox = {
       source  = "bpg/proxmox"
-      version = ">= 0.70.0"
+      # disk.import_from and content_type "import" need >= 0.79.0
-    }
+      version = ">= 0.79.0"
-    docker = {
-      source  = "kreuzwerker/docker"
-      version = ">= 3.0.0"
     }
   }
 }
@@ -18,11 +15,6 @@ provider "proxmox" {
   insecure  = var.proxmox_insecure
 }
 
-provider "docker" {
-  # Uses the Docker socket from the dev-infra container
-  # (mounted at /var/run/docker.sock)
-}
-
 resource "proxmox_virtual_environment_file" "auto_iso" {
   for_each     = var.pve_isos
   content_type = "iso"
@@ -91,9 +83,10 @@ resource "proxmox_virtual_environment_vm" "nested_pve" {
 
   started = true
 
-  # VMs must not boot until the HTTP answer server is running,
+  # VMs must not boot until the storage VM exists — it serves the HTTP answer
-  # otherwise the PVE auto-installer can't fetch its answer file.
+  # files the PVE auto-installer fetches. run-integration.sh additionally
-  depends_on = [docker_container.answer_server]
+  # configures the storage VM (phase one) before applying these resources.
+  depends_on = [proxmox_virtual_environment_vm.storage]
 
   lifecycle {
     ignore_changes = [started, cdrom]

tests/infrastructure/outputs.tf

@@ -8,17 +8,7 @@ output "pve_test_node_name" {
   description = "Default node name inside a fresh PVE install"
 }
 
-output "storage_ip" {
+output "storage_vm_ip" {
-  description = "IP address where storage services are reachable"
+  description = "Static address of the shared NFS/iSCSI storage VM"
-  value       = var.docker_host_ip
+  value       = var.storage_vm_ip
-}
-
-output "storage_iscsi_iqn" {
-  description = "iSCSI target IQN"
-  value       = var.storage_iscsi_iqn
-}
-
-output "storage_nfs_export" {
-  description = "NFS export path"
-  value       = "${var.docker_host_ip}:/srv/nfs/shared"
 }

tests/infrastructure/scripts/run-integration.sh

@@ -4,11 +4,12 @@
 # Single source of truth for the provision → test → cleanup lifecycle.
 # Called by both the GitHub Actions workflow and the local dev container.
 #
-# Provisions two PVE nodes per version (a/b) for cluster testing, plus
+# Provisions two PVE nodes per version (a/b) for cluster testing, plus a
-# Docker containers on the runner host for iSCSI/NFS shared storage.
+# small storage VM on the same isolated VLAN serving NFS, iSCSI, and the
+# auto-install answer files.
 #
 # Usage:
-#   run-integration.sh provision [8|9|all]         Provision nested PVE VMs + start storage containers
+#   run-integration.sh provision [8|9|all]         Provision storage VM + nested PVE VMs
 #   run-integration.sh test [8|9|all] [filter]    Run integration tests (default: all, no filter)
 #   run-integration.sh cleanup [8|9|all]           Destroy provisioned VMs
 #   run-integration.sh all [8|9|all]              Full lifecycle: provision → test → cleanup
@@ -42,6 +43,10 @@
 #   MODULE_ARTIFACT    Path to built module DLLs (default: ./publish/netstandard2.0)
 #   PVE_VERSIONS       Space-separated versions to provision (default: "9 8")
 #   STORAGE_ISCSI_IQN  iSCSI IQN for storage target (default: iqn.2024-01.local.test:storage)
+#   STORAGE_VM_IP      Static CIDR address for the storage VM on the CI VLAN,
+#                      outside the DHCP range (default: 172.16.60.60/24)
+#   STORAGE_VM_GATEWAY Gateway + DNS for the storage VM (default: 172.16.60.1)
+#   STORAGE_VMID       VMID for the storage VM (default: 5080)
 
 set -euo pipefail
 
@@ -50,7 +55,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 INFRA_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
 REPO_ROOT="$(cd "$INFRA_DIR/../.." && pwd)"
 
-# ── Defaults ────────────────────────────────────────────────────────
+# ── Defaults ──────────────────────────────────────────────────────
 CACHE_DIR="${CACHE_DIR:-/opt/pve-integration}"
 # Always use a path under CACHE_DIR (shared mount) so files are visible
 # to sibling Docker containers. Do NOT use RUNNER_TEMP — it's container-local
@@ -61,11 +66,18 @@ MODULE_ARTIFACT="${MODULE_ARTIFACT:-$REPO_ROOT/publish/netstandard2.0}"
 PVE_VERSIONS="${PVE_VERSIONS:-9 8}"
 SKIP_PROVISION="${SKIP_PROVISION:-false}"
 STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}"
-STORAGE_COMPOSE="$INFRA_DIR/docker-compose.storage.yml"
+STORAGE_VM_IP="${STORAGE_VM_IP:-172.16.60.60/24}"
+STORAGE_VM_GATEWAY="${STORAGE_VM_GATEWAY:-172.16.60.1}"
+STORAGE_VMID="${STORAGE_VMID:-5080}"
+STORAGE_VM_HOST="${STORAGE_VM_IP%%/*}"
+# Keypair for SSH to the storage VM (cloud images refuse password SSH)
+STORAGE_VM_SSH_KEY="${STORAGE_VM_SSH_KEY:-$WORK_DIR/storage-vm-key}"
+# Must match CLOUD_IMAGE_FILENAME in ensure-cloud-images.sh
+CLOUD_IMAGE_NAME="noble-server-cloudimg-amd64.qcow2"
 # Store Terraform state on the shared mount so it persists across CI jobs
 TF_STATE_FILE="$WORK_DIR/terraform.tfstate"
 
-# ── Node config ───────────────────────────────────────────────────
+# ── Node config ─────────────────────────────────────────────────
 # Each version gets two nodes: a (primary) and b (secondary).
 # ISOs are per-version; nodes within a version share the same base ISO.
 
@@ -78,6 +90,14 @@ pve_iso() {
     esac
 }
 
+# Generic auto-install ISO name; embeds the answer-server host so a cached
+# ISO baked with an older answer URL is never reused.
+pve_auto_iso() {
+    local base
+    base="$(pve_iso "$1")"
+    echo "${base%.iso}-auto-${STORAGE_VM_HOST//./-}.iso"
+}
+
 pve_vmid() {
     case "$1" in
         9a) echo "${PVE9A_VMID:-5091}" ;; 9b) echo "${PVE9B_VMID:-5092}" ;;
@@ -122,7 +142,7 @@ expand_nodes() {
 
 ALL_NODES="$(expand_nodes)"
 
-# ── CI helpers ──────────────────────────────────────────────────────
+# ── CI helpers ────────────────────────────────────────────────────
 ci_mask()  { [[ "${GITHUB_ACTIONS:-}" == "true" ]] && echo "::add-mask::$1" || true; }
 ci_error() { [[ "${GITHUB_ACTIONS:-}" == "true" ]] && echo "::error::$1" || echo "ERROR: $1" >&2; }
 
@@ -157,7 +177,8 @@ pve_free_nodes() {
 # the parent hypervisor.
 resolve_target_node() {
     local count="$1"
-    local need=$(( count * ${PVE_VM_MEM_GB:-8} + ${PVE_MEM_HEADROOM_GB:-8} ))
+    # +2 GiB for the storage VM provisioned alongside the nested nodes
+    local need=$(( count * ${PVE_VM_MEM_GB:-8} + 2 + ${PVE_MEM_HEADROOM_GB:-8} ))
     local nodes
     nodes="$(pve_free_nodes || true)"
 
@@ -186,7 +207,7 @@ resolve_target_node() {
     export PVE_TARGET_NODE
 }
 
-# ── Subcommands ─────────────────────────────────────────────────────
+# ── Subcommands ───────────────────────────────────────────────────
 
 cmd_provision() {
     local requested="${1:-all}"
@@ -203,7 +224,7 @@ cmd_provision() {
     log "Starting provisioning..."
     log "  Versions: $provision_versions"
     log "  Nodes:$provision_nodes"
-    log "  Storage: Docker containers (iSCSI + NFS)"
+    log "  Storage: dedicated VM at $STORAGE_VM_HOST (NFS + iSCSI + answer server)"
     require_env PVE_ENDPOINT
     require_env PVE_API_TOKEN
     require_env PVE_PASSWORD
@@ -225,19 +246,6 @@ cmd_provision() {
     CLOUD_IMAGE_PATH=$(echo "$cloud_output" | grep "^CLOUD_IMAGE_PATH=" | cut -d= -f2)
     OVA_PATH=$(echo "$cloud_output" | grep "^OVA_PATH=" | cut -d= -f2)
 
-    # Discover Docker host IP early — needed for the HTTP auto-install ISO URL
-    # and for the docker_host_ip Terraform variable.
-    local storage_ip
-    storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}')
-    if [ -z "$storage_ip" ]; then
-        storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1)
-    fi
-    if [ -z "$storage_ip" ]; then
-        ci_error "Could not determine Docker host IP for storage services"
-        exit 1
-    fi
-    log "Docker host IP: $storage_ip"
-
     # Generate per-MAC answer files for the HTTP answer server.
     # Each node gets a file named by its MAC address so the server can
     # route the correct answer to each VM during auto-install.
@@ -267,12 +275,12 @@ cmd_provision() {
     for v in $provision_versions; do
         local base_iso_name generic_iso
         base_iso_name="$(pve_iso "$v")"
-        generic_iso="$WORK_DIR/${base_iso_name%.iso}-http-auto.iso"
+        generic_iso="$WORK_DIR/$(pve_auto_iso "$v")"
         if [ ! -f "$generic_iso" ]; then
             log "Preparing HTTP auto-install ISO for PVE $v..."
             proxmox-auto-install-assistant prepare-iso \
                 --fetch-from http \
-                --url "http://${storage_ip}:8000/answer" \
+                --url "http://${STORAGE_VM_HOST}:8000/answer" \
                 --on-first-boot "$SCRIPT_DIR/first-boot.sh" \
                 --tmp "$WORK_DIR" \
                 --output "$generic_iso" \
@@ -297,9 +305,7 @@ cmd_provision() {
     # Build pve_isos map: version -> ISO path (one per version)
     local isos='{}'
     for v in $PVE_VERSIONS; do
-        local iso_name
+        local iso_path="$WORK_DIR/$(pve_auto_iso "$v")"
-        iso_name="$(pve_iso "$v")"
-        local iso_path="$WORK_DIR/${iso_name%.iso}-http-auto.iso"
         isos="$(jq --arg key "$v" --arg path "$iso_path" \
             '. + {($key): $path}' <<<"$isos")"
     done
@@ -325,6 +331,39 @@ cmd_provision() {
     jq -n --argjson pve_instances "$instances" --argjson pve_isos "$isos" \
         '{pve_instances: $pve_instances, pve_isos: $pve_isos}' > "$tfvars"
 
+    # The nested PVE installers fetch their answer files from the storage VM,
+    # so it must be provisioned and configured before they boot: phase one
+    # creates and configures the storage VM, phase two everything else.
+    if [[ ! -f "$STORAGE_VM_SSH_KEY" ]]; then
+        ssh-keygen -t ed25519 -f "$STORAGE_VM_SSH_KEY" -N '' -C 'pvetest-storage' >/dev/null
+    fi
+
+    tf_apply() {
+        # TMPDIR: use work dir to avoid filling the container's /tmp with
+        # multi-GB ISO uploads.
+        (cd "$INFRA_DIR" && \
+            TMPDIR="$WORK_DIR" \
+            TF_VAR_proxmox_endpoint="$PVE_ENDPOINT" \
+            TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \
+            TF_VAR_target_node="$PVE_TARGET_NODE" \
+            TF_VAR_test_vm_password="$PVE_PASSWORD" \
+            TF_VAR_cloud_image_path="$CLOUD_IMAGE_PATH" \
+            TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \
+            TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \
+            TF_VAR_storage_vmid="$STORAGE_VMID" \
+            TF_VAR_storage_vm_ssh_public_key="$(cat "${STORAGE_VM_SSH_KEY}.pub")" \
+            terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" "$@")
+    }
+
+    log "Running Terraform apply (storage VM)..."
+    tf_apply \
+        -target=proxmox_virtual_environment_file.storage_cloud_image \
+        -target=proxmox_virtual_environment_vm.storage
+
+    log "Configuring storage VM at $STORAGE_VM_HOST..."
+    bash "$SCRIPT_DIR/setup-storage-server.sh" \
+        "$STORAGE_VM_HOST" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server"
+
     log "Running Terraform apply (PVE nodes)..."
     local tf_targets=""
     if [[ "$requested" != "all" ]]; then
@@ -334,26 +373,10 @@ cmd_provision() {
         for node in $provision_nodes; do
             tf_targets="$tf_targets -target=proxmox_virtual_environment_vm.nested_pve[\"$node\"]"
         done
-        # Always include shared Docker storage and answer server resources
-        tf_targets="$tf_targets -target=docker_image.ubuntu"
-        tf_targets="$tf_targets -target=docker_container.iscsi_target"
-        tf_targets="$tf_targets -target=docker_container.nfs_server"
-        tf_targets="$tf_targets -target=docker_container.answer_server"
-        tf_targets="$tf_targets -target=docker_volume.iscsi_data"
-        tf_targets="$tf_targets -target=docker_volume.nfs_data"
         log "Terraform targets: $tf_targets"
     fi
 
-    # TMPDIR: use work dir to avoid filling the container's /tmp with multi-GB ISO uploads.
+    tf_apply $tf_targets
-    (cd "$INFRA_DIR" && \
-        TMPDIR="$WORK_DIR" \
-        TF_VAR_proxmox_endpoint="$PVE_ENDPOINT" \
-        TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \
-        TF_VAR_target_node="$PVE_TARGET_NODE" \
-        TF_VAR_test_vm_password="$PVE_PASSWORD" \
-        TF_VAR_docker_host_ip="$storage_ip" \
-        TF_VAR_answer_server_dir="$WORK_DIR/answer-server" \
-        terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets)
 
     # Wait for PVE instances to boot and discover IPs
     for node in $provision_nodes; do
@@ -401,7 +424,7 @@ cmd_provision() {
     config=$(jq \
         --arg cloud_image "${CLOUD_IMAGE_PATH:-}" \
         --arg ova "${OVA_PATH:-}" \
-        --arg storage_ip "$storage_ip" \
+        --arg storage_ip "$STORAGE_VM_HOST" \
         --arg storage_iqn "$STORAGE_ISCSI_IQN" \
         '. + {
             storage: {ip: $storage_ip, iscsi_iqn: $storage_iqn, nfs_export: ($storage_ip + ":/srv/nfs/shared")},
@@ -480,7 +503,7 @@ cmd_test() {
             export PVETEST_OVA_PATH=$(jq -r '.ova_path' "$CONFIG_FILE")
             # Secondary node (b)
             export PVETEST_HOST_B=$(jq -r ".pve${v}.nodes.b.host" "$CONFIG_FILE")
-            # Storage services (Docker on runner)
+            # Storage services (dedicated VM on the CI VLAN)
             export PVETEST_STORAGE_VM_IP=$(jq -r '.storage.ip' "$CONFIG_FILE")
             export PVETEST_ISCSI_IQN=$(jq -r '.storage.iscsi_iqn' "$CONFIG_FILE")
             export PVETEST_NFS_EXPORT=$(jq -r '.storage.nfs_export' "$CONFIG_FILE")
@@ -552,11 +575,19 @@ cmd_test() {
             }
 
             \$config.Filter.Tag = 'Integration'
+            \$config.Run.PassThru = \$true
             \$config.Output.Verbosity = 'Detailed'
             \$config.TestResult.Enabled = \$true
             \$config.TestResult.OutputFormat = 'NUnitXml'
             \$config.TestResult.OutputPath = \"TestResults/integration-results-pve\${PveVersion}.xml\"
-            Invoke-Pester -Configuration \$config
+            \$result = Invoke-Pester -Configuration \$config
+            if (-not \$result -or \$result.TotalCount -eq 0) {
+                Write-Error 'No integration tests were discovered or executed'
+                exit 1
+            }
+            if (\$result.FailedCount -gt 0) {
+                exit 1
+            }
         " || test_exit=$?
 
         if [[ $test_exit -ne 0 ]]; then
@@ -581,13 +612,6 @@ cmd_cleanup() {
     fi
     require_env PVE_TARGET_NODE
 
-    # Discover Docker host IP for the docker_host_ip variable
-    local storage_ip
-    storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}')
-    if [ -z "$storage_ip" ]; then
-        storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1)
-    fi
-
     # Build tfvars for all versions (Terraform needs the full variable map)
     local tfvars="$WORK_DIR/instances.tfvars.json"
     if [[ ! -f "$tfvars" ]]; then
@@ -616,10 +640,6 @@ cmd_cleanup() {
 
     (cd "$INFRA_DIR" && terraform init -input=false 2>/dev/null)
 
-    # Ensure answer server dir exists (terraform destroy validates host_path mounts)
-    mkdir -p "$WORK_DIR/answer-server/answers"
-    touch "$WORK_DIR/answer-server/default.toml"
-
     # Build -target flags when destroying a subset
     local tf_targets=""
     if [[ "$requested" != "all" ]]; then
@@ -641,8 +661,9 @@ cmd_cleanup() {
         TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \
         TF_VAR_target_node="$PVE_TARGET_NODE" \
         TF_VAR_test_vm_password="${PVE_PASSWORD:-placeholder}" \
-        TF_VAR_docker_host_ip="${storage_ip:-127.0.0.1}" \
+        TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \
-        TF_VAR_answer_server_dir="${WORK_DIR}/answer-server" \
+        TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \
+        TF_VAR_storage_vmid="$STORAGE_VMID" \
         terraform destroy -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets)
 
     # Clean up work directory when destroying all
@@ -665,6 +686,8 @@ cmd_force_cleanup() {
 
     log "Force cleanup — bypassing Terraform, using direct API calls..."
 
+    # preflight-cleanup.sh reads PVE_TARGET_NODE from the environment; in auto
+    # mode resolve it from the node provision picked.
     if [[ -z "${PVE_TARGET_NODE:-}" || "$PVE_TARGET_NODE" == "auto" ]]; then
         PVE_TARGET_NODE="$(cat "$TARGET_NODE_FILE" 2>/dev/null || true)"
         [[ -n "$PVE_TARGET_NODE" ]] && export PVE_TARGET_NODE
@@ -674,14 +697,13 @@ cmd_force_cleanup() {
     # Track which versions we've already cleaned up ISOs for (generic ISOs are shared)
     local cleaned_iso_versions=""
     for node in $cleanup_nodes; do
-        local vm_id v iso_name iso_file
+        local vm_id v iso_file
         vm_id="$(pve_vmid "$node")"
         v="${node%[ab]}"
-        iso_name="$(pve_iso "$v")"
         # Only clean up the generic ISO once per version
         iso_file=""
         if [[ ! " $cleaned_iso_versions " =~ " $v " ]]; then
-            iso_file="${iso_name%.iso}-http-auto.iso"
+            iso_file="$(pve_auto_iso "$v")"
             cleaned_iso_versions="$cleaned_iso_versions $v"
         fi
         log "Force cleaning $node (VMID $vm_id)..."
@@ -691,11 +713,13 @@ cmd_force_cleanup() {
             || true
     done
 
-    # Always stop Docker containers in force mode — leaving them causes
+    # Unconditional: the state wipe below is unconditional too, and a storage
-    # Terraform to fail on next provision (container already exists).
+    # VM surviving its state entry cannot be reclaimed by the next provision.
-    log "Stopping storage and answer server containers..."
+    log "Force cleaning storage VM (VMID $STORAGE_VMID)..."
-    docker rm -f pvetest-iscsi pvetest-nfs pvetest-answer-server 2>/dev/null || true
+    bash "$SCRIPT_DIR/preflight-cleanup.sh" \
-    docker volume rm pvetest-iscsi-data pvetest-nfs-data 2>/dev/null || true
+        "${PVE_ENDPOINT:-}" "${PVE_API_TOKEN:-}" \
+        "$STORAGE_VMID" "$CLOUD_IMAGE_NAME" "$INFRA_DIR" \
+        || true
 
     # Remove Terraform state (both local and shared mount) so next provision starts clean.
     # Keep .terraform.lock.hcl (provider version lock) for reproducibility.
@@ -704,8 +728,9 @@ cmd_force_cleanup() {
     rm -f "$TF_STATE_FILE" "${TF_STATE_FILE}.backup"
     rm -rf "$INFRA_DIR/.terraform"
 
-    # Remove work artifacts
+    # Remove work artifacts, including locally cached auto-install ISOs
     rm -f "$CONFIG_FILE" "$WORK_DIR"/instances.tfvars.json "$TARGET_NODE_FILE"
+    rm -f "$WORK_DIR"/*-auto-*.iso "$WORK_DIR"/*-http-auto.iso
 
     log "Force cleanup complete. Next provision will start from scratch."
 }
@@ -759,7 +784,7 @@ cmd_all() {
     return $test_exit
 }
 
-# ── Main ────────────────────────────────────────────────────────────
+# ── Main ───────────────────────────────────────────────────────────
 main() {
     local cmd="${1:-}"
     shift || true
@@ -775,7 +800,7 @@ main() {
             echo "Usage: $(basename "$0") {provision|test|cleanup|force-cleanup|taint|all} [8|9|all] [test-filter]"
             echo ""
             echo "Subcommands:"
-            echo "  provision [8|9|all]        Provision nested PVE VMs + storage containers"
+            echo "  provision [8|9|all]        Provision storage VM + nested PVE VMs"
             echo "  test [8|9|all] [filter]    Run integration tests (default: all versions, no filter)"
             echo "  cleanup [8|9|all]          Destroy resources via terraform destroy (default: all)"
             echo "  force-cleanup [8|9|all]    Bypass Terraform — destroy via API + wipe state (recovery)"

tests/infrastructure/scripts/setup-storage-server.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Configures the shared services VM inside the CI sandbox VLAN: NFS export,
+# iSCSI target, and the HTTP answer server the PVE auto-installers fetch
+# from. Idempotent — safe to re-run against an already-configured VM.
+#
+# Usage: setup-storage-server.sh <storage-vm-ip> <ssh-private-key-path> <iscsi-iqn> <answer-dir>
+#   <answer-dir> is the local directory holding default.toml and answers/.
+
+set -euo pipefail
+
+STORAGE_IP="${1:?Usage: setup-storage-server.sh <ip> <ssh-key> <iqn> <answer-dir>}"
+SSH_KEY="${2:?missing SSH private key path}"
+ISCSI_IQN="${3:?missing iSCSI IQN}"
+ANSWER_DIR="${4:?missing answer directory}"
+
+# Same pinned image the Terraform-managed runner container used previously.
+ANSWER_SERVER_IMAGE="slothcroissant/proxmox-auto-installer-server@sha256:0f45d7bfe6e3cc76aa00fc578e40b80b9054e377db18a79122866fe5522bc7ed"
+
+SSH_OPTS="-i ${SSH_KEY} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
+
+echo "=== Configuring storage services on ${STORAGE_IP} ==="
+
+# Cloud-init may still be running on first boot; wait for SSH to accept the
+# key login it installs. The last attempt's stderr is kept so an auth failure
+# is distinguishable from an unreachable host.
+for i in $(seq 1 60); do
+    if ssh ${SSH_OPTS} -o ConnectTimeout=5 "ubuntu@${STORAGE_IP}" true 2>/tmp/storage-ssh-err; then
+        break
+    fi
+    if [[ $i -eq 60 ]]; then
+        echo "ERROR: storage VM at ${STORAGE_IP} not reachable over SSH" >&2
+        cat /tmp/storage-ssh-err >&2
+        exit 1
+    fi
+    sleep 5
+done
+
+# sshd accepts logins before cloud-init's first-boot work (and its apt/dpkg
+# locks) is finished; a degraded cloud-init result is fine for our purposes,
+# and a hung cloud-init must fail fast, not eat the job timeout.
+timeout 300 ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
+    "sudo cloud-init status --wait" || true
+
+echo "Copying answer files..."
+ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
+    "sudo mkdir -p /opt/answer-server && sudo rm -rf /opt/answer-server/answers && sudo chown ubuntu /opt/answer-server"
+scp ${SSH_OPTS} -r \
+    "${ANSWER_DIR}/default.toml" "${ANSWER_DIR}/answers" \
+    "ubuntu@${STORAGE_IP}:/opt/answer-server/"
+
+ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
+    "sudo ISCSI_IQN='${ISCSI_IQN}' ANSWER_SERVER_IMAGE='${ANSWER_SERVER_IMAGE}' bash -s" <<'REMOTE'
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+# cloud-init's first-boot apt activity can hold the dpkg lock briefly
+for i in $(seq 1 30); do
+    apt-get update -qq >/dev/null 2>&1 && break
+    sleep 5
+done
+apt-get install -y -qq nfs-kernel-server tgt docker.io >/dev/null
+
+# Ubuntu's nfs-kernel-server package does not create /etc/exports.d
+mkdir -p /etc/exports.d /srv/nfs/shared /srv/iscsi
+echo '/srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)' > /etc/exports.d/pvetest.exports
+exportfs -ra
+
+if [ ! -f /srv/iscsi/lun0.img ]; then
+    truncate -s 10G /srv/iscsi/lun0.img
+fi
+cat > /etc/tgt/conf.d/pvetest.conf <<CONF
+<target ${ISCSI_IQN}>
+    backing-store /srv/iscsi/lun0.img
+</target>
+CONF
+
+systemctl enable --now nfs-kernel-server tgt >/dev/null || {
+    journalctl -u nfs-kernel-server -u tgt --no-pager -n 30 >&2
+    exit 1
+}
+systemctl restart tgt
+
+docker rm -f pvetest-answer-server >/dev/null 2>&1 || true
+docker run -d --name pvetest-answer-server --restart unless-stopped \
+    --network host \
+    -v /opt/answer-server/answers:/app/answers \
+    -v /opt/answer-server/default.toml:/app/default.toml \
+    "${ANSWER_SERVER_IMAGE}" >/dev/null
+
+# Ordering requirement: the PVE installers fetch answers as soon as this
+# script returns, so :8000 must accept connections before exit (any HTTP
+# status counts as listening).
+for i in $(seq 1 20); do
+    code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 2 http://127.0.0.1:8000/ || true)
+    [ "$code" != "000" ] && break
+    if [ "$i" -eq 20 ]; then
+        echo "ERROR: answer server did not start listening on :8000" >&2
+        docker logs pvetest-answer-server 2>&1 | tail -20 >&2
+        exit 1
+    fi
+    sleep 3
+done
+
+echo "Services ready: NFS /srv/nfs/shared, iSCSI ${ISCSI_IQN}, answer server :8000"
+REMOTE
+
+echo "Storage server configuration complete."

tests/infrastructure/storage-vm.tf

@@ -0,0 +1,80 @@
+# ── Shared storage VM ───────────────────────────────────────────────
+# One small Ubuntu guest in the CI pool serves NFS + iSCSI to every nested
+# PVE node. It lives on the same isolated VLAN as the nodes under test, so
+# no firewall path out of the CI sandbox is needed. Package install and
+# service configuration happen over SSH (setup-storage-server.sh) after boot.
+
+resource "proxmox_virtual_environment_file" "storage_cloud_image" {
+  content_type = "import"
+  datastore_id = var.iso_storage
+  node_name    = var.target_node
+  overwrite    = true
+
+  source_file {
+    path = var.cloud_image_path
+  }
+}
+
+resource "proxmox_virtual_environment_vm" "storage" {
+  name      = "pvetest-storage"
+  node_name = var.target_node
+  vm_id     = var.storage_vmid
+  pool_id   = var.pool_id
+
+  cpu {
+    type    = "host"
+    cores   = 2
+    sockets = 1
+  }
+
+  memory {
+    dedicated = 2048
+  }
+
+  disk {
+    datastore_id = var.disk_storage
+    interface    = "scsi0"
+    size         = 32
+    import_from  = proxmox_virtual_environment_file.storage_cloud_image.id
+  }
+
+  initialization {
+    datastore_id = var.disk_storage
+
+    ip_config {
+      ipv4 {
+        address = var.storage_vm_ip
+        gateway = var.storage_vm_gateway
+      }
+    }
+
+    dns {
+      servers = [var.storage_vm_gateway]
+    }
+
+    user_account {
+      username = "ubuntu"
+      # Password is console-only: Ubuntu cloud images refuse SSH password auth
+      # and PVE's cloud-init never sets ssh_pwauth. SSH uses the key.
+      password = var.test_vm_password
+      keys     = [trimspace(var.storage_vm_ssh_public_key)]
+    }
+  }
+
+  network_device {
+    bridge = var.network_bridge
+    model  = "virtio"
+  }
+
+  operating_system {
+    type = "l26"
+  }
+
+  # Resized Ubuntu cloud images kernel-panic on boot without a serial console
+  # (bpg/terraform-provider-proxmox documented issue).
+  serial_device {
+    device = "socket"
+  }
+
+  started = true
+}

tests/infrastructure/storage.tf

@@ -1,112 +0,0 @@
-# ── Answer server container ──────────────────────────────────────────
-
-resource "docker_container" "answer_server" {
-  name     = "pvetest-answer-server"
-  # Pin to digest for reproducibility. Update by pulling latest and running:
-  #   docker inspect slothcroissant/proxmox-auto-installer-server:latest --format '{{index .RepoDigests 0}}'
-  image    = "slothcroissant/proxmox-auto-installer-server@sha256:0f45d7bfe6e3cc76aa00fc578e40b80b9054e377db18a79122866fe5522bc7ed"
-  restart  = "unless-stopped"
-  must_run = true
-  start    = true
-
-  network_mode = "host"
-
-  volumes {
-    host_path      = "${var.answer_server_dir}/answers"
-    container_path = "/app/answers"
-  }
-
-  volumes {
-    host_path      = "${var.answer_server_dir}/default.toml"
-    container_path = "/app/default.toml"
-  }
-}
-
-# ── Docker images & volumes ──────────────────────────────────────────
-
-resource "docker_image" "ubuntu" {
-  # Pin to digest for reproducibility. Update:
-  #   docker pull ubuntu:24.04 && docker inspect ubuntu:24.04 --format '{{index .RepoDigests 0}}'
-  name = "ubuntu@sha256:186072bba1b2f436cbb91ef2567abca677337cfc786c86e107d25b7072feef0c"
-}
-
-resource "docker_volume" "iscsi_data" {
-  name = "pvetest-iscsi-data"
-}
-
-resource "docker_volume" "nfs_data" {
-  name = "pvetest-nfs-data"
-}
-
-# ── iSCSI target container ──────────────────────────────────────────
-
-resource "docker_container" "iscsi_target" {
-  name       = "pvetest-iscsi"
-  image      = docker_image.ubuntu.image_id
-  privileged = true
-  restart    = "unless-stopped"
-
-  network_mode = "host"
-
-  volumes {
-    volume_name    = docker_volume.iscsi_data.name
-    container_path = "/srv/iscsi"
-  }
-
-  env = [
-    "ISCSI_IQN=${var.storage_iscsi_iqn}",
-    "ISCSI_LUN_SIZE=${var.storage_iscsi_lun_size}",
-  ]
-
-  entrypoint = ["/bin/bash", "-c"]
-  command = [<<-EOT
-    set -e
-    apt-get update -qq && apt-get install -y -qq tgt >/dev/null 2>&1
-    mkdir -p /srv/iscsi
-    if [ ! -f /srv/iscsi/lun0.img ]; then
-      truncate -s $${ISCSI_LUN_SIZE} /srv/iscsi/lun0.img
-    fi
-    tgtd --foreground &
-    sleep 2
-    if ! tgtadm --lld iscsi --op show --mode target | grep -q "Target 1: $${ISCSI_IQN}"; then
-      tgtadm --lld iscsi --op new --mode target --tid 1 -T $${ISCSI_IQN}
-    fi
-    if ! tgtadm --lld iscsi --op show --mode logicalunit --tid 1 2>/dev/null | grep -qE "LUN:[[:space:]]*1($$|[^0-9])"; then
-      tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 --backing-store /srv/iscsi/lun0.img
-    fi
-    if ! tgtadm --lld iscsi --op show --mode target --tid 1 2>/dev/null | grep -q "Initiator-address: ALL"; then
-      tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL
-    fi
-    echo "iSCSI target ready: $${ISCSI_IQN} (port 3260)"
-    wait
-  EOT
-  ]
-}
-
-# ── NFS server container ────────────────────────────────────────────
-
-resource "docker_container" "nfs_server" {
-  name       = "pvetest-nfs"
-  # Pin to digest for reproducibility. Update:
-  #   docker pull erichough/nfs-server:2.2.1 && docker inspect erichough/nfs-server:2.2.1 --format '{{index .RepoDigests 0}}'
-  image      = "erichough/nfs-server@sha256:1efd4ece380c5ba27479417585224ef857006daa46ab84560a28c1224bc71e9e"
-  privileged = true
-  restart    = "unless-stopped"
-
-  network_mode = "host"
-
-  volumes {
-    volume_name    = docker_volume.nfs_data.name
-    container_path = "/srv/nfs/shared"
-  }
-
-  volumes {
-    host_path      = "/lib/modules"
-    container_path = "/lib/modules"
-    read_only      = true
-  }
-
-  env = [
-    "NFS_EXPORT_0=/srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)",
-  ]
-}

tests/infrastructure/variables.tf

@@ -61,7 +61,7 @@ variable "disk_storage" {
 }
 
 variable "iso_storage" {
-  description = "Proxmox storage pool for uploading the ISO (must accept ISO content type)"
+  description = "Proxmox storage pool for uploads (must accept the iso AND import content types — import is not enabled by default on most storages)"
   type        = string
   default     = "local"
 }
@@ -78,28 +78,40 @@ variable "test_vm_password" {
   sensitive   = true
 }
 
-variable "storage_iscsi_iqn" {
+variable "storage_vmid" {
-  description = "iSCSI target IQN for the test storage"
+  description = "VMID for the shared storage VM (must be inside the CI pool's reserved range)"
-  type        = string
+  type        = number
-  default     = "iqn.2024-01.local.test:storage"
+  default     = 5080
 }
 
-variable "storage_iscsi_lun_size" {
+variable "storage_vm_ip" {
-  description = "Size of the iSCSI LUN backing file"
+  description = "Static IPv4 address in CIDR form for the storage VM, outside the VLAN's DHCP range"
   type        = string
-  default     = "10G"
+  default     = "172.16.60.60/24"
+
+  validation {
+    condition     = can(cidrnetmask(var.storage_vm_ip))
+    error_message = "storage_vm_ip must be CIDR notation, e.g. 172.16.60.60/24."
+  }
 }
 
-variable "docker_host_ip" {
+variable "storage_vm_ssh_public_key" {
-  description = "IP of the Docker host, used by PVE nodes to reach storage containers"
+  description = "SSH public key granted to the storage VM's ubuntu user (cloud images refuse password SSH; required for provision, unused on destroy)"
   type        = string
+  default     = ""
 }
 
-variable "answer_server_dir" {
+variable "storage_vm_gateway" {
-  description = "Host path to the answer server root directory (contains default.toml and answers/ subdirectory)"
+  description = "Gateway (and DNS server) for the storage VM"
   type        = string
+  default     = "172.16.60.1"
 }
 
+variable "cloud_image_path" {
+  description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)"
+  type        = string
+  default     = ""
+}
 
 variable "pool_id" {
   description = "Resource pool the nested VMs are created in (a pool-scoped API token can only allocate here)"