ci: storage and answer services move to a VM inside the CI sandbox (1/2)

The nested PVE guests live on an isolated VLAN with no route to the
runner network (ADR 0032, homelab repo), and the runner-hosted Docker
services died with each ephemeral ARC pod anyway. A small cloud-image VM
in the ci pool now serves NFS, iSCSI, and the auto-install answer files
from inside the sandbox. Includes fixes from Codex and in-house review:
bpg provider >= 0.79.0, serial console for the resized cloud image,
key-based SSH (cloud images refuse password auth), CIDR validation,
storage VM in the headroom check, docker socket mounts removed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/integration-tests.yml

@@ -24,6 +24,9 @@ name: Integration Tests
 #                        node with the most free memory and refuses to run when
 #                        no node has enough headroom — the token's user needs
 #                        PVEAuditor on /nodes for the memory stats.
+#   STORAGE_VM_IP      - Static CIDR address for the shared storage/answer-server
+#                        VM on the CI VLAN (default: 172.16.60.60/24)
+#   STORAGE_VM_GATEWAY - Gateway + DNS for that VM (default: 172.16.60.1)
 #
 # Optional secrets (for skip_provision mode):
 #   PVETEST_HOST       - Hostname or IP of a pre-existing nested PVE
@@ -122,11 +125,6 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
       volumes:
         - /opt/pve-integration:/opt/pve-integration
-        # WARNING: The Docker socket is mounted to allow storage containers to be
-        # managed from within the job container. This grants root-equivalent access
-        # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted
-        # runners — never on shared runners.
-        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
       - uses: actions/checkout@v7
@@ -138,6 +136,8 @@ jobs:
           PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
           PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }}
           PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }}
+          STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }}
+          STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }}
           TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }}
           TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }}
           TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }}
@@ -201,11 +201,6 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
       volumes:
         - /opt/pve-integration:/opt/pve-integration
-        # WARNING: The Docker socket is mounted to allow storage containers to be
-        # stopped from within the job container. This grants root-equivalent access
-        # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted
-        # runners — never on shared runners.
-        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
       - uses: actions/checkout@v7
@@ -217,6 +212,8 @@ jobs:
           PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
           PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }}
           PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }}
+          STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }}
+          STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }}
           TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }}
           TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }}
           TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }}

tests/PSProxmoxVE.Tests/Integration/03a_SharedStorage.Tests.ps1

@@ -4,7 +4,7 @@
     Pester 5 integration tests for shared storage backends (NFS, iSCSI).
 
     These tests require the multi-node integration test infrastructure with
-    Docker-based storage services (iSCSI target + NFS server). They are
+    the shared storage VM (iSCSI target + NFS server on the CI VLAN). They are
     SKIPPED when the storage env vars are not set.
 
     Required environment variables (in addition to base integration vars):

tests/infrastructure/main.tf

@@ -3,11 +3,8 @@ terraform {
   required_providers {
     proxmox = {
       source  = "bpg/proxmox"
-      version = ">= 0.70.0"
-    }
-    docker = {
-      source  = "kreuzwerker/docker"
-      version = ">= 3.0.0"
+      # disk.import_from and content_type "import" need >= 0.79.0
+      version = ">= 0.79.0"
     }
   }
 }
@@ -18,11 +15,6 @@ provider "proxmox" {
   insecure  = var.proxmox_insecure
 }
 
-provider "docker" {
-  # Uses the Docker socket from the dev-infra container
-  # (mounted at /var/run/docker.sock)
-}
-
 resource "proxmox_virtual_environment_file" "auto_iso" {
   for_each     = var.pve_isos
   content_type = "iso"
@@ -91,9 +83,10 @@ resource "proxmox_virtual_environment_vm" "nested_pve" {
 
   started = true
 
-  # VMs must not boot until the HTTP answer server is running,
-  # otherwise the PVE auto-installer can't fetch its answer file.
-  depends_on = [docker_container.answer_server]
+  # VMs must not boot until the storage VM exists — it serves the HTTP answer
+  # files the PVE auto-installer fetches. run-integration.sh additionally
+  # configures the storage VM (phase one) before applying these resources.
+  depends_on = [proxmox_virtual_environment_vm.storage]
 
   lifecycle {
     ignore_changes = [started, cdrom]

tests/infrastructure/outputs.tf

@@ -8,17 +8,7 @@ output "pve_test_node_name" {
   description = "Default node name inside a fresh PVE install"
 }
 
-output "storage_ip" {
-  description = "IP address where storage services are reachable"
-  value       = var.docker_host_ip
-}
-
-output "storage_iscsi_iqn" {
-  description = "iSCSI target IQN"
-  value       = var.storage_iscsi_iqn
-}
-
-output "storage_nfs_export" {
-  description = "NFS export path"
-  value       = "${var.docker_host_ip}:/srv/nfs/shared"
+output "storage_vm_ip" {
+  description = "Static address of the shared NFS/iSCSI storage VM"
+  value       = var.storage_vm_ip
 }

tests/infrastructure/scripts/setup-storage-server.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Configures the shared services VM inside the CI sandbox VLAN: NFS export,
+# iSCSI target, and the HTTP answer server the PVE auto-installers fetch
+# from. Idempotent — safe to re-run against an already-configured VM.
+#
+# Usage: setup-storage-server.sh <storage-vm-ip> <ssh-private-key-path> <iscsi-iqn> <answer-dir>
+#   <answer-dir> is the local directory holding default.toml and answers/.
+
+set -euo pipefail
+
+STORAGE_IP="${1:?Usage: setup-storage-server.sh <ip> <ssh-key> <iqn> <answer-dir>}"
+SSH_KEY="${2:?missing SSH private key path}"
+ISCSI_IQN="${3:?missing iSCSI IQN}"
+ANSWER_DIR="${4:?missing answer directory}"
+
+# Same pinned image the Terraform-managed runner container used previously.
+ANSWER_SERVER_IMAGE="slothcroissant/proxmox-auto-installer-server@sha256:0f45d7bfe6e3cc76aa00fc578e40b80b9054e377db18a79122866fe5522bc7ed"
+
+SSH_OPTS="-i ${SSH_KEY} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
+
+echo "=== Configuring storage services on ${STORAGE_IP} ==="
+
+# Cloud-init may still be running on first boot; wait for SSH to accept the
+# key login it installs. The last attempt's stderr is kept so an auth failure
+# is distinguishable from an unreachable host.
+for i in $(seq 1 60); do
+    if ssh ${SSH_OPTS} -o ConnectTimeout=5 "ubuntu@${STORAGE_IP}" true 2>/tmp/storage-ssh-err; then
+        break
+    fi
+    if [[ $i -eq 60 ]]; then
+        echo "ERROR: storage VM at ${STORAGE_IP} not reachable over SSH" >&2
+        cat /tmp/storage-ssh-err >&2
+        exit 1
+    fi
+    sleep 5
+done
+
+# sshd accepts logins before cloud-init's first-boot work (and its apt/dpkg
+# locks) is finished; a degraded cloud-init result is fine for our purposes,
+# and a hung cloud-init must fail fast, not eat the job timeout.
+timeout 300 ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
+    "sudo cloud-init status --wait" || true
+
+echo "Copying answer files..."
+ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
+    "sudo mkdir -p /opt/answer-server && sudo rm -rf /opt/answer-server/answers && sudo chown ubuntu /opt/answer-server"
+scp ${SSH_OPTS} -r \
+    "${ANSWER_DIR}/default.toml" "${ANSWER_DIR}/answers" \
+    "ubuntu@${STORAGE_IP}:/opt/answer-server/"
+
+ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
+    "sudo ISCSI_IQN='${ISCSI_IQN}' ANSWER_SERVER_IMAGE='${ANSWER_SERVER_IMAGE}' bash -s" <<'REMOTE'
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+# cloud-init's first-boot apt activity can hold the dpkg lock briefly
+for i in $(seq 1 30); do
+    apt-get update -qq >/dev/null 2>&1 && break
+    sleep 5
+done
+apt-get install -y -qq nfs-kernel-server tgt docker.io >/dev/null
+
+# Ubuntu's nfs-kernel-server package does not create /etc/exports.d
+mkdir -p /etc/exports.d /srv/nfs/shared /srv/iscsi
+echo '/srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)' > /etc/exports.d/pvetest.exports
+exportfs -ra
+
+if [ ! -f /srv/iscsi/lun0.img ]; then
+    truncate -s 10G /srv/iscsi/lun0.img
+fi
+cat > /etc/tgt/conf.d/pvetest.conf <<CONF
+<target ${ISCSI_IQN}>
+    backing-store /srv/iscsi/lun0.img
+</target>
+CONF
+
+systemctl enable --now nfs-kernel-server tgt >/dev/null || {
+    journalctl -u nfs-kernel-server -u tgt --no-pager -n 30 >&2
+    exit 1
+}
+systemctl restart tgt
+
+docker rm -f pvetest-answer-server >/dev/null 2>&1 || true
+docker run -d --name pvetest-answer-server --restart unless-stopped \
+    --network host \
+    -v /opt/answer-server/answers:/app/answers \
+    -v /opt/answer-server/default.toml:/app/default.toml \
+    "${ANSWER_SERVER_IMAGE}" >/dev/null
+
+# Ordering requirement: the PVE installers fetch answers as soon as this
+# script returns, so :8000 must accept connections before exit (any HTTP
+# status counts as listening).
+for i in $(seq 1 20); do
+    code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 2 http://127.0.0.1:8000/ || true)
+    [ "$code" != "000" ] && break
+    if [ "$i" -eq 20 ]; then
+        echo "ERROR: answer server did not start listening on :8000" >&2
+        docker logs pvetest-answer-server 2>&1 | tail -20 >&2
+        exit 1
+    fi
+    sleep 3
+done
+
+echo "Services ready: NFS /srv/nfs/shared, iSCSI ${ISCSI_IQN}, answer server :8000"
+REMOTE
+
+echo "Storage server configuration complete."

tests/infrastructure/storage-vm.tf

@@ -0,0 +1,80 @@
+# ── Shared storage VM ───────────────────────────────────────────────
+# One small Ubuntu guest in the CI pool serves NFS + iSCSI to every nested
+# PVE node. It lives on the same isolated VLAN as the nodes under test, so
+# no firewall path out of the CI sandbox is needed. Package install and
+# service configuration happen over SSH (setup-storage-server.sh) after boot.
+
+resource "proxmox_virtual_environment_file" "storage_cloud_image" {
+  content_type = "import"
+  datastore_id = var.iso_storage
+  node_name    = var.target_node
+  overwrite    = true
+
+  source_file {
+    path = var.cloud_image_path
+  }
+}
+
+resource "proxmox_virtual_environment_vm" "storage" {
+  name      = "pvetest-storage"
+  node_name = var.target_node
+  vm_id     = var.storage_vmid
+  pool_id   = var.pool_id
+
+  cpu {
+    type    = "host"
+    cores   = 2
+    sockets = 1
+  }
+
+  memory {
+    dedicated = 2048
+  }
+
+  disk {
+    datastore_id = var.disk_storage
+    interface    = "scsi0"
+    size         = 32
+    import_from  = proxmox_virtual_environment_file.storage_cloud_image.id
+  }
+
+  initialization {
+    datastore_id = var.disk_storage
+
+    ip_config {
+      ipv4 {
+        address = var.storage_vm_ip
+        gateway = var.storage_vm_gateway
+      }
+    }
+
+    dns {
+      servers = [var.storage_vm_gateway]
+    }
+
+    user_account {
+      username = "ubuntu"
+      # Password is console-only: Ubuntu cloud images refuse SSH password auth
+      # and PVE's cloud-init never sets ssh_pwauth. SSH uses the key.
+      password = var.test_vm_password
+      keys     = [trimspace(var.storage_vm_ssh_public_key)]
+    }
+  }
+
+  network_device {
+    bridge = var.network_bridge
+    model  = "virtio"
+  }
+
+  operating_system {
+    type = "l26"
+  }
+
+  # Resized Ubuntu cloud images kernel-panic on boot without a serial console
+  # (bpg/terraform-provider-proxmox documented issue).
+  serial_device {
+    device = "socket"
+  }
+
+  started = true
+}

tests/infrastructure/variables.tf

@@ -61,7 +61,7 @@ variable "disk_storage" {
 }
 
 variable "iso_storage" {
-  description = "Proxmox storage pool for uploading the ISO (must accept ISO content type)"
+  description = "Proxmox storage pool for uploads (must accept the iso AND import content types — import is not enabled by default on most storages)"
   type        = string
   default     = "local"
 }
@@ -78,28 +78,40 @@ variable "test_vm_password" {
   sensitive   = true
 }
 
-variable "storage_iscsi_iqn" {
-  description = "iSCSI target IQN for the test storage"
-  type        = string
-  default     = "iqn.2024-01.local.test:storage"
+variable "storage_vmid" {
+  description = "VMID for the shared storage VM (must be inside the CI pool's reserved range)"
+  type        = number
+  default     = 5080
 }
 
-variable "storage_iscsi_lun_size" {
-  description = "Size of the iSCSI LUN backing file"
+variable "storage_vm_ip" {
+  description = "Static IPv4 address in CIDR form for the storage VM, outside the VLAN's DHCP range"
   type        = string
-  default     = "10G"
+  default     = "172.16.60.60/24"
+
+  validation {
+    condition     = can(cidrnetmask(var.storage_vm_ip))
+    error_message = "storage_vm_ip must be CIDR notation, e.g. 172.16.60.60/24."
+  }
 }
 
-variable "docker_host_ip" {
-  description = "IP of the Docker host, used by PVE nodes to reach storage containers"
+variable "storage_vm_ssh_public_key" {
+  description = "SSH public key granted to the storage VM's ubuntu user (cloud images refuse password SSH; required for provision, unused on destroy)"
   type        = string
+  default     = ""
 }
 
-variable "answer_server_dir" {
-  description = "Host path to the answer server root directory (contains default.toml and answers/ subdirectory)"
+variable "storage_vm_gateway" {
+  description = "Gateway (and DNS server) for the storage VM"
   type        = string
+  default     = "172.16.60.1"
 }
 
+variable "cloud_image_path" {
+  description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)"
+  type        = string
+  default     = ""
+}
 
 variable "pool_id" {
   description = "Resource pool the nested VMs are created in (a pool-scoped API token can only allocate here)"