From 5d0b384508a703360c62455256092f2e329850fb Mon Sep 17 00:00:00 2001 From: "goodolclint-claude[bot]" <323206664+goodolclint-claude[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:39:02 +0000 Subject: [PATCH 1/4] ci: storage VM setup diagnoses its own network and logs phases Run 168 burned 13 minutes to report 'unable to locate package tgt' when the real failure was dead egress from the CI VLAN (apt lists never updated). The setup script now probes gateway/internet/DNS from inside the VM and prints the verdicts, fails apt fast with the real error, and marks each phase. Co-Authored-By: Claude Fable 5 --- .../scripts/setup-storage-server.sh | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/infrastructure/scripts/setup-storage-server.sh b/tests/infrastructure/scripts/setup-storage-server.sh index 8e76149..654dda9 100644 --- a/tests/infrastructure/scripts/setup-storage-server.sh +++ b/tests/infrastructure/scripts/setup-storage-server.sh @@ -38,8 +38,21 @@ done # sshd accepts logins before cloud-init's first-boot work (and its apt/dpkg # locks) is finished; a degraded cloud-init result is fine for our purposes, # and a hung cloud-init must fail fast, not eat the job timeout. +echo "Waiting for cloud-init to finish (up to 300s)..." timeout 300 ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ - "sudo cloud-init status --wait" || true + "sudo cloud-init status --wait" \ + || echo "WARNING: cloud-init did not finish cleanly within 300s — continuing" + +echo "Network probe from the storage VM:" +ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" ' + ip -4 -brief addr; ip route show default + ping -c1 -W2 "$(ip route show default | awk "{print \$3; exit}")" >/dev/null 2>&1 \ + && echo "gateway ping: OK" || echo "gateway ping: FAIL" + ping -c1 -W2 1.1.1.1 >/dev/null 2>&1 \ + && echo "internet ping: OK" || echo "internet ping: FAIL" + getent hosts archive.ubuntu.com >/dev/null 2>&1 \ + && echo "DNS resolve: OK" || echo "DNS resolve: FAIL (resolv.conf: $(grep ^nameserver /etc/resolv.conf | tr "\n" " "))" +' || true echo "Copying answer files..." ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ @@ -53,11 +66,23 @@ ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ set -euo pipefail export DEBIAN_FRONTEND=noninteractive -# cloud-init's first-boot apt activity can hold the dpkg lock briefly -for i in $(seq 1 30); do - apt-get update -qq >/dev/null 2>&1 && break +# cloud-init's first-boot apt activity can hold the dpkg lock briefly; +# a network/DNS failure must fail fast with the real apt error, not fall +# through to a misleading "package not found" from empty lists. +echo "Updating apt package lists..." +for i in $(seq 1 12); do + if apt_out=$(apt-get update -qq 2>&1); then + break + fi + echo "apt-get update attempt $i/12 failed" + if [ "$i" -eq 12 ]; then + echo "$apt_out" >&2 + echo "ERROR: apt-get update never succeeded — check CI VLAN egress/DNS" >&2 + exit 1 + fi sleep 5 done +echo "Installing nfs-kernel-server, tgt, docker.io..." apt-get install -y -qq nfs-kernel-server tgt docker.io >/dev/null # Ubuntu's nfs-kernel-server package does not create /etc/exports.d @@ -80,6 +105,7 @@ systemctl enable --now nfs-kernel-server tgt >/dev/null || { } systemctl restart tgt +echo "Starting answer server container..." docker rm -f pvetest-answer-server >/dev/null 2>&1 || true docker run -d --name pvetest-answer-server --restart unless-stopped \ --network host \ From 7872f081024430e3cdb9c71ce807882b5cae65bf Mon Sep 17 00:00:00 2001 From: "goodolclint-claude[bot]" <323206664+goodolclint-claude[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:42:07 +0000 Subject: [PATCH 2/4] =?UTF-8?q?ci:=20storage=20VM=20uses=20public=20DNS=20?= =?UTF-8?q?=E2=80=94=20the=20CI=20VLAN=20gateway=20runs=20no=20resolver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- tests/infrastructure/storage-vm.tf | 2 +- tests/infrastructure/variables.tf | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/infrastructure/storage-vm.tf b/tests/infrastructure/storage-vm.tf index 40c76c6..7044154 100644 --- a/tests/infrastructure/storage-vm.tf +++ b/tests/infrastructure/storage-vm.tf @@ -49,7 +49,7 @@ resource "proxmox_virtual_environment_vm" "storage" { } dns { - servers = [var.storage_vm_gateway] + servers = [var.storage_vm_dns] } user_account { diff --git a/tests/infrastructure/variables.tf b/tests/infrastructure/variables.tf index 4c18863..7c58ac5 100644 --- a/tests/infrastructure/variables.tf +++ b/tests/infrastructure/variables.tf @@ -102,11 +102,17 @@ variable "storage_vm_ssh_public_key" { } variable "storage_vm_gateway" { - description = "Gateway (and DNS server) for the storage VM" + description = "Gateway for the storage VM" type = string default = "172.16.60.1" } +variable "storage_vm_dns" { + description = "DNS server for the storage VM. The CI VLAN gateway runs no resolver; public DNS works under the ci-to-internet rule and keeps the sandbox independent of fleet DNS" + type = string + default = "1.1.1.1" +} + variable "cloud_image_path" { description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)" type = string From 25746a94e3066986b9e003d630ec25618a5313b6 Mon Sep 17 00:00:00 2001 From: "goodolclint-claude[bot]" <323206664+goodolclint-claude[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:49:48 +0000 Subject: [PATCH 3/4] ci: storage VM moves to DHCP and is addressed by FQDN (1/2) The repo stops holding opinions about CI VLAN addressing: the VM boots via DHCP with hostname pvetest-storage (deterministic MAC for an optional reservation), registers in the operator's CI DNS zone, and everything addresses it by STORAGE_VM_FQDN. Replaces the static-IP + explicit-DNS variables. Co-Authored-By: Claude Fable 5 --- .github/workflows/integration-tests.yml | 13 +++++------ tests/infrastructure/outputs.tf | 5 ---- .../scripts/setup-storage-server.sh | 4 ++-- tests/infrastructure/storage-vm.tf | 11 ++++----- tests/infrastructure/variables.tf | 23 ------------------- 5 files changed, 13 insertions(+), 43 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b3b3ff7..ff6be01 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -24,9 +24,10 @@ name: Integration Tests # node with the most free memory and refuses to run when # no node has enough headroom — the token's user needs # PVEAuditor on /nodes for the memory stats. -# STORAGE_VM_IP - Static CIDR address for the shared storage/answer-server -# VM on the CI VLAN (default: 172.16.60.60/24) -# STORAGE_VM_GATEWAY - Gateway + DNS for that VM (default: 172.16.60.1) +# STORAGE_VM_FQDN - DNS name of the shared storage/answer-server VM; it +# boots via DHCP as hostname pvetest-storage and must +# resolve from the runner and the CI VLAN +# (default: pvetest-storage.test.local) # # Optional secrets (for skip_provision mode): # PVETEST_HOST - Hostname or IP of a pre-existing nested PVE @@ -139,8 +140,7 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} - STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} - STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} + STORAGE_VM_FQDN: ${{ vars.STORAGE_VM_FQDN }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} @@ -214,8 +214,7 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} - STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} - STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} + STORAGE_VM_FQDN: ${{ vars.STORAGE_VM_FQDN }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} diff --git a/tests/infrastructure/outputs.tf b/tests/infrastructure/outputs.tf index dea5cd4..4ecb5ee 100644 --- a/tests/infrastructure/outputs.tf +++ b/tests/infrastructure/outputs.tf @@ -7,8 +7,3 @@ output "pve_test_node_name" { value = "pve" description = "Default node name inside a fresh PVE install" } - -output "storage_vm_ip" { - description = "Static address of the shared NFS/iSCSI storage VM" - value = var.storage_vm_ip -} diff --git a/tests/infrastructure/scripts/setup-storage-server.sh b/tests/infrastructure/scripts/setup-storage-server.sh index 654dda9..7c1c55c 100644 --- a/tests/infrastructure/scripts/setup-storage-server.sh +++ b/tests/infrastructure/scripts/setup-storage-server.sh @@ -3,12 +3,12 @@ # iSCSI target, and the HTTP answer server the PVE auto-installers fetch # from. Idempotent — safe to re-run against an already-configured VM. # -# Usage: setup-storage-server.sh +# Usage: setup-storage-server.sh # is the local directory holding default.toml and answers/. set -euo pipefail -STORAGE_IP="${1:?Usage: setup-storage-server.sh }" +STORAGE_IP="${1:?Usage: setup-storage-server.sh }" SSH_KEY="${2:?missing SSH private key path}" ISCSI_IQN="${3:?missing iSCSI IQN}" ANSWER_DIR="${4:?missing answer directory}" diff --git a/tests/infrastructure/storage-vm.tf b/tests/infrastructure/storage-vm.tf index 7044154..e812f91 100644 --- a/tests/infrastructure/storage-vm.tf +++ b/tests/infrastructure/storage-vm.tf @@ -41,17 +41,14 @@ resource "proxmox_virtual_environment_vm" "storage" { initialization { datastore_id = var.disk_storage + # DHCP: pfSense owns addressing/DNS for the CI VLAN; the VM registers its + # hostname (= VM name) in the CI DNS zone and is addressed by FQDN. ip_config { ipv4 { - address = var.storage_vm_ip - gateway = var.storage_vm_gateway + address = "dhcp" } } - dns { - servers = [var.storage_vm_dns] - } - user_account { username = "ubuntu" # Password is console-only: Ubuntu cloud images refuse SSH password auth @@ -64,6 +61,8 @@ resource "proxmox_virtual_environment_vm" "storage" { network_device { bridge = var.network_bridge model = "virtio" + # Deterministic MAC so the operator can pin a DHCP reservation to it + mac_address = "aa:bb:cc:00:60:01" } operating_system { diff --git a/tests/infrastructure/variables.tf b/tests/infrastructure/variables.tf index 7c58ac5..86cc09f 100644 --- a/tests/infrastructure/variables.tf +++ b/tests/infrastructure/variables.tf @@ -84,35 +84,12 @@ variable "storage_vmid" { default = 5080 } -variable "storage_vm_ip" { - description = "Static IPv4 address in CIDR form for the storage VM, outside the VLAN's DHCP range" - type = string - default = "172.16.60.60/24" - - validation { - condition = can(cidrnetmask(var.storage_vm_ip)) - error_message = "storage_vm_ip must be CIDR notation, e.g. 172.16.60.60/24." - } -} - variable "storage_vm_ssh_public_key" { description = "SSH public key granted to the storage VM's ubuntu user (cloud images refuse password SSH; required for provision, unused on destroy)" type = string default = "" } -variable "storage_vm_gateway" { - description = "Gateway for the storage VM" - type = string - default = "172.16.60.1" -} - -variable "storage_vm_dns" { - description = "DNS server for the storage VM. The CI VLAN gateway runs no resolver; public DNS works under the ci-to-internet rule and keeps the sandbox independent of fleet DNS" - type = string - default = "1.1.1.1" -} - variable "cloud_image_path" { description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)" type = string From 59a47e3fab02fdd7159d312b2a3fe1cc5f6767ef Mon Sep 17 00:00:00 2001 From: "goodolclint-claude[bot]" <323206664+goodolclint-claude[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:54:14 +0000 Subject: [PATCH 4/4] ci: storage VM moves to DHCP and is addressed by FQDN (2/2) Co-Authored-By: Claude Fable 5 --- .../infrastructure/scripts/run-integration.sh | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tests/infrastructure/scripts/run-integration.sh b/tests/infrastructure/scripts/run-integration.sh index 3cef75d..87068a5 100644 --- a/tests/infrastructure/scripts/run-integration.sh +++ b/tests/infrastructure/scripts/run-integration.sh @@ -43,9 +43,9 @@ # MODULE_ARTIFACT Path to built module DLLs (default: ./publish/netstandard2.0) # PVE_VERSIONS Space-separated versions to provision (default: "9"; "9 8" still works) # STORAGE_ISCSI_IQN iSCSI IQN for storage target (default: iqn.2024-01.local.test:storage) -# STORAGE_VM_IP Static CIDR address for the storage VM on the CI VLAN, -# outside the DHCP range (default: 172.16.60.60/24) -# STORAGE_VM_GATEWAY Gateway + DNS for the storage VM (default: 172.16.60.1) +# STORAGE_VM_FQDN DNS name of the storage VM; it boots via DHCP with +# hostname pvetest-storage and must be resolvable from +# the runner and the CI VLAN (default: pvetest-storage.test.local) # STORAGE_VMID VMID for the storage VM (default: 5080) set -euo pipefail @@ -66,10 +66,8 @@ MODULE_ARTIFACT="${MODULE_ARTIFACT:-$REPO_ROOT/publish/netstandard2.0}" PVE_VERSIONS="${PVE_VERSIONS:-9}" SKIP_PROVISION="${SKIP_PROVISION:-false}" STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}" -STORAGE_VM_IP="${STORAGE_VM_IP:-172.16.60.60/24}" -STORAGE_VM_GATEWAY="${STORAGE_VM_GATEWAY:-172.16.60.1}" +STORAGE_VM_FQDN="${STORAGE_VM_FQDN:-pvetest-storage.test.local}" STORAGE_VMID="${STORAGE_VMID:-5080}" -STORAGE_VM_HOST="${STORAGE_VM_IP%%/*}" # Keypair for SSH to the storage VM (cloud images refuse password SSH) STORAGE_VM_SSH_KEY="${STORAGE_VM_SSH_KEY:-$WORK_DIR/storage-vm-key}" # Must match CLOUD_IMAGE_FILENAME in ensure-cloud-images.sh @@ -95,7 +93,7 @@ pve_iso() { pve_auto_iso() { local base base="$(pve_iso "$1")" - echo "${base%.iso}-auto-${STORAGE_VM_HOST//./-}.iso" + echo "${base%.iso}-auto-${STORAGE_VM_FQDN//./-}.iso" } pve_vmid() { @@ -224,7 +222,7 @@ cmd_provision() { log "Starting provisioning..." log " Versions: $provision_versions" log " Nodes:$provision_nodes" - log " Storage: dedicated VM at $STORAGE_VM_HOST (NFS + iSCSI + answer server)" + log " Storage: dedicated VM at $STORAGE_VM_FQDN (NFS + iSCSI + answer server)" require_env PVE_ENDPOINT require_env PVE_API_TOKEN require_env PVE_PASSWORD @@ -280,7 +278,7 @@ cmd_provision() { log "Preparing HTTP auto-install ISO for PVE $v..." proxmox-auto-install-assistant prepare-iso \ --fetch-from http \ - --url "http://${STORAGE_VM_HOST}:8000/answer" \ + --url "http://${STORAGE_VM_FQDN}:8000/answer" \ --on-first-boot "$SCRIPT_DIR/first-boot.sh" \ --tmp "$WORK_DIR" \ --output "$generic_iso" \ @@ -348,8 +346,6 @@ cmd_provision() { TF_VAR_target_node="$PVE_TARGET_NODE" \ TF_VAR_test_vm_password="$PVE_PASSWORD" \ TF_VAR_cloud_image_path="$CLOUD_IMAGE_PATH" \ - TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \ - TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \ TF_VAR_storage_vmid="$STORAGE_VMID" \ TF_VAR_storage_vm_ssh_public_key="$(cat "${STORAGE_VM_SSH_KEY}.pub")" \ terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" "$@") @@ -360,9 +356,9 @@ cmd_provision() { -target=proxmox_virtual_environment_file.storage_cloud_image \ -target=proxmox_virtual_environment_vm.storage - log "Configuring storage VM at $STORAGE_VM_HOST..." + log "Configuring storage VM at $STORAGE_VM_FQDN..." bash "$SCRIPT_DIR/setup-storage-server.sh" \ - "$STORAGE_VM_HOST" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server" + "$STORAGE_VM_FQDN" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server" log "Running Terraform apply (PVE nodes)..." local tf_targets="" @@ -424,7 +420,7 @@ cmd_provision() { config=$(jq \ --arg cloud_image "${CLOUD_IMAGE_PATH:-}" \ --arg ova "${OVA_PATH:-}" \ - --arg storage_ip "$STORAGE_VM_HOST" \ + --arg storage_ip "$STORAGE_VM_FQDN" \ --arg storage_iqn "$STORAGE_ISCSI_IQN" \ '. + { storage: {ip: $storage_ip, iscsi_iqn: $storage_iqn, nfs_export: ($storage_ip + ":/srv/nfs/shared")}, @@ -661,8 +657,6 @@ cmd_cleanup() { TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \ TF_VAR_target_node="$PVE_TARGET_NODE" \ TF_VAR_test_vm_password="${PVE_PASSWORD:-placeholder}" \ - TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \ - TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \ TF_VAR_storage_vmid="$STORAGE_VMID" \ terraform destroy -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets)