diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b3b3ff7..ff6be01 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -24,9 +24,10 @@ name: Integration Tests # node with the most free memory and refuses to run when # no node has enough headroom — the token's user needs # PVEAuditor on /nodes for the memory stats. -# STORAGE_VM_IP - Static CIDR address for the shared storage/answer-server -# VM on the CI VLAN (default: 172.16.60.60/24) -# STORAGE_VM_GATEWAY - Gateway + DNS for that VM (default: 172.16.60.1) +# STORAGE_VM_FQDN - DNS name of the shared storage/answer-server VM; it +# boots via DHCP as hostname pvetest-storage and must +# resolve from the runner and the CI VLAN +# (default: pvetest-storage.test.local) # # Optional secrets (for skip_provision mode): # PVETEST_HOST - Hostname or IP of a pre-existing nested PVE @@ -139,8 +140,7 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} - STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} - STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} + STORAGE_VM_FQDN: ${{ vars.STORAGE_VM_FQDN }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} @@ -214,8 +214,7 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} - STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }} - STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }} + STORAGE_VM_FQDN: ${{ vars.STORAGE_VM_FQDN }} TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }} TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }} TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }} diff --git a/tests/infrastructure/outputs.tf b/tests/infrastructure/outputs.tf index dea5cd4..4ecb5ee 100644 --- a/tests/infrastructure/outputs.tf +++ b/tests/infrastructure/outputs.tf @@ -7,8 +7,3 @@ output "pve_test_node_name" { value = "pve" description = "Default node name inside a fresh PVE install" } - -output "storage_vm_ip" { - description = "Static address of the shared NFS/iSCSI storage VM" - value = var.storage_vm_ip -} diff --git a/tests/infrastructure/scripts/run-integration.sh b/tests/infrastructure/scripts/run-integration.sh index 3cef75d..87068a5 100644 --- a/tests/infrastructure/scripts/run-integration.sh +++ b/tests/infrastructure/scripts/run-integration.sh @@ -43,9 +43,9 @@ # MODULE_ARTIFACT Path to built module DLLs (default: ./publish/netstandard2.0) # PVE_VERSIONS Space-separated versions to provision (default: "9"; "9 8" still works) # STORAGE_ISCSI_IQN iSCSI IQN for storage target (default: iqn.2024-01.local.test:storage) -# STORAGE_VM_IP Static CIDR address for the storage VM on the CI VLAN, -# outside the DHCP range (default: 172.16.60.60/24) -# STORAGE_VM_GATEWAY Gateway + DNS for the storage VM (default: 172.16.60.1) +# STORAGE_VM_FQDN DNS name of the storage VM; it boots via DHCP with +# hostname pvetest-storage and must be resolvable from +# the runner and the CI VLAN (default: pvetest-storage.test.local) # STORAGE_VMID VMID for the storage VM (default: 5080) set -euo pipefail @@ -66,10 +66,8 @@ MODULE_ARTIFACT="${MODULE_ARTIFACT:-$REPO_ROOT/publish/netstandard2.0}" PVE_VERSIONS="${PVE_VERSIONS:-9}" SKIP_PROVISION="${SKIP_PROVISION:-false}" STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}" -STORAGE_VM_IP="${STORAGE_VM_IP:-172.16.60.60/24}" -STORAGE_VM_GATEWAY="${STORAGE_VM_GATEWAY:-172.16.60.1}" +STORAGE_VM_FQDN="${STORAGE_VM_FQDN:-pvetest-storage.test.local}" STORAGE_VMID="${STORAGE_VMID:-5080}" -STORAGE_VM_HOST="${STORAGE_VM_IP%%/*}" # Keypair for SSH to the storage VM (cloud images refuse password SSH) STORAGE_VM_SSH_KEY="${STORAGE_VM_SSH_KEY:-$WORK_DIR/storage-vm-key}" # Must match CLOUD_IMAGE_FILENAME in ensure-cloud-images.sh @@ -95,7 +93,7 @@ pve_iso() { pve_auto_iso() { local base base="$(pve_iso "$1")" - echo "${base%.iso}-auto-${STORAGE_VM_HOST//./-}.iso" + echo "${base%.iso}-auto-${STORAGE_VM_FQDN//./-}.iso" } pve_vmid() { @@ -224,7 +222,7 @@ cmd_provision() { log "Starting provisioning..." log " Versions: $provision_versions" log " Nodes:$provision_nodes" - log " Storage: dedicated VM at $STORAGE_VM_HOST (NFS + iSCSI + answer server)" + log " Storage: dedicated VM at $STORAGE_VM_FQDN (NFS + iSCSI + answer server)" require_env PVE_ENDPOINT require_env PVE_API_TOKEN require_env PVE_PASSWORD @@ -280,7 +278,7 @@ cmd_provision() { log "Preparing HTTP auto-install ISO for PVE $v..." proxmox-auto-install-assistant prepare-iso \ --fetch-from http \ - --url "http://${STORAGE_VM_HOST}:8000/answer" \ + --url "http://${STORAGE_VM_FQDN}:8000/answer" \ --on-first-boot "$SCRIPT_DIR/first-boot.sh" \ --tmp "$WORK_DIR" \ --output "$generic_iso" \ @@ -348,8 +346,6 @@ cmd_provision() { TF_VAR_target_node="$PVE_TARGET_NODE" \ TF_VAR_test_vm_password="$PVE_PASSWORD" \ TF_VAR_cloud_image_path="$CLOUD_IMAGE_PATH" \ - TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \ - TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \ TF_VAR_storage_vmid="$STORAGE_VMID" \ TF_VAR_storage_vm_ssh_public_key="$(cat "${STORAGE_VM_SSH_KEY}.pub")" \ terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" "$@") @@ -360,9 +356,9 @@ cmd_provision() { -target=proxmox_virtual_environment_file.storage_cloud_image \ -target=proxmox_virtual_environment_vm.storage - log "Configuring storage VM at $STORAGE_VM_HOST..." + log "Configuring storage VM at $STORAGE_VM_FQDN..." bash "$SCRIPT_DIR/setup-storage-server.sh" \ - "$STORAGE_VM_HOST" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server" + "$STORAGE_VM_FQDN" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server" log "Running Terraform apply (PVE nodes)..." local tf_targets="" @@ -424,7 +420,7 @@ cmd_provision() { config=$(jq \ --arg cloud_image "${CLOUD_IMAGE_PATH:-}" \ --arg ova "${OVA_PATH:-}" \ - --arg storage_ip "$STORAGE_VM_HOST" \ + --arg storage_ip "$STORAGE_VM_FQDN" \ --arg storage_iqn "$STORAGE_ISCSI_IQN" \ '. + { storage: {ip: $storage_ip, iscsi_iqn: $storage_iqn, nfs_export: ($storage_ip + ":/srv/nfs/shared")}, @@ -661,8 +657,6 @@ cmd_cleanup() { TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \ TF_VAR_target_node="$PVE_TARGET_NODE" \ TF_VAR_test_vm_password="${PVE_PASSWORD:-placeholder}" \ - TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \ - TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \ TF_VAR_storage_vmid="$STORAGE_VMID" \ terraform destroy -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets) diff --git a/tests/infrastructure/scripts/setup-storage-server.sh b/tests/infrastructure/scripts/setup-storage-server.sh index 8e76149..7c1c55c 100644 --- a/tests/infrastructure/scripts/setup-storage-server.sh +++ b/tests/infrastructure/scripts/setup-storage-server.sh @@ -3,12 +3,12 @@ # iSCSI target, and the HTTP answer server the PVE auto-installers fetch # from. Idempotent — safe to re-run against an already-configured VM. # -# Usage: setup-storage-server.sh +# Usage: setup-storage-server.sh # is the local directory holding default.toml and answers/. set -euo pipefail -STORAGE_IP="${1:?Usage: setup-storage-server.sh }" +STORAGE_IP="${1:?Usage: setup-storage-server.sh }" SSH_KEY="${2:?missing SSH private key path}" ISCSI_IQN="${3:?missing iSCSI IQN}" ANSWER_DIR="${4:?missing answer directory}" @@ -38,8 +38,21 @@ done # sshd accepts logins before cloud-init's first-boot work (and its apt/dpkg # locks) is finished; a degraded cloud-init result is fine for our purposes, # and a hung cloud-init must fail fast, not eat the job timeout. +echo "Waiting for cloud-init to finish (up to 300s)..." timeout 300 ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ - "sudo cloud-init status --wait" || true + "sudo cloud-init status --wait" \ + || echo "WARNING: cloud-init did not finish cleanly within 300s — continuing" + +echo "Network probe from the storage VM:" +ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" ' + ip -4 -brief addr; ip route show default + ping -c1 -W2 "$(ip route show default | awk "{print \$3; exit}")" >/dev/null 2>&1 \ + && echo "gateway ping: OK" || echo "gateway ping: FAIL" + ping -c1 -W2 1.1.1.1 >/dev/null 2>&1 \ + && echo "internet ping: OK" || echo "internet ping: FAIL" + getent hosts archive.ubuntu.com >/dev/null 2>&1 \ + && echo "DNS resolve: OK" || echo "DNS resolve: FAIL (resolv.conf: $(grep ^nameserver /etc/resolv.conf | tr "\n" " "))" +' || true echo "Copying answer files..." ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ @@ -53,11 +66,23 @@ ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \ set -euo pipefail export DEBIAN_FRONTEND=noninteractive -# cloud-init's first-boot apt activity can hold the dpkg lock briefly -for i in $(seq 1 30); do - apt-get update -qq >/dev/null 2>&1 && break +# cloud-init's first-boot apt activity can hold the dpkg lock briefly; +# a network/DNS failure must fail fast with the real apt error, not fall +# through to a misleading "package not found" from empty lists. +echo "Updating apt package lists..." +for i in $(seq 1 12); do + if apt_out=$(apt-get update -qq 2>&1); then + break + fi + echo "apt-get update attempt $i/12 failed" + if [ "$i" -eq 12 ]; then + echo "$apt_out" >&2 + echo "ERROR: apt-get update never succeeded — check CI VLAN egress/DNS" >&2 + exit 1 + fi sleep 5 done +echo "Installing nfs-kernel-server, tgt, docker.io..." apt-get install -y -qq nfs-kernel-server tgt docker.io >/dev/null # Ubuntu's nfs-kernel-server package does not create /etc/exports.d @@ -80,6 +105,7 @@ systemctl enable --now nfs-kernel-server tgt >/dev/null || { } systemctl restart tgt +echo "Starting answer server container..." docker rm -f pvetest-answer-server >/dev/null 2>&1 || true docker run -d --name pvetest-answer-server --restart unless-stopped \ --network host \ diff --git a/tests/infrastructure/storage-vm.tf b/tests/infrastructure/storage-vm.tf index 40c76c6..e812f91 100644 --- a/tests/infrastructure/storage-vm.tf +++ b/tests/infrastructure/storage-vm.tf @@ -41,17 +41,14 @@ resource "proxmox_virtual_environment_vm" "storage" { initialization { datastore_id = var.disk_storage + # DHCP: pfSense owns addressing/DNS for the CI VLAN; the VM registers its + # hostname (= VM name) in the CI DNS zone and is addressed by FQDN. ip_config { ipv4 { - address = var.storage_vm_ip - gateway = var.storage_vm_gateway + address = "dhcp" } } - dns { - servers = [var.storage_vm_gateway] - } - user_account { username = "ubuntu" # Password is console-only: Ubuntu cloud images refuse SSH password auth @@ -64,6 +61,8 @@ resource "proxmox_virtual_environment_vm" "storage" { network_device { bridge = var.network_bridge model = "virtio" + # Deterministic MAC so the operator can pin a DHCP reservation to it + mac_address = "aa:bb:cc:00:60:01" } operating_system { diff --git a/tests/infrastructure/variables.tf b/tests/infrastructure/variables.tf index 4c18863..86cc09f 100644 --- a/tests/infrastructure/variables.tf +++ b/tests/infrastructure/variables.tf @@ -84,29 +84,12 @@ variable "storage_vmid" { default = 5080 } -variable "storage_vm_ip" { - description = "Static IPv4 address in CIDR form for the storage VM, outside the VLAN's DHCP range" - type = string - default = "172.16.60.60/24" - - validation { - condition = can(cidrnetmask(var.storage_vm_ip)) - error_message = "storage_vm_ip must be CIDR notation, e.g. 172.16.60.60/24." - } -} - variable "storage_vm_ssh_public_key" { description = "SSH public key granted to the storage VM's ubuntu user (cloud images refuse password SSH; required for provision, unused on destroy)" type = string default = "" } -variable "storage_vm_gateway" { - description = "Gateway (and DNS server) for the storage VM" - type = string - default = "172.16.60.1" -} - variable "cloud_image_path" { description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)" type = string