mirror of
https://github.com/GoodOlClint/PSProxmoxVE.git
synced 2026-09-06 12:09:01 +00:00
Merge pull request #90 from GoodOlClint/ci/storage-vm-diagnostics
ci: storage VM on DHCP + FQDN addressing, self-diagnosing setup
This commit is contained in:
GoodOlClint
@@ -24,9 +24,10 @@ name: Integration Tests
|
||||
# node with the most free memory and refuses to run when
|
||||
# no node has enough headroom — the token's user needs
|
||||
# PVEAuditor on /nodes for the memory stats.
|
||||
# STORAGE_VM_IP - Static CIDR address for the shared storage/answer-server
|
||||
# VM on the CI VLAN (default: 172.16.60.60/24)
|
||||
# STORAGE_VM_GATEWAY - Gateway + DNS for that VM (default: 172.16.60.1)
|
||||
# STORAGE_VM_FQDN - DNS name of the shared storage/answer-server VM; it
|
||||
# boots via DHCP as hostname pvetest-storage and must
|
||||
# resolve from the runner and the CI VLAN
|
||||
# (default: pvetest-storage.test.local)
|
||||
#
|
||||
# Optional secrets (for skip_provision mode):
|
||||
# PVETEST_HOST - Hostname or IP of a pre-existing nested PVE
|
||||
@@ -139,8 +140,7 @@ jobs:
|
||||
PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
|
||||
PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }}
|
||||
PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }}
|
||||
STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }}
|
||||
STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }}
|
||||
STORAGE_VM_FQDN: ${{ vars.STORAGE_VM_FQDN }}
|
||||
TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }}
|
||||
TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }}
|
||||
TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }}
|
||||
@@ -214,8 +214,7 @@ jobs:
|
||||
PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
|
||||
PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }}
|
||||
PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }}
|
||||
STORAGE_VM_IP: ${{ vars.STORAGE_VM_IP }}
|
||||
STORAGE_VM_GATEWAY: ${{ vars.STORAGE_VM_GATEWAY }}
|
||||
STORAGE_VM_FQDN: ${{ vars.STORAGE_VM_FQDN }}
|
||||
TF_VAR_disk_storage: ${{ vars.DISK_STORAGE }}
|
||||
TF_VAR_iso_storage: ${{ vars.ISO_STORAGE }}
|
||||
TF_VAR_network_bridge: ${{ vars.NETWORK_BRIDGE }}
|
||||
|
||||
@@ -7,8 +7,3 @@ output "pve_test_node_name" {
|
||||
value = "pve"
|
||||
description = "Default node name inside a fresh PVE install"
|
||||
}
|
||||
|
||||
output "storage_vm_ip" {
|
||||
description = "Static address of the shared NFS/iSCSI storage VM"
|
||||
value = var.storage_vm_ip
|
||||
}
|
||||
|
||||
@@ -43,9 +43,9 @@
|
||||
# MODULE_ARTIFACT Path to built module DLLs (default: ./publish/netstandard2.0)
|
||||
# PVE_VERSIONS Space-separated versions to provision (default: "9"; "9 8" still works)
|
||||
# STORAGE_ISCSI_IQN iSCSI IQN for storage target (default: iqn.2024-01.local.test:storage)
|
||||
# STORAGE_VM_IP Static CIDR address for the storage VM on the CI VLAN,
|
||||
# outside the DHCP range (default: 172.16.60.60/24)
|
||||
# STORAGE_VM_GATEWAY Gateway + DNS for the storage VM (default: 172.16.60.1)
|
||||
# STORAGE_VM_FQDN DNS name of the storage VM; it boots via DHCP with
|
||||
# hostname pvetest-storage and must be resolvable from
|
||||
# the runner and the CI VLAN (default: pvetest-storage.test.local)
|
||||
# STORAGE_VMID VMID for the storage VM (default: 5080)
|
||||
|
||||
set -euo pipefail
|
||||
@@ -66,10 +66,8 @@ MODULE_ARTIFACT="${MODULE_ARTIFACT:-$REPO_ROOT/publish/netstandard2.0}"
|
||||
PVE_VERSIONS="${PVE_VERSIONS:-9}"
|
||||
SKIP_PROVISION="${SKIP_PROVISION:-false}"
|
||||
STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}"
|
||||
STORAGE_VM_IP="${STORAGE_VM_IP:-172.16.60.60/24}"
|
||||
STORAGE_VM_GATEWAY="${STORAGE_VM_GATEWAY:-172.16.60.1}"
|
||||
STORAGE_VM_FQDN="${STORAGE_VM_FQDN:-pvetest-storage.test.local}"
|
||||
STORAGE_VMID="${STORAGE_VMID:-5080}"
|
||||
STORAGE_VM_HOST="${STORAGE_VM_IP%%/*}"
|
||||
# Keypair for SSH to the storage VM (cloud images refuse password SSH)
|
||||
STORAGE_VM_SSH_KEY="${STORAGE_VM_SSH_KEY:-$WORK_DIR/storage-vm-key}"
|
||||
# Must match CLOUD_IMAGE_FILENAME in ensure-cloud-images.sh
|
||||
@@ -95,7 +93,7 @@ pve_iso() {
|
||||
pve_auto_iso() {
|
||||
local base
|
||||
base="$(pve_iso "$1")"
|
||||
echo "${base%.iso}-auto-${STORAGE_VM_HOST//./-}.iso"
|
||||
echo "${base%.iso}-auto-${STORAGE_VM_FQDN//./-}.iso"
|
||||
}
|
||||
|
||||
pve_vmid() {
|
||||
@@ -224,7 +222,7 @@ cmd_provision() {
|
||||
log "Starting provisioning..."
|
||||
log " Versions: $provision_versions"
|
||||
log " Nodes:$provision_nodes"
|
||||
log " Storage: dedicated VM at $STORAGE_VM_HOST (NFS + iSCSI + answer server)"
|
||||
log " Storage: dedicated VM at $STORAGE_VM_FQDN (NFS + iSCSI + answer server)"
|
||||
require_env PVE_ENDPOINT
|
||||
require_env PVE_API_TOKEN
|
||||
require_env PVE_PASSWORD
|
||||
@@ -280,7 +278,7 @@ cmd_provision() {
|
||||
log "Preparing HTTP auto-install ISO for PVE $v..."
|
||||
proxmox-auto-install-assistant prepare-iso \
|
||||
--fetch-from http \
|
||||
--url "http://${STORAGE_VM_HOST}:8000/answer" \
|
||||
--url "http://${STORAGE_VM_FQDN}:8000/answer" \
|
||||
--on-first-boot "$SCRIPT_DIR/first-boot.sh" \
|
||||
--tmp "$WORK_DIR" \
|
||||
--output "$generic_iso" \
|
||||
@@ -348,8 +346,6 @@ cmd_provision() {
|
||||
TF_VAR_target_node="$PVE_TARGET_NODE" \
|
||||
TF_VAR_test_vm_password="$PVE_PASSWORD" \
|
||||
TF_VAR_cloud_image_path="$CLOUD_IMAGE_PATH" \
|
||||
TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \
|
||||
TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \
|
||||
TF_VAR_storage_vmid="$STORAGE_VMID" \
|
||||
TF_VAR_storage_vm_ssh_public_key="$(cat "${STORAGE_VM_SSH_KEY}.pub")" \
|
||||
terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" "$@")
|
||||
@@ -360,9 +356,9 @@ cmd_provision() {
|
||||
-target=proxmox_virtual_environment_file.storage_cloud_image \
|
||||
-target=proxmox_virtual_environment_vm.storage
|
||||
|
||||
log "Configuring storage VM at $STORAGE_VM_HOST..."
|
||||
log "Configuring storage VM at $STORAGE_VM_FQDN..."
|
||||
bash "$SCRIPT_DIR/setup-storage-server.sh" \
|
||||
"$STORAGE_VM_HOST" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server"
|
||||
"$STORAGE_VM_FQDN" "$STORAGE_VM_SSH_KEY" "$STORAGE_ISCSI_IQN" "$WORK_DIR/answer-server"
|
||||
|
||||
log "Running Terraform apply (PVE nodes)..."
|
||||
local tf_targets=""
|
||||
@@ -424,7 +420,7 @@ cmd_provision() {
|
||||
config=$(jq \
|
||||
--arg cloud_image "${CLOUD_IMAGE_PATH:-}" \
|
||||
--arg ova "${OVA_PATH:-}" \
|
||||
--arg storage_ip "$STORAGE_VM_HOST" \
|
||||
--arg storage_ip "$STORAGE_VM_FQDN" \
|
||||
--arg storage_iqn "$STORAGE_ISCSI_IQN" \
|
||||
'. + {
|
||||
storage: {ip: $storage_ip, iscsi_iqn: $storage_iqn, nfs_export: ($storage_ip + ":/srv/nfs/shared")},
|
||||
@@ -661,8 +657,6 @@ cmd_cleanup() {
|
||||
TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \
|
||||
TF_VAR_target_node="$PVE_TARGET_NODE" \
|
||||
TF_VAR_test_vm_password="${PVE_PASSWORD:-placeholder}" \
|
||||
TF_VAR_storage_vm_ip="$STORAGE_VM_IP" \
|
||||
TF_VAR_storage_vm_gateway="$STORAGE_VM_GATEWAY" \
|
||||
TF_VAR_storage_vmid="$STORAGE_VMID" \
|
||||
terraform destroy -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets)
|
||||
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
# iSCSI target, and the HTTP answer server the PVE auto-installers fetch
|
||||
# from. Idempotent — safe to re-run against an already-configured VM.
|
||||
#
|
||||
# Usage: setup-storage-server.sh <storage-vm-ip> <ssh-private-key-path> <iscsi-iqn> <answer-dir>
|
||||
# Usage: setup-storage-server.sh <storage-vm-host> <ssh-private-key-path> <iscsi-iqn> <answer-dir>
|
||||
# <answer-dir> is the local directory holding default.toml and answers/.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
STORAGE_IP="${1:?Usage: setup-storage-server.sh <ip> <ssh-key> <iqn> <answer-dir>}"
|
||||
STORAGE_IP="${1:?Usage: setup-storage-server.sh <host> <ssh-key> <iqn> <answer-dir>}"
|
||||
SSH_KEY="${2:?missing SSH private key path}"
|
||||
ISCSI_IQN="${3:?missing iSCSI IQN}"
|
||||
ANSWER_DIR="${4:?missing answer directory}"
|
||||
@@ -38,8 +38,21 @@ done
|
||||
# sshd accepts logins before cloud-init's first-boot work (and its apt/dpkg
|
||||
# locks) is finished; a degraded cloud-init result is fine for our purposes,
|
||||
# and a hung cloud-init must fail fast, not eat the job timeout.
|
||||
echo "Waiting for cloud-init to finish (up to 300s)..."
|
||||
timeout 300 ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
|
||||
"sudo cloud-init status --wait" || true
|
||||
"sudo cloud-init status --wait" \
|
||||
|| echo "WARNING: cloud-init did not finish cleanly within 300s — continuing"
|
||||
|
||||
echo "Network probe from the storage VM:"
|
||||
ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" '
|
||||
ip -4 -brief addr; ip route show default
|
||||
ping -c1 -W2 "$(ip route show default | awk "{print \$3; exit}")" >/dev/null 2>&1 \
|
||||
&& echo "gateway ping: OK" || echo "gateway ping: FAIL"
|
||||
ping -c1 -W2 1.1.1.1 >/dev/null 2>&1 \
|
||||
&& echo "internet ping: OK" || echo "internet ping: FAIL"
|
||||
getent hosts archive.ubuntu.com >/dev/null 2>&1 \
|
||||
&& echo "DNS resolve: OK" || echo "DNS resolve: FAIL (resolv.conf: $(grep ^nameserver /etc/resolv.conf | tr "\n" " "))"
|
||||
' || true
|
||||
|
||||
echo "Copying answer files..."
|
||||
ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
|
||||
@@ -53,11 +66,23 @@ ssh ${SSH_OPTS} "ubuntu@${STORAGE_IP}" \
|
||||
set -euo pipefail
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# cloud-init's first-boot apt activity can hold the dpkg lock briefly
|
||||
for i in $(seq 1 30); do
|
||||
apt-get update -qq >/dev/null 2>&1 && break
|
||||
# cloud-init's first-boot apt activity can hold the dpkg lock briefly;
|
||||
# a network/DNS failure must fail fast with the real apt error, not fall
|
||||
# through to a misleading "package not found" from empty lists.
|
||||
echo "Updating apt package lists..."
|
||||
for i in $(seq 1 12); do
|
||||
if apt_out=$(apt-get update -qq 2>&1); then
|
||||
break
|
||||
fi
|
||||
echo "apt-get update attempt $i/12 failed"
|
||||
if [ "$i" -eq 12 ]; then
|
||||
echo "$apt_out" >&2
|
||||
echo "ERROR: apt-get update never succeeded — check CI VLAN egress/DNS" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "Installing nfs-kernel-server, tgt, docker.io..."
|
||||
apt-get install -y -qq nfs-kernel-server tgt docker.io >/dev/null
|
||||
|
||||
# Ubuntu's nfs-kernel-server package does not create /etc/exports.d
|
||||
@@ -80,6 +105,7 @@ systemctl enable --now nfs-kernel-server tgt >/dev/null || {
|
||||
}
|
||||
systemctl restart tgt
|
||||
|
||||
echo "Starting answer server container..."
|
||||
docker rm -f pvetest-answer-server >/dev/null 2>&1 || true
|
||||
docker run -d --name pvetest-answer-server --restart unless-stopped \
|
||||
--network host \
|
||||
|
||||
@@ -41,17 +41,14 @@ resource "proxmox_virtual_environment_vm" "storage" {
|
||||
initialization {
|
||||
datastore_id = var.disk_storage
|
||||
|
||||
# DHCP: pfSense owns addressing/DNS for the CI VLAN; the VM registers its
|
||||
# hostname (= VM name) in the CI DNS zone and is addressed by FQDN.
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = var.storage_vm_ip
|
||||
gateway = var.storage_vm_gateway
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
|
||||
dns {
|
||||
servers = [var.storage_vm_gateway]
|
||||
}
|
||||
|
||||
user_account {
|
||||
username = "ubuntu"
|
||||
# Password is console-only: Ubuntu cloud images refuse SSH password auth
|
||||
@@ -64,6 +61,8 @@ resource "proxmox_virtual_environment_vm" "storage" {
|
||||
network_device {
|
||||
bridge = var.network_bridge
|
||||
model = "virtio"
|
||||
# Deterministic MAC so the operator can pin a DHCP reservation to it
|
||||
mac_address = "aa:bb:cc:00:60:01"
|
||||
}
|
||||
|
||||
operating_system {
|
||||
|
||||
@@ -84,29 +84,12 @@ variable "storage_vmid" {
|
||||
default = 5080
|
||||
}
|
||||
|
||||
variable "storage_vm_ip" {
|
||||
description = "Static IPv4 address in CIDR form for the storage VM, outside the VLAN's DHCP range"
|
||||
type = string
|
||||
default = "172.16.60.60/24"
|
||||
|
||||
validation {
|
||||
condition = can(cidrnetmask(var.storage_vm_ip))
|
||||
error_message = "storage_vm_ip must be CIDR notation, e.g. 172.16.60.60/24."
|
||||
}
|
||||
}
|
||||
|
||||
variable "storage_vm_ssh_public_key" {
|
||||
description = "SSH public key granted to the storage VM's ubuntu user (cloud images refuse password SSH; required for provision, unused on destroy)"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "storage_vm_gateway" {
|
||||
description = "Gateway (and DNS server) for the storage VM"
|
||||
type = string
|
||||
default = "172.16.60.1"
|
||||
}
|
||||
|
||||
variable "cloud_image_path" {
|
||||
description = "Local path to the Ubuntu cloud image imported as the storage VM's disk (required for provision; unused on destroy)"
|
||||
type = string
|
||||
|
||||
Reference in New Issue
Block a user