diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 484725a..8fce156 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -102,12 +102,12 @@ jobs: push: true tags: ${{ env.TEST_IMAGE }}:${{ github.sha }},${{ env.TEST_IMAGE }}:latest - # ── Provision all nested PVE VMs (self-hosted) ────────────────────── + # ── Provision all nested PVE VMs + storage VM (self-hosted) ───────── provision: if: inputs.skip_provision != true needs: [build, container-image] runs-on: [self-hosted, proxmox, integration] - timeout-minutes: 30 + timeout-minutes: 45 container: image: ghcr.io/goodolclint/psproxmoxve-integration:${{ github.sha }} credentials: @@ -115,6 +115,11 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} volumes: - /opt/pve-isos:/opt/pve-isos + # WARNING: The Docker socket is mounted to allow storage containers to be + # managed from within the job container. This grants root-equivalent access + # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted + # runners — never on shared runners. + - /var/run/docker.sock:/var/run/docker.sock steps: - uses: actions/checkout@v6 @@ -185,6 +190,11 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} volumes: - /opt/pve-isos:/opt/pve-isos + # WARNING: The Docker socket is mounted to allow storage containers to be + # stopped from within the job container. This grants root-equivalent access + # to the runner's Docker daemon. Only use on dedicated, isolated self-hosted + # runners — never on shared runners. + - /var/run/docker.sock:/var/run/docker.sock steps: - uses: actions/checkout@v6 diff --git a/tests/.env.test.example b/tests/.env.test.example index ec9883e..0f6e083 100644 --- a/tests/.env.test.example +++ b/tests/.env.test.example @@ -4,6 +4,11 @@ # These credentials point to the PARENT PVE host where nested test VMs # will be provisioned. Integration tests run against the nested VMs, # not against this host directly. +# +# Provisioned resources: +# - 2x PVE 9 nodes (pve9a, pve9b) for cluster testing +# - 2x PVE 8 nodes (pve8a, pve8b) for cluster testing +# - Shared storage provided by Docker containers on the runner host (iSCSI + NFS) # ── Required: parent PVE for provisioning ───────────────────────────── PVE_ENDPOINT=https://pve.example.com:8006 @@ -15,3 +20,9 @@ PVE_PASSWORD= # CACHE_DIR=/opt/pve-isos # WORK_DIR=/tmp/pve-integration # PVE_VERSIONS=9 8 +# +# ── VMID overrides ─────────────────────────────────────────────────── +# PVE9A_VMID=99091 +# PVE9B_VMID=99092 +# PVE8A_VMID=99081 +# PVE8B_VMID=99082 diff --git a/tests/Dockerfile.test b/tests/Dockerfile.test index e64e8e6..9261022 100644 --- a/tests/Dockerfile.test +++ b/tests/Dockerfile.test @@ -63,3 +63,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && apt-get update && apt-get install -y --no-install-recommends \ terraform proxmox-auto-install-assistant qemu-utils \ && rm -rf /var/lib/apt/lists/* + +# Install Docker CLI (for managing storage containers on the host via mounted socket) +RUN install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && chmod a+r /etc/apt/keyrings/docker.gpg \ + && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable" \ + > /etc/apt/sources.list.d/docker.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends docker-ce-cli docker-compose-plugin \ + && rm -rf /var/lib/apt/lists/* diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index b891467..196780c 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -38,6 +38,9 @@ services: volumes: - ..:/repo - /opt/pve-isos:/opt/pve-isos + # WARNING: Mounting the Docker socket grants this container root-equivalent + # access to the host's Docker daemon. Only use on trusted, isolated machines. + - /var/run/docker.sock:/var/run/docker.sock working_dir: /repo stdin_open: true tty: true diff --git a/tests/infrastructure/answer.toml.tftpl b/tests/infrastructure/answer.toml.tftpl index c7fea2b..fc8fbf8 100644 --- a/tests/infrastructure/answer.toml.tftpl +++ b/tests/infrastructure/answer.toml.tftpl @@ -1,7 +1,7 @@ [global] keyboard = "en-us" country = "us" -fqdn = "pve.test.local" +fqdn = "${fqdn}" mailto = "test@test.local" timezone = "UTC" root_password = "${root_password}" diff --git a/tests/infrastructure/docker-compose.storage.yml b/tests/infrastructure/docker-compose.storage.yml new file mode 100644 index 0000000..3993428 --- /dev/null +++ b/tests/infrastructure/docker-compose.storage.yml @@ -0,0 +1,62 @@ +## Shared storage services for integration testing. +## Provides iSCSI target and NFS server accessible from nested PVE nodes. +## +## These run on the GitHub runner (or any Docker host on the same network +## as the PVE nodes) using host networking so the PVE nodes can reach them. +## +## Usage: +## docker compose -f tests/infrastructure/docker-compose.storage.yml up -d +## docker compose -f tests/infrastructure/docker-compose.storage.yml down -v + +services: + iscsi-target: + image: ubuntu:24.04 + container_name: pvetest-iscsi + network_mode: host + privileged: true + volumes: + - iscsi-data:/srv/iscsi + environment: + ISCSI_IQN: ${ISCSI_IQN:-iqn.2024-01.local.test:storage} + ISCSI_LUN_SIZE: ${ISCSI_LUN_SIZE:-10G} + entrypoint: ["/bin/bash", "-c"] + command: + - | + set -e + apt-get update -qq && apt-get install -y -qq tgt >/dev/null 2>&1 + mkdir -p /srv/iscsi + if [ ! -f /srv/iscsi/lun0.img ]; then + truncate -s $$ISCSI_LUN_SIZE /srv/iscsi/lun0.img + fi + tgtd --foreground & + sleep 2 + # Create target if it does not already exist + if ! tgtadm --lld iscsi --op show --mode target | grep -q "Target 1: $$ISCSI_IQN"; then + tgtadm --lld iscsi --op new --mode target --tid 1 -T $$ISCSI_IQN + fi + # Create logical unit (LUN 1) if it does not already exist + if ! tgtadm --lld iscsi --op show --mode logicalunit --tid 1 2>/dev/null | grep -qE "LUN:[[:space:]]*1($|[^0-9])"; then + tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 --backing-store /srv/iscsi/lun0.img + fi + # Bind target to all initiators if not already bound + if ! tgtadm --lld iscsi --op show --mode target --tid 1 2>/dev/null | grep -q "Initiator-address: ALL"; then + tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL + fi + echo "iSCSI target ready: $$ISCSI_IQN (port 3260)" + wait + restart: unless-stopped + + nfs-server: + image: erichough/nfs-server:2.2.1 + container_name: pvetest-nfs + network_mode: host + privileged: true + volumes: + - nfs-data:/srv/nfs/shared + environment: + NFS_EXPORT_0: /srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash) + restart: unless-stopped + +volumes: + iscsi-data: + nfs-data: diff --git a/tests/infrastructure/main.tf b/tests/infrastructure/main.tf index 934113b..2199061 100644 --- a/tests/infrastructure/main.tf +++ b/tests/infrastructure/main.tf @@ -25,6 +25,8 @@ resource "proxmox_virtual_environment_file" "auto_iso" { } } +# ── Nested PVE VMs ──────────────────────────────────────────────────── + resource "proxmox_virtual_environment_vm" "nested_pve" { for_each = var.pve_instances name = each.value.vm_name diff --git a/tests/infrastructure/scripts/create-api-token.sh b/tests/infrastructure/scripts/create-api-token.sh index d89bfb9..932a911 100755 --- a/tests/infrastructure/scripts/create-api-token.sh +++ b/tests/infrastructure/scripts/create-api-token.sh @@ -88,7 +88,14 @@ if [ -z "$TICKET" ] || [ -z "$CSRF" ]; then exit 1 fi +# Delete existing token if present, then create fresh echo "Creating API token root@pam!integration..." +curl -sk \ + -b "PVEAuthCookie=${TICKET}" \ + -H "CSRFPreventionToken: ${CSRF}" \ + -X DELETE \ + "${NESTED_API}/access/users/root@pam/token/integration" >/dev/null 2>&1 || true + TOKEN_RESPONSE=$(curl -sk \ -b "PVEAuthCookie=${TICKET}" \ -H "CSRFPreventionToken: ${CSRF}" \ diff --git a/tests/infrastructure/scripts/first-boot.sh b/tests/infrastructure/scripts/first-boot.sh index fab10f5..2083737 100755 --- a/tests/infrastructure/scripts/first-boot.sh +++ b/tests/infrastructure/scripts/first-boot.sh @@ -19,5 +19,6 @@ fi echo "deb http://download.proxmox.com/debian/pve ${SUITE} pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list apt-get update -qq -apt-get install -y -qq qemu-guest-agent +apt-get install -y -qq qemu-guest-agent open-iscsi systemctl start qemu-guest-agent +systemctl enable open-iscsi diff --git a/tests/infrastructure/scripts/preflight-cleanup.sh b/tests/infrastructure/scripts/preflight-cleanup.sh index f4014f4..ef12e33 100755 --- a/tests/infrastructure/scripts/preflight-cleanup.sh +++ b/tests/infrastructure/scripts/preflight-cleanup.sh @@ -49,6 +49,9 @@ else fi # --- Clean up orphaned ISO --- +if [ -z "$ISO_FILENAME" ]; then + echo "No ISO filename specified, skipping ISO cleanup" +else ISO_EXISTS=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" \ "${API_BASE}/nodes/${NODE}/storage/local/content" 2>/dev/null \ | python3 -c " @@ -71,6 +74,7 @@ if [ -n "$ISO_EXISTS" ]; then else echo "No orphaned ISO found" fi +fi # end ISO_FILENAME check # --- Clean up stale Terraform state --- if [ -d "$TF_DIR" ]; then diff --git a/tests/infrastructure/scripts/run-integration.sh b/tests/infrastructure/scripts/run-integration.sh index 3f3af32..6a152c4 100755 --- a/tests/infrastructure/scripts/run-integration.sh +++ b/tests/infrastructure/scripts/run-integration.sh @@ -4,8 +4,11 @@ # Single source of truth for the provision → test → cleanup lifecycle. # Called by both the GitHub Actions workflow and the local dev container. # +# Provisions two PVE nodes per version (a/b) for cluster testing, plus +# Docker containers on the runner host for iSCSI/NFS shared storage. +# # Usage: -# run-integration.sh provision Provision nested PVE VMs +# run-integration.sh provision Provision nested PVE VMs + start storage containers # run-integration.sh test [8|9|all] Run integration tests (default: all) # run-integration.sh cleanup Destroy provisioned VMs # run-integration.sh all [8|9|all] Full lifecycle: provision → test → cleanup @@ -17,8 +20,8 @@ # PVE_PASSWORD Root password for nested PVE instances # # Required env vars (test with pre-existing PVE): -# PVETEST_HOST PVE host IP -# PVETEST_APITOKEN PVE API token +# PVETEST_HOST PVE host IP (node A) +# PVETEST_APITOKEN PVE API token (node A) # Set SKIP_PROVISION=true # # Optional env vars: @@ -27,6 +30,7 @@ # CONFIG_FILE Test config JSON path (default: $CACHE_DIR/test-config.json) # MODULE_ARTIFACT Path to built module DLLs (default: ./publish/netstandard2.0) # PVE_VERSIONS Space-separated versions to provision (default: "9 8") +# STORAGE_ISCSI_IQN iSCSI IQN for storage target (default: iqn.2024-01.local.test:storage) set -euo pipefail @@ -42,11 +46,52 @@ CONFIG_FILE="${CONFIG_FILE:-$CACHE_DIR/test-config.json}" MODULE_ARTIFACT="${MODULE_ARTIFACT:-$REPO_ROOT/publish/netstandard2.0}" PVE_VERSIONS="${PVE_VERSIONS:-9 8}" SKIP_PROVISION="${SKIP_PROVISION:-false}" +STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}" +STORAGE_COMPOSE="$INFRA_DIR/docker-compose.storage.yml" -# ── Version config ────────────────────────────────────────────────── -pve_iso() { case "$1" in 9) echo "${PVE9_ISO:-proxmox-ve_9.1-1.iso}";; 8) echo "${PVE8_ISO:-proxmox-ve_8.4-1.iso}";; esac; } -pve_vmid() { case "$1" in 9) echo "${PVE9_VMID:-99909}";; 8) echo "${PVE8_VMID:-99908}";; esac; } -pve_vmname() { case "$1" in 9) echo "pve-test-pve9";; 8) echo "pve-test-pve8";; esac; } +# ── Node config ─────────────────────────────────────────────────── +# Each version gets two nodes: a (primary) and b (secondary). +# ISOs are per-version; nodes within a version share the same base ISO. + +pve_iso() { + local ver="${1%%[ab]}" # strip suffix: "9a" -> "9" + case "$ver" in + 9) echo "${PVE9_ISO:-proxmox-ve_9.1-1.iso}" ;; + 8) echo "${PVE8_ISO:-proxmox-ve_8.4-1.iso}" ;; + esac +} + +pve_vmid() { + case "$1" in + 9a) echo "${PVE9A_VMID:-99091}" ;; 9b) echo "${PVE9B_VMID:-99092}" ;; + 8a) echo "${PVE8A_VMID:-99081}" ;; 8b) echo "${PVE8B_VMID:-99082}" ;; + esac +} + +pve_vmname() { + case "$1" in + 9a) echo "pve-test-9a" ;; 9b) echo "pve-test-9b" ;; + 8a) echo "pve-test-8a" ;; 8b) echo "pve-test-8b" ;; + esac +} + +pve_fqdn() { + case "$1" in + 9a) echo "pve9a.test.local" ;; 9b) echo "pve9b.test.local" ;; + 8a) echo "pve8a.test.local" ;; 8b) echo "pve8b.test.local" ;; + esac +} + +# Expand versions to node list: "9 8" -> "9a 9b 8a 8b" +expand_nodes() { + local nodes="" + for v in $PVE_VERSIONS; do + nodes="$nodes ${v}a ${v}b" + done + echo $nodes +} + +ALL_NODES="$(expand_nodes)" # ── CI helpers ────────────────────────────────────────────────────── ci_mask() { [[ "${GITHUB_ACTIONS:-}" == "true" ]] && echo "::add-mask::$1" || true; } @@ -66,6 +111,9 @@ require_env() { cmd_provision() { log "Starting provisioning..." + log " Versions: $PVE_VERSIONS" + log " Nodes: $ALL_NODES" + log " Storage: Docker containers (iSCSI + NFS)" require_env PVE_ENDPOINT require_env PVE_API_TOKEN require_env PVE_TARGET_NODE @@ -74,7 +122,7 @@ cmd_provision() { ci_mask "$PVE_PASSWORD" mkdir -p "$WORK_DIR" "$CACHE_DIR" - # Ensure base ISOs + # Ensure base ISOs (one per version, not per node) for v in $PVE_VERSIONS; do log "Ensuring base ISO for PVE $v..." bash "$SCRIPT_DIR/ensure-base-iso.sh" "$(pve_iso "$v")" "$CACHE_DIR" @@ -87,111 +135,146 @@ cmd_provision() { CLOUD_IMAGE_PATH=$(echo "$cloud_output" | grep "^CLOUD_IMAGE_PATH=" | cut -d= -f2) OVA_PATH=$(echo "$cloud_output" | grep "^OVA_PATH=" | cut -d= -f2) - # Pre-flight cleanup - for v in $PVE_VERSIONS; do - local iso_name - iso_name="$(pve_iso "$v")" - log "Pre-flight cleanup for PVE $v (VMID $(pve_vmid "$v"))..." - bash "$SCRIPT_DIR/preflight-cleanup.sh" \ - "$PVE_ENDPOINT" "$PVE_API_TOKEN" \ - "$(pve_vmid "$v")" "${iso_name%.iso}-auto.iso" "$INFRA_DIR" - done - - # Generate answer file - log "Generating answer file..." + # Generate per-node answer files (each needs unique FQDN for clustering) + log "Generating answer files..." local escaped_pve_password escaped_pve_password=$(printf '%s' "$PVE_PASSWORD" | sed 's/[\/&\\]/\\&/g') - sed "s/\${root_password}/${escaped_pve_password}/" \ - "$INFRA_DIR/answer.toml.tftpl" > "$WORK_DIR/answer.toml" + for node in $ALL_NODES; do + local fqdn + fqdn="$(pve_fqdn "$node")" + sed -e "s/\${root_password}/${escaped_pve_password}/" \ + -e "s/\${fqdn}/${fqdn}/" \ + "$INFRA_DIR/answer.toml.tftpl" > "$WORK_DIR/answer-${node}.toml" + done - # Prepare auto-install ISOs - for v in $PVE_VERSIONS; do + # Prepare auto-install ISOs (one per node — each has unique answer file) + for node in $ALL_NODES; do local iso_name - iso_name="$(pve_iso "$v")" - log "Preparing auto-install ISO for PVE $v..." + iso_name="$(pve_iso "$node")" + log "Preparing auto-install ISO for $node..." bash "$SCRIPT_DIR/prepare-auto-iso.sh" \ "$CACHE_DIR/$iso_name" \ - "$WORK_DIR/answer.toml" \ + "$WORK_DIR/answer-${node}.toml" \ "$SCRIPT_DIR/first-boot.sh" \ - "$WORK_DIR/${iso_name%.iso}-auto.iso" \ + "$WORK_DIR/${iso_name%.iso}-${node}-auto.iso" \ --cache-dir "$CACHE_DIR" done - # Terraform + # Terraform — remove any stale .tfvars from previous manual runs + rm -f "$INFRA_DIR/terraform.tfvars" + log "Running Terraform init..." (cd "$INFRA_DIR" && terraform init -input=false) log "Building Terraform vars..." local tfvars="$WORK_DIR/instances.tfvars.json" local instances='{}' - for v in $PVE_VERSIONS; do + for node in $ALL_NODES; do local iso_name - iso_name="$(pve_iso "$v")" - local iso_path="$WORK_DIR/${iso_name%.iso}-auto.iso" + iso_name="$(pve_iso "$node")" + local iso_path="$WORK_DIR/${iso_name%.iso}-${node}-auto.iso" local vm_id - vm_id="$(pve_vmid "$v")" + vm_id="$(pve_vmid "$node")" local vm_name - vm_name="$(pve_vmname "$v")" + vm_name="$(pve_vmname "$node")" instances="$(jq \ - --arg key "pve${v}" \ + --arg key "$node" \ --arg iso_local_path "$iso_path" \ --arg vm_name "$vm_name" \ --argjson vm_id "$vm_id" \ '. + {($key): {iso_local_path: $iso_local_path, vm_id: $vm_id, vm_name: $vm_name}}' \ <<<"$instances")" done - jq -n --argjson pve_instances "$instances" '{pve_instances: $pve_instances}' > "$tfvars" - log "Running Terraform apply..." + jq -n --argjson pve_instances "$instances" \ + '{pve_instances: $pve_instances}' > "$tfvars" + + log "Running Terraform apply (PVE nodes)..." + # TMPDIR: use work dir to avoid filling the container's /tmp with multi-GB ISO uploads. (cd "$INFRA_DIR" && \ + TMPDIR="$WORK_DIR" \ TF_VAR_proxmox_endpoint="$PVE_ENDPOINT" \ TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \ TF_VAR_target_node="$PVE_TARGET_NODE" \ TF_VAR_test_vm_password="$PVE_PASSWORD" \ terraform apply -auto-approve -input=false -var-file="$tfvars") - # Wait for PVE instances and create API tokens - for v in $PVE_VERSIONS; do - log "Waiting for PVE $v to boot and creating API token..." + # Start iSCSI/NFS storage containers on the Docker host + log "Starting storage containers (iSCSI + NFS)..." + # Get the Docker host's real IP (not the container's). The storage containers + # use host networking, so PVE nodes reach them via the host's IP. + local storage_ip + # Prefer the IP of the default route's interface on the Docker host. + storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}') + if [ -z "$storage_ip" ]; then + # Fallback: use the local node address from Docker info (not RemoteManagers, + # which can return addresses of remote Swarm managers). + storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1) + fi + if [ -z "$storage_ip" ]; then + ci_error "Could not determine Docker host IP for storage services" + exit 1 + fi + ISCSI_IQN="$STORAGE_ISCSI_IQN" \ + docker compose -f "$STORAGE_COMPOSE" up -d + log "Storage services ready at $storage_ip (iSCSI: $STORAGE_ISCSI_IQN)" + + # Wait for PVE instances and create API tokens (per node) + for node in $ALL_NODES; do + log "Waiting for $node to boot and creating API token..." local output output=$(bash "$SCRIPT_DIR/create-api-token.sh" \ "$PVE_ENDPOINT" "$PVE_API_TOKEN" \ - "$(pve_vmid "$v")" "$PVE_PASSWORD" 900) + "$(pve_vmid "$node")" "$PVE_PASSWORD" 900) local ip token ip=$(echo "$output" | grep "^IP=" | cut -d= -f2) token=$(echo "$output" | grep "^TOKEN=" | cut -d= -f2-) - log "PVE $v ready at $ip" - echo "{\"host\":\"$ip\",\"token\":\"$token\"}" > "$WORK_DIR/pve${v}.json" + # Node name = hostname portion of FQDN (e.g. pve9a.test.local -> pve9a) + local node_name + node_name="$(pve_fqdn "$node" | cut -d. -f1)" + log "$node ready at $ip (node: $node_name)" + jq -n --arg host "$ip" --arg token "$token" --arg node "$node_name" \ + '{host: $host, token: $token, node: $node}' > "$WORK_DIR/${node}.json" done - # Prepare test environments - for v in $PVE_VERSIONS; do + # Prepare test environments on all PVE nodes + for node in $ALL_NODES; do local ip - ip=$(jq -r .host "$WORK_DIR/pve${v}.json") - log "Preparing test environment on PVE $v ($ip)..." + ip=$(jq -r .host "$WORK_DIR/${node}.json") + log "Preparing test environment on $node ($ip)..." bash "$SCRIPT_DIR/prepare-test-environment.sh" "$ip" "$PVE_PASSWORD" done # Write test config log "Writing test config to $CONFIG_FILE..." - local jq_args=() - for v in $PVE_VERSIONS; do - jq_args+=(--argjson "pve${v}" "$(cat "$WORK_DIR/pve${v}.json")") - done - jq_args+=(--arg cloud_image "${CLOUD_IMAGE_PATH:-}") - jq_args+=(--arg ova "${OVA_PATH:-}") + local config='{}' - local jq_expr="{" - local first=true for v in $PVE_VERSIONS; do - $first || jq_expr+="," - first=false - jq_expr+="pve${v}: \$pve${v}" + local node_a="${v}a" + local node_b="${v}b" + local version_config + version_config=$(jq -n \ + --argjson a "$(cat "$WORK_DIR/${node_a}.json")" \ + --argjson b "$(cat "$WORK_DIR/${node_b}.json")" \ + '{nodes: {a: $a, b: $b}}') + config=$(jq --arg key "pve${v}" --argjson val "$version_config" \ + '. + {($key): $val}' <<<"$config") done - jq_expr+=", cloud_image_path: \$cloud_image, ova_path: \$ova}" - jq -n "${jq_args[@]}" "$jq_expr" > "$CONFIG_FILE" + config=$(jq \ + --arg cloud_image "${CLOUD_IMAGE_PATH:-}" \ + --arg ova "${OVA_PATH:-}" \ + --arg storage_ip "$storage_ip" \ + --arg storage_iqn "$STORAGE_ISCSI_IQN" \ + '. + { + storage: {ip: $storage_ip, iscsi_iqn: $storage_iqn, nfs_export: ($storage_ip + ":/srv/nfs/shared")}, + cloud_image_path: $cloud_image, + ova_path: $ova + }' <<<"$config") + + echo "$config" | jq . > "$CONFIG_FILE" log "Test config written to $CONFIG_FILE" + jq . "$CONFIG_FILE" log "Provisioning complete." } @@ -241,34 +324,60 @@ cmd_test() { export PVETEST_STORAGE="${PVETEST_STORAGE:-local}" export PVETEST_CLOUD_IMAGE_PATH="${PVETEST_CLOUD_IMAGE_PATH:-}" export PVETEST_OVA_PATH="${PVETEST_OVA_PATH:-}" + # Secondary node and storage VM may not be available in skip mode + export PVETEST_HOST_B="${PVETEST_HOST_B:-}" + export PVETEST_APITOKEN_B="${PVETEST_APITOKEN_B:-}" + export PVETEST_STORAGE_VM_IP="${PVETEST_STORAGE_VM_IP:-}" + export PVETEST_ISCSI_IQN="${PVETEST_ISCSI_IQN:-}" + export PVETEST_NFS_EXPORT="${PVETEST_NFS_EXPORT:-}" else if [[ ! -f "$CONFIG_FILE" ]]; then ci_error "No test config found at $CONFIG_FILE — run 'provision' first or set SKIP_PROVISION=true" exit 1 fi - export PVETEST_HOST=$(jq -r ".pve${v}.host" "$CONFIG_FILE") - export PVETEST_APITOKEN=$(jq -r ".pve${v}.token" "$CONFIG_FILE") + # Primary node (a) + export PVETEST_HOST=$(jq -r ".pve${v}.nodes.a.host" "$CONFIG_FILE") + export PVETEST_APITOKEN=$(jq -r ".pve${v}.nodes.a.token" "$CONFIG_FILE") export PVETEST_PORT=8006 - export PVETEST_NODE=pve + export PVETEST_NODE=$(jq -r ".pve${v}.nodes.a.node" "$CONFIG_FILE") export PVETEST_STORAGE=local export PVETEST_CLOUD_IMAGE_PATH=$(jq -r '.cloud_image_path' "$CONFIG_FILE") export PVETEST_OVA_PATH=$(jq -r '.ova_path' "$CONFIG_FILE") + # Secondary node (b) + export PVETEST_HOST_B=$(jq -r ".pve${v}.nodes.b.host" "$CONFIG_FILE") + export PVETEST_APITOKEN_B=$(jq -r ".pve${v}.nodes.b.token" "$CONFIG_FILE") + # Storage services (Docker on runner) + export PVETEST_STORAGE_VM_IP=$(jq -r '.storage.ip' "$CONFIG_FILE") + export PVETEST_ISCSI_IQN=$(jq -r '.storage.iscsi_iqn' "$CONFIG_FILE") + export PVETEST_NFS_EXPORT=$(jq -r '.storage.nfs_export' "$CONFIG_FILE") fi export PVETEST_ISO_PATH="$iso_path" export PVETEST_PVE_VERSION="$v" export PVETEST_PASSWORD="${PVETEST_PASSWORD:-${PVE_PASSWORD:-}}" - # Verify API reachable - log "Verifying PVE $v API at $PVETEST_HOST:$PVETEST_PORT..." + # Verify API reachable (node A) + log "Verifying PVE $v node A API at $PVETEST_HOST:$PVETEST_PORT..." if ! curl -sk --connect-timeout 10 \ -H "Authorization: PVEAPIToken=${PVETEST_APITOKEN}" \ "https://${PVETEST_HOST}:${PVETEST_PORT}/api2/json/nodes" | grep -q '"node"'; then - ci_error "Cannot reach PVE $v API at ${PVETEST_HOST}:${PVETEST_PORT}" + ci_error "Cannot reach PVE $v node A API at ${PVETEST_HOST}:${PVETEST_PORT}" overall_exit=3 continue fi + # Verify node B if available + if [[ -n "${PVETEST_HOST_B:-}" ]]; then + log "Verifying PVE $v node B API at $PVETEST_HOST_B:$PVETEST_PORT..." + if ! curl -sk --connect-timeout 10 \ + -H "Authorization: PVEAPIToken=${PVETEST_APITOKEN_B}" \ + "https://${PVETEST_HOST_B}:${PVETEST_PORT}/api2/json/nodes" | grep -q '"node"'; then + ci_error "Cannot reach PVE $v node B API at ${PVETEST_HOST_B}:${PVETEST_PORT}" + overall_exit=3 + continue + fi + fi + # Run Pester local test_exit=0 pwsh -NoProfile -Command ' @@ -296,15 +405,22 @@ cmd_test() { cmd_cleanup() { log "Starting cleanup..." - for v in $PVE_VERSIONS; do + + # Clean up PVE nodes + for node in $ALL_NODES; do local iso_name - iso_name="$(pve_iso "$v")" - log "Cleaning up PVE $v (VMID $(pve_vmid "$v"))..." + iso_name="$(pve_iso "$node")" + log "Cleaning up $node (VMID $(pve_vmid "$node"))..." bash "$SCRIPT_DIR/preflight-cleanup.sh" \ "${PVE_ENDPOINT:-}" "${PVE_API_TOKEN:-}" \ - "$(pve_vmid "$v")" "${iso_name%.iso}-auto.iso" "$INFRA_DIR" \ + "$(pve_vmid "$node")" "${iso_name%.iso}-${node}-auto.iso" "$INFRA_DIR" \ || true done + + # Stop storage containers + log "Stopping storage containers..." + docker compose -f "$STORAGE_COMPOSE" down -v 2>/dev/null || true + log "Cleanup complete." } @@ -339,9 +455,9 @@ main() { echo "Usage: $(basename "$0") {provision|test|cleanup|all} [8|9|all]" echo "" echo "Subcommands:" - echo " provision Provision nested PVE VMs via Terraform" + echo " provision Provision nested PVE VMs + start storage containers" echo " test [8|9|all] Run integration tests (default: all versions)" - echo " cleanup Destroy provisioned VMs" + echo " cleanup Destroy all provisioned VMs" echo " all [8|9|all] Full lifecycle: provision → test → cleanup" exit 1 ;; diff --git a/tests/infrastructure/variables.tf b/tests/infrastructure/variables.tf index 18864c3..afcdd07 100644 --- a/tests/infrastructure/variables.tf +++ b/tests/infrastructure/variables.tf @@ -70,3 +70,4 @@ variable "test_vm_password" { type = string sensitive = true } +