diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0549b27..24eac88 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -50,12 +50,14 @@ jobs: steps: - uses: actions/checkout@v4 - # ── Provision nested PVE ─────────────────────────────────────────── + # ── Pre-flight cleanup ───────────────────────────────────────────── - - name: Select PVE ISO + - name: Pre-flight cleanup if: inputs.skip_provision != true - id: iso shell: bash + id: iso + env: + PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} run: | VERSION="${{ inputs.pve_version || '9' }}" if [ "$VERSION" = "9" ]; then @@ -66,6 +68,15 @@ jobs: echo "filename=proxmox-ve_8.4-1-auto.iso" >> "$GITHUB_OUTPUT" fi + bash ${SCRIPTS_DIR}/preflight-cleanup.sh \ + "${{ secrets.PVE_ENDPOINT }}" \ + "${PVE_API_TOKEN}" \ + 99900 \ + "auto.iso" \ + "${INFRA_DIR}" + + # ── Provision nested PVE ─────────────────────────────────────────── + - name: Generate answer file if: inputs.skip_provision != true shell: bash @@ -113,9 +124,8 @@ jobs: env: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} run: | - PARENT_HOST=$(echo '${{ secrets.PVE_ENDPOINT }}' | sed -E 's|https?://||;s|:.*||') OUTPUT=$(bash ${SCRIPTS_DIR}/create-api-token.sh \ - "${PARENT_HOST}" \ + "${{ secrets.PVE_ENDPOINT }}" \ "${PVE_API_TOKEN}" \ "${{ steps.terraform.outputs.vm_id }}" \ "${PVE_PASSWORD}" \ @@ -223,11 +233,18 @@ jobs: TF_VAR_test_vm_password: ${{ env.PVE_PASSWORD }} run: | terraform init -input=false - # Clear stale lock from cancelled runs - if [ -f .terraform.tfstate.lock.info ]; then - LOCK_ID=$(python3 -c "import json; print(json.load(open('.terraform.tfstate.lock.info'))['ID'])") - terraform force-unlock -force "$LOCK_ID" || true - fi terraform destroy -auto-approve -input=false || true - # Clean up state so next run starts fresh rm -f terraform.tfstate terraform.tfstate.backup .terraform.tfstate.lock.info + + - name: Final cleanup + if: always() && inputs.skip_provision != true + shell: bash + env: + PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} + run: | + bash ${SCRIPTS_DIR}/preflight-cleanup.sh \ + "${{ secrets.PVE_ENDPOINT }}" \ + "${PVE_API_TOKEN}" \ + 99900 \ + "${{ steps.iso.outputs.filename }}" \ + "${INFRA_DIR}" || true diff --git a/tests/infrastructure/answer.toml.tftpl b/tests/infrastructure/answer.toml.tftpl index 68e6272..6521e99 100644 --- a/tests/infrastructure/answer.toml.tftpl +++ b/tests/infrastructure/answer.toml.tftpl @@ -14,3 +14,7 @@ source = "from-dhcp" [disk-setup] filesystem = "ext4" disk_list = ["sda"] + +[first-boot] +source = "from-iso" +ordering = "fully-up" diff --git a/tests/infrastructure/scripts/create-api-token.sh b/tests/infrastructure/scripts/create-api-token.sh index 0d409a6..cdad15c 100755 --- a/tests/infrastructure/scripts/create-api-token.sh +++ b/tests/infrastructure/scripts/create-api-token.sh @@ -2,21 +2,23 @@ # Wait for a fresh nested PVE instance to boot, discover its IP via the QEMU guest agent, # then wait for the PVE API and create an API token. # -# Usage: create-api-token.sh [max-wait-seconds] +# Usage: create-api-token.sh [max-wait-seconds] +# parent-pve-endpoint: Full URL e.g. https://172.16.100.150:8006 # Outputs two lines: # IP= # TOKEN=root@pam!integration= set -euo pipefail -PARENT_HOST="$1" +PARENT_ENDPOINT="${1%/}" PARENT_TOKEN="$2" VM_ID="$3" ROOT_PASSWORD="$4" MAX_WAIT="${5:-600}" INTERVAL=10 -PARENT_API="https://${PARENT_HOST}/api2/json" -PARENT_NODE=$(curl -sk -H "Authorization: PVEAPIToken=${PARENT_TOKEN}" \ - "${PARENT_API}/nodes" | python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])") +PARENT_API="${PARENT_ENDPOINT}/api2/json" +NODES_JSON=$(curl -sk -H "Authorization: PVEAPIToken=${PARENT_TOKEN}" \ + "${PARENT_API}/nodes") +PARENT_NODE=$(echo "$NODES_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])") # --- Phase 1: Discover IP via QEMU guest agent --- echo "Waiting for guest agent on VM ${VM_ID} (node: ${PARENT_NODE})..." @@ -60,7 +62,7 @@ fi NESTED_API="https://${VM_IP}:8006/api2/json" echo "Waiting for nested PVE API at ${NESTED_API}..." while [ $elapsed -lt $MAX_WAIT ]; do - if curl -sk --connect-timeout 5 "${NESTED_API}/version" 2>/dev/null | grep -q '"version"'; then + if curl -sk --connect-timeout 5 "${NESTED_API}/access/domains" 2>/dev/null | grep -q '"realm"'; then echo "Nested PVE API is responsive after ${elapsed}s" break fi diff --git a/tests/infrastructure/scripts/first-boot.sh b/tests/infrastructure/scripts/first-boot.sh index 0a3934e..34ddbcc 100755 --- a/tests/infrastructure/scripts/first-boot.sh +++ b/tests/infrastructure/scripts/first-boot.sh @@ -5,6 +5,15 @@ set -e +# Disable enterprise repos (no subscription) and enable the no-subscription repo +# PVE 8.x uses .list files, PVE 9.x uses .sources (DEB822 format) +rm -f /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/pve-enterprise.sources +rm -f /etc/apt/sources.list.d/ceph.list /etc/apt/sources.list.d/ceph.sources + +# Detect suite from existing debian.sources or default to trixie +SUITE=$(grep -oP 'Suites:\s*\K\S+' /etc/apt/sources.list.d/debian.sources 2>/dev/null | head -1 || echo "trixie") +echo "deb http://download.proxmox.com/debian/pve ${SUITE} pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list + apt-get update -qq apt-get install -y -qq qemu-guest-agent -systemctl enable --now qemu-guest-agent +systemctl start qemu-guest-agent diff --git a/tests/infrastructure/scripts/preflight-cleanup.sh b/tests/infrastructure/scripts/preflight-cleanup.sh new file mode 100755 index 0000000..f4014f4 --- /dev/null +++ b/tests/infrastructure/scripts/preflight-cleanup.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Pre-flight cleanup for CI runs. +# Ensures no leftover resources from a previous failed run before starting fresh. +# +# Usage: preflight-cleanup.sh +set -uo pipefail +# Note: not using -e — we want to attempt all cleanup steps even if some fail + +PVE_ENDPOINT="${1%/}" +API_TOKEN="$2" +VM_ID="$3" +ISO_FILENAME="$4" +TF_DIR="$5" +API_BASE="${PVE_ENDPOINT}/api2/json" + +# Discover node name +NODES_JSON=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" "${API_BASE}/nodes" 2>/dev/null) +NODE=$(echo "$NODES_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])" 2>/dev/null || echo "pve") + +echo "=== Pre-flight cleanup (node: ${NODE}, vmid: ${VM_ID}) ===" + +# --- Clean up orphaned VM --- +VM_STATUS=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" \ + "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}/status/current" 2>/dev/null \ + | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null || true) + +if [ -n "$VM_STATUS" ]; then + echo "Found orphaned VM ${VM_ID} (status: ${VM_STATUS})" + if [ "$VM_STATUS" = "running" ]; then + echo " Stopping VM..." + curl -sk -X POST -H "Authorization: PVEAPIToken=${API_TOKEN}" \ + "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}/status/stop" >/dev/null 2>&1 + # Wait for stop + for i in $(seq 1 12); do + sleep 5 + S=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" \ + "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}/status/current" 2>/dev/null \ + | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null || true) + if [ "$S" = "stopped" ]; then break; fi + done + fi + echo " Deleting VM..." + curl -sk -X DELETE -H "Authorization: PVEAPIToken=${API_TOKEN}" \ + "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}?destroy-unreferenced-disks=1&purge=1" >/dev/null 2>&1 + sleep 3 + echo " VM cleanup done" +else + echo "No orphaned VM ${VM_ID} found" +fi + +# --- Clean up orphaned ISO --- +ISO_EXISTS=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" \ + "${API_BASE}/nodes/${NODE}/storage/local/content" 2>/dev/null \ + | python3 -c " +import json, sys +data = json.load(sys.stdin).get('data', []) +for item in data: + if item.get('volid', '').endswith('/${ISO_FILENAME}'): + print(item['volid']) + break +" 2>/dev/null || true) + +if [ -n "$ISO_EXISTS" ]; then + echo "Found orphaned ISO: ${ISO_EXISTS}" + echo " Deleting..." + ENCODED=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${ISO_EXISTS}', safe=''))") + curl -sk -X DELETE -H "Authorization: PVEAPIToken=${API_TOKEN}" \ + "${API_BASE}/nodes/${NODE}/storage/local/content/${ENCODED}" >/dev/null 2>&1 + sleep 2 + echo " ISO cleanup done" +else + echo "No orphaned ISO found" +fi + +# --- Clean up stale Terraform state --- +if [ -d "$TF_DIR" ]; then + if [ -f "${TF_DIR}/.terraform.tfstate.lock.info" ]; then + echo "Found stale Terraform lock, removing..." + rm -f "${TF_DIR}/.terraform.tfstate.lock.info" + fi + if [ -f "${TF_DIR}/terraform.tfstate" ]; then + echo "Found stale Terraform state, removing..." + rm -f "${TF_DIR}/terraform.tfstate" "${TF_DIR}/terraform.tfstate.backup" + fi +else + echo "Terraform dir not found (clean checkout)" +fi + +echo "=== Pre-flight cleanup complete ==="