fix(ci): first-boot repo fix, robust cleanup, and script fixes

- first-boot.sh: Remove .sources files (PVE 9 DEB822 format) not just
  .list, and detect suite (trixie vs bookworm) from debian.sources
- create-api-token.sh: Accept full endpoint URL (fixes missing port),
  use /access/domains for API check (no auth needed on PVE 9),
  avoid pipefail on curl|python3 pipe
- preflight-cleanup.sh: New script that runs at start and end of every
  provisioning run — cleans orphaned VMs, ISOs, and stale terraform state
- Workflow: Add pre-flight and final cleanup steps so failed runs can't
  leave the runner/host in a broken state

All changes validated manually on the self-hosted runner.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.github/workflows/integration-tests.yml

@@ -50,12 +50,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      # ── Provision nested PVE ───────────────────────────────────────────
+      # ── Pre-flight cleanup ─────────────────────────────────────────────
 
-      - name: Select PVE ISO
+      - name: Pre-flight cleanup
         if: inputs.skip_provision != true
-        id: iso
         shell: bash
+        id: iso
+        env:
+          PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
         run: |
           VERSION="${{ inputs.pve_version || '9' }}"
           if [ "$VERSION" = "9" ]; then
@@ -66,6 +68,15 @@ jobs:
             echo "filename=proxmox-ve_8.4-1-auto.iso" >> "$GITHUB_OUTPUT"
           fi
 
+          bash ${SCRIPTS_DIR}/preflight-cleanup.sh \
+            "${{ secrets.PVE_ENDPOINT }}" \
+            "${PVE_API_TOKEN}" \
+            99900 \
+            "auto.iso" \
+            "${INFRA_DIR}"
+
+      # ── Provision nested PVE ───────────────────────────────────────────
+
       - name: Generate answer file
         if: inputs.skip_provision != true
         shell: bash
@@ -113,9 +124,8 @@ jobs:
         env:
           PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
         run: |
-          PARENT_HOST=$(echo '${{ secrets.PVE_ENDPOINT }}' | sed -E 's|https?://||;s|:.*||')
           OUTPUT=$(bash ${SCRIPTS_DIR}/create-api-token.sh \
-            "${PARENT_HOST}" \
+            "${{ secrets.PVE_ENDPOINT }}" \
             "${PVE_API_TOKEN}" \
             "${{ steps.terraform.outputs.vm_id }}" \
             "${PVE_PASSWORD}" \
@@ -223,11 +233,18 @@ jobs:
           TF_VAR_test_vm_password: ${{ env.PVE_PASSWORD }}
         run: |
           terraform init -input=false
-          # Clear stale lock from cancelled runs
-          if [ -f .terraform.tfstate.lock.info ]; then
-            LOCK_ID=$(python3 -c "import json; print(json.load(open('.terraform.tfstate.lock.info'))['ID'])")
-            terraform force-unlock -force "$LOCK_ID" || true
-          fi
           terraform destroy -auto-approve -input=false || true
-          # Clean up state so next run starts fresh
           rm -f terraform.tfstate terraform.tfstate.backup .terraform.tfstate.lock.info
+
+      - name: Final cleanup
+        if: always() && inputs.skip_provision != true
+        shell: bash
+        env:
+          PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }}
+        run: |
+          bash ${SCRIPTS_DIR}/preflight-cleanup.sh \
+            "${{ secrets.PVE_ENDPOINT }}" \
+            "${PVE_API_TOKEN}" \
+            99900 \
+            "${{ steps.iso.outputs.filename }}" \
+            "${INFRA_DIR}" || true

tests/infrastructure/answer.toml.tftpl

@@ -14,3 +14,7 @@ source = "from-dhcp"
 [disk-setup]
 filesystem = "ext4"
 disk_list = ["sda"]
+
+[first-boot]
+source = "from-iso"
+ordering = "fully-up"

tests/infrastructure/scripts/create-api-token.sh

@@ -2,21 +2,23 @@
 # Wait for a fresh nested PVE instance to boot, discover its IP via the QEMU guest agent,
 # then wait for the PVE API and create an API token.
 #
-# Usage: create-api-token.sh <parent-pve-host> <parent-api-token> <vm-id> <root-password> [max-wait-seconds]
+# Usage: create-api-token.sh <parent-pve-endpoint> <parent-api-token> <vm-id> <root-password> [max-wait-seconds]
+#   parent-pve-endpoint: Full URL e.g. https://172.16.100.150:8006
 #   Outputs two lines:
 #     IP=<discovered-ip>
 #     TOKEN=root@pam!integration=<secret>
 set -euo pipefail
 
-PARENT_HOST="$1"
+PARENT_ENDPOINT="${1%/}"
 PARENT_TOKEN="$2"
 VM_ID="$3"
 ROOT_PASSWORD="$4"
 MAX_WAIT="${5:-600}"
 INTERVAL=10
-PARENT_API="https://${PARENT_HOST}/api2/json"
-PARENT_NODE=$(curl -sk -H "Authorization: PVEAPIToken=${PARENT_TOKEN}" \
-    "${PARENT_API}/nodes" | python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])")
+PARENT_API="${PARENT_ENDPOINT}/api2/json"
+NODES_JSON=$(curl -sk -H "Authorization: PVEAPIToken=${PARENT_TOKEN}" \
+    "${PARENT_API}/nodes")
+PARENT_NODE=$(echo "$NODES_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])")
 
 # --- Phase 1: Discover IP via QEMU guest agent ---
 echo "Waiting for guest agent on VM ${VM_ID} (node: ${PARENT_NODE})..."
@@ -60,7 +62,7 @@ fi
 NESTED_API="https://${VM_IP}:8006/api2/json"
 echo "Waiting for nested PVE API at ${NESTED_API}..."
 while [ $elapsed -lt $MAX_WAIT ]; do
-    if curl -sk --connect-timeout 5 "${NESTED_API}/version" 2>/dev/null | grep -q '"version"'; then
+    if curl -sk --connect-timeout 5 "${NESTED_API}/access/domains" 2>/dev/null | grep -q '"realm"'; then
         echo "Nested PVE API is responsive after ${elapsed}s"
         break
     fi

tests/infrastructure/scripts/first-boot.sh

@@ -5,6 +5,15 @@
 
 set -e
 
+# Disable enterprise repos (no subscription) and enable the no-subscription repo
+# PVE 8.x uses .list files, PVE 9.x uses .sources (DEB822 format)
+rm -f /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/pve-enterprise.sources
+rm -f /etc/apt/sources.list.d/ceph.list /etc/apt/sources.list.d/ceph.sources
+
+# Detect suite from existing debian.sources or default to trixie
+SUITE=$(grep -oP 'Suites:\s*\K\S+' /etc/apt/sources.list.d/debian.sources 2>/dev/null | head -1 || echo "trixie")
+echo "deb http://download.proxmox.com/debian/pve ${SUITE} pve-no-subscription" > /etc/apt/sources.list.d/pve-no-subscription.list
+
 apt-get update -qq
 apt-get install -y -qq qemu-guest-agent
-systemctl enable --now qemu-guest-agent
+systemctl start qemu-guest-agent

tests/infrastructure/scripts/preflight-cleanup.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Pre-flight cleanup for CI runs.
+# Ensures no leftover resources from a previous failed run before starting fresh.
+#
+# Usage: preflight-cleanup.sh <pve-endpoint> <api-token> <vm-id> <iso-filename> <terraform-dir>
+set -uo pipefail
+# Note: not using -e — we want to attempt all cleanup steps even if some fail
+
+PVE_ENDPOINT="${1%/}"
+API_TOKEN="$2"
+VM_ID="$3"
+ISO_FILENAME="$4"
+TF_DIR="$5"
+API_BASE="${PVE_ENDPOINT}/api2/json"
+
+# Discover node name
+NODES_JSON=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" "${API_BASE}/nodes" 2>/dev/null)
+NODE=$(echo "$NODES_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])" 2>/dev/null || echo "pve")
+
+echo "=== Pre-flight cleanup (node: ${NODE}, vmid: ${VM_ID}) ==="
+
+# --- Clean up orphaned VM ---
+VM_STATUS=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" \
+    "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}/status/current" 2>/dev/null \
+    | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null || true)
+
+if [ -n "$VM_STATUS" ]; then
+    echo "Found orphaned VM ${VM_ID} (status: ${VM_STATUS})"
+    if [ "$VM_STATUS" = "running" ]; then
+        echo "  Stopping VM..."
+        curl -sk -X POST -H "Authorization: PVEAPIToken=${API_TOKEN}" \
+            "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}/status/stop" >/dev/null 2>&1
+        # Wait for stop
+        for i in $(seq 1 12); do
+            sleep 5
+            S=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" \
+                "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}/status/current" 2>/dev/null \
+                | python3 -c "import json,sys; print(json.load(sys.stdin).get('data',{}).get('status',''))" 2>/dev/null || true)
+            if [ "$S" = "stopped" ]; then break; fi
+        done
+    fi
+    echo "  Deleting VM..."
+    curl -sk -X DELETE -H "Authorization: PVEAPIToken=${API_TOKEN}" \
+        "${API_BASE}/nodes/${NODE}/qemu/${VM_ID}?destroy-unreferenced-disks=1&purge=1" >/dev/null 2>&1
+    sleep 3
+    echo "  VM cleanup done"
+else
+    echo "No orphaned VM ${VM_ID} found"
+fi
+
+# --- Clean up orphaned ISO ---
+ISO_EXISTS=$(curl -sk -H "Authorization: PVEAPIToken=${API_TOKEN}" \
+    "${API_BASE}/nodes/${NODE}/storage/local/content" 2>/dev/null \
+    | python3 -c "
+import json, sys
+data = json.load(sys.stdin).get('data', [])
+for item in data:
+    if item.get('volid', '').endswith('/${ISO_FILENAME}'):
+        print(item['volid'])
+        break
+" 2>/dev/null || true)
+
+if [ -n "$ISO_EXISTS" ]; then
+    echo "Found orphaned ISO: ${ISO_EXISTS}"
+    echo "  Deleting..."
+    ENCODED=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${ISO_EXISTS}', safe=''))")
+    curl -sk -X DELETE -H "Authorization: PVEAPIToken=${API_TOKEN}" \
+        "${API_BASE}/nodes/${NODE}/storage/local/content/${ENCODED}" >/dev/null 2>&1
+    sleep 2
+    echo "  ISO cleanup done"
+else
+    echo "No orphaned ISO found"
+fi
+
+# --- Clean up stale Terraform state ---
+if [ -d "$TF_DIR" ]; then
+    if [ -f "${TF_DIR}/.terraform.tfstate.lock.info" ]; then
+        echo "Found stale Terraform lock, removing..."
+        rm -f "${TF_DIR}/.terraform.tfstate.lock.info"
+    fi
+    if [ -f "${TF_DIR}/terraform.tfstate" ]; then
+        echo "Found stale Terraform state, removing..."
+        rm -f "${TF_DIR}/terraform.tfstate" "${TF_DIR}/terraform.tfstate.backup"
+    fi
+else
+    echo "Terraform dir not found (clean checkout)"
+fi
+
+echo "=== Pre-flight cleanup complete ==="