From 94d5fe99444b9b30394f71c52b516068fedab635 Mon Sep 17 00:00:00 2001 From: Clint Branham Date: Wed, 25 Mar 2026 17:01:45 -0500 Subject: [PATCH 1/3] fix: PVE VMs depend on answer server container Terraform creates resources in parallel. Without depends_on, PVE VMs could boot and start the auto-installer before the HTTP answer server container is running, causing "could not find answer file" errors. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/infrastructure/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/infrastructure/main.tf b/tests/infrastructure/main.tf index df66f34..a1e07f2 100644 --- a/tests/infrastructure/main.tf +++ b/tests/infrastructure/main.tf @@ -90,6 +90,10 @@ resource "proxmox_virtual_environment_vm" "nested_pve" { started = true + # VMs must not boot until the HTTP answer server is running, + # otherwise the PVE auto-installer can't fetch its answer file. + depends_on = [docker_container.answer_server] + lifecycle { ignore_changes = [started, cdrom] } From 226f9d8a1283ef634a1404d09f37b67a429c6060 Mon Sep 17 00:00:00 2001 From: Clint Branham Date: Wed, 25 Mar 2026 17:04:38 -0500 Subject: [PATCH 2/3] fix: store Terraform state on shared mount for CI persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Terraform state was stored in the container's working directory (fresh checkout), so it was lost between CI jobs. The cleanup job couldn't destroy resources because it had no state. Now stores state at /opt/pve-integration/work/terraform.tfstate via -state flag on all terraform commands. This persists across the provision → test → cleanup job chain in GitHub Actions. Also: - Force cleanup now removes state from both local and shared paths - Added -reconfigure to terraform init (avoids backend mismatch errors) Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/integration-tests.yml | 2 ++ tests/infrastructure/scripts/run-integration.sh | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 714b97d..ae63190 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -209,6 +209,8 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} + # Terraform state is stored on the shared mount (/opt/pve-integration/work/) + # so it persists between CI jobs. run: bash ${SCRIPTS_DIR}/run-integration.sh cleanup # ── Clean up old container images from GHCR ────────────────────────── diff --git a/tests/infrastructure/scripts/run-integration.sh b/tests/infrastructure/scripts/run-integration.sh index 9d84554..01bc779 100755 --- a/tests/infrastructure/scripts/run-integration.sh +++ b/tests/infrastructure/scripts/run-integration.sh @@ -54,6 +54,8 @@ PVE_VERSIONS="${PVE_VERSIONS:-9 8}" SKIP_PROVISION="${SKIP_PROVISION:-false}" STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}" STORAGE_COMPOSE="$INFRA_DIR/docker-compose.storage.yml" +# Store Terraform state on the shared mount so it persists across CI jobs +TF_STATE_FILE="$WORK_DIR/terraform.tfstate" # ── Node config ─────────────────────────────────────────────────── # Each version gets two nodes: a (primary) and b (secondary). @@ -226,7 +228,7 @@ cmd_provision() { rm -f "$INFRA_DIR/terraform.tfvars" log "Running Terraform init..." - (cd "$INFRA_DIR" && terraform init -input=false) + (cd "$INFRA_DIR" && terraform init -input=false -reconfigure) # Always build tfvars for ALL versions to keep Terraform state consistent. # When provisioning a subset, we use -target to limit the apply. @@ -296,7 +298,7 @@ cmd_provision() { TF_VAR_docker_host_ip="$storage_ip" \ TF_VAR_answer_files_dir="$WORK_DIR/answers" \ TF_VAR_default_answer_file="$WORK_DIR/default-answer.toml" \ - terraform apply -auto-approve -input=false -var-file="$tfvars" $tf_targets) + terraform apply -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets) # Wait for PVE instances to boot and discover IPs for node in $provision_nodes; do @@ -584,7 +586,7 @@ cmd_cleanup() { TF_VAR_docker_host_ip="${storage_ip:-127.0.0.1}" \ TF_VAR_answer_files_dir="${WORK_DIR}/answers" \ TF_VAR_default_answer_file="${WORK_DIR}/default-answer.toml" \ - terraform destroy -auto-approve -input=false -var-file="$tfvars" $tf_targets) + terraform destroy -auto-approve -input=false -state="$TF_STATE_FILE" -var-file="$tfvars" $tf_targets) # Clean up work directory when destroying all if [[ "$requested" == "all" ]]; then @@ -633,10 +635,11 @@ cmd_force_cleanup() { docker rm -f pvetest-iscsi pvetest-nfs pvetest-answer-server 2>/dev/null || true docker volume rm pvetest-iscsi-data pvetest-nfs-data 2>/dev/null || true - # Remove Terraform state so next provision starts clean. + # Remove Terraform state (both local and shared mount) so next provision starts clean. # Keep .terraform.lock.hcl (provider version lock) for reproducibility. log "Removing Terraform state..." rm -f "$INFRA_DIR/terraform.tfstate" "$INFRA_DIR/terraform.tfstate.backup" + rm -f "$TF_STATE_FILE" "${TF_STATE_FILE}.backup" rm -rf "$INFRA_DIR/.terraform" # Remove work artifacts @@ -658,20 +661,20 @@ cmd_taint() { fi log "Tainting PVE VMs for reprovisioning..." - (cd "$INFRA_DIR" && terraform init -input=false 2>/dev/null) + (cd "$INFRA_DIR" && terraform init -input=false -reconfigure 2>/dev/null) # Taint ISOs (keyed by version, e.g. "9") for v in $taint_versions; do log " Tainting ISO: PVE $v" (cd "$INFRA_DIR" && \ - terraform taint "proxmox_virtual_environment_file.auto_iso[\"$v\"]") 2>/dev/null || true + terraform taint -state="$TF_STATE_FILE" "proxmox_virtual_environment_file.auto_iso[\"$v\"]") 2>/dev/null || true done # Taint VMs (keyed by node, e.g. "9a") for node in $taint_nodes; do log " Tainting VM: $node" (cd "$INFRA_DIR" && \ - terraform taint "proxmox_virtual_environment_vm.nested_pve[\"$node\"]") 2>/dev/null || true + terraform taint -state="$TF_STATE_FILE" "proxmox_virtual_environment_vm.nested_pve[\"$node\"]") 2>/dev/null || true done log "Taint complete. Next 'provision' will recreate these VMs." From e90fb760f6c262e4d2e6862f8e398a787669ee2b Mon Sep 17 00:00:00 2001 From: Clint Branham Date: Wed, 25 Mar 2026 17:06:22 -0500 Subject: [PATCH 3/3] fix: use force-cleanup in CI for reliable teardown CI cleanup should always succeed regardless of Terraform state (cancelled runs, timeouts, partial failures). force-cleanup uses direct API calls + docker rm instead of terraform destroy. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/integration-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ae63190..d54e279 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -209,9 +209,9 @@ jobs: PVE_API_TOKEN: ${{ secrets.PVE_API_TOKEN }} PVE_TARGET_NODE: ${{ vars.PVE_TARGET_NODE }} PVE_PASSWORD: ${{ secrets.PVE_TEST_PASSWORD }} - # Terraform state is stored on the shared mount (/opt/pve-integration/work/) - # so it persists between CI jobs. - run: bash ${SCRIPTS_DIR}/run-integration.sh cleanup + # Always use force-cleanup in CI — handles corrupted state from + # cancelled runs, timeouts, or partial provisioning failures. + run: bash ${SCRIPTS_DIR}/run-integration.sh force-cleanup # ── Clean up old container images from GHCR ────────────────────────── cleanup-images: