diff --git a/tests/dev.ps1 b/tests/dev.ps1 index a93efbc..19cfa32 100644 --- a/tests/dev.ps1 +++ b/tests/dev.ps1 @@ -44,6 +44,12 @@ before applying, forcing them to be destroyed and recreated. Useful when the VMs are in a bad state (e.g. clustered, broken). +.PARAMETER Force + When used with -Cleanup, bypasses Terraform and destroys VMs + directly via the PVE API. Useful when Terraform state is corrupted + (e.g. after an interrupted provision). Also removes Terraform + state files so the next provision starts clean. + .PARAMETER Tests Filter integration tests by area name. Comma-separated list of test area names that match the numbered file prefixes. Examples: @@ -92,6 +98,7 @@ param( [switch] $Stop, [switch] $Rebuild, [switch] $Reprovision, + [switch] $Force, [string[]] $Tests, @@ -258,7 +265,11 @@ if ($Integration) { if ($Cleanup) { Start-InfraContainer - docker exec $InfraContainer bash $RunIntegration cleanup $Version + if ($Force) { + docker exec $InfraContainer bash $RunIntegration force-cleanup $Version + } else { + docker exec $InfraContainer bash $RunIntegration cleanup $Version + } if ($LASTEXITCODE -ne 0) { throw "Cleanup failed (exit code $LASTEXITCODE)" } } diff --git a/tests/infrastructure/scripts/run-integration.sh b/tests/infrastructure/scripts/run-integration.sh index 36af906..a8c2afd 100755 --- a/tests/infrastructure/scripts/run-integration.sh +++ b/tests/infrastructure/scripts/run-integration.sh @@ -535,6 +535,50 @@ cmd_cleanup() { log "Cleanup complete." } +cmd_force_cleanup() { + local requested="${1:-all}" + local cleanup_nodes="$ALL_NODES" + if [[ "$requested" != "all" ]]; then + cleanup_nodes="" + for v in $requested; do + cleanup_nodes="$cleanup_nodes ${v}a ${v}b" + done + fi + + log "Force cleanup — bypassing Terraform, using direct API calls..." + + # Destroy VMs via the PVE API (works even with broken Terraform state) + for node in $cleanup_nodes; do + local vm_id + vm_id="$(pve_vmid "$node")" + local iso_name + iso_name="$(pve_iso "$node")" + log "Force cleaning $node (VMID $vm_id)..." + bash "$SCRIPT_DIR/preflight-cleanup.sh" \ + "${PVE_ENDPOINT:-}" "${PVE_API_TOKEN:-}" \ + "$vm_id" "${iso_name%.iso}-${node}-auto.iso" "$INFRA_DIR" \ + || true + done + + # Stop Docker storage containers + if [[ "$requested" == "all" ]]; then + log "Stopping storage containers..." + docker rm -f pvetest-iscsi pvetest-nfs 2>/dev/null || true + docker volume rm pvetest-iscsi-data pvetest-nfs-data 2>/dev/null || true + fi + + # Remove Terraform state so next provision starts clean + log "Removing Terraform state..." + rm -f "$INFRA_DIR/terraform.tfstate" "$INFRA_DIR/terraform.tfstate.backup" + rm -f "$INFRA_DIR/.terraform.lock.hcl" + rm -rf "$INFRA_DIR/.terraform" + + # Remove work artifacts + rm -f "$CONFIG_FILE" "$WORK_DIR"/instances.tfvars.json + + log "Force cleanup complete. Next provision will start from scratch." +} + cmd_taint() { local requested="${1:-all}" local taint_nodes="$ALL_NODES" @@ -585,6 +629,7 @@ main() { provision) cmd_provision "$@" ;; test) cmd_test "$@" ;; cleanup) cmd_cleanup "$@" ;; + force-cleanup) cmd_force_cleanup "$@" ;; taint) cmd_taint "$@" ;; all) cmd_all "$@" ;; *)