From 943a9e61dd7e9017960ef8a1cd16d691a2e7d448 Mon Sep 17 00:00:00 2001
From: Clint Branham <goodolclint@gmail.com>
Date: Wed, 25 Mar 2026 14:27:43 -0500
Subject: [PATCH] feat: add -Force cleanup that bypasses Terraform

When Terraform state is corrupted (e.g. interrupted provision),
-Cleanup -Force bypasses Terraform and:
1. Destroys VMs via direct PVE API calls (preflight-cleanup.sh)
2. Force-removes Docker storage containers and volumes
3. Deletes Terraform state files so next provision starts clean

Usage: dev.ps1 -Cleanup -Force -DockerHost 172.16.40.113

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 tests/dev.ps1                                 | 13 +++++-
 .../infrastructure/scripts/run-integration.sh | 45 +++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/tests/dev.ps1 b/tests/dev.ps1
index a93efbc..19cfa32 100644
--- a/tests/dev.ps1
+++ b/tests/dev.ps1
@@ -44,6 +44,12 @@
     before applying, forcing them to be destroyed and recreated.
     Useful when the VMs are in a bad state (e.g. clustered, broken).
 
+.PARAMETER Force
+    When used with -Cleanup, bypasses Terraform and destroys VMs
+    directly via the PVE API. Useful when Terraform state is corrupted
+    (e.g. after an interrupted provision). Also removes Terraform
+    state files so the next provision starts clean.
+
 .PARAMETER Tests
     Filter integration tests by area name. Comma-separated list of test
     area names that match the numbered file prefixes. Examples:
@@ -92,6 +98,7 @@ param(
     [switch] $Stop,
     [switch] $Rebuild,
     [switch] $Reprovision,
+    [switch] $Force,
 
     [string[]] $Tests,
 
@@ -258,7 +265,11 @@ if ($Integration) {
 
 if ($Cleanup) {
     Start-InfraContainer
-    docker exec $InfraContainer bash $RunIntegration cleanup $Version
+    if ($Force) {
+        docker exec $InfraContainer bash $RunIntegration force-cleanup $Version
+    } else {
+        docker exec $InfraContainer bash $RunIntegration cleanup $Version
+    }
     if ($LASTEXITCODE -ne 0) { throw "Cleanup failed (exit code $LASTEXITCODE)" }
 }
 
diff --git a/tests/infrastructure/scripts/run-integration.sh b/tests/infrastructure/scripts/run-integration.sh
index 36af906..a8c2afd 100755
--- a/tests/infrastructure/scripts/run-integration.sh
+++ b/tests/infrastructure/scripts/run-integration.sh
@@ -535,6 +535,50 @@ cmd_cleanup() {
     log "Cleanup complete."
 }
 
+cmd_force_cleanup() {
+    local requested="${1:-all}"
+    local cleanup_nodes="$ALL_NODES"
+    if [[ "$requested" != "all" ]]; then
+        cleanup_nodes=""
+        for v in $requested; do
+            cleanup_nodes="$cleanup_nodes ${v}a ${v}b"
+        done
+    fi
+
+    log "Force cleanup — bypassing Terraform, using direct API calls..."
+
+    # Destroy VMs via the PVE API (works even with broken Terraform state)
+    for node in $cleanup_nodes; do
+        local vm_id
+        vm_id="$(pve_vmid "$node")"
+        local iso_name
+        iso_name="$(pve_iso "$node")"
+        log "Force cleaning $node (VMID $vm_id)..."
+        bash "$SCRIPT_DIR/preflight-cleanup.sh" \
+            "${PVE_ENDPOINT:-}" "${PVE_API_TOKEN:-}" \
+            "$vm_id" "${iso_name%.iso}-${node}-auto.iso" "$INFRA_DIR" \
+            || true
+    done
+
+    # Stop Docker storage containers
+    if [[ "$requested" == "all" ]]; then
+        log "Stopping storage containers..."
+        docker rm -f pvetest-iscsi pvetest-nfs 2>/dev/null || true
+        docker volume rm pvetest-iscsi-data pvetest-nfs-data 2>/dev/null || true
+    fi
+
+    # Remove Terraform state so next provision starts clean
+    log "Removing Terraform state..."
+    rm -f "$INFRA_DIR/terraform.tfstate" "$INFRA_DIR/terraform.tfstate.backup"
+    rm -f "$INFRA_DIR/.terraform.lock.hcl"
+    rm -rf "$INFRA_DIR/.terraform"
+
+    # Remove work artifacts
+    rm -f "$CONFIG_FILE" "$WORK_DIR"/instances.tfvars.json
+
+    log "Force cleanup complete. Next provision will start from scratch."
+}
+
 cmd_taint() {
     local requested="${1:-all}"
     local taint_nodes="$ALL_NODES"
@@ -585,6 +629,7 @@ main() {
         provision)    cmd_provision "$@" ;;
         test)         cmd_test "$@" ;;
         cleanup)      cmd_cleanup "$@" ;;
+        force-cleanup) cmd_force_cleanup "$@" ;;
         taint)        cmd_taint "$@" ;;
         all)          cmd_all "$@" ;;
         *)