feat: add -Force cleanup that bypasses Terraform

When Terraform state is corrupted (e.g. interrupted provision),
-Cleanup -Force bypasses Terraform and:
1. Destroys VMs via direct PVE API calls (preflight-cleanup.sh)
2. Force-removes Docker storage containers and volumes
3. Deletes Terraform state files so next provision starts clean

Usage: dev.ps1 -Cleanup -Force -DockerHost 172.16.40.113

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Clint Branham
2026-03-25 14:27:43 -05:00
parent eb0ffdfeba
commit 943a9e61dd
2 changed files with 57 additions and 1 deletions
+12 -1
View File
@@ -44,6 +44,12 @@
before applying, forcing them to be destroyed and recreated. before applying, forcing them to be destroyed and recreated.
Useful when the VMs are in a bad state (e.g. clustered, broken). Useful when the VMs are in a bad state (e.g. clustered, broken).
.PARAMETER Force
When used with -Cleanup, bypasses Terraform and destroys VMs
directly via the PVE API. Useful when Terraform state is corrupted
(e.g. after an interrupted provision). Also removes Terraform
state files so the next provision starts clean.
.PARAMETER Tests .PARAMETER Tests
Filter integration tests by area name. Comma-separated list of test Filter integration tests by area name. Comma-separated list of test
area names that match the numbered file prefixes. Examples: area names that match the numbered file prefixes. Examples:
@@ -92,6 +98,7 @@ param(
[switch] $Stop, [switch] $Stop,
[switch] $Rebuild, [switch] $Rebuild,
[switch] $Reprovision, [switch] $Reprovision,
[switch] $Force,
[string[]] $Tests, [string[]] $Tests,
@@ -258,7 +265,11 @@ if ($Integration) {
if ($Cleanup) { if ($Cleanup) {
Start-InfraContainer Start-InfraContainer
docker exec $InfraContainer bash $RunIntegration cleanup $Version if ($Force) {
docker exec $InfraContainer bash $RunIntegration force-cleanup $Version
} else {
docker exec $InfraContainer bash $RunIntegration cleanup $Version
}
if ($LASTEXITCODE -ne 0) { throw "Cleanup failed (exit code $LASTEXITCODE)" } if ($LASTEXITCODE -ne 0) { throw "Cleanup failed (exit code $LASTEXITCODE)" }
} }
@@ -535,6 +535,50 @@ cmd_cleanup() {
log "Cleanup complete." log "Cleanup complete."
} }
cmd_force_cleanup() {
local requested="${1:-all}"
local cleanup_nodes="$ALL_NODES"
if [[ "$requested" != "all" ]]; then
cleanup_nodes=""
for v in $requested; do
cleanup_nodes="$cleanup_nodes ${v}a ${v}b"
done
fi
log "Force cleanup — bypassing Terraform, using direct API calls..."
# Destroy VMs via the PVE API (works even with broken Terraform state)
for node in $cleanup_nodes; do
local vm_id
vm_id="$(pve_vmid "$node")"
local iso_name
iso_name="$(pve_iso "$node")"
log "Force cleaning $node (VMID $vm_id)..."
bash "$SCRIPT_DIR/preflight-cleanup.sh" \
"${PVE_ENDPOINT:-}" "${PVE_API_TOKEN:-}" \
"$vm_id" "${iso_name%.iso}-${node}-auto.iso" "$INFRA_DIR" \
|| true
done
# Stop Docker storage containers
if [[ "$requested" == "all" ]]; then
log "Stopping storage containers..."
docker rm -f pvetest-iscsi pvetest-nfs 2>/dev/null || true
docker volume rm pvetest-iscsi-data pvetest-nfs-data 2>/dev/null || true
fi
# Remove Terraform state so next provision starts clean
log "Removing Terraform state..."
rm -f "$INFRA_DIR/terraform.tfstate" "$INFRA_DIR/terraform.tfstate.backup"
rm -f "$INFRA_DIR/.terraform.lock.hcl"
rm -rf "$INFRA_DIR/.terraform"
# Remove work artifacts
rm -f "$CONFIG_FILE" "$WORK_DIR"/instances.tfvars.json
log "Force cleanup complete. Next provision will start from scratch."
}
cmd_taint() { cmd_taint() {
local requested="${1:-all}" local requested="${1:-all}"
local taint_nodes="$ALL_NODES" local taint_nodes="$ALL_NODES"
@@ -585,6 +629,7 @@ main() {
provision) cmd_provision "$@" ;; provision) cmd_provision "$@" ;;
test) cmd_test "$@" ;; test) cmd_test "$@" ;;
cleanup) cmd_cleanup "$@" ;; cleanup) cmd_cleanup "$@" ;;
force-cleanup) cmd_force_cleanup "$@" ;;
taint) cmd_taint "$@" ;; taint) cmd_taint "$@" ;;
all) cmd_all "$@" ;; all) cmd_all "$@" ;;
*) *)