mirror of
https://github.com/GoodOlClint/PSProxmoxVE.git
synced 2026-07-26 07:58:14 +00:00
feat: consolidate infrastructure into Terraform + terraform destroy cleanup
Major provisioning pipeline changes: Terraform: - Add kreuzwerker/docker provider to manage iSCSI and NFS storage containers alongside PVE VMs in a single Terraform config - New storage.tf with Docker container, image, and volume resources - docker_host_ip variable for PVE nodes to reach storage services Provisioning: - Replace docker-compose storage management with Terraform - Replace create-api-token.sh with wait-for-pve.sh (IP discovery + API readiness + auth verification only — no token creation) - Tests use root@pam credentials, not API tokens Cleanup: - Replace preflight-cleanup.sh loop with terraform destroy - Supports version filtering: cleanup 9 destroys only PVE 9 resources - Full cleanup also removes config.json and tfvars New commands: - taint [8|9|all]: marks VMs for recreation on next provision - dev.ps1 -Reprovision: runs taint before provision to force VM rebuild Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -39,6 +39,11 @@
|
||||
.PARAMETER Rebuild
|
||||
Rebuild container images from scratch.
|
||||
|
||||
.PARAMETER Reprovision
|
||||
When used with -Provision, taints the PVE VMs in Terraform state
|
||||
before applying, forcing them to be destroyed and recreated.
|
||||
Useful when the VMs are in a bad state (e.g. clustered, broken).
|
||||
|
||||
.PARAMETER Tests
|
||||
Filter integration tests by area name. Comma-separated list of test
|
||||
area names that match the numbered file prefixes. Examples:
|
||||
@@ -86,6 +91,7 @@ param(
|
||||
[switch] $Cleanup,
|
||||
[switch] $Stop,
|
||||
[switch] $Rebuild,
|
||||
[switch] $Reprovision,
|
||||
|
||||
[string[]] $Tests,
|
||||
|
||||
@@ -229,6 +235,10 @@ if ($Test) {
|
||||
|
||||
if ($Provision) {
|
||||
Start-InfraContainer
|
||||
if ($Reprovision) {
|
||||
docker exec $InfraContainer bash $RunIntegration taint $Version
|
||||
if ($LASTEXITCODE -ne 0) { throw "Taint failed (exit code $LASTEXITCODE)" }
|
||||
}
|
||||
docker exec $InfraContainer bash $RunIntegration provision $Version
|
||||
if ($LASTEXITCODE -ne 0) { throw "Provisioning failed (exit code $LASTEXITCODE)" }
|
||||
}
|
||||
|
||||
@@ -5,6 +5,10 @@ terraform {
|
||||
source = "bpg/proxmox"
|
||||
version = ">= 0.70.0"
|
||||
}
|
||||
docker = {
|
||||
source = "kreuzwerker/docker"
|
||||
version = ">= 3.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +18,11 @@ provider "proxmox" {
|
||||
insecure = var.proxmox_insecure
|
||||
}
|
||||
|
||||
provider "docker" {
|
||||
# Uses the Docker socket from the dev-infra container
|
||||
# (mounted at /var/run/docker.sock)
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_file" "auto_iso" {
|
||||
for_each = var.pve_instances
|
||||
content_type = "iso"
|
||||
|
||||
@@ -7,3 +7,18 @@ output "pve_test_node_name" {
|
||||
value = "pve"
|
||||
description = "Default node name inside a fresh PVE install"
|
||||
}
|
||||
|
||||
output "storage_ip" {
|
||||
description = "IP address where storage services are reachable"
|
||||
value = var.docker_host_ip
|
||||
}
|
||||
|
||||
output "storage_iscsi_iqn" {
|
||||
description = "iSCSI target IQN"
|
||||
value = var.storage_iscsi_iqn
|
||||
}
|
||||
|
||||
output "storage_nfs_export" {
|
||||
description = "NFS export path"
|
||||
value = "${var.docker_host_ip}:/srv/nfs/shared"
|
||||
}
|
||||
|
||||
@@ -180,6 +180,18 @@ cmd_provision() {
|
||||
# Terraform — remove any stale .tfvars from previous manual runs
|
||||
rm -f "$INFRA_DIR/terraform.tfvars"
|
||||
|
||||
# Discover Docker host IP before Terraform apply (needed for docker_host_ip var)
|
||||
local storage_ip
|
||||
storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}')
|
||||
if [ -z "$storage_ip" ]; then
|
||||
storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1)
|
||||
fi
|
||||
if [ -z "$storage_ip" ]; then
|
||||
ci_error "Could not determine Docker host IP for storage services"
|
||||
exit 1
|
||||
fi
|
||||
log "Docker host IP: $storage_ip"
|
||||
|
||||
log "Running Terraform init..."
|
||||
(cd "$INFRA_DIR" && terraform init -input=false)
|
||||
|
||||
@@ -228,44 +240,22 @@ cmd_provision() {
|
||||
TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \
|
||||
TF_VAR_target_node="$PVE_TARGET_NODE" \
|
||||
TF_VAR_test_vm_password="$PVE_PASSWORD" \
|
||||
TF_VAR_docker_host_ip="$storage_ip" \
|
||||
terraform apply -auto-approve -input=false -var-file="$tfvars" $tf_targets)
|
||||
|
||||
# Start iSCSI/NFS storage containers on the Docker host
|
||||
log "Starting storage containers (iSCSI + NFS)..."
|
||||
# Get the Docker host's real IP (not the container's). The storage containers
|
||||
# use host networking, so PVE nodes reach them via the host's IP.
|
||||
local storage_ip
|
||||
# Prefer the IP of the default route's interface on the Docker host.
|
||||
storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}')
|
||||
if [ -z "$storage_ip" ]; then
|
||||
# Fallback: use the local node address from Docker info (not RemoteManagers,
|
||||
# which can return addresses of remote Swarm managers).
|
||||
storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1)
|
||||
fi
|
||||
if [ -z "$storage_ip" ]; then
|
||||
ci_error "Could not determine Docker host IP for storage services"
|
||||
exit 1
|
||||
fi
|
||||
ISCSI_IQN="$STORAGE_ISCSI_IQN" \
|
||||
docker compose -f "$STORAGE_COMPOSE" up -d
|
||||
log "Storage services ready at $storage_ip (iSCSI: $STORAGE_ISCSI_IQN)"
|
||||
|
||||
# Wait for PVE instances and create API tokens (per node)
|
||||
# Wait for PVE instances to boot and discover IPs
|
||||
for node in $provision_nodes; do
|
||||
log "Waiting for $node to boot and creating API token..."
|
||||
log "Waiting for $node to boot..."
|
||||
local output
|
||||
output=$(bash "$SCRIPT_DIR/create-api-token.sh" \
|
||||
output=$(bash "$SCRIPT_DIR/wait-for-pve.sh" \
|
||||
"$PVE_ENDPOINT" "$PVE_API_TOKEN" \
|
||||
"$(pve_vmid "$node")" "$PVE_PASSWORD" 900)
|
||||
local ip token
|
||||
local ip node_name
|
||||
ip=$(echo "$output" | grep "^IP=" | cut -d= -f2)
|
||||
token=$(echo "$output" | grep "^TOKEN=" | cut -d= -f2-)
|
||||
# Node name = hostname portion of FQDN (e.g. pve9a.test.local -> pve9a)
|
||||
local node_name
|
||||
node_name="$(pve_fqdn "$node" | cut -d. -f1)"
|
||||
node_name=$(echo "$output" | grep "^NODE=" | cut -d= -f2)
|
||||
log "$node ready at $ip (node: $node_name)"
|
||||
jq -n --arg host "$ip" --arg token "$token" --arg node "$node_name" \
|
||||
'{host: $host, token: $token, node: $node}' > "$WORK_DIR/${node}.json"
|
||||
jq -n --arg host "$ip" --arg node "$node_name" \
|
||||
'{host: $host, node: $node}' > "$WORK_DIR/${node}.json"
|
||||
done
|
||||
|
||||
# Prepare test environments on provisioned PVE nodes
|
||||
@@ -354,15 +344,13 @@ cmd_test() {
|
||||
# Set env vars from config or from environment
|
||||
if [[ "$SKIP_PROVISION" == "true" ]]; then
|
||||
: "${PVETEST_HOST:?Set PVETEST_HOST when using SKIP_PROVISION}"
|
||||
: "${PVETEST_APITOKEN:?Set PVETEST_APITOKEN when using SKIP_PROVISION}"
|
||||
: "${PVETEST_PASSWORD:?Set PVETEST_PASSWORD when using SKIP_PROVISION}"
|
||||
export PVETEST_PORT="${PVETEST_PORT:-8006}"
|
||||
export PVETEST_NODE="${PVETEST_NODE:-pve}"
|
||||
export PVETEST_STORAGE="${PVETEST_STORAGE:-local}"
|
||||
export PVETEST_CLOUD_IMAGE_PATH="${PVETEST_CLOUD_IMAGE_PATH:-}"
|
||||
export PVETEST_OVA_PATH="${PVETEST_OVA_PATH:-}"
|
||||
# Secondary node and storage VM may not be available in skip mode
|
||||
export PVETEST_HOST_B="${PVETEST_HOST_B:-}"
|
||||
export PVETEST_APITOKEN_B="${PVETEST_APITOKEN_B:-}"
|
||||
export PVETEST_STORAGE_VM_IP="${PVETEST_STORAGE_VM_IP:-}"
|
||||
export PVETEST_ISCSI_IQN="${PVETEST_ISCSI_IQN:-}"
|
||||
export PVETEST_NFS_EXPORT="${PVETEST_NFS_EXPORT:-}"
|
||||
@@ -373,7 +361,6 @@ cmd_test() {
|
||||
fi
|
||||
# Primary node (a)
|
||||
export PVETEST_HOST=$(jq -r ".pve${v}.nodes.a.host" "$CONFIG_FILE")
|
||||
export PVETEST_APITOKEN=$(jq -r ".pve${v}.nodes.a.token" "$CONFIG_FILE")
|
||||
export PVETEST_PORT=8006
|
||||
export PVETEST_NODE=$(jq -r ".pve${v}.nodes.a.node" "$CONFIG_FILE")
|
||||
export PVETEST_STORAGE=local
|
||||
@@ -381,7 +368,6 @@ cmd_test() {
|
||||
export PVETEST_OVA_PATH=$(jq -r '.ova_path' "$CONFIG_FILE")
|
||||
# Secondary node (b)
|
||||
export PVETEST_HOST_B=$(jq -r ".pve${v}.nodes.b.host" "$CONFIG_FILE")
|
||||
export PVETEST_APITOKEN_B=$(jq -r ".pve${v}.nodes.b.token" "$CONFIG_FILE")
|
||||
# Storage services (Docker on runner)
|
||||
export PVETEST_STORAGE_VM_IP=$(jq -r '.storage.ip' "$CONFIG_FILE")
|
||||
export PVETEST_ISCSI_IQN=$(jq -r '.storage.iscsi_iqn' "$CONFIG_FILE")
|
||||
@@ -392,12 +378,12 @@ cmd_test() {
|
||||
export PVETEST_PVE_VERSION="$v"
|
||||
export PVETEST_PASSWORD="${PVETEST_PASSWORD:-${PVE_PASSWORD:-}}"
|
||||
|
||||
# Verify API reachable (node A)
|
||||
# Verify API reachable (node A) using ticket auth
|
||||
log "Verifying PVE $v node A API at $PVETEST_HOST:$PVETEST_PORT..."
|
||||
if ! curl -sk --connect-timeout 10 \
|
||||
-H "Authorization: PVEAPIToken=${PVETEST_APITOKEN}" \
|
||||
"https://${PVETEST_HOST}:${PVETEST_PORT}/api2/json/nodes" | grep -q '"node"'; then
|
||||
ci_error "Cannot reach PVE $v node A API at ${PVETEST_HOST}:${PVETEST_PORT}"
|
||||
-d "username=root@pam&password=${PVETEST_PASSWORD}" \
|
||||
"https://${PVETEST_HOST}:${PVETEST_PORT}/api2/json/access/ticket" | grep -q '"ticket"'; then
|
||||
ci_error "Cannot authenticate to PVE $v node A at ${PVETEST_HOST}:${PVETEST_PORT}"
|
||||
overall_exit=3
|
||||
continue
|
||||
fi
|
||||
@@ -406,9 +392,9 @@ cmd_test() {
|
||||
if [[ -n "${PVETEST_HOST_B:-}" ]]; then
|
||||
log "Verifying PVE $v node B API at $PVETEST_HOST_B:$PVETEST_PORT..."
|
||||
if ! curl -sk --connect-timeout 10 \
|
||||
-H "Authorization: PVEAPIToken=${PVETEST_APITOKEN_B}" \
|
||||
"https://${PVETEST_HOST_B}:${PVETEST_PORT}/api2/json/nodes" | grep -q '"node"'; then
|
||||
ci_error "Cannot reach PVE $v node B API at ${PVETEST_HOST_B}:${PVETEST_PORT}"
|
||||
-d "username=root@pam&password=${PVETEST_PASSWORD}" \
|
||||
"https://${PVETEST_HOST_B}:${PVETEST_PORT}/api2/json/access/ticket" | grep -q '"ticket"'; then
|
||||
ci_error "Cannot authenticate to PVE $v node B at ${PVETEST_HOST_B}:${PVETEST_PORT}"
|
||||
overall_exit=3
|
||||
continue
|
||||
fi
|
||||
@@ -474,35 +460,99 @@ cmd_test() {
|
||||
|
||||
cmd_cleanup() {
|
||||
local requested="${1:-all}"
|
||||
local cleanup_nodes="$ALL_NODES"
|
||||
log "Starting cleanup..."
|
||||
|
||||
require_env PVE_ENDPOINT
|
||||
require_env PVE_API_TOKEN
|
||||
require_env PVE_TARGET_NODE
|
||||
|
||||
# Discover Docker host IP for the docker_host_ip variable
|
||||
local storage_ip
|
||||
storage_ip=$(docker run --rm --net=host alpine ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}')
|
||||
if [ -z "$storage_ip" ]; then
|
||||
storage_ip=$(docker info --format '{{.Swarm.NodeAddr}}' 2>/dev/null | cut -d: -f1)
|
||||
fi
|
||||
|
||||
# Build tfvars for all versions (Terraform needs the full variable map)
|
||||
local tfvars="$WORK_DIR/instances.tfvars.json"
|
||||
if [[ ! -f "$tfvars" ]]; then
|
||||
# Generate minimal tfvars if none exist (cleanup without prior provision)
|
||||
local instances='{}'
|
||||
for node in $ALL_NODES; do
|
||||
local vm_id vm_name
|
||||
vm_id="$(pve_vmid "$node")"
|
||||
vm_name="$(pve_vmname "$node")"
|
||||
instances="$(jq \
|
||||
--arg key "$node" \
|
||||
--arg vm_name "$vm_name" \
|
||||
--argjson vm_id "$vm_id" \
|
||||
--arg iso_local_path "/dev/null" \
|
||||
'. + {($key): {iso_local_path: $iso_local_path, vm_id: $vm_id, vm_name: $vm_name}}' \
|
||||
<<<"$instances")"
|
||||
done
|
||||
mkdir -p "$WORK_DIR"
|
||||
jq -n --argjson pve_instances "$instances" \
|
||||
'{pve_instances: $pve_instances}' > "$tfvars"
|
||||
fi
|
||||
|
||||
(cd "$INFRA_DIR" && terraform init -input=false 2>/dev/null)
|
||||
|
||||
# Build -target flags when destroying a subset
|
||||
local tf_targets=""
|
||||
if [[ "$requested" != "all" ]]; then
|
||||
cleanup_nodes=""
|
||||
local cleanup_nodes=""
|
||||
for v in $requested; do
|
||||
cleanup_nodes="$cleanup_nodes ${v}a ${v}b"
|
||||
done
|
||||
for node in $cleanup_nodes; do
|
||||
tf_targets="$tf_targets -target=proxmox_virtual_environment_vm.nested_pve[\"$node\"]"
|
||||
tf_targets="$tf_targets -target=proxmox_virtual_environment_file.auto_iso[\"$node\"]"
|
||||
done
|
||||
log "Destroying PVE $requested nodes only..."
|
||||
else
|
||||
log "Destroying all resources..."
|
||||
fi
|
||||
log "Starting cleanup..."
|
||||
|
||||
# Clean up PVE nodes
|
||||
for node in $cleanup_nodes; do
|
||||
local iso_name
|
||||
iso_name="$(pve_iso "$node")"
|
||||
log "Cleaning up $node (VMID $(pve_vmid "$node"))..."
|
||||
bash "$SCRIPT_DIR/preflight-cleanup.sh" \
|
||||
"${PVE_ENDPOINT:-}" "${PVE_API_TOKEN:-}" \
|
||||
"$(pve_vmid "$node")" "${iso_name%.iso}-${node}-auto.iso" "$INFRA_DIR" \
|
||||
|| true
|
||||
done
|
||||
(cd "$INFRA_DIR" && \
|
||||
TF_VAR_proxmox_endpoint="$PVE_ENDPOINT" \
|
||||
TF_VAR_proxmox_api_token="$PVE_API_TOKEN" \
|
||||
TF_VAR_target_node="$PVE_TARGET_NODE" \
|
||||
TF_VAR_test_vm_password="${PVE_PASSWORD:-placeholder}" \
|
||||
TF_VAR_docker_host_ip="${storage_ip:-127.0.0.1}" \
|
||||
terraform destroy -auto-approve -input=false -var-file="$tfvars" $tf_targets) || true
|
||||
|
||||
# Stop storage containers only when cleaning all versions
|
||||
# Clean up work directory when destroying all
|
||||
if [[ "$requested" == "all" ]]; then
|
||||
log "Stopping storage containers..."
|
||||
docker compose -f "$STORAGE_COMPOSE" down -v 2>/dev/null || true
|
||||
rm -f "$CONFIG_FILE" "$WORK_DIR"/instances.tfvars.json
|
||||
fi
|
||||
|
||||
log "Cleanup complete."
|
||||
}
|
||||
|
||||
cmd_taint() {
|
||||
local requested="${1:-all}"
|
||||
local taint_nodes="$ALL_NODES"
|
||||
if [[ "$requested" != "all" ]]; then
|
||||
taint_nodes=""
|
||||
for v in $requested; do
|
||||
taint_nodes="$taint_nodes ${v}a ${v}b"
|
||||
done
|
||||
fi
|
||||
|
||||
log "Tainting PVE VMs for reprovisioning..."
|
||||
(cd "$INFRA_DIR" && terraform init -input=false 2>/dev/null)
|
||||
|
||||
for node in $taint_nodes; do
|
||||
log " Tainting VM: $node"
|
||||
(cd "$INFRA_DIR" && \
|
||||
terraform taint "proxmox_virtual_environment_vm.nested_pve[\"$node\"]") 2>/dev/null || true
|
||||
(cd "$INFRA_DIR" && \
|
||||
terraform taint "proxmox_virtual_environment_file.auto_iso[\"$node\"]") 2>/dev/null || true
|
||||
done
|
||||
|
||||
log "Taint complete. Next 'provision' will recreate these VMs."
|
||||
}
|
||||
|
||||
cmd_all() {
|
||||
local test_versions="${1:-all}"
|
||||
local test_exit=0
|
||||
@@ -529,14 +579,16 @@ main() {
|
||||
provision) cmd_provision "$@" ;;
|
||||
test) cmd_test "$@" ;;
|
||||
cleanup) cmd_cleanup "$@" ;;
|
||||
taint) cmd_taint "$@" ;;
|
||||
all) cmd_all "$@" ;;
|
||||
*)
|
||||
echo "Usage: $(basename "$0") {provision|test|cleanup|all} [8|9|all] [test-filter]"
|
||||
echo "Usage: $(basename "$0") {provision|test|cleanup|taint|all} [8|9|all] [test-filter]"
|
||||
echo ""
|
||||
echo "Subcommands:"
|
||||
echo " provision [8|9|all] Provision nested PVE VMs + start storage containers"
|
||||
echo " provision [8|9|all] Provision nested PVE VMs + storage containers"
|
||||
echo " test [8|9|all] [filter] Run integration tests (default: all versions, no filter)"
|
||||
echo " cleanup [8|9|all] Destroy provisioned VMs (default: all)"
|
||||
echo " cleanup [8|9|all] Destroy resources via terraform destroy (default: all)"
|
||||
echo " taint [8|9|all] Mark VMs for recreation on next provision"
|
||||
echo " all [8|9|all] Full lifecycle: provision → test → cleanup"
|
||||
echo ""
|
||||
echo "Test filter: comma-separated area names matching test filenames."
|
||||
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
# Wait for a fresh nested PVE instance to boot, discover its IP via the QEMU guest agent,
|
||||
# then wait for the PVE API to become responsive.
|
||||
#
|
||||
# Usage: wait-for-pve.sh <parent-pve-endpoint> <parent-api-token> <vm-id> <root-password> [max-wait-seconds]
|
||||
# Outputs:
|
||||
# IP=<discovered-ip>
|
||||
# NODE=<pve-hostname>
|
||||
set -euo pipefail
|
||||
|
||||
PARENT_ENDPOINT="${1%/}"
|
||||
PARENT_TOKEN="$2"
|
||||
VM_ID="$3"
|
||||
ROOT_PASSWORD="$4"
|
||||
MAX_WAIT="${5:-600}"
|
||||
INTERVAL=10
|
||||
PARENT_API="${PARENT_ENDPOINT}/api2/json"
|
||||
NODES_JSON=$(curl -sk -H "Authorization: PVEAPIToken=${PARENT_TOKEN}" \
|
||||
"${PARENT_API}/nodes")
|
||||
PARENT_NODE=$(echo "$NODES_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])")
|
||||
|
||||
# --- Phase 1: Discover IP via QEMU guest agent ---
|
||||
echo "Waiting for guest agent on VM ${VM_ID} (node: ${PARENT_NODE})..."
|
||||
VM_IP=""
|
||||
elapsed=0
|
||||
while [ $elapsed -lt $MAX_WAIT ]; do
|
||||
AGENT_RESPONSE=$(curl -sk \
|
||||
-H "Authorization: PVEAPIToken=${PARENT_TOKEN}" \
|
||||
"${PARENT_API}/nodes/${PARENT_NODE}/qemu/${VM_ID}/agent/network-get-interfaces" 2>/dev/null || true)
|
||||
|
||||
VM_IP=$(echo "$AGENT_RESPONSE" | python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
data = json.load(sys.stdin).get('data', {}).get('result', [])
|
||||
for iface in data:
|
||||
if iface.get('name') == 'lo':
|
||||
continue
|
||||
for addr in iface.get('ip-addresses', []):
|
||||
if addr.get('ip-address-type') == 'ipv4' and not addr['ip-address'].startswith('127.'):
|
||||
print(addr['ip-address'])
|
||||
sys.exit(0)
|
||||
except:
|
||||
pass
|
||||
" 2>/dev/null || true)
|
||||
|
||||
if [ -n "$VM_IP" ]; then
|
||||
echo "Discovered VM IP: $VM_IP (after ${elapsed}s)"
|
||||
break
|
||||
fi
|
||||
echo " Guest agent not ready yet (${elapsed}s elapsed)..."
|
||||
sleep $INTERVAL
|
||||
elapsed=$((elapsed + INTERVAL))
|
||||
done
|
||||
|
||||
if [ -z "$VM_IP" ]; then
|
||||
echo "ERROR: Could not discover VM IP via guest agent after ${MAX_WAIT}s" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Phase 2: Wait for PVE API on the nested instance ---
|
||||
NESTED_API="https://${VM_IP}:8006/api2/json"
|
||||
echo "Waiting for nested PVE API at ${NESTED_API}..."
|
||||
while [ $elapsed -lt $MAX_WAIT ]; do
|
||||
if curl -sk --connect-timeout 5 "${NESTED_API}/access/domains" 2>/dev/null | grep -q '"realm"'; then
|
||||
echo "Nested PVE API is responsive after ${elapsed}s"
|
||||
break
|
||||
fi
|
||||
echo " API not ready yet (${elapsed}s elapsed)..."
|
||||
sleep $INTERVAL
|
||||
elapsed=$((elapsed + INTERVAL))
|
||||
done
|
||||
|
||||
if [ $elapsed -ge $MAX_WAIT ]; then
|
||||
echo "ERROR: Nested PVE API not responsive after ${MAX_WAIT}s" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Phase 3: Verify authentication works ---
|
||||
echo "Verifying root@pam authentication..."
|
||||
AUTH_RESPONSE=$(curl -sk -d "username=root@pam&password=${ROOT_PASSWORD}" \
|
||||
"${NESTED_API}/access/ticket" 2>/dev/null || true)
|
||||
TICKET=$(echo "$AUTH_RESPONSE" | python3 -c "import json,sys; print(json.load(sys.stdin)['data']['ticket'])" 2>/dev/null || true)
|
||||
|
||||
if [ -z "$TICKET" ]; then
|
||||
echo "ERROR: root@pam authentication failed on ${VM_IP}. Response: $AUTH_RESPONSE" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Authentication verified."
|
||||
|
||||
# Discover the nested node's hostname
|
||||
NODE_NAME=$(curl -sk -b "PVEAuthCookie=${TICKET}" \
|
||||
"${NESTED_API}/nodes" 2>/dev/null \
|
||||
| python3 -c "import json,sys; print(json.load(sys.stdin)['data'][0]['node'])" 2>/dev/null || echo "unknown")
|
||||
|
||||
echo "IP=${VM_IP}"
|
||||
echo "NODE=${NODE_NAME}"
|
||||
@@ -0,0 +1,84 @@
|
||||
# ── Docker images & volumes ──────────────────────────────────────────
|
||||
|
||||
resource "docker_image" "ubuntu" {
|
||||
name = "ubuntu:24.04"
|
||||
}
|
||||
|
||||
resource "docker_volume" "iscsi_data" {
|
||||
name = "pvetest-iscsi-data"
|
||||
}
|
||||
|
||||
resource "docker_volume" "nfs_data" {
|
||||
name = "pvetest-nfs-data"
|
||||
}
|
||||
|
||||
# ── iSCSI target container ──────────────────────────────────────────
|
||||
|
||||
resource "docker_container" "iscsi_target" {
|
||||
name = "pvetest-iscsi"
|
||||
image = docker_image.ubuntu.image_id
|
||||
privileged = true
|
||||
restart = "unless-stopped"
|
||||
|
||||
network_mode = "host"
|
||||
|
||||
volumes {
|
||||
volume_name = docker_volume.iscsi_data.name
|
||||
container_path = "/srv/iscsi"
|
||||
}
|
||||
|
||||
env = [
|
||||
"ISCSI_IQN=${var.storage_iscsi_iqn}",
|
||||
"ISCSI_LUN_SIZE=${var.storage_iscsi_lun_size}",
|
||||
]
|
||||
|
||||
entrypoint = ["/bin/bash", "-c"]
|
||||
command = [<<-EOT
|
||||
set -e
|
||||
apt-get update -qq && apt-get install -y -qq tgt >/dev/null 2>&1
|
||||
mkdir -p /srv/iscsi
|
||||
if [ ! -f /srv/iscsi/lun0.img ]; then
|
||||
truncate -s $${ISCSI_LUN_SIZE} /srv/iscsi/lun0.img
|
||||
fi
|
||||
tgtd --foreground &
|
||||
sleep 2
|
||||
if ! tgtadm --lld iscsi --op show --mode target | grep -q "Target 1: $${ISCSI_IQN}"; then
|
||||
tgtadm --lld iscsi --op new --mode target --tid 1 -T $${ISCSI_IQN}
|
||||
fi
|
||||
if ! tgtadm --lld iscsi --op show --mode logicalunit --tid 1 2>/dev/null | grep -qE "LUN:[[:space:]]*1($$|[^0-9])"; then
|
||||
tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 --backing-store /srv/iscsi/lun0.img
|
||||
fi
|
||||
if ! tgtadm --lld iscsi --op show --mode target --tid 1 2>/dev/null | grep -q "Initiator-address: ALL"; then
|
||||
tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL
|
||||
fi
|
||||
echo "iSCSI target ready: $${ISCSI_IQN} (port 3260)"
|
||||
wait
|
||||
EOT
|
||||
]
|
||||
}
|
||||
|
||||
# ── NFS server container ────────────────────────────────────────────
|
||||
|
||||
resource "docker_container" "nfs_server" {
|
||||
name = "pvetest-nfs"
|
||||
image = "erichough/nfs-server:2.2.1"
|
||||
privileged = true
|
||||
restart = "unless-stopped"
|
||||
|
||||
network_mode = "host"
|
||||
|
||||
volumes {
|
||||
volume_name = docker_volume.nfs_data.name
|
||||
container_path = "/srv/nfs/shared"
|
||||
}
|
||||
|
||||
volumes {
|
||||
host_path = "/lib/modules"
|
||||
container_path = "/lib/modules"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
env = [
|
||||
"NFS_EXPORT_0=/srv/nfs/shared *(rw,sync,no_subtree_check,no_root_squash)",
|
||||
]
|
||||
}
|
||||
@@ -71,3 +71,20 @@ variable "test_vm_password" {
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "storage_iscsi_iqn" {
|
||||
description = "iSCSI target IQN for the test storage"
|
||||
type = string
|
||||
default = "iqn.2024-01.local.test:storage"
|
||||
}
|
||||
|
||||
variable "storage_iscsi_lun_size" {
|
||||
description = "Size of the iSCSI LUN backing file"
|
||||
type = string
|
||||
default = "10G"
|
||||
}
|
||||
|
||||
variable "docker_host_ip" {
|
||||
description = "IP of the Docker host, used by PVE nodes to reach storage containers"
|
||||
type = string
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user