spinifex/tests/e2e/run-tofu-examples-e2e.sh

#!/bin/bash
# run-tofu-examples-e2e.sh — smoke-test the public terraform workbooks.
#
# Invoked by the nightly matrix (cell #17) after bootstrap-install.sh has
# brought up a single-node cluster. For each workbook, the driver does a
# clean tofu init/apply, runs one smoke assertion, then tofu destroy.
#
# All assertions are minimal by design (one check per workbook) so the
# maintenance cost of keeping them in sync with workbook edits stays low.
#
# Expects the workbooks to be available at $WORKBOOK_DIR (defaults to the
# tarball-included ./workbooks/ directory beside this script).

set -u
cd "$(dirname "$0")"
SCRIPT_DIR="$(pwd)"

export AWS_PROFILE=spinifex
WORKBOOK_DIR="${WORKBOOK_DIR:-${SCRIPT_DIR}/workbooks}"
OPENTOFU_VERSION="${OPENTOFU_VERSION:-1.11.5}"

# awsgw and predastore are bound to the WAN IP (bootstrap-install passes --bind
# ${PRIMARY_WAN}), not loopback. Workbooks need the WAN IP for both the tofu
# provider endpoints and any CLI assertions that talk to S3 (:8443).
WAN_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}')
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR)

# A guest's public address, reached from where a customer would reach it. Empty
# means this host is already outside the cluster's own network, which is true of
# every hypervisor runner. Set it to host:port for a SOCKS5 proxy whose egress is
# off-network when the host is not: a cloud may refuse a public address from inside
# its own network, or admit only listed sources, and either fails a healthy cluster
# from here however well the datapath works.
PUBLIC_PROXY="${E2E_PUBLIC_PROXY:-}"

CURRENT_WORKBOOK=""

log() { echo "[$(date +%H:%M:%S)] $*"; }

# curl, from outside the cluster's network when PUBLIC_PROXY says it has to be.
# socks5h, not socks5: the proxy resolves, so a name only the far side knows
# still works.
curl_public() {
    if [ -n "$PUBLIC_PROXY" ]; then
        curl --proxy "socks5h://${PUBLIC_PROXY}" "$@"
        return
    fi
    curl "$@"
}

# PUBLIC_PROXY is for public addresses only, so this is deliberately not in
# SSH_OPTS itself: a hop added to every ssh would also divert the bastion's
# inner ssh to a private address, which no proxy can reach.
ssh_public() {
    if [ -n "$PUBLIC_PROXY" ]; then
        ssh -o "ProxyCommand=nc -X 5 -x ${PUBLIC_PROXY} %h %p" "$@"
        return
    fi
    ssh "$@"
}

# Fail here rather than letting every assertion time out in turn: a proxy that
# is named and does not work is indistinguishable, from the far end, from a
# cluster that is not serving.
check_public_proxy() {
    [ -n "$PUBLIC_PROXY" ] || return 0
    command -v nc >/dev/null 2>&1 || {
        sudo apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || {
            log "public-address probes need nc (netcat-openbsd) for ssh over SOCKS"
            return 1
        }
    }
    # Something off-network that is not ours, so the answer is about the proxy
    # and not about the cluster.
    if ! curl_public -sf -o /dev/null --max-time 20 https://api.github.com/; then
        log "public-address probes go through SOCKS ${PUBLIC_PROXY}, which cannot reach the internet"
        return 1
    fi
    log "public-address probes go through SOCKS ${PUBLIC_PROXY} (outside this host's network)"
}

install_tofu() {
    command -v tofu >/dev/null 2>&1 && return 0
    local arch
    case "$(uname -m)" in
        x86_64)  arch=amd64 ;;
        aarch64) arch=arm64 ;;
        *) log "unsupported arch: $(uname -m)"; return 1 ;;
    esac
    local url="https://github.com/opentofu/opentofu/releases/download/v${OPENTOFU_VERSION}/tofu_${OPENTOFU_VERSION}_linux_${arch}.tar.gz"
    log "Installing OpenTofu ${OPENTOFU_VERSION} (${arch})"
    local tmp
    tmp=$(mktemp -d)
    curl -fsSL "$url" | tar -xz -C "$tmp" tofu
    sudo install -m 0755 "$tmp/tofu" /usr/local/bin/tofu
    rm -rf "$tmp"
}

# dump_ovn_state captures OVN NB port_group/ACL state and the SB Address_Set
# rows ovn-northd derives from port-group membership. Called inside run_workbook
# on assertion failure (before destroy, while VMs still exist) and again from
# the EXIT trap (after destroy, mostly for parity).
#
# SG-to-SG ACL matches like `ip4.src == $sg_<id>_ip4` resolve against SB
# Address_Sets that ovn-northd auto-derives from port-group LSP addresses; if
# this dump shows port groups with members but no matching `<pg>_ip4` address
# set (or an empty one), that pinpoints the SG enforcement break.
dump_ovn_state() {
    local label="${1:-ovn-state}"
    log "--- ${label}: ovn-nbctl ls-list ---"
    sudo ovn-nbctl --no-leader-only ls-list 2>&1 | head -40 || true
    log "--- ${label}: ovn-nbctl lr-list ---"
    sudo ovn-nbctl --no-leader-only lr-list 2>&1 | head -40 || true
    log "--- ${label}: port groups (name, ports, ACL count) ---"
    sudo ovn-nbctl --no-leader-only --bare --columns=name,ports list port_group 2>&1 | head -200 || true
    log "--- ${label}: logical_switch_port name/addresses/port_security ---"
    sudo ovn-nbctl --no-leader-only --bare --columns=name,addresses,port_security \
        list logical_switch_port 2>&1 | head -200 || true
    log "--- ${label}: NB ACLs (priority, direction, match, action) ---"
    sudo ovn-nbctl --no-leader-only --bare \
        --columns=priority,direction,match,action,name list acl 2>&1 | head -200 || true
    log "--- ${label}: SB address_sets (auto-derived from port groups) ---"
    sudo ovn-sbctl --no-leader-only --bare --columns=name,addresses list address_set 2>&1 | head -200 || true
    log "--- ${label}: SB port bindings (logical_port, mac, chassis, up) ---"
    sudo ovn-sbctl --no-leader-only --bare \
        --columns=logical_port,mac,chassis,up list port_binding 2>&1 | head -80 || true
}

cleanup() {
    EXIT_CODE=$?
    if [ "$EXIT_CODE" -ne 0 ]; then
        log "=== FAIL (workbook=${CURRENT_WORKBOOK:-<none>}) — dumping diagnostics ==="
        log "--- spx get vms ---"
        sudo -u spinifex-daemon spx get vms 2>&1 | head -80 || \
            spx get vms 2>&1 | head -80 || true
        log "--- aws ec2 describe-instances ---"
        aws ec2 describe-instances \
            --query 'Reservations[].Instances[].[InstanceId,State.Name,StateReason.Message,PrivateIpAddress,PublicIpAddress,Tags[?Key==`Name`].Value|[0]]' \
            --output table 2>&1 | head -60 || true
        log "--- spinifex service errors (last 15 min) ---"
        for svc in spinifex-daemon spinifex-awsgw spinifex-vpcd spinifex-viperblock spinifex-predastore; do
            sudo journalctl -u "$svc" --since '15 min ago' --no-pager 2>/dev/null | \
                grep -iE 'panic|fatal|level=error|ERROR' | tail -30 | \
                sed "s|^|    [${svc}] |" || true
        done
        log "--- service journals (last 200 lines each) ---"
        for svc in spinifex-nats spinifex-predastore spinifex-viperblock \
                   spinifex-daemon spinifex-awsgw spinifex-vpcd; do
            echo "=== ${svc} ==="
            sudo journalctl -u "${svc}" --no-pager -n 200 2>/dev/null || true
        done
    fi
    exit "$EXIT_CODE"
}
trap cleanup EXIT

# --- Per-workbook assertions ---

# Wait up to 150s for instance SSH readiness.
wait_for_ssh() {
    local key="$1" host="$2"
    for _ in $(seq 1 30); do
        if ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" true 2>/dev/null; then
            return 0
        fi
        sleep 5
    done
    return 1
}

# Wait up to $2 seconds (default 150) for a 200 response from $1.
wait_for_http_200() {
    local url="$1" budget="${2:-150}"
    local attempts=$((budget / 5))
    for _ in $(seq 1 "$attempts"); do
        local status
        status=$(curl_public -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null || echo "000")
        if [ "$status" = "200" ]; then
            return 0
        fi
        sleep 5
    done
    return 1
}

# Wait up to $2 seconds for the target group $1 to report at least one healthy target.
wait_for_alb_healthy() {
    local tg_arn="$1" budget="${2:-300}"
    local attempts=$((budget / 10))
    for _ in $(seq 1 "$attempts"); do
        local healthy
        healthy=$(aws elbv2 describe-target-health --target-group-arn "$tg_arn" \
            --query 'length(TargetHealthDescriptions[?TargetHealth.State==`healthy`])' \
            --output text 2>/dev/null || echo "0")
        if [ "$healthy" -gt 0 ] 2>/dev/null; then
            return 0
        fi
        sleep 10
    done
    return 1
}

assert_bastion_private_subnet() {
    local bastion private key
    bastion=$(tofu output -raw bastion_public_ip)
    private=$(tofu output -raw private_instance_ip)
    key="$(pwd)/bastion-demo.pem"
    chmod 600 "$key"

    wait_for_ssh "$key" "$bastion" || {
        log "  bastion SSH not ready"
        return 1
    }
    # sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem.
    # Wait up to 180s for user_data to drop the key before hopping.
    if ! ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" \
        'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1'; then
        log "  bastion: ~/.ssh/bastion-demo.pem never appeared (cloud-init stalled?)"
        return 1
    fi
    local inner_ssh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes -i ~/.ssh/bastion-demo.pem ubuntu@${private}"

    local attempt
    for attempt in $(seq 1 30); do
        if ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} true" 2>/dev/null; then
            ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} id" | grep -q '^uid='
            return $?
        fi
        sleep 5
    done
    log "  bastion→private: SSH never reachable after 150s"
    return 1
}

assert_nginx_alb() {
    # ALB DNS (*.elb.spinifex.local) isn't resolvable from the host — README
    # documents fetching the public IP via elbv2 describe-load-balancers.
    local name ip tg_arn attempt
    name=$(tofu output -raw alb_name)

    # Retry describe-load-balancers: the gateway intermittently returns
    # InternalError during ALB provisioning while the lb-agent VM is booting.
    ip=""
    for attempt in 1 2 3 4 5 6; do
        ip=$(aws elbv2 describe-load-balancers --names "$name" \
            --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
            --output text 2>/dev/null | awk '{print $1}')
        if [ -n "$ip" ] && [ "$ip" != "None" ]; then
            break
        fi
        log "  nginx-alb: describe-load-balancers attempt ${attempt} returned no IP, retrying in 10s"
        sleep 10
    done
    if [ -z "$ip" ] || [ "$ip" = "None" ]; then
        log "  nginx-alb: no public IP for ${name} after retries"
        return 1
    fi
    log "  nginx-alb: ALB ${name} public IP ${ip}"

    tg_arn=$(aws elbv2 describe-target-groups --names nginx-alb-tg \
        --query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null)
    if [ -z "$tg_arn" ] || [ "$tg_arn" = "None" ]; then
        log "  nginx-alb: could not resolve target group ARN"
        return 1
    fi

    wait_for_alb_healthy "$tg_arn" 300 || {
        log "  nginx-alb: no healthy targets after 300s"
        return 1
    }

    wait_for_http_200 "http://${ip}/" 300 || {
        log "  nginx-alb: targets healthy but no HTTP 200 after 300s"
        return 1
    }
}

assert_nginx_webserver() {
    local ip
    ip=$(tofu output -raw public_ip)
    wait_for_http_200 "http://${ip}/"
}

# The DB endpoint is a private VPC address and there is no publicly_accessible
# mode, so nothing on the runner can reach it. The assertion therefore runs psql
# from the workbook's client VM over SSH — which is also what proves the
# security-group rule admitting :5432 from the client group is enforced.
assert_rds_quickstart() {
    local ip key endpoint query_output
    ip=$(tofu output -raw client_public_ip)
    endpoint=$(tofu output -raw db_address)
    key="$(pwd)/rds-quickstart-client.pem"
    chmod 600 "$key"

    log "  rds-quickstart: endpoint ${endpoint}, client ${ip}"

    wait_for_ssh "$key" "$ip" || {
        log "  rds-quickstart: client SSH not ready"
        return 1
    }

    # cloud-init installs psql and writes .pgpass after sshd starts. Bound the
    # wait so a stalled guest still produces diagnostics within the cell budget.
    if ! ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \
        'timeout 300 cloud-init status --wait >/dev/null && command -v psql >/dev/null && test -s "$HOME/.pgpass"'; then
        log "  rds-quickstart: client bootstrap incomplete after 300s"
        ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \
            'cloud-init status --long; sudo tail -n 40 /var/log/cloud-init-output.log' 2>&1 | \
            sed "s|^|    |" || true
        return 1
    fi

    # A round-trip rather than a bare connect also proves the engine can write.
    # Use a login shell so /etc/profile.d supplies the PostgreSQL connection.
    if ! query_output=$(printf '%s\n' \
        "CREATE TABLE IF NOT EXISTS smoke (id int primary key, note text);" \
        "INSERT INTO smoke VALUES (1, 'nightly') ON CONFLICT (id) DO UPDATE SET note = 'nightly';" \
        "SELECT note FROM smoke WHERE id = 1;" |
        ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \
            'bash -lc "psql -v ON_ERROR_STOP=1 -tA"' 2>&1); then
        log "  rds-quickstart: SQL round-trip failed"
        printf '%s\n' "$query_output" | sed 's|^|    |'
        return 1
    fi

    if [ "$(printf '%s\n' "$query_output" | tail -1)" != "nightly" ]; then
        log "  rds-quickstart: SQL round-trip returned an unexpected result"
        printf '%s\n' "$query_output" | sed 's|^|    |'
        return 1
    fi
}

# dump_s3_webapp_guest SSHes into the webapp instance and tails cloud-init's
# output log plus the s3-webapp unit status, so a first-boot apt/pip/egress
# failure is distinguishable from a control-plane regression in the bundle.
# Best-effort: a missing key or unreachable SSH just logs and returns.
dump_s3_webapp_guest() {
    local host="$1" key="$(pwd)/s3-webapp-demo.pem"
    if [ ! -f "$key" ]; then
        log "  s3-webapp: ${key} missing, skipping guest cloud-init dump"
        return 0
    fi
    chmod 600 "$key" 2>/dev/null || true
    log "--- s3-webapp guest: cloud-init status + output log + unit status ---"
    ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" '
        echo "== cloud-init status =="; cloud-init status --long 2>/dev/null || true
        echo "== /var/log/cloud-init-output.log (tail 80) =="
        sudo tail -n 80 /var/log/cloud-init-output.log 2>/dev/null || true
        echo "== s3-webapp.service =="
        systemctl status s3-webapp --no-pager 2>/dev/null | head -20 || true
    ' 2>&1 | sed "s|^|    |" || log "  s3-webapp: guest SSH unreachable for cloud-init dump"
}

assert_s3_webapp() {
    # Prove the INSTANCE wrote to S3 with IMDS-sourced STS creds: upload a file
    # through the app (PutObject), then read it back from the listing (ListBucket
    # + GetObject link). This exercises the full IMDS -> STS -> IAM ->
    # predastore-authz path and fails closed if any link regresses — a
    # harness-side CLI upload would still pass with IMDS broken.
    local ip sentinel tmp
    ip=$(tofu output -raw public_ip)
    sentinel="spinifex-nightly-$(date +%s).txt"

    # 420s budget: this is the heaviest first-boot of any workbook — cloud-init
    # runs apt-get install python3-pip/venv + pip install flask boto3 before the
    # listener binds :80. The default 150s isn't enough on a slow runner / PyPI.
    wait_for_http_200 "http://${ip}/" 420 || {
        log "  s3-webapp: webapp not reachable on http://${ip}/ after 420s"
        dump_s3_webapp_guest "$ip"
        return 1
    }

    tmp=$(mktemp)
    echo "nightly-smoke" > "$tmp"
    if ! curl_public -sf -F "file=@${tmp};filename=${sentinel}" "http://${ip}/upload" >/dev/null; then
        rm -f "$tmp"
        log "  s3-webapp: upload via app failed (IMDS -> STS -> S3 PutObject?)"
        return 1
    fi
    rm -f "$tmp"

    curl_public -sf "http://${ip}/" | grep -q "$sentinel" || return 1

    # The bucket carries tags, and a clean plan over them is the gate: the AWS
    # provider reads them through S3 Control at https://{account}.{endpoint},
    # a name no IP endpoint can resolve, so this is what catches that regressing.
    local bucket tags
    bucket=$(tofu output -raw bucket_name)
    tags=$(aws s3api get-bucket-tagging --bucket "$bucket" \
        --endpoint-url "https://${WAN_IP}:8443" --output text 2>&1) || {
        log "  s3-webapp: get-bucket-tagging failed for ${bucket}: ${tags}"
        return 1
    }
    grep -q "s3-webapp" <<<"$tags" || {
        log "  s3-webapp: bucket ${bucket} did not carry the tags Terraform set: ${tags}"
        return 1
    }
}

# A container instance registers only after its agent has booted, read IMDS, and
# reached the gateway, so the service reaching its desired count proves the whole
# ECS path rather than just that CreateService was accepted. The ALB check is the
# same shape as nginx-alb's: .elb.spinifex.local does not resolve here, so the
# public address comes from describe-load-balancers.
assert_ecs_quickstart() {
    local cluster service name ip tg_arn
    cluster=$(tofu output -raw cluster_arn)
    cluster=${cluster##*/}
    service=$(tofu output -raw service_name)

    local budget=420 running desired
    while [ "$budget" -gt 0 ]; do
        read -r running desired <<<"$(aws ecs describe-services --cluster "$cluster" --services "$service" \
            --query 'services[0].[runningCount,desiredCount]' --output text 2>/dev/null)"
        if [ -n "${desired:-}" ] && [ "$desired" != "None" ] && [ "$running" = "$desired" ] && [ "$running" != "0" ]; then
            break
        fi
        sleep 10
        budget=$((budget - 10))
    done
    if [ "$budget" -le 0 ]; then
        log "  ecs-quickstart: service ${service} never reached its desired count (${running:-?}/${desired:-?})"
        aws ecs list-container-instances --cluster "$cluster" --output text 2>&1 | sed 's|^|    |' || true
        return 1
    fi
    log "  ecs-quickstart: service ${service} running ${running}/${desired}"

    name=$(tofu output -raw alb_name)
    ip=$(aws elbv2 describe-load-balancers --names "$name" \
        --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
        --output text 2>/dev/null | awk '{print $1}')
    if [ -z "$ip" ] || [ "$ip" = "None" ]; then
        log "  ecs-quickstart: no public IP for ${name}"
        return 1
    fi
    tg_arn=$(aws elbv2 describe-target-groups --names "${cluster}-tg" \
        --query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null)
    if [ -n "$tg_arn" ] && [ "$tg_arn" != "None" ]; then
        wait_for_alb_healthy "$tg_arn" 300 || {
            log "  ecs-quickstart: no healthy targets after 300s"
            return 1
        }
    fi
    wait_for_http_200 "http://${ip}/" 300 || {
        log "  ecs-quickstart: ALB ${ip} never returned 200"
        return 1
    }
}

# The cluster module stops at a running control plane and worker node group; the
# demo app lives in the nested workloads/ module and needs an image pushed to ECR
# first, which is outside what this harness builds. So the assertion is that the
# cluster is ACTIVE, the node group is ACTIVE, and every worker registered with
# the API server as Ready — the part that proves EKS on this cluster works.
#
# Shared by every EKS workbook: they differ in what they layer on top, and all
# three stand or fall on the same three facts underneath.
assert_eks_cluster_ready() {
    local label="$1" name desired budget=900 status
    name=$(tofu output -raw cluster_name)
    desired=$(tofu output -raw node_desired_size)

    while [ "$budget" -gt 0 ]; do
        status=$(aws eks describe-cluster --name "$name" --query 'cluster.status' --output text 2>/dev/null)
        [ "$status" = "ACTIVE" ] && break
        [ "$status" = "FAILED" ] && { log "  ${label}: cluster ${name} FAILED"; return 1; }
        sleep 15
        budget=$((budget - 15))
    done
    if [ "$status" != "ACTIVE" ]; then
        log "  ${label}: cluster ${name} still ${status:-unknown} after 900s"
        return 1
    fi
    log "  ${label}: cluster ${name} ACTIVE"

    if ! command -v kubectl >/dev/null 2>&1; then
        local kver
        kver=$(curl -fsSL https://dl.k8s.io/release/stable.txt 2>/dev/null || echo v1.32.0)
        curl -fsSL -o /tmp/kubectl "https://dl.k8s.io/release/${kver}/bin/linux/amd64/kubectl" &&
            sudo install -m 0755 /tmp/kubectl /usr/local/bin/kubectl || {
            log "  ${label}: kubectl unavailable, cannot check node readiness"
            return 1
        }
    fi
    aws eks update-kubeconfig --name "$name" >/dev/null 2>&1 || {
        log "  ${label}: update-kubeconfig failed"
        return 1
    }

    # Count agents, not every node. The control plane is three K3s servers, so
    # an unfiltered count reaches a one-worker cluster's desired size before any
    # worker joins — the assertion passes on exactly the failure it exists to
    # catch. Node-group workers boot as k3s-agent and carry no control-plane
    # role label (scripts/images/eks-node/eks-node-role.sh).
    budget=600
    local ready
    while [ "$budget" -gt 0 ]; do
        ready=$(kubectl get nodes --no-headers -l '!node-role.kubernetes.io/control-plane' 2>/dev/null |
            awk '$2=="Ready"' | wc -l)
        [ "$ready" -ge "$desired" ] 2>/dev/null && break
        sleep 15
        budget=$((budget - 15))
    done
    if [ "${ready:-0}" -lt "$desired" ] 2>/dev/null; then
        log "  ${label}: ${ready:-0}/${desired} workers Ready after 600s"
        kubectl get nodes 2>&1 | sed 's|^|    |' || true
        return 1
    fi
    log "  ${label}: ${ready}/${desired} workers Ready"
}

assert_eks_quickstart() {
    assert_eks_cluster_ready eks-quickstart
}

# Same cluster assertion, over a harder network: these two put their workers in
# private subnets behind a NAT gateway, so a worker only registers if its egress
# to the cluster endpoint and to ECR survives the whole NAT path. A worker that
# never reaches Ready here, with eks-quickstart passing, is a NAT gateway fault
# and not an EKS one.
assert_eks_https_ingress() {
    assert_eks_cluster_ready eks-https-ingress
}

assert_eks_gitops_argocd() {
    assert_eks_cluster_ready eks-gitops-argocd
}

# Workbooks whose describe path has to round-trip every attribute their config
# sets. A second plan on these must be empty; a diff is a describe response that
# failed to echo back a field Terraform sent, which makes the provider unusable
# while every individual API call still looks correct.
#
# Opt-in rather than universal: the four original workbooks were never written
# against this assertion, and a pre-existing diff in one of them would fail a
# suite that is otherwise reporting on something else.
CLEAN_PLAN_WORKBOOKS=" rds-quickstart "

assert_clean_plan() {
    local example="$1"
    shift
    case "$CLEAN_PLAN_WORKBOOKS" in
        *" ${example} "*) ;;
        *) return 0 ;;
    esac

    local out rc
    out=$(tofu plan -detailed-exitcode -input=false -no-color "$@" 2>&1)
    rc=$?
    case "$rc" in
        0) log "  ${example}: second plan is clean" ; return 0 ;;
        2) log "  ${example}: second plan is not empty — a describe is not echoing back what apply sent" ;;
        *) log "  ${example}: second plan errored (exit ${rc})" ;;
    esac
    echo "$out" | tail -60 | sed "s|^|    |"
    return 1
}

# RDS resolves its database VM image from all three tags. Check this before
# applying any workbook so a missing operator prerequisite does not surface as
# provider retries during the final, most expensive example.
require_rds_image() {
    local image_ids
    if ! image_ids=$(aws ec2 describe-images \
        --filters \
            'Name=tag:spinifex:managed-by,Values=rds' \
            'Name=tag:engine,Values=postgres' \
            'Name=tag:engine-version,Values=18' \
        --query 'Images[].ImageId' --output text); then
        log "RDS AMI preflight: describe-images failed"
        return 1
    fi
    if [ -z "$image_ids" ] || [ "$image_ids" = "None" ]; then
        log "RDS AMI preflight: spinifex-rds-postgres is not registered"
        log "Run: spx admin images import --name spinifex-rds-postgres"
        return 1
    fi
    log "RDS AMI preflight: found ${image_ids}"
}

# Pick an instance type available on this cluster. Workbooks default to
# t3.small (Intel); on AMD-only hosts t3 isn't registered, so we query and
# fall back to the smallest type meeting the floor (matches the nginx-alb
# README's documented approach). Defaults to 2 vCPU / 1 GiB; callers that need
# more headroom, such as an EKS worker, pass their own.
detect_instance_type() {
    local vcpus="${1:-2}" mib="${2:-1024}"
    local endpoint="https://${WAN_IP}:9999"
    local picked
    picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \
        --query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus>=\`${vcpus}\` && MemoryInfo.SizeInMiB>=\`${mib}\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
        --output text 2>/dev/null || true)
    if [ -z "$picked" ] || [ "$picked" = "None" ]; then
        picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \
            --query 'InstanceTypes[0].InstanceType' --output text 2>/dev/null || true)
    fi
    echo "$picked"
}

INSTANCE_TYPE=""

# --- Workbook driver ---

run_workbook() {
    local example="$1"
    local path="${WORKBOOK_DIR}/${example}"

    if [ ! -d "$path" ]; then
        log "SKIP ${example}: ${path} not found"
        return 1
    fi

    CURRENT_WORKBOOK="$example"
    log "=== ${example} ==="
    cd "$path"
    rm -rf .terraform terraform.tfstate* .terraform.lock.hcl

    # EKS workers run a kubelet, a container runtime and the demo pods, so the
    # smallest type the cluster offers is not enough; the workbook's own default
    # is t3.medium and 4 GiB is the floor that boots reliably.
    local instance_type="$INSTANCE_TYPE"
    case "$example" in
        eks-*) instance_type=$(detect_instance_type 2 4096) ;;
    esac

    # Workbooks name the worker size differently — the EKS ones that split
    # public and private subnets call it node_instance_type. Read the name from
    # the config rather than keeping a list here: tofu rejects a -var for an
    # undeclared variable outright, so a stale list fails the apply in seconds
    # and reads like a product defect.
    local size_var=instance_type
    grep -qE '^variable "node_instance_type"' main.tf 2>/dev/null && size_var=node_instance_type

    local apply_args=(
        -input=false -no-color
        "-var=spinifex_endpoint=https://${WAN_IP}:9999"
        "-var=${size_var}=${instance_type}"
    )

    # ecs-quickstart's gateway_url is required and has no default: the container
    # instances read it from their own user-data, so it must be the address a
    # guest can reach, never 127.0.0.1.
    if [ "$example" = "ecs-quickstart" ]; then
        apply_args+=("-var=gateway_url=https://${WAN_IP}:9999")
    fi

    # s3-webapp has three required-no-default vars. The s3_access/secret keys are
    # the operator creds the provider uses to create the bucket + IAM role on
    # predastore; the instance authenticates to S3 via IMDS, not these keys.
    if [ "$example" = "s3-webapp" ]; then
        local access_key secret_key
        access_key=$(aws configure get aws_access_key_id --profile spinifex)
        secret_key=$(aws configure get aws_secret_access_key --profile spinifex)
        apply_args+=(
            "-var=predastore_endpoint=https://${WAN_IP}:8443"
            "-var=predastore_host=${WAN_IP}:8443"
            "-var=s3_access_key=${access_key}"
            "-var=s3_secret_key=${secret_key}"
        )
    fi

    if ! tofu init -input=false -no-color; then
        log "  FAIL ${example}: tofu init"
        return 1
    fi

    if ! tofu apply -auto-approve "${apply_args[@]}"; then
        log "  FAIL ${example}: tofu apply"
        # Bounded like the teardown below: this one only tidies up after a
        # workbook that has already failed, so it must not hold the suite open.
        timeout "${DESTROY_TIMEOUT:-900}" tofu destroy -auto-approve "${apply_args[@]}" >/dev/null 2>&1 || true
        return 1
    fi

    local rc=0 destroy_log
    destroy_log=$(mktemp)
    if ! assert_"${example//-/_}" || ! assert_clean_plan "$example" "${apply_args[@]}"; then
        log "  FAIL ${example}: assertion"
        # Capture OVN state while VMs still exist — the EXIT trap fires after
        # destroy, by which point port groups, address sets, and ACLs are gone.
        dump_ovn_state "post-fail ${example} (pre-destroy)"
        rc=1
    fi

    # The teardown is part of what the workbook proves: a customer who cannot
    # remove what they created is as stuck as one who could not create it. It
    # was a WARN, and a bucket that could never be emptied passed for weeks.
    #
    # Bounded because the failure mode is a wedge rather than an error — the
    # provider retries DeleteBucket against a bucket it cannot empty forever.
    if ! timeout "${DESTROY_TIMEOUT:-900}" tofu destroy -auto-approve "${apply_args[@]}" > "$destroy_log" 2>&1; then
        log "  FAIL ${example}: tofu destroy"
        sed 's|^|    |' "$destroy_log" | tail -40
        rc=1
    fi
    rm -f "$destroy_log"

    # After the teardown, not before it. The verdict now depends on it, so a
    # workbook that printed PASS and then failed to delete what it built would
    # have reported both.
    if [ "$rc" -eq 0 ]; then
        log "  PASS ${example}"
    fi

    cd "$SCRIPT_DIR"
    return "$rc"
}

# --- Main ---

install_tofu || { log "tofu install failed"; exit 1; }
check_public_proxy || { log "the public-address vantage point is unusable" ; exit 1; }
require_rds_image || { log "required RDS AMI is unavailable"; exit 1; }

INSTANCE_TYPE=$(detect_instance_type)
if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then
    log "no instance type available from describe-instance-types"
    exit 1
fi
log "Using instance_type=${INSTANCE_TYPE}"

# Each workbook is emitted as a top-level `go test -v` testcase
# (=== RUN / --- PASS|FAIL / package trailer) so go-junit-report converts the
# tee'd log into junit-tofu.xml and the e2e-analyze action produces the same
# RCA bundle the Go suites do (nightly cell 17). The per-workbook log() output
# between RUN and the result line is captured as the failure diagnostics.
#
# rds-quickstart runs last because it is the most expensive: its apply carries a
# DB VM boot, initdb and the first healthy agent heartbeat before Terraform even
# launches the client, then the client's own boot and an apt-get. Budget it at
# ~15 minutes against the ~7 the other four share, and note that its assertion
# needs roughly 2 GiB of free guest memory for the two VMs.
#
# WORKBOOKS selects the list. A failure no longer aborts the rest: s3-webapp
# failing says nothing about whether RDS works, and four nights of rds-quickstart
# coverage were lost that way. STOP_ON_FAIL=1 restores the early abort for a
# cluster you already know is broken and do not want to spend an hour on.
SUITE_RC=0
for workbook in ${WORKBOOKS:-nginx-alb bastion-private-subnet nginx-webserver s3-webapp rds-quickstart}; do
    tname="TestTofuWorkbook_${workbook//-/_}"
    wb_start=$SECONDS
    echo "=== RUN   ${tname}"
    if run_workbook "$workbook"; then
        printf -- '--- PASS: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))"
    else
        printf -- '--- FAIL: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))"
        SUITE_RC=1
        if [ "${STOP_ON_FAIL:-0}" = "1" ]; then
            log "FAIL ${workbook} — aborting remaining workbooks (STOP_ON_FAIL=1)"
            break
        fi
        log "FAIL ${workbook} — continuing with the remaining workbooks"
    fi
done

CURRENT_WORKBOOK=""
if [ "$SUITE_RC" -eq 0 ]; then
    log "All workbooks passed"
    echo "PASS"
    printf 'ok  \ttofu-examples\t%d.000s\n' "$SECONDS"
else
    echo "FAIL"
    printf 'FAIL\ttofu-examples\t%d.000s\n' "$SECONDS"
fi
exit "$SUITE_RC"