mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-21 03:36:35 +00:00
Ben Duncan
fcda031117
* test(e2e): make the workbook teardown a gate, not a warning A customer who cannot remove what they created is as stuck as one who could not create it, so the teardown is part of what a workbook proves. It logged WARN and passed the suite regardless, and a bucket that could never be emptied went unnoticed for weeks behind exactly that line. The failure it hides is a wedge rather than an error: the AWS provider retries DeleteBucket against a bucket it cannot empty indefinitely. So the run is bounded by DESTROY_TIMEOUT, defaulting to 900s, and the last forty lines of its output are printed when it fails. The cleanup after a failed apply is bounded the same way for the same reason. * test(e2e): print a workbook's PASS after its teardown, not before The verdict now depends on the teardown, so a workbook that built everything correctly and then could not delete it printed PASS and FAIL for the same run.
715 lines
29 KiB
Bash
Executable File
715 lines
29 KiB
Bash
Executable File
#!/bin/bash
|
|
# run-tofu-examples-e2e.sh — smoke-test the public terraform workbooks.
|
|
#
|
|
# Invoked by the nightly matrix (cell #17) after bootstrap-install.sh has
|
|
# brought up a single-node cluster. For each workbook, the driver does a
|
|
# clean tofu init/apply, runs one smoke assertion, then tofu destroy.
|
|
#
|
|
# All assertions are minimal by design (one check per workbook) so the
|
|
# maintenance cost of keeping them in sync with workbook edits stays low.
|
|
#
|
|
# Expects the workbooks to be available at $WORKBOOK_DIR (defaults to the
|
|
# tarball-included ./workbooks/ directory beside this script).
|
|
|
|
set -u
|
|
cd "$(dirname "$0")"
|
|
SCRIPT_DIR="$(pwd)"
|
|
|
|
export AWS_PROFILE=spinifex
|
|
WORKBOOK_DIR="${WORKBOOK_DIR:-${SCRIPT_DIR}/workbooks}"
|
|
OPENTOFU_VERSION="${OPENTOFU_VERSION:-1.11.5}"
|
|
|
|
# awsgw and predastore are bound to the WAN IP (bootstrap-install passes --bind
|
|
# ${PRIMARY_WAN}), not loopback. Workbooks need the WAN IP for both the tofu
|
|
# provider endpoints and any CLI assertions that talk to S3 (:8443).
|
|
WAN_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}')
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR)
|
|
|
|
CURRENT_WORKBOOK=""
|
|
|
|
log() { echo "[$(date +%H:%M:%S)] $*"; }
|
|
|
|
install_tofu() {
|
|
command -v tofu >/dev/null 2>&1 && return 0
|
|
local arch
|
|
case "$(uname -m)" in
|
|
x86_64) arch=amd64 ;;
|
|
aarch64) arch=arm64 ;;
|
|
*) log "unsupported arch: $(uname -m)"; return 1 ;;
|
|
esac
|
|
local url="https://github.com/opentofu/opentofu/releases/download/v${OPENTOFU_VERSION}/tofu_${OPENTOFU_VERSION}_linux_${arch}.tar.gz"
|
|
log "Installing OpenTofu ${OPENTOFU_VERSION} (${arch})"
|
|
local tmp
|
|
tmp=$(mktemp -d)
|
|
curl -fsSL "$url" | tar -xz -C "$tmp" tofu
|
|
sudo install -m 0755 "$tmp/tofu" /usr/local/bin/tofu
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
# dump_ovn_state captures OVN NB port_group/ACL state and the SB Address_Set
|
|
# rows ovn-northd derives from port-group membership. Called inside run_workbook
|
|
# on assertion failure (before destroy, while VMs still exist) and again from
|
|
# the EXIT trap (after destroy, mostly for parity).
|
|
#
|
|
# SG-to-SG ACL matches like `ip4.src == $sg_<id>_ip4` resolve against SB
|
|
# Address_Sets that ovn-northd auto-derives from port-group LSP addresses; if
|
|
# this dump shows port groups with members but no matching `<pg>_ip4` address
|
|
# set (or an empty one), that pinpoints the SG enforcement break.
|
|
dump_ovn_state() {
|
|
local label="${1:-ovn-state}"
|
|
log "--- ${label}: ovn-nbctl ls-list ---"
|
|
sudo ovn-nbctl --no-leader-only ls-list 2>&1 | head -40 || true
|
|
log "--- ${label}: ovn-nbctl lr-list ---"
|
|
sudo ovn-nbctl --no-leader-only lr-list 2>&1 | head -40 || true
|
|
log "--- ${label}: port groups (name, ports, ACL count) ---"
|
|
sudo ovn-nbctl --no-leader-only --bare --columns=name,ports list port_group 2>&1 | head -200 || true
|
|
log "--- ${label}: logical_switch_port name/addresses/port_security ---"
|
|
sudo ovn-nbctl --no-leader-only --bare --columns=name,addresses,port_security \
|
|
list logical_switch_port 2>&1 | head -200 || true
|
|
log "--- ${label}: NB ACLs (priority, direction, match, action) ---"
|
|
sudo ovn-nbctl --no-leader-only --bare \
|
|
--columns=priority,direction,match,action,name list acl 2>&1 | head -200 || true
|
|
log "--- ${label}: SB address_sets (auto-derived from port groups) ---"
|
|
sudo ovn-sbctl --no-leader-only --bare --columns=name,addresses list address_set 2>&1 | head -200 || true
|
|
log "--- ${label}: SB port bindings (logical_port, mac, chassis, up) ---"
|
|
sudo ovn-sbctl --no-leader-only --bare \
|
|
--columns=logical_port,mac,chassis,up list port_binding 2>&1 | head -80 || true
|
|
}
|
|
|
|
cleanup() {
|
|
EXIT_CODE=$?
|
|
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
log "=== FAIL (workbook=${CURRENT_WORKBOOK:-<none>}) — dumping diagnostics ==="
|
|
log "--- spx get vms ---"
|
|
sudo -u spinifex-daemon spx get vms 2>&1 | head -80 || \
|
|
spx get vms 2>&1 | head -80 || true
|
|
log "--- aws ec2 describe-instances ---"
|
|
aws ec2 describe-instances \
|
|
--query 'Reservations[].Instances[].[InstanceId,State.Name,StateReason.Message,PrivateIpAddress,PublicIpAddress,Tags[?Key==`Name`].Value|[0]]' \
|
|
--output table 2>&1 | head -60 || true
|
|
log "--- spinifex service errors (last 15 min) ---"
|
|
for svc in spinifex-daemon spinifex-awsgw spinifex-vpcd spinifex-viperblock spinifex-predastore; do
|
|
sudo journalctl -u "$svc" --since '15 min ago' --no-pager 2>/dev/null | \
|
|
grep -iE 'panic|fatal|level=error|ERROR' | tail -30 | \
|
|
sed "s|^| [${svc}] |" || true
|
|
done
|
|
log "--- service journals (last 200 lines each) ---"
|
|
for svc in spinifex-nats spinifex-predastore spinifex-viperblock \
|
|
spinifex-daemon spinifex-awsgw spinifex-vpcd; do
|
|
echo "=== ${svc} ==="
|
|
sudo journalctl -u "${svc}" --no-pager -n 200 2>/dev/null || true
|
|
done
|
|
fi
|
|
exit "$EXIT_CODE"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# --- Per-workbook assertions ---
|
|
|
|
# Wait up to 150s for instance SSH readiness.
|
|
wait_for_ssh() {
|
|
local key="$1" host="$2"
|
|
for _ in $(seq 1 30); do
|
|
if ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" true 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Wait up to $2 seconds (default 150) for a 200 response from $1.
|
|
wait_for_http_200() {
|
|
local url="$1" budget="${2:-150}"
|
|
local attempts=$((budget / 5))
|
|
for _ in $(seq 1 "$attempts"); do
|
|
local status
|
|
status=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null || echo "000")
|
|
if [ "$status" = "200" ]; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Wait up to $2 seconds for the target group $1 to report at least one healthy target.
|
|
wait_for_alb_healthy() {
|
|
local tg_arn="$1" budget="${2:-300}"
|
|
local attempts=$((budget / 10))
|
|
for _ in $(seq 1 "$attempts"); do
|
|
local healthy
|
|
healthy=$(aws elbv2 describe-target-health --target-group-arn "$tg_arn" \
|
|
--query 'length(TargetHealthDescriptions[?TargetHealth.State==`healthy`])' \
|
|
--output text 2>/dev/null || echo "0")
|
|
if [ "$healthy" -gt 0 ] 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 10
|
|
done
|
|
return 1
|
|
}
|
|
|
|
assert_bastion_private_subnet() {
|
|
local bastion private key
|
|
bastion=$(tofu output -raw bastion_public_ip)
|
|
private=$(tofu output -raw private_instance_ip)
|
|
key="$(pwd)/bastion-demo.pem"
|
|
chmod 600 "$key"
|
|
|
|
wait_for_ssh "$key" "$bastion" || {
|
|
log " bastion SSH not ready"
|
|
return 1
|
|
}
|
|
# sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem.
|
|
# Wait up to 180s for user_data to drop the key before hopping.
|
|
if ! ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" \
|
|
'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1'; then
|
|
log " bastion: ~/.ssh/bastion-demo.pem never appeared (cloud-init stalled?)"
|
|
return 1
|
|
fi
|
|
local inner_ssh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes -i ~/.ssh/bastion-demo.pem ubuntu@${private}"
|
|
|
|
local attempt
|
|
for attempt in $(seq 1 30); do
|
|
if ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} true" 2>/dev/null; then
|
|
ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} id" | grep -q '^uid='
|
|
return $?
|
|
fi
|
|
sleep 5
|
|
done
|
|
log " bastion→private: SSH never reachable after 150s"
|
|
return 1
|
|
}
|
|
|
|
assert_nginx_alb() {
|
|
# ALB DNS (*.elb.spinifex.local) isn't resolvable from the host — README
|
|
# documents fetching the public IP via elbv2 describe-load-balancers.
|
|
local name ip tg_arn attempt
|
|
name=$(tofu output -raw alb_name)
|
|
|
|
# Retry describe-load-balancers: the gateway intermittently returns
|
|
# InternalError during ALB provisioning while the lb-agent VM is booting.
|
|
ip=""
|
|
for attempt in 1 2 3 4 5 6; do
|
|
ip=$(aws elbv2 describe-load-balancers --names "$name" \
|
|
--query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
|
|
--output text 2>/dev/null | awk '{print $1}')
|
|
if [ -n "$ip" ] && [ "$ip" != "None" ]; then
|
|
break
|
|
fi
|
|
log " nginx-alb: describe-load-balancers attempt ${attempt} returned no IP, retrying in 10s"
|
|
sleep 10
|
|
done
|
|
if [ -z "$ip" ] || [ "$ip" = "None" ]; then
|
|
log " nginx-alb: no public IP for ${name} after retries"
|
|
return 1
|
|
fi
|
|
log " nginx-alb: ALB ${name} public IP ${ip}"
|
|
|
|
tg_arn=$(aws elbv2 describe-target-groups --names nginx-alb-tg \
|
|
--query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null)
|
|
if [ -z "$tg_arn" ] || [ "$tg_arn" = "None" ]; then
|
|
log " nginx-alb: could not resolve target group ARN"
|
|
return 1
|
|
fi
|
|
|
|
wait_for_alb_healthy "$tg_arn" 300 || {
|
|
log " nginx-alb: no healthy targets after 300s"
|
|
return 1
|
|
}
|
|
|
|
wait_for_http_200 "http://${ip}/" 300 || {
|
|
log " nginx-alb: targets healthy but no HTTP 200 after 300s"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
assert_nginx_webserver() {
|
|
local ip
|
|
ip=$(tofu output -raw public_ip)
|
|
wait_for_http_200 "http://${ip}/"
|
|
}
|
|
|
|
# The DB endpoint is a private VPC address and there is no publicly_accessible
|
|
# mode, so nothing on the runner can reach it. The assertion therefore runs psql
|
|
# from the workbook's client VM over SSH — which is also what proves the
|
|
# security-group rule admitting :5432 from the client group is enforced.
|
|
assert_rds_quickstart() {
|
|
local ip key endpoint query_output
|
|
ip=$(tofu output -raw client_public_ip)
|
|
endpoint=$(tofu output -raw db_address)
|
|
key="$(pwd)/rds-quickstart-client.pem"
|
|
chmod 600 "$key"
|
|
|
|
log " rds-quickstart: endpoint ${endpoint}, client ${ip}"
|
|
|
|
wait_for_ssh "$key" "$ip" || {
|
|
log " rds-quickstart: client SSH not ready"
|
|
return 1
|
|
}
|
|
|
|
# cloud-init installs psql and writes .pgpass after sshd starts. Bound the
|
|
# wait so a stalled guest still produces diagnostics within the cell budget.
|
|
if ! ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \
|
|
'timeout 300 cloud-init status --wait >/dev/null && command -v psql >/dev/null && test -s "$HOME/.pgpass"'; then
|
|
log " rds-quickstart: client bootstrap incomplete after 300s"
|
|
ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \
|
|
'cloud-init status --long; sudo tail -n 40 /var/log/cloud-init-output.log' 2>&1 | \
|
|
sed "s|^| |" || true
|
|
return 1
|
|
fi
|
|
|
|
# A round-trip rather than a bare connect also proves the engine can write.
|
|
# Use a login shell so /etc/profile.d supplies the PostgreSQL connection.
|
|
if ! query_output=$(printf '%s\n' \
|
|
"CREATE TABLE IF NOT EXISTS smoke (id int primary key, note text);" \
|
|
"INSERT INTO smoke VALUES (1, 'nightly') ON CONFLICT (id) DO UPDATE SET note = 'nightly';" \
|
|
"SELECT note FROM smoke WHERE id = 1;" |
|
|
ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \
|
|
'bash -lc "psql -v ON_ERROR_STOP=1 -tA"' 2>&1); then
|
|
log " rds-quickstart: SQL round-trip failed"
|
|
printf '%s\n' "$query_output" | sed 's|^| |'
|
|
return 1
|
|
fi
|
|
|
|
if [ "$(printf '%s\n' "$query_output" | tail -1)" != "nightly" ]; then
|
|
log " rds-quickstart: SQL round-trip returned an unexpected result"
|
|
printf '%s\n' "$query_output" | sed 's|^| |'
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# dump_s3_webapp_guest SSHes into the webapp instance and tails cloud-init's
|
|
# output log plus the s3-webapp unit status, so a first-boot apt/pip/egress
|
|
# failure is distinguishable from a control-plane regression in the bundle.
|
|
# Best-effort: a missing key or unreachable SSH just logs and returns.
|
|
dump_s3_webapp_guest() {
|
|
local host="$1" key="$(pwd)/s3-webapp-demo.pem"
|
|
if [ ! -f "$key" ]; then
|
|
log " s3-webapp: ${key} missing, skipping guest cloud-init dump"
|
|
return 0
|
|
fi
|
|
chmod 600 "$key" 2>/dev/null || true
|
|
log "--- s3-webapp guest: cloud-init status + output log + unit status ---"
|
|
ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" '
|
|
echo "== cloud-init status =="; cloud-init status --long 2>/dev/null || true
|
|
echo "== /var/log/cloud-init-output.log (tail 80) =="
|
|
sudo tail -n 80 /var/log/cloud-init-output.log 2>/dev/null || true
|
|
echo "== s3-webapp.service =="
|
|
systemctl status s3-webapp --no-pager 2>/dev/null | head -20 || true
|
|
' 2>&1 | sed "s|^| |" || log " s3-webapp: guest SSH unreachable for cloud-init dump"
|
|
}
|
|
|
|
assert_s3_webapp() {
|
|
# Prove the INSTANCE wrote to S3 with IMDS-sourced STS creds: upload a file
|
|
# through the app (PutObject), then read it back from the listing (ListBucket
|
|
# + GetObject link). This exercises the full IMDS -> STS -> IAM ->
|
|
# predastore-authz path and fails closed if any link regresses — a
|
|
# harness-side CLI upload would still pass with IMDS broken.
|
|
local ip sentinel tmp
|
|
ip=$(tofu output -raw public_ip)
|
|
sentinel="spinifex-nightly-$(date +%s).txt"
|
|
|
|
# 420s budget: this is the heaviest first-boot of any workbook — cloud-init
|
|
# runs apt-get install python3-pip/venv + pip install flask boto3 before the
|
|
# listener binds :80. The default 150s isn't enough on a slow runner / PyPI.
|
|
wait_for_http_200 "http://${ip}/" 420 || {
|
|
log " s3-webapp: webapp not reachable on http://${ip}/ after 420s"
|
|
dump_s3_webapp_guest "$ip"
|
|
return 1
|
|
}
|
|
|
|
tmp=$(mktemp)
|
|
echo "nightly-smoke" > "$tmp"
|
|
if ! curl -sf -F "file=@${tmp};filename=${sentinel}" "http://${ip}/upload" >/dev/null; then
|
|
rm -f "$tmp"
|
|
log " s3-webapp: upload via app failed (IMDS -> STS -> S3 PutObject?)"
|
|
return 1
|
|
fi
|
|
rm -f "$tmp"
|
|
|
|
curl -sf "http://${ip}/" | grep -q "$sentinel" || return 1
|
|
|
|
# The bucket carries tags, and a clean plan over them is the gate: the AWS
|
|
# provider reads them through S3 Control at https://{account}.{endpoint},
|
|
# a name no IP endpoint can resolve, so this is what catches that regressing.
|
|
local bucket tags
|
|
bucket=$(tofu output -raw bucket_name)
|
|
tags=$(aws s3api get-bucket-tagging --bucket "$bucket" \
|
|
--endpoint-url "https://${WAN_IP}:8443" --output text 2>&1) || {
|
|
log " s3-webapp: get-bucket-tagging failed for ${bucket}: ${tags}"
|
|
return 1
|
|
}
|
|
grep -q "s3-webapp" <<<"$tags" || {
|
|
log " s3-webapp: bucket ${bucket} did not carry the tags Terraform set: ${tags}"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
# A container instance registers only after its agent has booted, read IMDS, and
|
|
# reached the gateway, so the service reaching its desired count proves the whole
|
|
# ECS path rather than just that CreateService was accepted. The ALB check is the
|
|
# same shape as nginx-alb's: .elb.spinifex.local does not resolve here, so the
|
|
# public address comes from describe-load-balancers.
|
|
assert_ecs_quickstart() {
|
|
local cluster service name ip tg_arn
|
|
cluster=$(tofu output -raw cluster_arn)
|
|
cluster=${cluster##*/}
|
|
service=$(tofu output -raw service_name)
|
|
|
|
local budget=420 running desired
|
|
while [ "$budget" -gt 0 ]; do
|
|
read -r running desired <<<"$(aws ecs describe-services --cluster "$cluster" --services "$service" \
|
|
--query 'services[0].[runningCount,desiredCount]' --output text 2>/dev/null)"
|
|
if [ -n "${desired:-}" ] && [ "$desired" != "None" ] && [ "$running" = "$desired" ] && [ "$running" != "0" ]; then
|
|
break
|
|
fi
|
|
sleep 10
|
|
budget=$((budget - 10))
|
|
done
|
|
if [ "$budget" -le 0 ]; then
|
|
log " ecs-quickstart: service ${service} never reached its desired count (${running:-?}/${desired:-?})"
|
|
aws ecs list-container-instances --cluster "$cluster" --output text 2>&1 | sed 's|^| |' || true
|
|
return 1
|
|
fi
|
|
log " ecs-quickstart: service ${service} running ${running}/${desired}"
|
|
|
|
name=$(tofu output -raw alb_name)
|
|
ip=$(aws elbv2 describe-load-balancers --names "$name" \
|
|
--query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
|
|
--output text 2>/dev/null | awk '{print $1}')
|
|
if [ -z "$ip" ] || [ "$ip" = "None" ]; then
|
|
log " ecs-quickstart: no public IP for ${name}"
|
|
return 1
|
|
fi
|
|
tg_arn=$(aws elbv2 describe-target-groups --names "${cluster}-tg" \
|
|
--query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null)
|
|
if [ -n "$tg_arn" ] && [ "$tg_arn" != "None" ]; then
|
|
wait_for_alb_healthy "$tg_arn" 300 || {
|
|
log " ecs-quickstart: no healthy targets after 300s"
|
|
return 1
|
|
}
|
|
fi
|
|
wait_for_http_200 "http://${ip}/" 300 || {
|
|
log " ecs-quickstart: ALB ${ip} never returned 200"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
# The cluster module stops at a running control plane and worker node group; the
|
|
# demo app lives in the nested workloads/ module and needs an image pushed to ECR
|
|
# first, which is outside what this harness builds. So the assertion is that the
|
|
# cluster is ACTIVE, the node group is ACTIVE, and every worker registered with
|
|
# the API server as Ready — the part that proves EKS on this cluster works.
|
|
#
|
|
# Shared by every EKS workbook: they differ in what they layer on top, and all
|
|
# three stand or fall on the same three facts underneath.
|
|
assert_eks_cluster_ready() {
|
|
local label="$1" name desired budget=900 status
|
|
name=$(tofu output -raw cluster_name)
|
|
desired=$(tofu output -raw node_desired_size)
|
|
|
|
while [ "$budget" -gt 0 ]; do
|
|
status=$(aws eks describe-cluster --name "$name" --query 'cluster.status' --output text 2>/dev/null)
|
|
[ "$status" = "ACTIVE" ] && break
|
|
[ "$status" = "FAILED" ] && { log " ${label}: cluster ${name} FAILED"; return 1; }
|
|
sleep 15
|
|
budget=$((budget - 15))
|
|
done
|
|
if [ "$status" != "ACTIVE" ]; then
|
|
log " ${label}: cluster ${name} still ${status:-unknown} after 900s"
|
|
return 1
|
|
fi
|
|
log " ${label}: cluster ${name} ACTIVE"
|
|
|
|
if ! command -v kubectl >/dev/null 2>&1; then
|
|
local kver
|
|
kver=$(curl -fsSL https://dl.k8s.io/release/stable.txt 2>/dev/null || echo v1.32.0)
|
|
curl -fsSL -o /tmp/kubectl "https://dl.k8s.io/release/${kver}/bin/linux/amd64/kubectl" &&
|
|
sudo install -m 0755 /tmp/kubectl /usr/local/bin/kubectl || {
|
|
log " ${label}: kubectl unavailable, cannot check node readiness"
|
|
return 1
|
|
}
|
|
fi
|
|
aws eks update-kubeconfig --name "$name" >/dev/null 2>&1 || {
|
|
log " ${label}: update-kubeconfig failed"
|
|
return 1
|
|
}
|
|
|
|
# Count agents, not every node. The control plane is three K3s servers, so
|
|
# an unfiltered count reaches a one-worker cluster's desired size before any
|
|
# worker joins — the assertion passes on exactly the failure it exists to
|
|
# catch. Node-group workers boot as k3s-agent and carry no control-plane
|
|
# role label (scripts/images/eks-node/eks-node-role.sh).
|
|
budget=600
|
|
local ready
|
|
while [ "$budget" -gt 0 ]; do
|
|
ready=$(kubectl get nodes --no-headers -l '!node-role.kubernetes.io/control-plane' 2>/dev/null |
|
|
awk '$2=="Ready"' | wc -l)
|
|
[ "$ready" -ge "$desired" ] 2>/dev/null && break
|
|
sleep 15
|
|
budget=$((budget - 15))
|
|
done
|
|
if [ "${ready:-0}" -lt "$desired" ] 2>/dev/null; then
|
|
log " ${label}: ${ready:-0}/${desired} workers Ready after 600s"
|
|
kubectl get nodes 2>&1 | sed 's|^| |' || true
|
|
return 1
|
|
fi
|
|
log " ${label}: ${ready}/${desired} workers Ready"
|
|
}
|
|
|
|
assert_eks_quickstart() {
|
|
assert_eks_cluster_ready eks-quickstart
|
|
}
|
|
|
|
# Same cluster assertion, over a harder network: these two put their workers in
|
|
# private subnets behind a NAT gateway, so a worker only registers if its egress
|
|
# to the cluster endpoint and to ECR survives the whole NAT path. A worker that
|
|
# never reaches Ready here, with eks-quickstart passing, is a NAT gateway fault
|
|
# and not an EKS one.
|
|
assert_eks_https_ingress() {
|
|
assert_eks_cluster_ready eks-https-ingress
|
|
}
|
|
|
|
assert_eks_gitops_argocd() {
|
|
assert_eks_cluster_ready eks-gitops-argocd
|
|
}
|
|
|
|
# Workbooks whose describe path has to round-trip every attribute their config
|
|
# sets. A second plan on these must be empty; a diff is a describe response that
|
|
# failed to echo back a field Terraform sent, which makes the provider unusable
|
|
# while every individual API call still looks correct.
|
|
#
|
|
# Opt-in rather than universal: the four original workbooks were never written
|
|
# against this assertion, and a pre-existing diff in one of them would fail a
|
|
# suite that is otherwise reporting on something else.
|
|
CLEAN_PLAN_WORKBOOKS=" rds-quickstart "
|
|
|
|
assert_clean_plan() {
|
|
local example="$1"
|
|
shift
|
|
case "$CLEAN_PLAN_WORKBOOKS" in
|
|
*" ${example} "*) ;;
|
|
*) return 0 ;;
|
|
esac
|
|
|
|
local out rc
|
|
out=$(tofu plan -detailed-exitcode -input=false -no-color "$@" 2>&1)
|
|
rc=$?
|
|
case "$rc" in
|
|
0) log " ${example}: second plan is clean" ; return 0 ;;
|
|
2) log " ${example}: second plan is not empty — a describe is not echoing back what apply sent" ;;
|
|
*) log " ${example}: second plan errored (exit ${rc})" ;;
|
|
esac
|
|
echo "$out" | tail -60 | sed "s|^| |"
|
|
return 1
|
|
}
|
|
|
|
# RDS resolves its database VM image from all three tags. Check this before
|
|
# applying any workbook so a missing operator prerequisite does not surface as
|
|
# provider retries during the final, most expensive example.
|
|
require_rds_image() {
|
|
local image_ids
|
|
if ! image_ids=$(aws ec2 describe-images \
|
|
--filters \
|
|
'Name=tag:spinifex:managed-by,Values=rds' \
|
|
'Name=tag:engine,Values=postgres' \
|
|
'Name=tag:engine-version,Values=18' \
|
|
--query 'Images[].ImageId' --output text); then
|
|
log "RDS AMI preflight: describe-images failed"
|
|
return 1
|
|
fi
|
|
if [ -z "$image_ids" ] || [ "$image_ids" = "None" ]; then
|
|
log "RDS AMI preflight: spinifex-rds-postgres is not registered"
|
|
log "Run: spx admin images import --name spinifex-rds-postgres"
|
|
return 1
|
|
fi
|
|
log "RDS AMI preflight: found ${image_ids}"
|
|
}
|
|
|
|
# Pick an instance type available on this cluster. Workbooks default to
|
|
# t3.small (Intel); on AMD-only hosts t3 isn't registered, so we query and
|
|
# fall back to the smallest type meeting the floor (matches the nginx-alb
|
|
# README's documented approach). Defaults to 2 vCPU / 1 GiB; callers that need
|
|
# more headroom, such as an EKS worker, pass their own.
|
|
detect_instance_type() {
|
|
local vcpus="${1:-2}" mib="${2:-1024}"
|
|
local endpoint="https://${WAN_IP}:9999"
|
|
local picked
|
|
picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \
|
|
--query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus>=\`${vcpus}\` && MemoryInfo.SizeInMiB>=\`${mib}\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
|
|
--output text 2>/dev/null || true)
|
|
if [ -z "$picked" ] || [ "$picked" = "None" ]; then
|
|
picked=$(aws --endpoint-url "$endpoint" ec2 describe-instance-types \
|
|
--query 'InstanceTypes[0].InstanceType' --output text 2>/dev/null || true)
|
|
fi
|
|
echo "$picked"
|
|
}
|
|
|
|
INSTANCE_TYPE=""
|
|
|
|
# --- Workbook driver ---
|
|
|
|
run_workbook() {
|
|
local example="$1"
|
|
local path="${WORKBOOK_DIR}/${example}"
|
|
|
|
if [ ! -d "$path" ]; then
|
|
log "SKIP ${example}: ${path} not found"
|
|
return 1
|
|
fi
|
|
|
|
CURRENT_WORKBOOK="$example"
|
|
log "=== ${example} ==="
|
|
cd "$path"
|
|
rm -rf .terraform terraform.tfstate* .terraform.lock.hcl
|
|
|
|
# EKS workers run a kubelet, a container runtime and the demo pods, so the
|
|
# smallest type the cluster offers is not enough; the workbook's own default
|
|
# is t3.medium and 4 GiB is the floor that boots reliably.
|
|
local instance_type="$INSTANCE_TYPE"
|
|
case "$example" in
|
|
eks-*) instance_type=$(detect_instance_type 2 4096) ;;
|
|
esac
|
|
|
|
# Workbooks name the worker size differently — the EKS ones that split
|
|
# public and private subnets call it node_instance_type. Read the name from
|
|
# the config rather than keeping a list here: tofu rejects a -var for an
|
|
# undeclared variable outright, so a stale list fails the apply in seconds
|
|
# and reads like a product defect.
|
|
local size_var=instance_type
|
|
grep -qE '^variable "node_instance_type"' main.tf 2>/dev/null && size_var=node_instance_type
|
|
|
|
local apply_args=(
|
|
-input=false -no-color
|
|
"-var=spinifex_endpoint=https://${WAN_IP}:9999"
|
|
"-var=${size_var}=${instance_type}"
|
|
)
|
|
|
|
# ecs-quickstart's gateway_url is required and has no default: the container
|
|
# instances read it from their own user-data, so it must be the address a
|
|
# guest can reach, never 127.0.0.1.
|
|
if [ "$example" = "ecs-quickstart" ]; then
|
|
apply_args+=("-var=gateway_url=https://${WAN_IP}:9999")
|
|
fi
|
|
|
|
# s3-webapp has three required-no-default vars. The s3_access/secret keys are
|
|
# the operator creds the provider uses to create the bucket + IAM role on
|
|
# predastore; the instance authenticates to S3 via IMDS, not these keys.
|
|
if [ "$example" = "s3-webapp" ]; then
|
|
local access_key secret_key
|
|
access_key=$(aws configure get aws_access_key_id --profile spinifex)
|
|
secret_key=$(aws configure get aws_secret_access_key --profile spinifex)
|
|
apply_args+=(
|
|
"-var=predastore_endpoint=https://${WAN_IP}:8443"
|
|
"-var=predastore_host=${WAN_IP}:8443"
|
|
"-var=s3_access_key=${access_key}"
|
|
"-var=s3_secret_key=${secret_key}"
|
|
)
|
|
fi
|
|
|
|
if ! tofu init -input=false -no-color; then
|
|
log " FAIL ${example}: tofu init"
|
|
return 1
|
|
fi
|
|
|
|
if ! tofu apply -auto-approve "${apply_args[@]}"; then
|
|
log " FAIL ${example}: tofu apply"
|
|
# Bounded like the teardown below: this one only tidies up after a
|
|
# workbook that has already failed, so it must not hold the suite open.
|
|
timeout "${DESTROY_TIMEOUT:-900}" tofu destroy -auto-approve "${apply_args[@]}" >/dev/null 2>&1 || true
|
|
return 1
|
|
fi
|
|
|
|
local rc=0 destroy_log
|
|
destroy_log=$(mktemp)
|
|
if ! assert_"${example//-/_}" || ! assert_clean_plan "$example" "${apply_args[@]}"; then
|
|
log " FAIL ${example}: assertion"
|
|
# Capture OVN state while VMs still exist — the EXIT trap fires after
|
|
# destroy, by which point port groups, address sets, and ACLs are gone.
|
|
dump_ovn_state "post-fail ${example} (pre-destroy)"
|
|
rc=1
|
|
fi
|
|
|
|
# The teardown is part of what the workbook proves: a customer who cannot
|
|
# remove what they created is as stuck as one who could not create it. It
|
|
# was a WARN, and a bucket that could never be emptied passed for weeks.
|
|
#
|
|
# Bounded because the failure mode is a wedge rather than an error — the
|
|
# provider retries DeleteBucket against a bucket it cannot empty forever.
|
|
if ! timeout "${DESTROY_TIMEOUT:-900}" tofu destroy -auto-approve "${apply_args[@]}" > "$destroy_log" 2>&1; then
|
|
log " FAIL ${example}: tofu destroy"
|
|
sed 's|^| |' "$destroy_log" | tail -40
|
|
rc=1
|
|
fi
|
|
rm -f "$destroy_log"
|
|
|
|
# After the teardown, not before it. The verdict now depends on it, so a
|
|
# workbook that printed PASS and then failed to delete what it built would
|
|
# have reported both.
|
|
if [ "$rc" -eq 0 ]; then
|
|
log " PASS ${example}"
|
|
fi
|
|
|
|
cd "$SCRIPT_DIR"
|
|
return "$rc"
|
|
}
|
|
|
|
# --- Main ---
|
|
|
|
install_tofu || { log "tofu install failed"; exit 1; }
|
|
require_rds_image || { log "required RDS AMI is unavailable"; exit 1; }
|
|
|
|
INSTANCE_TYPE=$(detect_instance_type)
|
|
if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then
|
|
log "no instance type available from describe-instance-types"
|
|
exit 1
|
|
fi
|
|
log "Using instance_type=${INSTANCE_TYPE}"
|
|
|
|
# Each workbook is emitted as a top-level `go test -v` testcase
|
|
# (=== RUN / --- PASS|FAIL / package trailer) so go-junit-report converts the
|
|
# tee'd log into junit-tofu.xml and the e2e-analyze action produces the same
|
|
# RCA bundle the Go suites do (nightly cell 17). The per-workbook log() output
|
|
# between RUN and the result line is captured as the failure diagnostics.
|
|
#
|
|
# rds-quickstart runs last because it is the most expensive: its apply carries a
|
|
# DB VM boot, initdb and the first healthy agent heartbeat before Terraform even
|
|
# launches the client, then the client's own boot and an apt-get. Budget it at
|
|
# ~15 minutes against the ~7 the other four share, and note that its assertion
|
|
# needs roughly 2 GiB of free guest memory for the two VMs.
|
|
#
|
|
# WORKBOOKS selects the list. A failure no longer aborts the rest: s3-webapp
|
|
# failing says nothing about whether RDS works, and four nights of rds-quickstart
|
|
# coverage were lost that way. STOP_ON_FAIL=1 restores the early abort for a
|
|
# cluster you already know is broken and do not want to spend an hour on.
|
|
SUITE_RC=0
|
|
for workbook in ${WORKBOOKS:-nginx-alb bastion-private-subnet nginx-webserver s3-webapp rds-quickstart}; do
|
|
tname="TestTofuWorkbook_${workbook//-/_}"
|
|
wb_start=$SECONDS
|
|
echo "=== RUN ${tname}"
|
|
if run_workbook "$workbook"; then
|
|
printf -- '--- PASS: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))"
|
|
else
|
|
printf -- '--- FAIL: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))"
|
|
SUITE_RC=1
|
|
if [ "${STOP_ON_FAIL:-0}" = "1" ]; then
|
|
log "FAIL ${workbook} — aborting remaining workbooks (STOP_ON_FAIL=1)"
|
|
break
|
|
fi
|
|
log "FAIL ${workbook} — continuing with the remaining workbooks"
|
|
fi
|
|
done
|
|
|
|
CURRENT_WORKBOOK=""
|
|
if [ "$SUITE_RC" -eq 0 ]; then
|
|
log "All workbooks passed"
|
|
echo "PASS"
|
|
printf 'ok \ttofu-examples\t%d.000s\n' "$SECONDS"
|
|
else
|
|
echo "FAIL"
|
|
printf 'FAIL\ttofu-examples\t%d.000s\n' "$SECONDS"
|
|
fi
|
|
exit "$SUITE_RC"
|