|
|
|
@@ -0,0 +1,798 @@
|
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
# deploy-latitude.sh — build a one-node Spinifex deployment on latitude.sh bare metal, or tear it down.
|
|
|
|
|
#
|
|
|
|
|
# deploy-latitude.sh up one m4.metal.medium with an Elastic IP
|
|
|
|
|
# deploy-latitude.sh status nodes, addresses, what the deployment holds, steps done
|
|
|
|
|
# deploy-latitude.sh credentials print the AWS access key and secret
|
|
|
|
|
# deploy-latitude.sh ssh [nodeN] [command] SSH to a node, node1 by default
|
|
|
|
|
# deploy-latitude.sh destroy
|
|
|
|
|
# deploy-latitude.sh resolve-uncertain clear the uncertain-create record after checking the dashboard
|
|
|
|
|
#
|
|
|
|
|
# Run it on your laptop with LATITUDESH_AUTH_TOKEN exported. OpenTofu, or Terraform when OpenTofu is
|
|
|
|
|
# not installed, creates the project (unless --project), SSH key, user data, server and Elastic IP.
|
|
|
|
|
# This script then works over SSH as the ubuntu user: it formats and mounts the blank data disk,
|
|
|
|
|
# and drives install-node.sh (install, ovn, firewall arm, start, verify). Each step is recorded
|
|
|
|
|
# under the state directory, so a rerun of up resumes at the step that failed.
|
|
|
|
|
#
|
|
|
|
|
# The token is read by the Terraform provider and by this script's own API calls from the
|
|
|
|
|
# environment. It is never a Terraform variable, never on a command line (curl reads it from stdin),
|
|
|
|
|
# and never sent to the node, so it is in no state file, log or user data.
|
|
|
|
|
#
|
|
|
|
|
# The AWS profile is "spinifex-latitude-<name>" (default name "default"). An existing profile of that
|
|
|
|
|
# name that this script did not write is refused unless --overwrite-profile is given.
|
|
|
|
|
|
|
|
|
|
set -euo pipefail
|
|
|
|
|
shopt -s inherit_errexit
|
|
|
|
|
# A provider trace log would carry the Authorization header.
|
|
|
|
|
unset TF_LOG TF_LOG_PATH TF_LOG_CORE TF_LOG_PROVIDER
|
|
|
|
|
|
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
|
SPINIFEX_SRC="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|
|
|
|
LAT_API="https://api.latitude.sh"
|
|
|
|
|
|
|
|
|
|
CMD="${1:-}"
|
|
|
|
|
if [ $# -gt 0 ]; then shift; fi
|
|
|
|
|
NODE_ARG=""
|
|
|
|
|
NAME="default"
|
|
|
|
|
SITE="DAL"
|
|
|
|
|
PLAN="m4-metal-medium"
|
|
|
|
|
OS="ubuntu_26_04_x64_lts"
|
|
|
|
|
PROJECT=""
|
|
|
|
|
DISTRO=""
|
|
|
|
|
CHANNEL="latest"
|
|
|
|
|
VERSION=""
|
|
|
|
|
SPX_REGION="us-east-1"
|
|
|
|
|
SPX_AZ="us-east-1a"
|
|
|
|
|
SSH_KEY="$HOME/.ssh/id_ed25519"
|
|
|
|
|
IMAGE="ubuntu-26.04-x86_64"
|
|
|
|
|
DATA_DISK_ID=""
|
|
|
|
|
STATE_ROOT="$HOME/.spinifex/latitude"
|
|
|
|
|
NODES=1
|
|
|
|
|
YES=0
|
|
|
|
|
OVERWRITE_PROFILE=0
|
|
|
|
|
|
|
|
|
|
usage() {
|
|
|
|
|
sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//'
|
|
|
|
|
cat <<EOF
|
|
|
|
|
|
|
|
|
|
Options:
|
|
|
|
|
--name NAME deployment name (default $NAME), 1-16 characters; names every resource
|
|
|
|
|
--site SLUG latitude.sh site (default $SITE); needs stock and Elastic IPs, not SAO
|
|
|
|
|
--plan SLUG server plan (default $PLAN)
|
|
|
|
|
--os SLUG operating system slug (default $OS, unverified)
|
|
|
|
|
--project ID|SLUG deploy into this existing project; by default one is created and removed on destroy
|
|
|
|
|
--distro TARBALL install this local release tarball
|
|
|
|
|
--channel CHANNEL install from a release channel: latest (default) or dev
|
|
|
|
|
--version TAG install this release instead of a channel
|
|
|
|
|
--spx-region NAME Spinifex region (default $SPX_REGION)
|
|
|
|
|
--spx-az NAME Spinifex availability zone (default $SPX_AZ)
|
|
|
|
|
--ssh-key PATH private key; PATH.pub is installed (default $SSH_KEY)
|
|
|
|
|
--image NAME AMI to import (default $IMAGE)
|
|
|
|
|
--no-image import no AMI
|
|
|
|
|
--data-disk-id PATH choose the data disk by its /dev/disk/by-id/nvme-eui.* path when several are blank
|
|
|
|
|
--nodes N must be 1 for now
|
|
|
|
|
--overwrite-profile replace an AWS profile of the same name that this script did not create
|
|
|
|
|
--yes destroy without asking
|
|
|
|
|
EOF
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
while [ $# -gt 0 ]; do
|
|
|
|
|
case "$1" in
|
|
|
|
|
--name) NAME="$2"; shift ;;
|
|
|
|
|
--site) SITE="$2"; shift ;;
|
|
|
|
|
--plan) PLAN="$2"; shift ;;
|
|
|
|
|
--os) OS="$2"; shift ;;
|
|
|
|
|
--project) PROJECT="$2"; shift ;;
|
|
|
|
|
--distro) DISTRO="$2"; shift ;;
|
|
|
|
|
--channel) CHANNEL="$2"; shift ;;
|
|
|
|
|
--version) VERSION="$2"; shift ;;
|
|
|
|
|
--spx-region) SPX_REGION="$2"; shift ;;
|
|
|
|
|
--spx-az) SPX_AZ="$2"; shift ;;
|
|
|
|
|
--ssh-key) SSH_KEY="$2"; shift ;;
|
|
|
|
|
--image) IMAGE="$2"; shift ;;
|
|
|
|
|
--no-image) IMAGE="" ;;
|
|
|
|
|
--data-disk-id) DATA_DISK_ID="$2"; shift ;;
|
|
|
|
|
--nodes) NODES="$2"; shift ;;
|
|
|
|
|
--overwrite-profile) OVERWRITE_PROFILE=1 ;;
|
|
|
|
|
--yes) YES=1 ;;
|
|
|
|
|
-h|--help) usage; exit 0 ;;
|
|
|
|
|
node[1-3])
|
|
|
|
|
case "$CMD" in ssh) ;; *) usage >&2; exit 2 ;; esac
|
|
|
|
|
[ -z "$NODE_ARG" ] || { usage >&2; exit 2; }
|
|
|
|
|
NODE_ARG="$1" ;;
|
|
|
|
|
*) if [ "$CMD" = ssh ]; then break; fi
|
|
|
|
|
echo "unknown option: $1" >&2; usage >&2; exit 2 ;;
|
|
|
|
|
esac
|
|
|
|
|
shift
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
log() { printf '[deploy %s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
|
|
|
|
|
die() { log "ERROR: $*"; exit 1; }
|
|
|
|
|
|
|
|
|
|
[[ "$NAME" =~ ^[a-z0-9][a-z0-9-]{0,15}$ ]] || die "--name must be 1-16 lowercase letters, digits or hyphens"
|
|
|
|
|
[ "$NODES" = 1 ] || die "--nodes must be 1: three nodes need the VLAN and the Elastic IP move, which are not built yet"
|
|
|
|
|
[[ "$SITE" =~ ^[A-Za-z0-9]+$ ]] || die "--site must be a site slug such as DAL"
|
|
|
|
|
[[ "$SPX_REGION" =~ ^[a-z0-9-]+$ ]] || die "--spx-region must be lowercase letters, digits and hyphens"
|
|
|
|
|
[[ "$SPX_AZ" =~ ^[a-z0-9-]+$ ]] || die "--spx-az must be lowercase letters, digits and hyphens"
|
|
|
|
|
[[ "$CHANNEL" =~ ^[a-z0-9.-]+$ ]] || die "--channel must be lowercase letters, digits, dots and hyphens"
|
|
|
|
|
[ -z "$VERSION" ] || [[ "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]] || die "--version has characters a release tag does not"
|
|
|
|
|
[ -z "$IMAGE" ] || [[ "$IMAGE" =~ ^[A-Za-z0-9._-]+$ ]] || die "--image has characters an image name does not"
|
|
|
|
|
[ -z "$DATA_DISK_ID" ] || [[ "$DATA_DISK_ID" =~ ^/dev/disk/by-id/nvme-eui\.[A-Za-z0-9]+$ ]] || die "--data-disk-id must look like /dev/disk/by-id/nvme-eui.<hex>"
|
|
|
|
|
case "$CMD" in up|destroy|status) [ -n "${LATITUDESH_AUTH_TOKEN:-}" ] || die "LATITUDESH_AUTH_TOKEN is not set" ;; esac
|
|
|
|
|
if command -v tofu >/dev/null; then TF_BIN=tofu
|
|
|
|
|
elif command -v terraform >/dev/null; then TF_BIN=terraform
|
|
|
|
|
else die "neither OpenTofu (tofu) nor Terraform is installed"; fi
|
|
|
|
|
|
|
|
|
|
STATE_DIR="$STATE_ROOT/$NAME"
|
|
|
|
|
TF_STATE="$STATE_DIR/terraform.tfstate"
|
|
|
|
|
KNOWN_HOSTS="$STATE_DIR/known_hosts"
|
|
|
|
|
INPUTS="$STATE_DIR/inputs"
|
|
|
|
|
OWNED="$STATE_DIR/owned"
|
|
|
|
|
# PENDING exists while an apply runs. If the apply fails or is interrupted it is appended to UNCERTAIN, which
|
|
|
|
|
# only resolve-uncertain removes: a create that reached latitude.sh but not the state cannot be attributed,
|
|
|
|
|
# so destroy gives no all-clear while UNCERTAIN exists.
|
|
|
|
|
UNCERTAIN="$STATE_DIR/uncertain"
|
|
|
|
|
PENDING="$STATE_DIR/uncertain.pending"
|
|
|
|
|
HOST_PREFIX="spx-$NAME-node"
|
|
|
|
|
PROFILE="spinifex-latitude-$NAME"
|
|
|
|
|
CA_FILE="$HOME/.spinifex/latitude-$NAME-ca.pem"
|
|
|
|
|
SSH_OPTS=(-i "$SSH_KEY" -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=10
|
|
|
|
|
-o StrictHostKeyChecking=accept-new -o "UserKnownHostsFile=$KNOWN_HOSTS")
|
|
|
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
|
chmod 0700 "$STATE_DIR"
|
|
|
|
|
|
|
|
|
|
# After the first up, destroy, status and ssh use the project choice that up recorded.
|
|
|
|
|
if [ -z "$PROJECT" ] && [ -f "$INPUTS" ]; then PROJECT="$(sed -n 's/^project=//p' "$INPUTS")"; fi
|
|
|
|
|
|
|
|
|
|
# latitude METHOD PATH prints the response body and fails on a non-2xx. The token goes to curl on
|
|
|
|
|
# stdin as a config line, so it is on no command line; printf is a shell builtin.
|
|
|
|
|
latitude() {
|
|
|
|
|
local out code
|
|
|
|
|
out="$(printf 'header = "Authorization: Bearer %s"\n' "$LATITUDESH_AUTH_TOKEN" |
|
|
|
|
|
curl -sS -g -K - -w '\n%{http_code}' -X "$1" -H 'Accept: application/json' "$LAT_API$2")" || return 1
|
|
|
|
|
code="${out##*$'\n'}"
|
|
|
|
|
out="${out%$'\n'*}"
|
|
|
|
|
case "$code" in
|
|
|
|
|
2??) printf '%s' "$out" ;;
|
|
|
|
|
*) log "latitude.sh $1 $2 returned $code: $(printf '%s' "$out" | head -c 400)"; return 1 ;;
|
|
|
|
|
esac
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# lat_list PATH [QUERY] prints a JSON array of every item of a paginated list. It follows links.next
|
|
|
|
|
# until it is null; where the body gives no next key (the documented API has no pagination metadata, only page[size] and page[number]) it reads page numbers until an empty page, so a server-capped page cannot pass as the last. It fails on
|
|
|
|
|
# an API error, on a body that is not {"data": [objects with an id and attributes]}, on a next link to
|
|
|
|
|
# another host, on a page that repeats an earlier one, and past 200 pages.
|
|
|
|
|
lat_list() {
|
|
|
|
|
local path="$1?${2:+$2&}page[size]=100&page[number]=1" out info n next page=1 all='[]'
|
|
|
|
|
while :; do
|
|
|
|
|
out="$(latitude GET "$path")" || return 1
|
|
|
|
|
info="$(python3 -I -c '
|
|
|
|
|
import json,sys
|
|
|
|
|
d = json.load(sys.stdin)
|
|
|
|
|
data = d.get("data") if isinstance(d, dict) else None
|
|
|
|
|
if not isinstance(data, list) or not all(isinstance(i, dict) and isinstance(i.get("id"), str) and isinstance(i.get("attributes"), dict) for i in data):
|
|
|
|
|
sys.exit("unexpected list schema")
|
|
|
|
|
links = d.get("links")
|
|
|
|
|
nxt = "?" if not isinstance(links, dict) or "next" not in links else (links["next"] or "-")
|
|
|
|
|
print(len(data)); print(nxt)' <<<"$out")" || { log "latitude.sh list $1 returned an unexpected body"; return 1; }
|
|
|
|
|
n="$(sed -n 1p <<<"$info")" next="$(sed -n 2p <<<"$info")"
|
|
|
|
|
all="$(python3 -I -c '
|
|
|
|
|
import json,sys
|
|
|
|
|
a = json.loads(sys.argv[1]); new = json.load(sys.stdin)["data"]
|
|
|
|
|
if new and any(i["id"] == new[0]["id"] for i in a): sys.exit("page repeats")
|
|
|
|
|
print(json.dumps(a + new))' "$all" <<<"$out")" || { log "latitude.sh list $1 does not advance between pages"; return 1; }
|
|
|
|
|
page=$((page + 1))
|
|
|
|
|
[ "$page" -le 200 ] || { log "latitude.sh list $1 has more than 200 pages"; return 1; }
|
|
|
|
|
case "$next" in
|
|
|
|
|
-) break ;;
|
|
|
|
|
"?") if [ "$n" = 0 ]; then break; fi
|
|
|
|
|
path="$1?${2:+$2&}page[size]=100&page[number]=$page" ;;
|
|
|
|
|
"$LAT_API"/*) path="${next#"$LAT_API"}" ;;
|
|
|
|
|
/*) path="$next" ;;
|
|
|
|
|
*) log "latitude.sh list $1 gave a next link to another host"; return 1 ;;
|
|
|
|
|
esac
|
|
|
|
|
done
|
|
|
|
|
printf '%s' "$all"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
tf() {
|
|
|
|
|
"$TF_BIN" -chdir="$SCRIPT_DIR" "$@" -state="$TF_STATE" \
|
|
|
|
|
-var "deployment_name=$NAME" -var "site=$SITE" -var "plan=$PLAN" -var "operating_system=$OS" \
|
|
|
|
|
-var "project=$PROJECT" -var "ssh_public_key_path=$SSH_KEY.pub" -var "node_count=$NODES"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
done_step() { [ -f "$STATE_DIR/step.$1" ]; }
|
|
|
|
|
mark_step() { date -u +%FT%TZ > "$STATE_DIR/step.$1"; }
|
|
|
|
|
|
|
|
|
|
# node_field FIELD prints FIELD of every node, in node order.
|
|
|
|
|
node_field() {
|
|
|
|
|
"$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -json nodes |
|
|
|
|
|
python3 -c 'import json,sys; [print(n[sys.argv[1]]) for n in json.load(sys.stdin)]' "$1"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# for_nodes_always FN runs FN on every node and records nothing.
|
|
|
|
|
for_nodes_always() {
|
|
|
|
|
local fn="$1" i
|
|
|
|
|
for i in "${!IPS[@]}"; do
|
|
|
|
|
IP="${IPS[$i]}" SERVER_ID="${IDS[$i]}" NODE_NAME="node$((i + 1))"
|
|
|
|
|
"$fn"
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# for_nodes STEP FN runs FN once per node with IP, SERVER_ID and NODE_NAME set, recording
|
|
|
|
|
# STEP.<node> so a rerun skips the nodes already done.
|
|
|
|
|
for_nodes() {
|
|
|
|
|
local step="$1" fn="$2" i
|
|
|
|
|
for i in "${!IPS[@]}"; do
|
|
|
|
|
IP="${IPS[$i]}" SERVER_ID="${IDS[$i]}" NODE_NAME="node$((i + 1))"
|
|
|
|
|
if done_step "$step.$NODE_NAME"; then continue; fi
|
|
|
|
|
"$fn"
|
|
|
|
|
mark_step "$step.$NODE_NAME"
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# on_tty needs -tt before the destination so passwd gets a terminal.
|
|
|
|
|
on_tty() { ssh -tt "${SSH_OPTS[@]}" "ubuntu@$IP" "$@"; }
|
|
|
|
|
# shellcheck disable=SC2029 # the command is meant to expand here
|
|
|
|
|
on() { ssh "${SSH_OPTS[@]}" "ubuntu@$IP" "$@"; }
|
|
|
|
|
|
|
|
|
|
# acquire_lock stops two runs on one deployment, so resolve-uncertain cannot clear a record while an
|
|
|
|
|
# apply is writing it. mkdir is atomic and works on macOS, which has no flock; a lock whose pid is gone is
|
|
|
|
|
# stale and is taken over. Holding the lock means no apply is running, so a pending entry left by an
|
|
|
|
|
# interrupted one is made permanent here.
|
|
|
|
|
acquire_lock() {
|
|
|
|
|
local lock="$STATE_DIR/lock.d" pid
|
|
|
|
|
if ! mkdir "$lock" 2>/dev/null; then
|
|
|
|
|
pid="$(cat "$lock/pid" 2>/dev/null || true)"
|
|
|
|
|
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then die "another run (pid $pid) is working on deployment '$NAME'"; fi
|
|
|
|
|
# A lock with no pid yet may be a run that has just made it: stale only once it is a minute old.
|
|
|
|
|
if [ -z "$pid" ] && [ -z "$(find "$lock" -maxdepth 0 -mmin +1 2>/dev/null)" ]; then die "another run is starting on deployment '$NAME'"; fi
|
|
|
|
|
rm -rf "$lock"
|
|
|
|
|
mkdir "$lock" || die "could not take the lock $lock"
|
|
|
|
|
fi
|
|
|
|
|
printf '%s\n' "$$" > "$lock/pid"
|
|
|
|
|
trap 'rm -rf "$STATE_DIR/lock.d"' EXIT
|
|
|
|
|
if [ -f "$PENDING" ]; then cat "$PENDING" >> "$UNCERTAIN"; rm -f "$PENDING"; fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# ── up ───────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
record_inputs() {
|
|
|
|
|
local now
|
|
|
|
|
now="$(printf 'site=%s\nplan=%s\nos=%s\nproject=%s\nnodes=%s' "$SITE" "$PLAN" "$OS" "$PROJECT" "$NODES")"
|
|
|
|
|
if [ -f "$INPUTS" ]; then
|
|
|
|
|
[ "$(cat "$INPUTS")" = "$now" ] || die "inputs differ from the first up of '$NAME' (recorded in $INPUTS); use the same inputs, or destroy it first"
|
|
|
|
|
else
|
|
|
|
|
printf '%s\n' "$now" > "$INPUTS"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
preflight() {
|
|
|
|
|
[ -r "$SSH_KEY" ] || die "SSH key $SSH_KEY not readable"
|
|
|
|
|
[ -r "$SSH_KEY.pub" ] || die "SSH key $SSH_KEY.pub not readable"
|
|
|
|
|
command -v aws >/dev/null || die "the AWS CLI is not installed"
|
|
|
|
|
command -v python3 >/dev/null || die "python3 is not installed"
|
|
|
|
|
if [ -n "$DISTRO" ]; then [ -r "$DISTRO" ] || die "--distro $DISTRO not readable"; fi
|
|
|
|
|
latitude GET '/projects?page[size]=1' >/dev/null || die "the API refused the token (LATITUDESH_AUTH_TOKEN)"
|
|
|
|
|
if [ -n "$PROJECT" ]; then
|
|
|
|
|
lat_list /projects "filter[slug]=$PROJECT" | python3 -c '
|
|
|
|
|
import json,sys
|
|
|
|
|
sys.exit(0 if any(i["attributes"]["slug"] == sys.argv[1] for i in json.load(sys.stdin)) else 1)' "$PROJECT" ||
|
|
|
|
|
die "no project with slug $PROJECT; use its slug, or leave --project out to create one"
|
|
|
|
|
fi
|
|
|
|
|
record_inputs
|
|
|
|
|
# A fresh deployment refuses an existing profile now; a rerun is checked again, with the real values,
|
|
|
|
|
# right before the profile is written.
|
|
|
|
|
if [ ! -f "$TF_STATE" ] && [ "$OVERWRITE_PROFILE" != 1 ] && [ "$(profile_state)" != none ]; then
|
|
|
|
|
die "AWS profile $PROFILE already has a section in $(AWS_CRED_FILE) or $(AWS_CONF_FILE); pass --overwrite-profile to replace it (both files are saved first), or use another --name"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
provision() {
|
|
|
|
|
log "terraform apply ($PLAN at $SITE)"
|
|
|
|
|
"$TF_BIN" -chdir="$SCRIPT_DIR" init -input=false >/dev/null
|
|
|
|
|
# A pending entry left by an interrupted apply becomes a permanent one. Entries are only appended.
|
|
|
|
|
if [ -f "$PENDING" ]; then cat "$PENDING" >> "$UNCERTAIN"; rm -f "$PENDING"; fi
|
|
|
|
|
if [ -f "$UNCERTAIN" ]; then
|
|
|
|
|
log "an earlier create failed or was interrupted ($UNCERTAIN); destroy gives no all-clear until you check the dashboard and run: $0 resolve-uncertain --name $NAME"
|
|
|
|
|
fi
|
|
|
|
|
date -u +%FT%TZ > "$PENDING"
|
|
|
|
|
tf apply -input=false -auto-approve >"$STATE_DIR/apply.log" 2>&1 || {
|
|
|
|
|
tail -20 "$STATE_DIR/apply.log" >&2
|
|
|
|
|
{ cat "$PENDING" 2>/dev/null || true; date -u +%FT%TZ; } >> "$UNCERTAIN"
|
|
|
|
|
rm -f "$PENDING"
|
|
|
|
|
record_owned || true
|
|
|
|
|
die "terraform apply failed; log in $STATE_DIR/apply.log. What was created is recorded in $OWNED and destroy removes it. A plan with no stock at the site fails here: try another --site"
|
|
|
|
|
}
|
|
|
|
|
record_owned
|
|
|
|
|
rm -f "$PENDING"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# record_owned adds the IDs of every latitude.sh object in the Terraform state to $OWNED, as
|
|
|
|
|
# "kind id label". Terraform writes state as each object is created, so this also covers a partly
|
|
|
|
|
# failed apply. destroy deletes only what is listed here.
|
|
|
|
|
record_owned() {
|
|
|
|
|
[ -f "$TF_STATE" ] || return 0
|
|
|
|
|
local out
|
|
|
|
|
out="$("$TF_BIN" -chdir="$SCRIPT_DIR" show -json "$TF_STATE" | python3 -I -c '
|
|
|
|
|
import json,sys
|
|
|
|
|
kinds = {"latitudesh_server": "server", "latitudesh_elastic_ip": "elastic-ip", "latitudesh_ssh_key": "ssh-key",
|
|
|
|
|
"latitudesh_user_data": "user-data", "latitudesh_project": "project"}
|
|
|
|
|
st = json.load(sys.stdin)
|
|
|
|
|
root = (st.get("values") or {}).get("root_module") or {}
|
|
|
|
|
for r in root.get("resources", []):
|
|
|
|
|
k = kinds.get(r.get("type"))
|
|
|
|
|
v = r.get("values") or {}
|
|
|
|
|
if k and v.get("id"):
|
|
|
|
|
print(k, v["id"], v.get("hostname") or v.get("address") or v.get("name") or v.get("description") or "-")')" ||
|
|
|
|
|
die "could not read the Terraform state to record what exists"
|
|
|
|
|
{ [ ! -f "$OWNED" ] || cat "$OWNED"; printf '%s\n' "$out"; } | sed '/^$/d' | sort -u > "$OWNED.new"
|
|
|
|
|
chmod 0600 "$OWNED.new"
|
|
|
|
|
mv "$OWNED.new" "$OWNED"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# wait_host pins the host key only once the marker cloud-init writes exists.
|
|
|
|
|
wait_host() {
|
|
|
|
|
log "waiting for SSH and cloud-init on $NODE_NAME"
|
|
|
|
|
local i
|
|
|
|
|
for i in $(seq 1 120); do
|
|
|
|
|
if ssh -i "$SSH_KEY" -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no \
|
|
|
|
|
-o UserKnownHostsFile=/dev/null -o LogLevel=ERROR "ubuntu@$IP" \
|
|
|
|
|
'test -f /var/lib/cloud/spinifex-host-ready' 2>/dev/null; then break; fi
|
|
|
|
|
[ "$i" != 120 ] || die "host $IP was not ready after 20 minutes"
|
|
|
|
|
sleep 10
|
|
|
|
|
done
|
|
|
|
|
ssh-keygen -R "$IP" -f "$KNOWN_HOSTS" >/dev/null 2>&1 || true
|
|
|
|
|
on 'cloud-init status --wait >/dev/null; test -e /dev/kvm' || die "no /dev/kvm on $NODE_NAME"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# host_prep is the first change to the node, so it runs after the console gate. The image's resolv.conf
|
|
|
|
|
# is not a link to systemd-resolved, so the node would bypass the split DNS Spinifex sets there for its
|
|
|
|
|
# load balancer and EKS names.
|
|
|
|
|
host_prep() {
|
|
|
|
|
on 'if [ ! -L /etc/resolv.conf ] && systemctl is-active -q systemd-resolved; then
|
|
|
|
|
sudo ln -sf ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf; fi' ||
|
|
|
|
|
die "could not point /etc/resolv.conf at systemd-resolved on $NODE_NAME"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# confirm_if_pending asks for the console only when a step that changes the node is still to run.
|
|
|
|
|
confirm_if_pending() {
|
|
|
|
|
local st
|
|
|
|
|
for st in prep disk install ovn firewall start; do
|
|
|
|
|
if ! done_step "$st.$NODE_NAME"; then console_gate; return 0; fi
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# console_gate runs before the first change to the node's disks, network or firewall. The person sets
|
|
|
|
|
# the ubuntu password here (typed into passwd on the node, never seen by this script), then proves the
|
|
|
|
|
# serial console works from another window, because that console is the only way back in.
|
|
|
|
|
console_gate() {
|
|
|
|
|
log "console recovery check for $NODE_NAME, before any change to its disks or firewall"
|
|
|
|
|
print_recovery >&2
|
|
|
|
|
[ -r /dev/tty ] || die "no terminal: the console check needs a typed confirmation"
|
|
|
|
|
printf '\nSet the ubuntu password now (typed on the node, not seen by this script):\n' >&2
|
|
|
|
|
on_tty 'sudo passwd ubuntu' < /dev/tty || die "setting the ubuntu password failed"
|
|
|
|
|
printf '\nIn ANOTHER window run: ssh %s@%s -p 2222 and log in as ubuntu with that password.\n' "$SERVER_ID" "$IP" >&2
|
|
|
|
|
printf 'Type "console-ok" once you are logged in over the serial console: ' >&2
|
|
|
|
|
local answer
|
|
|
|
|
read -r answer < /dev/tty || die "no answer"
|
|
|
|
|
[ "$answer" = console-ok ] || die "not confirmed; nothing has been changed on $NODE_NAME. Rerun up once the console works"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# data_disk formats the blank data disk, chosen by content on the node, and records its UUID.
|
|
|
|
|
data_disk() {
|
|
|
|
|
log "preparing the data disk on $NODE_NAME"
|
|
|
|
|
local tmp rc=0 uuid
|
|
|
|
|
scp -q "${SSH_OPTS[@]}" "$SCRIPT_DIR/host-data-disk.sh" "ubuntu@$IP:/tmp/host-data-disk.sh" || die "copy to $NODE_NAME failed"
|
|
|
|
|
tmp="$(mktemp "$STATE_DIR/data-disk.XXXXXX")"
|
|
|
|
|
on "sudo bash /tmp/host-data-disk.sh $DATA_DISK_ID" >"$tmp" 2>&1 || rc=$?
|
|
|
|
|
cat "$tmp" >&2
|
|
|
|
|
cat "$tmp" >> "$STATE_DIR/data-disk.$NODE_NAME.log"
|
|
|
|
|
uuid="$(sed -n 's/^DATA_UUID=//p' "$tmp" | tail -n1)"
|
|
|
|
|
rm -f "$tmp"
|
|
|
|
|
[ "$rc" = 0 ] || die "the data disk step failed (exit $rc); it may already have formatted the disk, so read the output above and $STATE_DIR/data-disk.$NODE_NAME.log before rerunning"
|
|
|
|
|
[[ "$uuid" =~ ^[0-9a-fA-F-]{16,36}$ ]] || die "the data disk step gave no UUID"
|
|
|
|
|
printf '%s\n' "$uuid" > "$STATE_DIR/data_uuid.$NODE_NAME"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# push_installer copies install-node.sh (and a local tarball with its setup.sh) to the node and
|
|
|
|
|
# proves the installer arrived intact.
|
|
|
|
|
push_installer() {
|
|
|
|
|
local want got
|
|
|
|
|
scp -q "${SSH_OPTS[@]}" "$SCRIPT_DIR/install-node.sh" "ubuntu@$IP:/tmp/install-node.sh" || die "copy to $NODE_NAME failed"
|
|
|
|
|
want="$(sha256sum "$SCRIPT_DIR/install-node.sh" | awk '{print $1}')"
|
|
|
|
|
got="$(on 'sha256sum /tmp/install-node.sh' | awk '{print $1}')"
|
|
|
|
|
[ "$want" = "$got" ] || die "install-node.sh changed in transit: local $want, node $got"
|
|
|
|
|
if [ -n "$DISTRO" ]; then
|
|
|
|
|
scp -q "${SSH_OPTS[@]}" "$DISTRO" "$SPINIFEX_SRC/scripts/setup.sh" "ubuntu@$IP:/tmp/" || die "copy of the tarball to $NODE_NAME failed"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# install_args is every input install-node.sh records and a later phase must repeat.
|
|
|
|
|
install_args() {
|
|
|
|
|
local uuid eip
|
|
|
|
|
uuid="$(cat "$STATE_DIR/data_uuid.$NODE_NAME")"
|
|
|
|
|
eip="$(cat "$STATE_DIR/eip")"
|
|
|
|
|
printf -- '--eip %s --data-uuid %s --node-name %s --spx-region %s --spx-az %s --channel %s' \
|
|
|
|
|
"$eip" "$uuid" "$NODE_NAME" "$SPX_REGION" "$SPX_AZ" "$CHANNEL"
|
|
|
|
|
if [ -n "$VERSION" ]; then printf -- ' --version %s' "$VERSION"; fi
|
|
|
|
|
if [ -n "$DISTRO" ]; then printf -- ' --distro /tmp/%s --setup-script /tmp/setup.sh' "$(basename "$DISTRO")"; fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# run_phase PHASE [EXTRA...] runs one install-node.sh phase on the node as root.
|
|
|
|
|
run_phase() {
|
|
|
|
|
local phase="$1"
|
|
|
|
|
shift
|
|
|
|
|
push_installer
|
|
|
|
|
log "install-node.sh --phase $phase on $NODE_NAME"
|
|
|
|
|
on "sudo bash /tmp/install-node.sh --phase $phase $(install_args) $*" 2>&1 | tee -a "$STATE_DIR/install-node.$NODE_NAME.log" >&2 ||
|
|
|
|
|
die "install-node.sh --phase $phase failed; the output above and $STATE_DIR/install-node.$NODE_NAME.log have it, and the node has /root/<step>.log"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
phase_install() { run_phase install; }
|
|
|
|
|
phase_ovn() { run_phase ovn; }
|
|
|
|
|
|
|
|
|
|
# arm_firewall runs after the ovn phase and before Spinifex first starts, so the daemon never runs
|
|
|
|
|
# unfiltered. The daemon writes peers.nft and loads the policy during its start. The start phase
|
|
|
|
|
# refuses to go on unless the generated policy allows ssh: it reads the active rule, the defined ports
|
|
|
|
|
# against sshd -T, and the 0.0.0.0/0 trust.
|
|
|
|
|
arm_firewall() {
|
|
|
|
|
log "arming the firewall on $NODE_NAME before the first start"
|
|
|
|
|
print_recovery >&2
|
|
|
|
|
log "Do not reboot $NODE_NAME until this run has finished, or until the console disarm above has been used"
|
|
|
|
|
on 'sudo bash -c "env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on >/root/firewall.log 2>&1"' ||
|
|
|
|
|
die "arming the firewall failed; see /root/firewall.log on $NODE_NAME. Spinifex was not started"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
phase_start() {
|
|
|
|
|
if [ -n "$IMAGE" ]; then run_phase start --armed --image "$IMAGE"; else run_phase start --armed; fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
phase_verify() { run_phase verify; }
|
|
|
|
|
|
|
|
|
|
# check_access proves the firewalled node still lets a NEW ssh login and the API through from here.
|
|
|
|
|
check_access() {
|
|
|
|
|
log "checking a new ssh login and the API through the firewall"
|
|
|
|
|
local i code=""
|
|
|
|
|
ssh "${SSH_OPTS[@]}" -o ControlMaster=no -o ControlPath=none "ubuntu@$IP" true ||
|
|
|
|
|
die "a new ssh login to $NODE_NAME failed after the firewall loaded; recover from the serial console (printed at the end of a good run; also: ssh $SERVER_ID@$IP -p 2222)"
|
|
|
|
|
for i in $(seq 1 12); do
|
|
|
|
|
code="$(curl -sk -m 10 -o /dev/null -w '%{http_code}' "https://$IP:9999/" || true)"
|
|
|
|
|
case "$code" in 2??|3??|4??) break ;; esac
|
|
|
|
|
sleep 5
|
|
|
|
|
done
|
|
|
|
|
case "$code" in
|
|
|
|
|
2??|3??|4??) ;;
|
|
|
|
|
*) die "the API on https://$IP:9999 did not answer from here (last status '$code'); ssh works, so check the firewall rules for tcp 9999" ;;
|
|
|
|
|
esac
|
|
|
|
|
on 'sudo bash -c "grep -qx on /etc/spinifex/firewall/mode && nft list table inet spinifex_filter >/dev/null"' ||
|
|
|
|
|
die "the firewall is not armed (mode not on or table inet spinifex_filter missing)"
|
|
|
|
|
log "access proven with the firewall armed"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
AWS_CRED_FILE() { printf '%s' "${AWS_SHARED_CREDENTIALS_FILE:-$HOME/.aws/credentials}"; }
|
|
|
|
|
AWS_CONF_FILE() { printf '%s' "${AWS_CONFIG_FILE:-$HOME/.aws/config}"; }
|
|
|
|
|
|
|
|
|
|
# profile_state [KEY SECRET ENDPOINT] prints none when the profile has no section in either AWS file
|
|
|
|
|
# (credentials: [name], config: [profile name]) whatever its type (static keys, SSO, credential_process).
|
|
|
|
|
# With the values up would write it prints match when the sections equal exactly those, else conflict;
|
|
|
|
|
# without them it prints exists. The secret reaches python on stdin.
|
|
|
|
|
profile_state() {
|
|
|
|
|
printf '%s\n%s\n%s\n' "${1:-}" "${2:-}" "${3:-}" | python3 -I -c '
|
|
|
|
|
import configparser, os, sys
|
|
|
|
|
cred, conf, name, ca, region = sys.argv[1:6]
|
|
|
|
|
key, secret, endpoint = sys.stdin.read().split("\n")[:3]
|
|
|
|
|
def sect(path, sec):
|
|
|
|
|
cp = configparser.RawConfigParser()
|
|
|
|
|
cp.optionxform = str
|
|
|
|
|
if os.path.exists(path):
|
|
|
|
|
cp.read(path)
|
|
|
|
|
return dict(cp.items(sec)) if cp.has_section(sec) else None
|
|
|
|
|
c = sect(cred, name)
|
|
|
|
|
g = sect(conf, "profile " + name)
|
|
|
|
|
if c is None and g is None:
|
|
|
|
|
print("none")
|
|
|
|
|
elif not key:
|
|
|
|
|
print("exists")
|
|
|
|
|
else:
|
|
|
|
|
want_c = {"aws_access_key_id": key, "aws_secret_access_key": secret}
|
|
|
|
|
want_g = {"region": region, "endpoint_url": endpoint, "ca_bundle": ca}
|
|
|
|
|
print("match" if c == want_c and g == want_g else "conflict")' "$(AWS_CRED_FILE)" "$(AWS_CONF_FILE)" "$PROFILE" "$CA_FILE" "$SPX_REGION"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# write_credentials KEY SECRET sets the profile's keys in the shared credentials file without putting
|
|
|
|
|
# the secret in any process argument: it reaches python on stdin. Other profiles are kept (comments in
|
|
|
|
|
# the file are not), the old file is saved beside it, and the new one is 0600.
|
|
|
|
|
write_credentials() {
|
|
|
|
|
local code
|
|
|
|
|
code='
|
|
|
|
|
import configparser, os, shutil, sys, time
|
|
|
|
|
profile = sys.argv[1]
|
|
|
|
|
key, secret = sys.stdin.read().split("\n")[:2]
|
|
|
|
|
path = os.environ.get("AWS_SHARED_CREDENTIALS_FILE") or os.path.expanduser("~/.aws/credentials")
|
|
|
|
|
os.umask(0o077)
|
|
|
|
|
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
|
|
|
|
cp = configparser.RawConfigParser()
|
|
|
|
|
cp.optionxform = str
|
|
|
|
|
if os.path.exists(path):
|
|
|
|
|
cp.read(path)
|
|
|
|
|
shutil.copy2(path, "%s.pre-latitude.%d" % (path, int(time.time())))
|
|
|
|
|
if not cp.has_section(profile):
|
|
|
|
|
cp.add_section(profile)
|
|
|
|
|
cp.set(profile, "aws_access_key_id", key)
|
|
|
|
|
cp.set(profile, "aws_secret_access_key", secret)
|
|
|
|
|
cp.remove_option(profile, "aws_session_token")
|
|
|
|
|
tmp = path + ".tmp"
|
|
|
|
|
with open(tmp, "w") as f:
|
|
|
|
|
cp.write(f)
|
|
|
|
|
os.chmod(tmp, 0o600)
|
|
|
|
|
os.replace(tmp, path)
|
|
|
|
|
'
|
|
|
|
|
printf '%s\n%s\n' "$1" "$2" | python3 -I -c "$code" "$PROFILE" || die "could not write the credentials file"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
write_profile() {
|
|
|
|
|
log "writing the CA to $CA_FILE and AWS profile $PROFILE"
|
|
|
|
|
mkdir -p "$(dirname "$CA_FILE")"
|
|
|
|
|
local creds key secret ca_new
|
|
|
|
|
# The CA is fetched to a temporary file and checked first; the existing file is replaced only after
|
|
|
|
|
# the profile check passes, with the old one kept beside it.
|
|
|
|
|
ca_new="$(mktemp "$CA_FILE.new.XXXXXX")"
|
|
|
|
|
on 'sudo cat /etc/spinifex/ca.pem' > "$ca_new" || { rm -f "$ca_new"; die "could not read the node's CA"; }
|
|
|
|
|
grep -q -- '-----BEGIN CERTIFICATE-----' "$ca_new" || { rm -f "$ca_new"; die "the node's CA is empty or not a PEM certificate"; }
|
|
|
|
|
creds="$(on 'awk "/^\[spinifex\]/{f=1;next} /^\[/{f=0} f" ~/.aws/credentials')"
|
|
|
|
|
key="$(awk -F' *= *' '/aws_access_key_id/{print $2}' <<<"$creds")"
|
|
|
|
|
secret="$(awk -F' *= *' '/aws_secret_access_key/{print $2}' <<<"$creds")"
|
|
|
|
|
[ -n "$key" ] || die "no access key in the ubuntu user's ~/.aws/credentials on $NODE_NAME"
|
|
|
|
|
[ -n "$secret" ] || die "no [spinifex] credentials in the ubuntu user's ~/.aws/credentials on $NODE_NAME"
|
|
|
|
|
# Rechecked here, with the real values, immediately before anything is written.
|
|
|
|
|
case "$(profile_state "$key" "$secret" "https://$IP:9999")" in
|
|
|
|
|
none | match) ;;
|
|
|
|
|
*) [ "$OVERWRITE_PROFILE" = 1 ] || { rm -f "$ca_new"; die "AWS profile $PROFILE has a section in $(AWS_CRED_FILE) or $(AWS_CONF_FILE) that differs from what this run would write; pass --overwrite-profile to replace it (both files are saved first), or use another --name"; } ;;
|
|
|
|
|
esac
|
|
|
|
|
if [ -f "$CA_FILE" ] && ! cmp -s "$CA_FILE" "$ca_new"; then cp -p "$CA_FILE" "$CA_FILE.pre-latitude.$(date +%s)"; fi
|
|
|
|
|
chmod 0644 "$ca_new"
|
|
|
|
|
mv -f "$ca_new" "$CA_FILE"
|
|
|
|
|
write_credentials "$key" "$secret"
|
|
|
|
|
local cfg
|
|
|
|
|
cfg="$(AWS_CONF_FILE)"
|
|
|
|
|
if [ -f "$cfg" ]; then (umask 077; cp -p "$cfg" "$cfg.pre-latitude.$(date +%s)"); fi
|
|
|
|
|
aws configure set --profile "$PROFILE" region "$SPX_REGION"
|
|
|
|
|
aws configure set --profile "$PROFILE" endpoint_url "https://$IP:9999"
|
|
|
|
|
aws configure set --profile "$PROFILE" ca_bundle "$CA_FILE"
|
|
|
|
|
# The certificate must name the node's address for this to pass; that is not verified on latitude.sh before this script.
|
|
|
|
|
aws --profile "$PROFILE" sts get-caller-identity --query Account --output text >/dev/null ||
|
|
|
|
|
die "profile $PROFILE does not work against https://$IP:9999; check the API certificate names $IP"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
print_recovery() {
|
|
|
|
|
printf '\nRecovery if ssh is ever lost (the firewall is on):\n'
|
|
|
|
|
printf ' Serial console: ssh %s@%s -p 2222 (needs an account SSH key and an OS user password: set one now with: %s ssh %s sudo passwd ubuntu)\n' \
|
|
|
|
|
"$SERVER_ID" "$IP" "$0" "$NODE_NAME"
|
|
|
|
|
printf ' Temporary disarm, in this order, from the console:\n'
|
|
|
|
|
printf ' sudo systemctl stop spinifex-daemon\n'
|
|
|
|
|
printf ' systemctl is-active spinifex-daemon (expect: inactive)\n'
|
|
|
|
|
printf ' sudo nft delete table inet spinifex_filter\n'
|
|
|
|
|
printf ' sudo mv /etc/spinifex/firewall/peers.nft /root/peers.nft.disarmed\n'
|
|
|
|
|
printf ' sudo nft list tables (must not list spinifex_filter)\n'
|
|
|
|
|
printf ' Do not reboot or restart Spinifex afterwards until a durable disarm is reviewed.\n'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd_up() {
|
|
|
|
|
preflight
|
|
|
|
|
if ! done_step terraform; then provision; mark_step terraform; fi
|
|
|
|
|
mapfile -t IPS < <(node_field public_ip)
|
|
|
|
|
mapfile -t IDS < <(node_field provider_host_id)
|
|
|
|
|
[ "${#IPS[@]}" = "$NODES" ] || die "the state holds ${#IPS[@]} node(s), not $NODES; destroy it or pass --nodes ${#IPS[@]}"
|
|
|
|
|
local eip
|
|
|
|
|
eip="$("$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -raw elastic_ip)"
|
|
|
|
|
[[ "$eip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || die "Terraform reported no usable Elastic IP address ('$eip')"
|
|
|
|
|
printf '%s\n' "$eip" > "$STATE_DIR/eip"
|
|
|
|
|
for_nodes host wait_host
|
|
|
|
|
# The console is confirmed on every up that still has a risky step to run, not once per deployment.
|
|
|
|
|
for_nodes_always confirm_if_pending
|
|
|
|
|
for_nodes prep host_prep
|
|
|
|
|
for_nodes disk data_disk
|
|
|
|
|
for_nodes install phase_install
|
|
|
|
|
for_nodes ovn phase_ovn
|
|
|
|
|
for_nodes firewall arm_firewall
|
|
|
|
|
for_nodes start phase_start
|
|
|
|
|
# Never skipped: the claim of success is always checked live by this run.
|
|
|
|
|
for_nodes_always phase_verify
|
|
|
|
|
for_nodes_always check_access
|
|
|
|
|
IP="${IPS[0]}" SERVER_ID="${IDS[0]}" NODE_NAME=node1
|
|
|
|
|
write_profile
|
|
|
|
|
log "ready: https://$IP:9999 (profile $PROFILE)"
|
|
|
|
|
printf 'endpoint=https://%s:9999\nprofile=%s\nca=%s\nconsole=https://%s:3000/\nelastic_ip=%s\n' "$IP" "$PROFILE" "$CA_FILE" "$IP" "$eip"
|
|
|
|
|
local i
|
|
|
|
|
for i in "${!IPS[@]}"; do printf 'node%d=%s\n' "$((i + 1))" "${IPS[$i]}"; done
|
|
|
|
|
print_recovery
|
|
|
|
|
printf '\nVerified by this run: the install phases, the firewall policy loaded, the ssh login and API reachable from this host, and a working AWS profile. Not shown: a guest launch and public reachability of the Elastic IP; test those.\n'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# ── destroy ──────────────────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
# inventory prints "owned kind id label" for every object recorded in $OWNED that still exists, and
|
|
|
|
|
# "untracked kind id label" for every object whose name matches this deployment but is not recorded.
|
|
|
|
|
# With UNCERTAIN present it also prints "unattributed kind id label" for everything else in the account.
|
|
|
|
|
# It reads full lists, fails on any API or schema error, and never reads a failure as empty.
|
|
|
|
|
inventory() {
|
|
|
|
|
local d eip="" rc=0 uncertain=0
|
|
|
|
|
d="$(mktemp -d "$STATE_DIR/inv.XXXXXX")"
|
|
|
|
|
if [ -f "$STATE_DIR/eip" ]; then eip="$(cat "$STATE_DIR/eip")"; fi
|
|
|
|
|
if [ -f "$UNCERTAIN" ] || [ -f "$PENDING" ]; then uncertain=1; fi
|
|
|
|
|
{
|
|
|
|
|
lat_list /servers > "$d/server" &&
|
|
|
|
|
lat_list /elastic_ips > "$d/elastic-ip" &&
|
|
|
|
|
lat_list /ssh_keys > "$d/ssh-key" &&
|
|
|
|
|
lat_list /user_data > "$d/user-data" &&
|
|
|
|
|
lat_list /projects > "$d/project"
|
|
|
|
|
} || rc=1
|
|
|
|
|
if [ "$rc" = 0 ]; then
|
|
|
|
|
touch "$OWNED"
|
|
|
|
|
python3 -I -c '
|
|
|
|
|
import json,sys,os
|
|
|
|
|
d, owned_path, prefix, name, eip, own_project, uncertain = sys.argv[1:8]
|
|
|
|
|
owned = set()
|
|
|
|
|
for line in open(owned_path):
|
|
|
|
|
f = line.split()
|
|
|
|
|
if len(f) >= 2: owned.add((f[0], f[1]))
|
|
|
|
|
seen = set()
|
|
|
|
|
def label(a):
|
|
|
|
|
return a.get("hostname") or a.get("address") or a.get("name") or a.get("description") or "-"
|
|
|
|
|
for kind in ("server", "elastic-ip", "ssh-key", "user-data", "project"):
|
|
|
|
|
for it in json.load(open(os.path.join(d, kind))):
|
|
|
|
|
a = it["attributes"]
|
|
|
|
|
if (kind, it["id"]) in owned:
|
|
|
|
|
seen.add((kind, it["id"]))
|
|
|
|
|
print("owned", kind, it["id"], label(a)); continue
|
|
|
|
|
match = False
|
|
|
|
|
if kind == "server": match = str(a.get("hostname", "")).startswith(prefix)
|
|
|
|
|
elif kind == "elastic-ip":
|
|
|
|
|
host = ((a.get("server") or {}).get("hostname")) or ""
|
|
|
|
|
match = bool(eip and a.get("address") == eip) or host.startswith(prefix)
|
|
|
|
|
elif kind == "ssh-key": match = a.get("name") == "spinifex-" + name
|
|
|
|
|
elif kind == "project": match = own_project == "1" and a.get("name") == "spinifex-" + name
|
|
|
|
|
elif kind == "user-data": match = a.get("description") == "spinifex-" + name
|
|
|
|
|
if match: print("untracked", kind, it["id"], label(a))
|
|
|
|
|
elif uncertain == "1": print("unattributed", kind, it["id"], label(a))
|
|
|
|
|
for line in open(owned_path):
|
|
|
|
|
f = line.split()
|
|
|
|
|
if len(f) >= 2 and (f[0], f[1]) not in seen: print("absent", f[0], f[1], " ".join(f[2:]) or "-")
|
|
|
|
|
' "$d" "$OWNED" "$HOST_PREFIX" "$NAME" "$eip" "$([ -z "$PROJECT" ] && echo 1 || echo 0)" "$uncertain" || rc=1
|
|
|
|
|
fi
|
|
|
|
|
find "$d" -type f -delete
|
|
|
|
|
rmdir "$d"
|
|
|
|
|
return "$rc"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# state_sum identifies the Terraform state that destroy will act on.
|
|
|
|
|
state_sum() { if [ -f "$TF_STATE" ]; then sha256sum < "$TF_STATE"; else echo none; fi; }
|
|
|
|
|
|
|
|
|
|
# owned_sum identifies the recorded set, so a change after the confirmation is noticed.
|
|
|
|
|
owned_sum() { if [ -f "$OWNED" ]; then sha256sum < "$OWNED"; else echo none; fi; }
|
|
|
|
|
|
|
|
|
|
cmd_destroy() {
|
|
|
|
|
local inv set before ssum kind id label i answer failed=0
|
|
|
|
|
# The deletion set is the recorded IDs plus the IDs in the Terraform state, built once and shown.
|
|
|
|
|
record_owned
|
|
|
|
|
ssum="$(state_sum)"
|
|
|
|
|
set="$(inventory)" || die "could not list the account; nothing was deleted and nothing is proven"
|
|
|
|
|
before="$(owned_sum)"
|
|
|
|
|
if [ "$YES" != 1 ]; then
|
|
|
|
|
log "deployment '$NAME': every recorded object will be deleted ('owned' exists, 'absent' is recorded but not in the listing, and terraform destroy acts on it too). 'untracked' and 'unattributed' ones will NOT be:"
|
|
|
|
|
printf '%s\n' "$set" >&2
|
|
|
|
|
read -r -p "Type the deployment name to confirm: " answer < /dev/tty
|
|
|
|
|
[ "$answer" = "$NAME" ] || die "not confirmed"
|
|
|
|
|
fi
|
|
|
|
|
record_owned
|
|
|
|
|
[ "$(state_sum)" = "$ssum" ] || die "the Terraform state changed after the preview; run destroy again to confirm what it holds now"
|
|
|
|
|
[ "$(owned_sum)" = "$before" ] || die "the set to delete changed after the preview; run destroy again to confirm the new set"
|
|
|
|
|
if [ -f "$TF_STATE" ]; then
|
|
|
|
|
log "terraform destroy"
|
|
|
|
|
"$TF_BIN" -chdir="$SCRIPT_DIR" init -input=false >/dev/null
|
|
|
|
|
tf destroy -input=false -auto-approve >"$STATE_DIR/destroy.log" 2>&1 || {
|
|
|
|
|
tail -20 "$STATE_DIR/destroy.log" >&2
|
|
|
|
|
die "terraform destroy failed; log in $STATE_DIR/destroy.log"
|
|
|
|
|
}
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# What Terraform did not remove: delete only members of the shown set, in dependency order. The paths
|
|
|
|
|
# are from the API reference (servers, elastic_ips, ssh_keys, user_data, projects), untested live.
|
|
|
|
|
for i in 1 2 3 4 5 6; do
|
|
|
|
|
inv="$(inventory)" || die "could not list what remains; treat everything as still billing and check the dashboard"
|
|
|
|
|
grep -q '^owned ' <<<"$inv" || break
|
|
|
|
|
for kind in server elastic-ip ssh-key user-data project; do
|
|
|
|
|
while read -r _ k id label; do
|
|
|
|
|
[ "$k" = "$kind" ] || continue
|
|
|
|
|
grep -q "^owned $kind $id " <<<"$set" || die "$kind $id is recorded but was not in the confirmed set; run destroy again"
|
|
|
|
|
log "deleting $kind $label ($id)"
|
|
|
|
|
case "$kind" in
|
|
|
|
|
server) latitude DELETE "/servers/$id" >/dev/null || failed=1 ;;
|
|
|
|
|
elastic-ip) latitude DELETE "/elastic_ips/$id" >/dev/null || failed=1 ;;
|
|
|
|
|
ssh-key) latitude DELETE "/ssh_keys/$id" >/dev/null || failed=1 ;;
|
|
|
|
|
user-data) latitude DELETE "/user_data/$id" >/dev/null || failed=1 ;;
|
|
|
|
|
project) latitude DELETE "/projects/$id" >/dev/null || failed=1 ;;
|
|
|
|
|
esac
|
|
|
|
|
done < <(grep '^owned ' <<<"$inv")
|
|
|
|
|
done
|
|
|
|
|
sleep 15
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
inv="$(inventory)" || die "could not list what remains; treat everything as still billing and check the dashboard"
|
|
|
|
|
if grep -q '^owned ' <<<"$inv"; then
|
|
|
|
|
log "these recorded objects still exist and may be billing:"
|
|
|
|
|
grep '^owned ' <<<"$inv" >&2
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
[ "$failed" = 0 ] || die "a DELETE was refused (see above) although the objects are now absent; check the dashboard, then run destroy again"
|
|
|
|
|
if grep -q '^untracked ' <<<"$inv"; then
|
|
|
|
|
log "recorded objects are gone, but these name-matched objects are not recorded for this deployment. NOT deleted; check each in the dashboard and remove it by hand if it is yours:"
|
|
|
|
|
grep '^untracked ' <<<"$inv" >&2
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
if [ -f "$UNCERTAIN" ] || [ -f "$PENDING" ]; then
|
|
|
|
|
log "an earlier create failed or was interrupted before its IDs were recorded ($UNCERTAIN), so an Elastic IP or server may exist that this script cannot attribute. Everything it cannot attribute:"
|
|
|
|
|
grep '^unattributed ' <<<"$inv" >&2 || true
|
|
|
|
|
log "check the latitude.sh dashboard; delete anything of this deployment by hand, then run: $0 resolve-uncertain --name $NAME. No all-clear is given."
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
rm -f "$STATE_DIR"/step.* "$STATE_DIR"/eip "$STATE_DIR"/data_uuid.* "$INPUTS" "$OWNED" "$TF_STATE" "$TF_STATE.backup"
|
|
|
|
|
log "deployment '$NAME' destroyed: every recorded object is absent from complete account listings and none is name-matched. Objects never recorded and named differently are not covered by this check"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd_status() {
|
|
|
|
|
local nodes inv
|
|
|
|
|
if [ -f "$TF_STATE" ] && nodes="$("$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -json nodes 2>/dev/null)"; then
|
|
|
|
|
python3 -I -c 'import json,sys; [print(n["name"], n["public_ip"]) for n in json.load(sys.stdin)]' <<<"$nodes"
|
|
|
|
|
fi
|
|
|
|
|
if inv="$(inventory)"; then
|
|
|
|
|
printf '%s\n' "$inv"
|
|
|
|
|
else
|
|
|
|
|
echo "UNKNOWN: the account could not be listed"
|
|
|
|
|
fi
|
|
|
|
|
find "$STATE_DIR" -maxdepth 1 -name 'step.*' | sort | sed 's/.*step\./done: /'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd_resolve_uncertain() {
|
|
|
|
|
local answer
|
|
|
|
|
[ -f "$UNCERTAIN" ] || [ -f "$PENDING" ] || { log "nothing uncertain is recorded for '$NAME'"; return 0; }
|
|
|
|
|
log "recorded uncertain creates:"
|
|
|
|
|
cat "$UNCERTAIN" "$PENDING" 2>/dev/null >&2 || true
|
|
|
|
|
[ -r /dev/tty ] || die "no terminal for the confirmation"
|
|
|
|
|
read -r -p 'Type "checked" once you have looked in the latitude.sh dashboard (servers, Elastic IPs, keys, user data, projects) and removed anything of this deployment that was not recorded: ' answer < /dev/tty
|
|
|
|
|
[ "$answer" = checked ] || die "not confirmed"
|
|
|
|
|
rm -f "$UNCERTAIN" "$PENDING"
|
|
|
|
|
log "uncertainty cleared for '$NAME'"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# node_ip NODE prints the public address of nodeN from the Terraform state.
|
|
|
|
|
node_ip() {
|
|
|
|
|
[ -f "$TF_STATE" ] || die "no deployment '$NAME'; run up first"
|
|
|
|
|
local ip
|
|
|
|
|
ip="$(node_field public_ip | sed -n "${1#node}p")"
|
|
|
|
|
[ -n "$ip" ] || die "deployment '$NAME' has no $1"
|
|
|
|
|
printf '%s' "$ip"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd_credentials() {
|
|
|
|
|
IP="$(node_ip node1)"
|
|
|
|
|
on 'awk "/^\[spinifex\]/{f=1} /^\[/&&!/^\[spinifex\]/{f=0} f" ~/.aws/credentials'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd_ssh() {
|
|
|
|
|
IP="$(node_ip "${NODE_ARG:-node1}")"
|
|
|
|
|
# shellcheck disable=SC2029
|
|
|
|
|
ssh "${SSH_OPTS[@]}" "ubuntu@$IP" "$@"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case "$CMD" in up | destroy | resolve-uncertain) acquire_lock ;; esac
|
|
|
|
|
case "$CMD" in
|
|
|
|
|
up) cmd_up ;;
|
|
|
|
|
destroy) cmd_destroy ;;
|
|
|
|
|
status) cmd_status ;;
|
|
|
|
|
credentials) cmd_credentials ;;
|
|
|
|
|
resolve-uncertain) cmd_resolve_uncertain ;;
|
|
|
|
|
ssh) cmd_ssh "$@" ;;
|
|
|
|
|
*) usage >&2; exit 2 ;;
|
|
|
|
|
esac
|
joshsiv-mulga