mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
feat(eks): eks-server + eks-agent AMI manifests (#273)
* feat(eks): eks-server + eks-agent AMI manifests + builder INSTALL_FILES
Manifest-driven AMI build for the two EKS image variants (eks-v1.md Q12).
Lifted forward from cs-eks-6b so the K3s boot path can be smoke-tested in
isolation before CreateCluster orchestration lands.
Builder:
- build-system-image.sh gains optional INSTALL_FILES (src:dst pairs,
mode 0644, parent dirs auto-created). Symmetric with INSTALL_BINARIES.
- ENABLE_SERVICES moves after INSTALL_FILES + SETUP_SCRIPT so init scripts
the manifest references are on disk before rc-update enables them.
eks-server image (Alpine 3.21.7, q35, 4G):
- K3s v1.32.5+k3s1 (downloaded at bake time, SHA256 verified against the
upstream sha256sum-amd64.txt trust anchor).
- nats-cli v0.4.0 (not in Alpine main/community — pulled from upstream
release zip; unzip installed + purged in same setup step).
- OpenRC init scripts: k3s, eks-token-webhook, k3s-first-boot.
- Cron periodics under /etc/periodic/{15min,daily}/ for state-report +
etcd-snapshot.
- k3s-first-boot publishes node-token + admin kubeconfig over NATS on
first successful K3s healthz, then self-disables (sentinel + rc-update
del).
- config.yaml.skel without webhook-token-auth — wiring it before the real
webhook lands (cs-eks-6c) would block anonymous probes against /healthz.
Orchestrator drops the live config + restarts k3s.
eks-agent image (Alpine 3.21.7, q35, 2G):
- Same pinned K3s release for server/agent lock-step.
- k3s-agent OpenRC service refuses to start without K3S_URL+K3S_TOKEN —
fail-closed pre-orchestration.
eks-token-webhook stub (cmd/eks-token-webhook/main.go):
- 503 + Retry-After:30 on every path. Statically linked. Lets the AMI
build succeed and kube-apiserver boot cleanly. Replaced wholesale by
cs-eks-6c.
Init system divergence from spec wording: Alpine ships OpenRC, not
systemd. Unit files use .initd extensions and cron periodics instead of
.service / .timer. Bead-level intent (auto-enable on boot, periodic
execution) preserved.
Smoke-tested:
- eks-server: builds 300M qcow2; qemu q35 boot reaches K3s apiserver on
6443 within ~120s; HTTPS probes return valid k8s Status JSON (the 401
is K3s v1.32 default RBAC behaviour against anonymous, not a regression).
- eks-agent: builds 280M qcow2; qemu boot mounts root cleanly.
* feat(eks): make targets for eks-server + eks-agent AMI builds
- build-system-image gains subdir-manifest fallback: tries
scripts/images/$(IMAGE).conf first, then scripts/images/$(IMAGE)/manifest.conf
so the new EKS layout works alongside the existing flat lb manifest.
- build-eks-server-image / build-eks-agent-image — explicit convenience
targets that delegate to the generic one.
This commit is contained in:
Josh Siviour
@@ -73,11 +73,24 @@ build-lb-agent:
|
||||
@echo -e "\n....Building lb-agent (static)"
|
||||
CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
|
||||
|
||||
build-system-image: ## Build a system image from manifest (use IMAGE=lb)
|
||||
build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-server)
|
||||
ifndef IMAGE
|
||||
$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
|
||||
endif
|
||||
./scripts/build-system-image.sh scripts/images/$(IMAGE).conf
|
||||
@if [ -f scripts/images/$(IMAGE).conf ]; then \
|
||||
./scripts/build-system-image.sh scripts/images/$(IMAGE).conf; \
|
||||
elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \
|
||||
./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf; \
|
||||
else \
|
||||
echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
build-eks-server-image: ## Build the eks-server AMI (K3s control-plane variant)
|
||||
$(MAKE) build-system-image IMAGE=eks-server
|
||||
|
||||
build-eks-agent-image: ## Build the eks-agent AMI (K3s worker variant)
|
||||
$(MAKE) build-system-image IMAGE=eks-agent
|
||||
|
||||
MICROVM_OUT_DIR := build/microvm
|
||||
MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
@@ -325,7 +338,7 @@ ansible-dev-version:
|
||||
ansible-dev-vpc:
|
||||
cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA)
|
||||
|
||||
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
|
||||
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-server-image build-eks-agent-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck \
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// eks-token-webhook is the K8s TokenReview webhook authenticator baked into
|
||||
// the eks-server AMI. It decodes the SigV4-presigned GetCallerIdentity URL
|
||||
// produced by `aws eks get-token`, enforces the X-K8s-Aws-Id cross-cluster
|
||||
// pin, calls Mulga STS over NATS, and resolves the principal to a TokenReview
|
||||
// response via the cluster's AccessEntry KV.
|
||||
//
|
||||
// This is a placeholder build that returns 503 Service Unavailable on every
|
||||
// path. It exists so the eks-server AMI build succeeds (INSTALL_BINARIES is
|
||||
// satisfied) and the K3s server VM boots cleanly. Real implementation lands
|
||||
// with bead mulga-cs-eks-6c.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
addr := flag.String("addr", "127.0.0.1:8443", "listen address")
|
||||
flag.Parse()
|
||||
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})))
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "30")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
http.Error(w, `{"error":"eks-token-webhook stub: real implementation not yet deployed"}`, http.StatusServiceUnavailable)
|
||||
})
|
||||
|
||||
slog.Info("eks-token-webhook stub starting", "addr", *addr)
|
||||
if err := http.ListenAndServe(*addr, mux); err != nil {
|
||||
slog.Error("listen failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -326,25 +326,6 @@ apt-get install -y --no-install-recommends ${APT_PACKAGES}
|
||||
"
|
||||
fi
|
||||
|
||||
# Enable services
|
||||
if [[ -n "${ENABLE_SERVICES:-}" ]]; then
|
||||
echo "Enabling services: ${ENABLE_SERVICES}..."
|
||||
IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
if [[ "$DISTRO" == "alpine" ]]; then
|
||||
if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
|
||||
echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then
|
||||
echo "ERROR: Failed to enable service '${svc}'"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Step 5: Copy binaries into the image (before setup script, which may reference them)
|
||||
if [[ -n "${INSTALL_BINARIES:-}" ]]; then
|
||||
echo "Installing binaries..."
|
||||
@@ -359,6 +340,27 @@ if [[ -n "${INSTALL_BINARIES:-}" ]]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# Copy auxiliary files (systemd units, OpenRC initd scripts, cron entries, configs).
|
||||
# Same src:dst pair grammar as INSTALL_BINARIES; mode 0644, parent dirs auto-created.
|
||||
# Optional — manifests without INSTALL_FILES (e.g. existing GPU images) skip this step.
|
||||
if [[ -n "${INSTALL_FILES:-}" ]]; then
|
||||
echo "Installing files..."
|
||||
IFS=' ' read -ra FILE_PAIRS <<< "$INSTALL_FILES"
|
||||
for pair in "${FILE_PAIRS[@]}"; do
|
||||
src="${pair%%:*}"
|
||||
dst="${pair#*:}"
|
||||
src_path="${PROJECT_DIR}/${src}"
|
||||
if [[ ! -f "$src_path" ]]; then
|
||||
echo "ERROR: INSTALL_FILES source not found: $src_path"
|
||||
exit 1
|
||||
fi
|
||||
echo " ${src} -> ${dst}"
|
||||
sudo mkdir -p "${MOUNT_DIR}$(dirname "$dst")"
|
||||
sudo cp "$src_path" "${MOUNT_DIR}${dst}"
|
||||
sudo chmod 0644 "${MOUNT_DIR}${dst}"
|
||||
done
|
||||
fi
|
||||
|
||||
# Run custom setup script inside chroot (after binaries are installed)
|
||||
if [[ -n "${SETUP_SCRIPT:-}" ]]; then
|
||||
setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
|
||||
@@ -381,6 +383,26 @@ if [[ -n "${SETUP_SCRIPT:-}" ]]; then
|
||||
sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
|
||||
fi
|
||||
|
||||
# Enable services (runs after INSTALL_FILES + SETUP_SCRIPT so init scripts the
|
||||
# manifest references are already on disk).
|
||||
if [[ -n "${ENABLE_SERVICES:-}" ]]; then
|
||||
echo "Enabling services: ${ENABLE_SERVICES}..."
|
||||
IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
if [[ "$DISTRO" == "alpine" ]]; then
|
||||
if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
|
||||
echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then
|
||||
echo "ERROR: Failed to enable service '${svc}'"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Step 6: Clean up and unmount
|
||||
echo "Cleaning up image..."
|
||||
if [[ "$DISTRO" == "alpine" ]]; then
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/sbin/openrc-run
|
||||
# K3s agent (worker node). Env vars K3S_URL + K3S_TOKEN come from cloud-init
|
||||
# user-data — the orchestrator embeds them at RunInstances time per the
|
||||
# Nodegroup bootstrap flow (eks-v1.md Q14).
|
||||
#
|
||||
# K3S_URL points at the cluster's NLB endpoint (https://{cluster}.{account}
|
||||
# .eks.{region}.{suffix}:443) so the agent join path stays stable across the
|
||||
# v2 HA migration (single-VM v1 → 3-node etcd quorum v2). No AMI rebuild
|
||||
# needed when the control plane expands.
|
||||
|
||||
description="K3s Kubernetes agent (Mulga EKS worker)"
|
||||
command="/usr/local/bin/k3s"
|
||||
command_args="agent"
|
||||
command_background="yes"
|
||||
pidfile="/run/k3s-agent.pid"
|
||||
output_log="/var/log/k3s-agent.log"
|
||||
error_log="/var/log/k3s-agent.log"
|
||||
|
||||
start_stop_daemon_args="--wait 5000"
|
||||
|
||||
depend() {
|
||||
need net cgroups
|
||||
after firewall
|
||||
}
|
||||
|
||||
start_pre() {
|
||||
# Cloud-init drops these into /etc/spinifex-eks/agent.env. We source them
|
||||
# here so OpenRC injects them into k3s' environment.
|
||||
if [ -f /etc/spinifex-eks/agent.env ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/spinifex-eks/agent.env
|
||||
export K3S_URL K3S_TOKEN K3S_NODE_NAME K3S_NODE_LABEL
|
||||
fi
|
||||
if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then
|
||||
eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
IMAGE_NAME="eks-agent"
|
||||
IMAGE_DESCRIPTION="EKS K3s agent VM — Alpine + K3s v1.32.5+k3s1 (worker node)"
|
||||
DISTRO="alpine"
|
||||
ALPINE_VERSION="3.21.7"
|
||||
|
||||
# 2G for the worker — kubelet + containerd + node images. Customer workload
|
||||
# storage attaches via separate EBS volumes (managed by viperblock); this is
|
||||
# the OS root only.
|
||||
IMAGE_SIZE="2G"
|
||||
|
||||
APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools openrc"
|
||||
|
||||
# K3s binary downloaded by setup.sh (same pinned version + SHA path as
|
||||
# eks-server) to keep server / agent in lock-step.
|
||||
|
||||
INSTALL_FILES="\
|
||||
scripts/images/eks-agent/k3s-agent.initd:/etc/init.d/k3s-agent \
|
||||
scripts/images/eks-agent/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report"
|
||||
|
||||
ENABLE_SERVICES="crond k3s-agent"
|
||||
|
||||
SETUP_SCRIPT="scripts/images/eks-agent/setup.sh"
|
||||
|
||||
AMI_NAME="spinifex-eks-agent"
|
||||
SYSTEM_TAG="spinifex:managed-by=eks"
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# mulga-eks-state-report (agent variant) — periodic NATS publish of agent
|
||||
# liveness signal. Subject: eks.state.{accountID}.{clusterName}.agent.{nodeName}
|
||||
# Payload: JSON { kubelet: "ok"|"fail", ts: <unix> }
|
||||
#
|
||||
# The agent has no API server to query for its own status; we shell-check the
|
||||
# local kubelet socket via crictl, and fall back to a process check.
|
||||
|
||||
ENVFILE=/etc/spinifex-eks/state-report.env
|
||||
[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing"; exit 0; }
|
||||
# shellcheck disable=SC1090
|
||||
. "${ENVFILE}"
|
||||
|
||||
: "${SPINIFEX_NATS_URL:?}"
|
||||
: "${EKS_ACCOUNT_ID:?}"
|
||||
: "${EKS_CLUSTER_NAME:?}"
|
||||
: "${EKS_NODE_NAME:?}"
|
||||
|
||||
if pgrep -f 'k3s agent' >/dev/null 2>&1; then
|
||||
KUBELET=ok
|
||||
else
|
||||
KUBELET=fail
|
||||
fi
|
||||
|
||||
TS=$(date +%s)
|
||||
PAYLOAD=$(printf '{"kubelet":"%s","ts":%s}' "${KUBELET}" "${TS}")
|
||||
SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.agent.${EKS_NODE_NAME}"
|
||||
|
||||
NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
|
||||
if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
|
||||
NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \
|
||||
| logger -t mulga-eks-state-report
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# setup.sh — chroot customisation for the eks-agent AMI.
|
||||
# Pulls the same K3s release as eks-server (lock-step versioning per Q12) and
|
||||
# wires the k3s-agent OpenRC service.
|
||||
|
||||
K3S_VERSION="v1.32.5+k3s1"
|
||||
K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}"
|
||||
|
||||
echo "[eks-agent-setup] fetching k3s checksums ${K3S_VERSION}"
|
||||
curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt"
|
||||
K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt)
|
||||
if [ -z "${K3S_SHA256}" ]; then
|
||||
echo "[eks-agent-setup] could not parse k3s sha256 from upstream checksums"
|
||||
cat /tmp/k3s-checksums.txt
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[eks-agent-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})"
|
||||
curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s"
|
||||
echo "${K3S_SHA256} /usr/local/bin/k3s" > /tmp/k3s.sha256
|
||||
if ! sha256sum -c /tmp/k3s.sha256; then
|
||||
echo "[eks-agent-setup] k3s SHA256 mismatch — refusing to bake AMI"
|
||||
exit 1
|
||||
fi
|
||||
chmod 0755 /usr/local/bin/k3s
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
|
||||
|
||||
# nats-cli — see eks-server setup.sh for rationale.
|
||||
NATS_CLI_VERSION="0.4.0"
|
||||
NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip"
|
||||
echo "[eks-agent-setup] downloading nats-cli ${NATS_CLI_VERSION}"
|
||||
curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}"
|
||||
apk add --no-cache unzip
|
||||
unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip
|
||||
install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats
|
||||
rm -rf /tmp/nats-cli /tmp/nats-cli.zip
|
||||
apk del --no-cache unzip
|
||||
|
||||
chmod 0755 /etc/init.d/k3s-agent
|
||||
chmod 0755 /etc/periodic/15min/mulga-eks-state-report
|
||||
|
||||
mkdir -p /etc/spinifex-eks
|
||||
echo "[eks-agent-setup] done"
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/sbin/openrc-run
|
||||
# eks-token-webhook — K8s TokenReview authenticator for IAM-backed kubectl.
|
||||
# Binds 127.0.0.1:8443 only; kube-apiserver is the sole client (loopback path).
|
||||
# Stub binary returns 503 on every request until cs-eks-6c lands.
|
||||
|
||||
description="EKS Token Review webhook (Mulga)"
|
||||
command="/usr/local/bin/eks-token-webhook"
|
||||
command_args="--addr=127.0.0.1:8443"
|
||||
command_background="yes"
|
||||
pidfile="/run/eks-token-webhook.pid"
|
||||
output_log="/var/log/eks-token-webhook.log"
|
||||
error_log="/var/log/eks-token-webhook.log"
|
||||
|
||||
depend() {
|
||||
need net
|
||||
before k3s
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/sbin/openrc-run
|
||||
# k3s-first-boot — one-shot OpenRC service that runs the first-boot publisher
|
||||
# after K3s is up. Self-disables by removing itself from the default runlevel
|
||||
# once the sentinel file is gone.
|
||||
|
||||
description="K3s first-boot publisher (node-token + admin kubeconfig)"
|
||||
command="/usr/local/sbin/k3s-first-boot"
|
||||
command_background="no"
|
||||
pidfile="/run/k3s-first-boot.pid"
|
||||
output_log="/var/log/k3s-first-boot.log"
|
||||
error_log="/var/log/k3s-first-boot.log"
|
||||
|
||||
depend() {
|
||||
need k3s
|
||||
after k3s
|
||||
}
|
||||
|
||||
start_pre() {
|
||||
if [ ! -f /var/lib/spinifex-eks/first-boot.pending ]; then
|
||||
einfo "first-boot already complete; nothing to do"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# k3s-first-boot — runs once after the K3s server reaches a healthy state.
|
||||
# Reads the bootstrap node-token and admin kubeconfig that K3s writes during
|
||||
# `--cluster-init`, rewrites the kubeconfig server address to the cluster's
|
||||
# NLB endpoint (so workers and external kubectl can use it), and publishes
|
||||
# both as one-shot NATS messages for the spinifex cluster reconciler to
|
||||
# consume into KV.
|
||||
#
|
||||
# Required env (from cloud-init user-data /etc/spinifex-eks/first-boot.env):
|
||||
# SPINIFEX_NATS_URL nats://...
|
||||
# SPINIFEX_NATS_CREDS_FILE /etc/spinifex-eks/nats.creds
|
||||
# EKS_ACCOUNT_ID
|
||||
# EKS_CLUSTER_NAME
|
||||
# EKS_NLB_ENDPOINT https://{cluster}.{accountID}.eks.{region}.{suffix}
|
||||
#
|
||||
# Idempotent: a sentinel file at /var/lib/spinifex-eks/first-boot.pending gates
|
||||
# execution. On success the sentinel is removed and the OpenRC service is
|
||||
# pulled from the default runlevel so it does not retry on subsequent boots.
|
||||
|
||||
SENTINEL=/var/lib/spinifex-eks/first-boot.pending
|
||||
ENVFILE=/etc/spinifex-eks/first-boot.env
|
||||
LOGTAG="k3s-first-boot"
|
||||
|
||||
log() { echo "[${LOGTAG}] $*"; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
if [ ! -f "${SENTINEL}" ]; then
|
||||
log "sentinel missing — first boot already complete"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -f "${ENVFILE}" ]; then
|
||||
die "${ENVFILE} not found — cloud-init did not seed first-boot env"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
. "${ENVFILE}"
|
||||
|
||||
for v in SPINIFEX_NATS_URL EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do
|
||||
eval "val=\${$v:-}"
|
||||
[ -n "${val}" ] || die "env ${v} not set"
|
||||
done
|
||||
|
||||
# 1. Wait for K3s /healthz. K3s self-signs initially; --insecure-skip is fine
|
||||
# here, we are on loopback.
|
||||
log "waiting for K3s /healthz on 127.0.0.1:6443"
|
||||
i=0
|
||||
while [ "${i}" -lt 300 ]; do
|
||||
if curl -sk --max-time 2 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then
|
||||
log "K3s healthz ok after ${i}s"
|
||||
break
|
||||
fi
|
||||
i=$((i + 2))
|
||||
sleep 2
|
||||
done
|
||||
[ "${i}" -lt 300 ] || die "K3s did not become healthy within 5 minutes"
|
||||
|
||||
# 2. Read node-token + admin kubeconfig.
|
||||
TOKEN_FILE=/var/lib/rancher/k3s/server/node-token
|
||||
KUBECONFIG_FILE=/etc/rancher/k3s/k3s.yaml
|
||||
[ -r "${TOKEN_FILE}" ] || die "${TOKEN_FILE} unreadable"
|
||||
[ -r "${KUBECONFIG_FILE}" ] || die "${KUBECONFIG_FILE} unreadable"
|
||||
|
||||
NODE_TOKEN=$(cat "${TOKEN_FILE}")
|
||||
# K3s ships kubeconfig with server: https://127.0.0.1:6443 — rewrite to the
|
||||
# NLB endpoint so it works from outside the control plane VM.
|
||||
KUBECONFIG_REWRITTEN=$(sed "s|server: https://127\.0\.0\.1:6443|server: ${EKS_NLB_ENDPOINT}|" "${KUBECONFIG_FILE}")
|
||||
|
||||
# 3. Publish one-shot NATS messages. Subjects per eks-v1.md Q14 + Q15.
|
||||
TOKEN_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-bootstrap-token"
|
||||
KUBE_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-admin-kubeconfig"
|
||||
|
||||
NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
|
||||
if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
|
||||
NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
|
||||
fi
|
||||
|
||||
log "publishing node-token to ${TOKEN_SUBJ}"
|
||||
# shellcheck disable=SC2086
|
||||
printf '%s' "${NODE_TOKEN}" | nats ${NATS_ARGS} pub "${TOKEN_SUBJ}" --stdin
|
||||
|
||||
log "publishing admin kubeconfig to ${KUBE_SUBJ}"
|
||||
# shellcheck disable=SC2086
|
||||
printf '%s' "${KUBECONFIG_REWRITTEN}" | nats ${NATS_ARGS} pub "${KUBE_SUBJ}" --stdin
|
||||
|
||||
# 4. Self-disable. Remove sentinel, pull from runlevel.
|
||||
rm -f "${SENTINEL}"
|
||||
rc-update del k3s-first-boot default 2>/dev/null || true
|
||||
log "first boot complete"
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/sbin/openrc-run
|
||||
# K3s server control plane. v1 = single-node embedded etcd (eks-v1.md Q6).
|
||||
# --cluster-init bootstraps a new etcd quorum on first start; subsequent
|
||||
# starts pick up the existing data dir at /var/lib/rancher/k3s. v2 HA grows
|
||||
# the cluster by launching additional eks-server VMs with --server pointing
|
||||
# at the existing NLB endpoint (Q17) — no AMI change required.
|
||||
|
||||
description="K3s Kubernetes server (Mulga EKS control plane)"
|
||||
command="/usr/local/bin/k3s"
|
||||
command_args="server --config /etc/rancher/k3s/config.yaml"
|
||||
command_background="yes"
|
||||
pidfile="/run/k3s.pid"
|
||||
output_log="/var/log/k3s.log"
|
||||
error_log="/var/log/k3s.log"
|
||||
|
||||
# Generous start timeout — K3s bootstraps containerd, runs etcd cluster-init,
|
||||
# pulls system images. First boot can take 2-3 minutes on cold cache.
|
||||
start_stop_daemon_args="--wait 5000"
|
||||
|
||||
depend() {
|
||||
need net cgroups
|
||||
after firewall
|
||||
}
|
||||
|
||||
start_pre() {
|
||||
# config.yaml is written by k3s-first-boot from the skeleton + cloud-init
|
||||
# data. Refuse to start if neither exists.
|
||||
if [ ! -f /etc/rancher/k3s/config.yaml ]; then
|
||||
if [ -f /etc/rancher/k3s/config.yaml.skel ]; then
|
||||
cp /etc/rancher/k3s/config.yaml.skel /etc/rancher/k3s/config.yaml
|
||||
else
|
||||
eerror "no /etc/rancher/k3s/config.yaml and no .skel to derive from"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
IMAGE_NAME="eks-server"
|
||||
IMAGE_DESCRIPTION="EKS K3s server VM — Alpine + K3s v1.32.5+k3s1 + eks-token-webhook + state-report + etcd-snapshot"
|
||||
DISTRO="alpine"
|
||||
ALPINE_VERSION="3.21.7"
|
||||
|
||||
# 4G fits Alpine + K3s server binary (~80MB) + etcd snapshot working set.
|
||||
# K3s embedded etcd lives on the data volume the orchestrator attaches at
|
||||
# LaunchSystemInstance time; root disk only carries the image + logs.
|
||||
IMAGE_SIZE="4G"
|
||||
|
||||
# Packages required at runtime (K3s pulls cgroups + iptables; nats-cli used by
|
||||
# first-boot publisher + state-report; etcd-client used by snapshot script).
|
||||
APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools etcd-ctl openrc"
|
||||
|
||||
# k3s binary downloaded by setup.sh (network access inside chroot); SHA256
|
||||
# verified at install time per spec Q12.
|
||||
BUILD_COMMANDS="CGO_ENABLED=0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook"
|
||||
|
||||
INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook"
|
||||
|
||||
INSTALL_FILES="\
|
||||
scripts/images/eks-server/k3s.initd:/etc/init.d/k3s \
|
||||
scripts/images/eks-server/eks-token-webhook.initd:/etc/init.d/eks-token-webhook \
|
||||
scripts/images/eks-server/k3s-first-boot.initd:/etc/init.d/k3s-first-boot \
|
||||
scripts/images/eks-server/k3s-first-boot.sh:/usr/local/sbin/k3s-first-boot \
|
||||
scripts/images/eks-server/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report \
|
||||
scripts/images/eks-server/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"
|
||||
|
||||
ENABLE_SERVICES="crond k3s eks-token-webhook k3s-first-boot"
|
||||
|
||||
SETUP_SCRIPT="scripts/images/eks-server/setup.sh"
|
||||
|
||||
AMI_NAME="spinifex-eks-server"
|
||||
SYSTEM_TAG="spinifex:managed-by=eks"
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# mulga-eks-etcd-snapshot — nightly K3s embedded etcd snapshot to predastore.
|
||||
# Run from /etc/periodic/daily (Alpine crond).
|
||||
# K3s has a built-in `k3s etcd-snapshot` command that handles the snapshot
|
||||
# semantics; this wrapper exposes it on cron and uploads to the system-bucket
|
||||
# (see eks-v1.md Q6 + Q17).
|
||||
#
|
||||
# Bucket layout (Q17):
|
||||
# eks-backups-system/{accountID}/{clusterName}/etcd-{timestamp}.snap
|
||||
# Retention: managed via S3 lifecycle policy on the bucket; this script does
|
||||
# not prune.
|
||||
|
||||
ENVFILE=/etc/spinifex-eks/etcd-snapshot.env
|
||||
[ -f "${ENVFILE}" ] || { logger -t mulga-eks-etcd-snapshot "${ENVFILE} missing"; exit 0; }
|
||||
# shellcheck disable=SC1090
|
||||
. "${ENVFILE}"
|
||||
|
||||
: "${EKS_ACCOUNT_ID:?}"
|
||||
: "${EKS_CLUSTER_NAME:?}"
|
||||
: "${SPINIFEX_PREDASTORE_ENDPOINT:?}"
|
||||
: "${SPINIFEX_PREDASTORE_AKID:?}"
|
||||
: "${SPINIFEX_PREDASTORE_SECRET:?}"
|
||||
|
||||
SNAPSHOT_DIR=/var/lib/rancher/k3s/server/db/snapshots
|
||||
TS=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
NAME="etcd-${TS}"
|
||||
BUCKET="eks-backups-system"
|
||||
KEY="${EKS_ACCOUNT_ID}/${EKS_CLUSTER_NAME}/${NAME}.snap"
|
||||
|
||||
/usr/local/bin/k3s etcd-snapshot save --name "${NAME}" 2>&1 | logger -t mulga-eks-etcd-snapshot
|
||||
|
||||
SNAPSHOT_FILE=$(ls -t "${SNAPSHOT_DIR}/${NAME}"* 2>/dev/null | head -1 || true)
|
||||
if [ -z "${SNAPSHOT_FILE}" ]; then
|
||||
logger -t mulga-eks-etcd-snapshot "snapshot file for ${NAME} not found in ${SNAPSHOT_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Upload via curl SigV4. predastore implements the S3 v4 wire so the standard
|
||||
# AWS PUT request signs cleanly. aws-cli is not in the image (kept small) — we
|
||||
# hand-roll the signed request with curl --aws-sigv4.
|
||||
#
|
||||
# Note: --aws-sigv4 requires curl >= 7.75 (Alpine 3.21 ships 8.x).
|
||||
URL="${SPINIFEX_PREDASTORE_ENDPOINT%/}/${BUCKET}/${KEY}"
|
||||
curl -fsSL \
|
||||
--aws-sigv4 "aws:amz:${AWS_REGION:-au-mel-1}:s3" \
|
||||
--user "${SPINIFEX_PREDASTORE_AKID}:${SPINIFEX_PREDASTORE_SECRET}" \
|
||||
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
||||
--upload-file "${SNAPSHOT_FILE}" \
|
||||
"${URL}" 2>&1 | logger -t mulga-eks-etcd-snapshot
|
||||
|
||||
# Remove the local snapshot to bound disk usage; the canonical copy lives in
|
||||
# predastore. Restore path (`spx admin eks restore-snapshot`) pulls from there.
|
||||
rm -f "${SNAPSHOT_FILE}"
|
||||
logger -t mulga-eks-etcd-snapshot "uploaded ${KEY}"
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# mulga-eks-state-report — periodic NATS publish of K3s server health.
|
||||
# Invoked every 15 minutes via /etc/periodic/15min (Alpine crond). One-shot
|
||||
# best-effort; failures log but do not retry — the spinifex reconciler also
|
||||
# does its own NLB healthcheck so this is supplementary signal.
|
||||
#
|
||||
# Subject: eks.state.{accountID}.{clusterName}.server
|
||||
# Payload: JSON { healthz: "ok"|"fail", node_count: N, ts: <unix-seconds> }
|
||||
|
||||
ENVFILE=/etc/spinifex-eks/state-report.env
|
||||
[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing — exiting"; exit 0; }
|
||||
# shellcheck disable=SC1090
|
||||
. "${ENVFILE}"
|
||||
|
||||
: "${SPINIFEX_NATS_URL:?}"
|
||||
: "${EKS_ACCOUNT_ID:?}"
|
||||
: "${EKS_CLUSTER_NAME:?}"
|
||||
|
||||
KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
|
||||
export KUBECONFIG
|
||||
|
||||
if curl -sk --max-time 3 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then
|
||||
HEALTH=ok
|
||||
else
|
||||
HEALTH=fail
|
||||
fi
|
||||
|
||||
if [ "${HEALTH}" = "ok" ]; then
|
||||
NODE_COUNT=$(kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
||||
else
|
||||
NODE_COUNT=0
|
||||
fi
|
||||
|
||||
TS=$(date +%s)
|
||||
PAYLOAD=$(printf '{"healthz":"%s","node_count":%s,"ts":%s}' "${HEALTH}" "${NODE_COUNT}" "${TS}")
|
||||
SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.server"
|
||||
|
||||
NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
|
||||
if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
|
||||
NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \
|
||||
| logger -t mulga-eks-state-report
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# setup.sh — chroot customisation for the eks-server AMI.
|
||||
#
|
||||
# Runs inside an Alpine 3.21 chroot under build-system-image.sh after packages
|
||||
# and binaries are installed. Downloads the pinned K3s binary, verifies its
|
||||
# SHA256, drops it into /usr/local/bin, sets executable bits on init scripts +
|
||||
# cron entries, and writes the K3s server config skeleton.
|
||||
#
|
||||
# Network access inside the chroot is provided by the host's /etc/resolv.conf
|
||||
# (build-system-image.sh copies it in). curl is in APK_PACKAGES.
|
||||
|
||||
K3S_VERSION="v1.32.5+k3s1"
|
||||
K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}"
|
||||
|
||||
# Pull the upstream signed checksums file and pin the amd64 line. A tampered
|
||||
# release replacing both files would still produce a self-consistent download,
|
||||
# so the SHA file URL is the trust anchor — anyone forging a Mulga AMI build
|
||||
# would need to compromise the k3s-io GitHub release artefacts.
|
||||
echo "[eks-server-setup] fetching k3s checksums ${K3S_VERSION}"
|
||||
curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt"
|
||||
K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt)
|
||||
if [ -z "${K3S_SHA256}" ]; then
|
||||
echo "[eks-server-setup] could not parse k3s sha256 from upstream checksums"
|
||||
cat /tmp/k3s-checksums.txt
|
||||
exit 1
|
||||
fi
|
||||
echo "[eks-server-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})"
|
||||
curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s"
|
||||
echo "${K3S_SHA256} /usr/local/bin/k3s" > /tmp/k3s.sha256
|
||||
if ! sha256sum -c /tmp/k3s.sha256; then
|
||||
echo "[eks-server-setup] k3s SHA256 mismatch — refusing to bake AMI"
|
||||
exit 1
|
||||
fi
|
||||
chmod 0755 /usr/local/bin/k3s
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
|
||||
ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
|
||||
|
||||
# nats-cli is not in Alpine main/community — pull the upstream release binary.
|
||||
# Used by k3s-first-boot.sh (one-shot publishes) and mulga-eks-state-report.sh
|
||||
# (periodic publishes).
|
||||
NATS_CLI_VERSION="0.4.0"
|
||||
NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip"
|
||||
echo "[eks-server-setup] downloading nats-cli ${NATS_CLI_VERSION}"
|
||||
curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}"
|
||||
apk add --no-cache unzip
|
||||
unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip
|
||||
install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats
|
||||
rm -rf /tmp/nats-cli /tmp/nats-cli.zip
|
||||
apk del --no-cache unzip
|
||||
|
||||
# Init scripts ship as 0644 from INSTALL_FILES; OpenRC requires 0755.
|
||||
chmod 0755 /etc/init.d/k3s /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot
|
||||
chmod 0755 /usr/local/sbin/k3s-first-boot
|
||||
chmod 0755 /etc/periodic/15min/mulga-eks-state-report
|
||||
chmod 0755 /etc/periodic/daily/mulga-eks-etcd-snapshot
|
||||
|
||||
# K3s server config — empty skeleton; cloud-init / first-boot fills in the
|
||||
# per-cluster fields (cluster-cidr, service-cidr, token-file, etc).
|
||||
# Webhook-token-auth is NOT wired in the skeleton: enabling it before the
|
||||
# eks-token-webhook binary is the real implementation (currently a 503 stub,
|
||||
# replaced by cs-eks-6c) would block /healthz from anonymous callers. The
|
||||
# Sprint 6c orchestrator drops a webhook-config.yaml + restarts k3s once
|
||||
# the binary is the real one.
|
||||
mkdir -p /etc/rancher/k3s
|
||||
cat > /etc/rancher/k3s/config.yaml.skel <<'EOF'
|
||||
# Populated at first boot by cloud-init user-data via k3s-first-boot.sh.
|
||||
# Fields documented at https://docs.k3s.io/installation/configuration.
|
||||
cluster-init: true
|
||||
disable:
|
||||
- traefik
|
||||
- servicelb
|
||||
- local-storage
|
||||
EOF
|
||||
|
||||
# Sentinel file marker — k3s-first-boot self-disables after first success by
|
||||
# checking for this path. Initial state is "not yet run".
|
||||
touch /var/lib/spinifex-eks/first-boot.pending 2>/dev/null || {
|
||||
mkdir -p /var/lib/spinifex-eks
|
||||
touch /var/lib/spinifex-eks/first-boot.pending
|
||||
}
|
||||
|
||||
echo "[eks-server-setup] done"
|
||||
Reference in New Issue
Block a user