diff --git a/Makefile b/Makefile index 4f3b900c8..484acfdd4 100644 --- a/Makefile +++ b/Makefile @@ -73,11 +73,24 @@ build-lb-agent: @echo -e "\n....Building lb-agent (static)" CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go -build-system-image: ## Build a system image from manifest (use IMAGE=lb) +build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-server) ifndef IMAGE $(error IMAGE is required. Usage: make build-system-image IMAGE=lb) endif - ./scripts/build-system-image.sh scripts/images/$(IMAGE).conf + @if [ -f scripts/images/$(IMAGE).conf ]; then \ + ./scripts/build-system-image.sh scripts/images/$(IMAGE).conf; \ + elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \ + ./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf; \ + else \ + echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \ + exit 1; \ + fi + +build-eks-server-image: ## Build the eks-server AMI (K3s control-plane variant) + $(MAKE) build-system-image IMAGE=eks-server + +build-eks-agent-image: ## Build the eks-agent AMI (K3s worker variant) + $(MAKE) build-system-image IMAGE=eks-agent MICROVM_OUT_DIR := build/microvm MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz @@ -325,7 +338,7 @@ ansible-dev-version: ansible-dev-vpc: cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA) -.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \ +.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-server-image build-eks-agent-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \ deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck \ diff --git a/cmd/eks-token-webhook/main.go b/cmd/eks-token-webhook/main.go new file mode 100644 index 000000000..b2b28ba55 --- /dev/null +++ b/cmd/eks-token-webhook/main.go @@ -0,0 +1,38 @@ +// eks-token-webhook is the K8s TokenReview webhook authenticator baked into +// the eks-server AMI. It decodes the SigV4-presigned GetCallerIdentity URL +// produced by `aws eks get-token`, enforces the X-K8s-Aws-Id cross-cluster +// pin, calls Mulga STS over NATS, and resolves the principal to a TokenReview +// response via the cluster's AccessEntry KV. +// +// This is a placeholder build that returns 503 Service Unavailable on every +// path. It exists so the eks-server AMI build succeeds (INSTALL_BINARIES is +// satisfied) and the K3s server VM boots cleanly. Real implementation lands +// with bead mulga-cs-eks-6c. +package main + +import ( + "flag" + "log/slog" + "net/http" + "os" +) + +func main() { + addr := flag.String("addr", "127.0.0.1:8443", "listen address") + flag.Parse() + + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "30") + w.Header().Set("Content-Type", "application/json") + http.Error(w, `{"error":"eks-token-webhook stub: real implementation not yet deployed"}`, http.StatusServiceUnavailable) + }) + + slog.Info("eks-token-webhook stub starting", "addr", *addr) + if err := http.ListenAndServe(*addr, mux); err != nil { + slog.Error("listen failed", "err", err) + os.Exit(1) + } +} diff --git a/scripts/build-system-image.sh b/scripts/build-system-image.sh index 77b96f45e..eaff2b855 100755 --- a/scripts/build-system-image.sh +++ b/scripts/build-system-image.sh @@ -326,25 +326,6 @@ apt-get install -y --no-install-recommends ${APT_PACKAGES} " fi -# Enable services -if [[ -n "${ENABLE_SERVICES:-}" ]]; then - echo "Enabling services: ${ENABLE_SERVICES}..." - IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES" - for svc in "${SERVICES[@]}"; do - if [[ "$DISTRO" == "alpine" ]]; then - if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then - echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?" - exit 1 - fi - else - if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then - echo "ERROR: Failed to enable service '${svc}'" - exit 1 - fi - fi - done -fi - # Step 5: Copy binaries into the image (before setup script, which may reference them) if [[ -n "${INSTALL_BINARIES:-}" ]]; then echo "Installing binaries..." @@ -359,6 +340,27 @@ if [[ -n "${INSTALL_BINARIES:-}" ]]; then done fi +# Copy auxiliary files (systemd units, OpenRC initd scripts, cron entries, configs). +# Same src:dst pair grammar as INSTALL_BINARIES; mode 0644, parent dirs auto-created. +# Optional — manifests without INSTALL_FILES (e.g. existing GPU images) skip this step. +if [[ -n "${INSTALL_FILES:-}" ]]; then + echo "Installing files..." + IFS=' ' read -ra FILE_PAIRS <<< "$INSTALL_FILES" + for pair in "${FILE_PAIRS[@]}"; do + src="${pair%%:*}" + dst="${pair#*:}" + src_path="${PROJECT_DIR}/${src}" + if [[ ! -f "$src_path" ]]; then + echo "ERROR: INSTALL_FILES source not found: $src_path" + exit 1 + fi + echo " ${src} -> ${dst}" + sudo mkdir -p "${MOUNT_DIR}$(dirname "$dst")" + sudo cp "$src_path" "${MOUNT_DIR}${dst}" + sudo chmod 0644 "${MOUNT_DIR}${dst}" + done +fi + # Run custom setup script inside chroot (after binaries are installed) if [[ -n "${SETUP_SCRIPT:-}" ]]; then setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}" @@ -381,6 +383,26 @@ if [[ -n "${SETUP_SCRIPT:-}" ]]; then sudo rm -f "${MOUNT_DIR}/tmp/setup.sh" fi +# Enable services (runs after INSTALL_FILES + SETUP_SCRIPT so init scripts the +# manifest references are already on disk). +if [[ -n "${ENABLE_SERVICES:-}" ]]; then + echo "Enabling services: ${ENABLE_SERVICES}..." + IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES" + for svc in "${SERVICES[@]}"; do + if [[ "$DISTRO" == "alpine" ]]; then + if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then + echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?" + exit 1 + fi + else + if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then + echo "ERROR: Failed to enable service '${svc}'" + exit 1 + fi + fi + done +fi + # Step 6: Clean up and unmount echo "Cleaning up image..." if [[ "$DISTRO" == "alpine" ]]; then diff --git a/scripts/images/eks-agent/k3s-agent.initd b/scripts/images/eks-agent/k3s-agent.initd new file mode 100644 index 000000000..4c7c51e79 --- /dev/null +++ b/scripts/images/eks-agent/k3s-agent.initd @@ -0,0 +1,38 @@ +#!/sbin/openrc-run +# K3s agent (worker node). Env vars K3S_URL + K3S_TOKEN come from cloud-init +# user-data — the orchestrator embeds them at RunInstances time per the +# Nodegroup bootstrap flow (eks-v1.md Q14). +# +# K3S_URL points at the cluster's NLB endpoint (https://{cluster}.{account} +# .eks.{region}.{suffix}:443) so the agent join path stays stable across the +# v2 HA migration (single-VM v1 → 3-node etcd quorum v2). No AMI rebuild +# needed when the control plane expands. + +description="K3s Kubernetes agent (Mulga EKS worker)" +command="/usr/local/bin/k3s" +command_args="agent" +command_background="yes" +pidfile="/run/k3s-agent.pid" +output_log="/var/log/k3s-agent.log" +error_log="/var/log/k3s-agent.log" + +start_stop_daemon_args="--wait 5000" + +depend() { + need net cgroups + after firewall +} + +start_pre() { + # Cloud-init drops these into /etc/spinifex-eks/agent.env. We source them + # here so OpenRC injects them into k3s' environment. + if [ -f /etc/spinifex-eks/agent.env ]; then + # shellcheck disable=SC1091 + . /etc/spinifex-eks/agent.env + export K3S_URL K3S_TOKEN K3S_NODE_NAME K3S_NODE_LABEL + fi + if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then + eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env" + return 1 + fi +} diff --git a/scripts/images/eks-agent/manifest.conf b/scripts/images/eks-agent/manifest.conf new file mode 100644 index 000000000..fc8b5c7bb --- /dev/null +++ b/scripts/images/eks-agent/manifest.conf @@ -0,0 +1,25 @@ +IMAGE_NAME="eks-agent" +IMAGE_DESCRIPTION="EKS K3s agent VM — Alpine + K3s v1.32.5+k3s1 (worker node)" +DISTRO="alpine" +ALPINE_VERSION="3.21.7" + +# 2G for the worker — kubelet + containerd + node images. Customer workload +# storage attaches via separate EBS volumes (managed by viperblock); this is +# the OS root only. +IMAGE_SIZE="2G" + +APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools openrc" + +# K3s binary downloaded by setup.sh (same pinned version + SHA path as +# eks-server) to keep server / agent in lock-step. + +INSTALL_FILES="\ +scripts/images/eks-agent/k3s-agent.initd:/etc/init.d/k3s-agent \ +scripts/images/eks-agent/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report" + +ENABLE_SERVICES="crond k3s-agent" + +SETUP_SCRIPT="scripts/images/eks-agent/setup.sh" + +AMI_NAME="spinifex-eks-agent" +SYSTEM_TAG="spinifex:managed-by=eks" diff --git a/scripts/images/eks-agent/mulga-eks-state-report.sh b/scripts/images/eks-agent/mulga-eks-state-report.sh new file mode 100644 index 000000000..bb382bf38 --- /dev/null +++ b/scripts/images/eks-agent/mulga-eks-state-report.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +# mulga-eks-state-report (agent variant) — periodic NATS publish of agent +# liveness signal. Subject: eks.state.{accountID}.{clusterName}.agent.{nodeName} +# Payload: JSON { kubelet: "ok"|"fail", ts: } +# +# The agent has no API server to query for its own status; we shell-check the +# local kubelet socket via crictl, and fall back to a process check. + +ENVFILE=/etc/spinifex-eks/state-report.env +[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing"; exit 0; } +# shellcheck disable=SC1090 +. "${ENVFILE}" + +: "${SPINIFEX_NATS_URL:?}" +: "${EKS_ACCOUNT_ID:?}" +: "${EKS_CLUSTER_NAME:?}" +: "${EKS_NODE_NAME:?}" + +if pgrep -f 'k3s agent' >/dev/null 2>&1; then + KUBELET=ok +else + KUBELET=fail +fi + +TS=$(date +%s) +PAYLOAD=$(printf '{"kubelet":"%s","ts":%s}' "${KUBELET}" "${TS}") +SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.agent.${EKS_NODE_NAME}" + +NATS_ARGS="-s ${SPINIFEX_NATS_URL}" +if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then + NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}" +fi + +# shellcheck disable=SC2086 +printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \ + | logger -t mulga-eks-state-report diff --git a/scripts/images/eks-agent/setup.sh b/scripts/images/eks-agent/setup.sh new file mode 100644 index 000000000..b1ad1897a --- /dev/null +++ b/scripts/images/eks-agent/setup.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu + +# setup.sh — chroot customisation for the eks-agent AMI. +# Pulls the same K3s release as eks-server (lock-step versioning per Q12) and +# wires the k3s-agent OpenRC service. + +K3S_VERSION="v1.32.5+k3s1" +K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}" + +echo "[eks-agent-setup] fetching k3s checksums ${K3S_VERSION}" +curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt" +K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt) +if [ -z "${K3S_SHA256}" ]; then + echo "[eks-agent-setup] could not parse k3s sha256 from upstream checksums" + cat /tmp/k3s-checksums.txt + exit 1 +fi + +echo "[eks-agent-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})" +curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s" +echo "${K3S_SHA256} /usr/local/bin/k3s" > /tmp/k3s.sha256 +if ! sha256sum -c /tmp/k3s.sha256; then + echo "[eks-agent-setup] k3s SHA256 mismatch — refusing to bake AMI" + exit 1 +fi +chmod 0755 /usr/local/bin/k3s +ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl +ln -sf /usr/local/bin/k3s /usr/local/bin/crictl +ln -sf /usr/local/bin/k3s /usr/local/bin/ctr + +# nats-cli — see eks-server setup.sh for rationale. +NATS_CLI_VERSION="0.4.0" +NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip" +echo "[eks-agent-setup] downloading nats-cli ${NATS_CLI_VERSION}" +curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}" +apk add --no-cache unzip +unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip +install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats +rm -rf /tmp/nats-cli /tmp/nats-cli.zip +apk del --no-cache unzip + +chmod 0755 /etc/init.d/k3s-agent +chmod 0755 /etc/periodic/15min/mulga-eks-state-report + +mkdir -p /etc/spinifex-eks +echo "[eks-agent-setup] done" diff --git a/scripts/images/eks-server/eks-token-webhook.initd b/scripts/images/eks-server/eks-token-webhook.initd new file mode 100644 index 000000000..9f18d2147 --- /dev/null +++ b/scripts/images/eks-server/eks-token-webhook.initd @@ -0,0 +1,17 @@ +#!/sbin/openrc-run +# eks-token-webhook — K8s TokenReview authenticator for IAM-backed kubectl. +# Binds 127.0.0.1:8443 only; kube-apiserver is the sole client (loopback path). +# Stub binary returns 503 on every request until cs-eks-6c lands. + +description="EKS Token Review webhook (Mulga)" +command="/usr/local/bin/eks-token-webhook" +command_args="--addr=127.0.0.1:8443" +command_background="yes" +pidfile="/run/eks-token-webhook.pid" +output_log="/var/log/eks-token-webhook.log" +error_log="/var/log/eks-token-webhook.log" + +depend() { + need net + before k3s +} diff --git a/scripts/images/eks-server/k3s-first-boot.initd b/scripts/images/eks-server/k3s-first-boot.initd new file mode 100644 index 000000000..ab7029a06 --- /dev/null +++ b/scripts/images/eks-server/k3s-first-boot.initd @@ -0,0 +1,23 @@ +#!/sbin/openrc-run +# k3s-first-boot — one-shot OpenRC service that runs the first-boot publisher +# after K3s is up. Self-disables by removing itself from the default runlevel +# once the sentinel file is gone. + +description="K3s first-boot publisher (node-token + admin kubeconfig)" +command="/usr/local/sbin/k3s-first-boot" +command_background="no" +pidfile="/run/k3s-first-boot.pid" +output_log="/var/log/k3s-first-boot.log" +error_log="/var/log/k3s-first-boot.log" + +depend() { + need k3s + after k3s +} + +start_pre() { + if [ ! -f /var/lib/spinifex-eks/first-boot.pending ]; then + einfo "first-boot already complete; nothing to do" + return 0 + fi +} diff --git a/scripts/images/eks-server/k3s-first-boot.sh b/scripts/images/eks-server/k3s-first-boot.sh new file mode 100644 index 000000000..bd871fefc --- /dev/null +++ b/scripts/images/eks-server/k3s-first-boot.sh @@ -0,0 +1,91 @@ +#!/bin/sh +set -eu + +# k3s-first-boot — runs once after the K3s server reaches a healthy state. +# Reads the bootstrap node-token and admin kubeconfig that K3s writes during +# `--cluster-init`, rewrites the kubeconfig server address to the cluster's +# NLB endpoint (so workers and external kubectl can use it), and publishes +# both as one-shot NATS messages for the spinifex cluster reconciler to +# consume into KV. +# +# Required env (from cloud-init user-data /etc/spinifex-eks/first-boot.env): +# SPINIFEX_NATS_URL nats://... +# SPINIFEX_NATS_CREDS_FILE /etc/spinifex-eks/nats.creds +# EKS_ACCOUNT_ID +# EKS_CLUSTER_NAME +# EKS_NLB_ENDPOINT https://{cluster}.{accountID}.eks.{region}.{suffix} +# +# Idempotent: a sentinel file at /var/lib/spinifex-eks/first-boot.pending gates +# execution. On success the sentinel is removed and the OpenRC service is +# pulled from the default runlevel so it does not retry on subsequent boots. + +SENTINEL=/var/lib/spinifex-eks/first-boot.pending +ENVFILE=/etc/spinifex-eks/first-boot.env +LOGTAG="k3s-first-boot" + +log() { echo "[${LOGTAG}] $*"; } +die() { log "ERROR: $*"; exit 1; } + +if [ ! -f "${SENTINEL}" ]; then + log "sentinel missing — first boot already complete" + exit 0 +fi + +if [ ! -f "${ENVFILE}" ]; then + die "${ENVFILE} not found — cloud-init did not seed first-boot env" +fi + +# shellcheck disable=SC1090 +. "${ENVFILE}" + +for v in SPINIFEX_NATS_URL EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do + eval "val=\${$v:-}" + [ -n "${val}" ] || die "env ${v} not set" +done + +# 1. Wait for K3s /healthz. K3s self-signs initially; --insecure-skip is fine +# here, we are on loopback. +log "waiting for K3s /healthz on 127.0.0.1:6443" +i=0 +while [ "${i}" -lt 300 ]; do + if curl -sk --max-time 2 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then + log "K3s healthz ok after ${i}s" + break + fi + i=$((i + 2)) + sleep 2 +done +[ "${i}" -lt 300 ] || die "K3s did not become healthy within 5 minutes" + +# 2. Read node-token + admin kubeconfig. +TOKEN_FILE=/var/lib/rancher/k3s/server/node-token +KUBECONFIG_FILE=/etc/rancher/k3s/k3s.yaml +[ -r "${TOKEN_FILE}" ] || die "${TOKEN_FILE} unreadable" +[ -r "${KUBECONFIG_FILE}" ] || die "${KUBECONFIG_FILE} unreadable" + +NODE_TOKEN=$(cat "${TOKEN_FILE}") +# K3s ships kubeconfig with server: https://127.0.0.1:6443 — rewrite to the +# NLB endpoint so it works from outside the control plane VM. +KUBECONFIG_REWRITTEN=$(sed "s|server: https://127\.0\.0\.1:6443|server: ${EKS_NLB_ENDPOINT}|" "${KUBECONFIG_FILE}") + +# 3. Publish one-shot NATS messages. Subjects per eks-v1.md Q14 + Q15. +TOKEN_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-bootstrap-token" +KUBE_SUBJ="eks.bus.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.k3s-admin-kubeconfig" + +NATS_ARGS="-s ${SPINIFEX_NATS_URL}" +if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then + NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}" +fi + +log "publishing node-token to ${TOKEN_SUBJ}" +# shellcheck disable=SC2086 +printf '%s' "${NODE_TOKEN}" | nats ${NATS_ARGS} pub "${TOKEN_SUBJ}" --stdin + +log "publishing admin kubeconfig to ${KUBE_SUBJ}" +# shellcheck disable=SC2086 +printf '%s' "${KUBECONFIG_REWRITTEN}" | nats ${NATS_ARGS} pub "${KUBE_SUBJ}" --stdin + +# 4. Self-disable. Remove sentinel, pull from runlevel. +rm -f "${SENTINEL}" +rc-update del k3s-first-boot default 2>/dev/null || true +log "first boot complete" diff --git a/scripts/images/eks-server/k3s.initd b/scripts/images/eks-server/k3s.initd new file mode 100644 index 000000000..ca69f2269 --- /dev/null +++ b/scripts/images/eks-server/k3s.initd @@ -0,0 +1,36 @@ +#!/sbin/openrc-run +# K3s server control plane. v1 = single-node embedded etcd (eks-v1.md Q6). +# --cluster-init bootstraps a new etcd quorum on first start; subsequent +# starts pick up the existing data dir at /var/lib/rancher/k3s. v2 HA grows +# the cluster by launching additional eks-server VMs with --server pointing +# at the existing NLB endpoint (Q17) — no AMI change required. + +description="K3s Kubernetes server (Mulga EKS control plane)" +command="/usr/local/bin/k3s" +command_args="server --config /etc/rancher/k3s/config.yaml" +command_background="yes" +pidfile="/run/k3s.pid" +output_log="/var/log/k3s.log" +error_log="/var/log/k3s.log" + +# Generous start timeout — K3s bootstraps containerd, runs etcd cluster-init, +# pulls system images. First boot can take 2-3 minutes on cold cache. +start_stop_daemon_args="--wait 5000" + +depend() { + need net cgroups + after firewall +} + +start_pre() { + # config.yaml is written by k3s-first-boot from the skeleton + cloud-init + # data. Refuse to start if neither exists. + if [ ! -f /etc/rancher/k3s/config.yaml ]; then + if [ -f /etc/rancher/k3s/config.yaml.skel ]; then + cp /etc/rancher/k3s/config.yaml.skel /etc/rancher/k3s/config.yaml + else + eerror "no /etc/rancher/k3s/config.yaml and no .skel to derive from" + return 1 + fi + fi +} diff --git a/scripts/images/eks-server/manifest.conf b/scripts/images/eks-server/manifest.conf new file mode 100644 index 000000000..aca0fab02 --- /dev/null +++ b/scripts/images/eks-server/manifest.conf @@ -0,0 +1,34 @@ +IMAGE_NAME="eks-server" +IMAGE_DESCRIPTION="EKS K3s server VM — Alpine + K3s v1.32.5+k3s1 + eks-token-webhook + state-report + etcd-snapshot" +DISTRO="alpine" +ALPINE_VERSION="3.21.7" + +# 4G fits Alpine + K3s server binary (~80MB) + etcd snapshot working set. +# K3s embedded etcd lives on the data volume the orchestrator attaches at +# LaunchSystemInstance time; root disk only carries the image + logs. +IMAGE_SIZE="4G" + +# Packages required at runtime (K3s pulls cgroups + iptables; nats-cli used by +# first-boot publisher + state-report; etcd-client used by snapshot script). +APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools etcd-ctl openrc" + +# k3s binary downloaded by setup.sh (network access inside chroot); SHA256 +# verified at install time per spec Q12. +BUILD_COMMANDS="CGO_ENABLED=0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook" + +INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook" + +INSTALL_FILES="\ +scripts/images/eks-server/k3s.initd:/etc/init.d/k3s \ +scripts/images/eks-server/eks-token-webhook.initd:/etc/init.d/eks-token-webhook \ +scripts/images/eks-server/k3s-first-boot.initd:/etc/init.d/k3s-first-boot \ +scripts/images/eks-server/k3s-first-boot.sh:/usr/local/sbin/k3s-first-boot \ +scripts/images/eks-server/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report \ +scripts/images/eks-server/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot" + +ENABLE_SERVICES="crond k3s eks-token-webhook k3s-first-boot" + +SETUP_SCRIPT="scripts/images/eks-server/setup.sh" + +AMI_NAME="spinifex-eks-server" +SYSTEM_TAG="spinifex:managed-by=eks" diff --git a/scripts/images/eks-server/mulga-eks-etcd-snapshot.sh b/scripts/images/eks-server/mulga-eks-etcd-snapshot.sh new file mode 100644 index 000000000..3798b258f --- /dev/null +++ b/scripts/images/eks-server/mulga-eks-etcd-snapshot.sh @@ -0,0 +1,56 @@ +#!/bin/sh +set -eu + +# mulga-eks-etcd-snapshot — nightly K3s embedded etcd snapshot to predastore. +# Run from /etc/periodic/daily (Alpine crond). +# K3s has a built-in `k3s etcd-snapshot` command that handles the snapshot +# semantics; this wrapper exposes it on cron and uploads to the system-bucket +# (see eks-v1.md Q6 + Q17). +# +# Bucket layout (Q17): +# eks-backups-system/{accountID}/{clusterName}/etcd-{timestamp}.snap +# Retention: managed via S3 lifecycle policy on the bucket; this script does +# not prune. + +ENVFILE=/etc/spinifex-eks/etcd-snapshot.env +[ -f "${ENVFILE}" ] || { logger -t mulga-eks-etcd-snapshot "${ENVFILE} missing"; exit 0; } +# shellcheck disable=SC1090 +. "${ENVFILE}" + +: "${EKS_ACCOUNT_ID:?}" +: "${EKS_CLUSTER_NAME:?}" +: "${SPINIFEX_PREDASTORE_ENDPOINT:?}" +: "${SPINIFEX_PREDASTORE_AKID:?}" +: "${SPINIFEX_PREDASTORE_SECRET:?}" + +SNAPSHOT_DIR=/var/lib/rancher/k3s/server/db/snapshots +TS=$(date -u +%Y%m%dT%H%M%SZ) +NAME="etcd-${TS}" +BUCKET="eks-backups-system" +KEY="${EKS_ACCOUNT_ID}/${EKS_CLUSTER_NAME}/${NAME}.snap" + +/usr/local/bin/k3s etcd-snapshot save --name "${NAME}" 2>&1 | logger -t mulga-eks-etcd-snapshot + +SNAPSHOT_FILE=$(ls -t "${SNAPSHOT_DIR}/${NAME}"* 2>/dev/null | head -1 || true) +if [ -z "${SNAPSHOT_FILE}" ]; then + logger -t mulga-eks-etcd-snapshot "snapshot file for ${NAME} not found in ${SNAPSHOT_DIR}" + exit 1 +fi + +# Upload via curl SigV4. predastore implements the S3 v4 wire so the standard +# AWS PUT request signs cleanly. aws-cli is not in the image (kept small) — we +# hand-roll the signed request with curl --aws-sigv4. +# +# Note: --aws-sigv4 requires curl >= 7.75 (Alpine 3.21 ships 8.x). +URL="${SPINIFEX_PREDASTORE_ENDPOINT%/}/${BUCKET}/${KEY}" +curl -fsSL \ + --aws-sigv4 "aws:amz:${AWS_REGION:-au-mel-1}:s3" \ + --user "${SPINIFEX_PREDASTORE_AKID}:${SPINIFEX_PREDASTORE_SECRET}" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + --upload-file "${SNAPSHOT_FILE}" \ + "${URL}" 2>&1 | logger -t mulga-eks-etcd-snapshot + +# Remove the local snapshot to bound disk usage; the canonical copy lives in +# predastore. Restore path (`spx admin eks restore-snapshot`) pulls from there. +rm -f "${SNAPSHOT_FILE}" +logger -t mulga-eks-etcd-snapshot "uploaded ${KEY}" diff --git a/scripts/images/eks-server/mulga-eks-state-report.sh b/scripts/images/eks-server/mulga-eks-state-report.sh new file mode 100644 index 000000000..1427e7e43 --- /dev/null +++ b/scripts/images/eks-server/mulga-eks-state-report.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu + +# mulga-eks-state-report — periodic NATS publish of K3s server health. +# Invoked every 15 minutes via /etc/periodic/15min (Alpine crond). One-shot +# best-effort; failures log but do not retry — the spinifex reconciler also +# does its own NLB healthcheck so this is supplementary signal. +# +# Subject: eks.state.{accountID}.{clusterName}.server +# Payload: JSON { healthz: "ok"|"fail", node_count: N, ts: } + +ENVFILE=/etc/spinifex-eks/state-report.env +[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing — exiting"; exit 0; } +# shellcheck disable=SC1090 +. "${ENVFILE}" + +: "${SPINIFEX_NATS_URL:?}" +: "${EKS_ACCOUNT_ID:?}" +: "${EKS_CLUSTER_NAME:?}" + +KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml} +export KUBECONFIG + +if curl -sk --max-time 3 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then + HEALTH=ok +else + HEALTH=fail +fi + +if [ "${HEALTH}" = "ok" ]; then + NODE_COUNT=$(kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ') +else + NODE_COUNT=0 +fi + +TS=$(date +%s) +PAYLOAD=$(printf '{"healthz":"%s","node_count":%s,"ts":%s}' "${HEALTH}" "${NODE_COUNT}" "${TS}") +SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.server" + +NATS_ARGS="-s ${SPINIFEX_NATS_URL}" +if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then + NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}" +fi + +# shellcheck disable=SC2086 +printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \ + | logger -t mulga-eks-state-report diff --git a/scripts/images/eks-server/setup.sh b/scripts/images/eks-server/setup.sh new file mode 100644 index 000000000..5803b0a3b --- /dev/null +++ b/scripts/images/eks-server/setup.sh @@ -0,0 +1,85 @@ +#!/bin/sh +set -eu + +# setup.sh — chroot customisation for the eks-server AMI. +# +# Runs inside an Alpine 3.21 chroot under build-system-image.sh after packages +# and binaries are installed. Downloads the pinned K3s binary, verifies its +# SHA256, drops it into /usr/local/bin, sets executable bits on init scripts + +# cron entries, and writes the K3s server config skeleton. +# +# Network access inside the chroot is provided by the host's /etc/resolv.conf +# (build-system-image.sh copies it in). curl is in APK_PACKAGES. + +K3S_VERSION="v1.32.5+k3s1" +K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}" + +# Pull the upstream signed checksums file and pin the amd64 line. A tampered +# release replacing both files would still produce a self-consistent download, +# so the SHA file URL is the trust anchor — anyone forging a Mulga AMI build +# would need to compromise the k3s-io GitHub release artefacts. +echo "[eks-server-setup] fetching k3s checksums ${K3S_VERSION}" +curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt" +K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt) +if [ -z "${K3S_SHA256}" ]; then + echo "[eks-server-setup] could not parse k3s sha256 from upstream checksums" + cat /tmp/k3s-checksums.txt + exit 1 +fi +echo "[eks-server-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})" +curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s" +echo "${K3S_SHA256} /usr/local/bin/k3s" > /tmp/k3s.sha256 +if ! sha256sum -c /tmp/k3s.sha256; then + echo "[eks-server-setup] k3s SHA256 mismatch — refusing to bake AMI" + exit 1 +fi +chmod 0755 /usr/local/bin/k3s +ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl +ln -sf /usr/local/bin/k3s /usr/local/bin/crictl +ln -sf /usr/local/bin/k3s /usr/local/bin/ctr + +# nats-cli is not in Alpine main/community — pull the upstream release binary. +# Used by k3s-first-boot.sh (one-shot publishes) and mulga-eks-state-report.sh +# (periodic publishes). +NATS_CLI_VERSION="0.4.0" +NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip" +echo "[eks-server-setup] downloading nats-cli ${NATS_CLI_VERSION}" +curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}" +apk add --no-cache unzip +unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip +install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats +rm -rf /tmp/nats-cli /tmp/nats-cli.zip +apk del --no-cache unzip + +# Init scripts ship as 0644 from INSTALL_FILES; OpenRC requires 0755. +chmod 0755 /etc/init.d/k3s /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot +chmod 0755 /usr/local/sbin/k3s-first-boot +chmod 0755 /etc/periodic/15min/mulga-eks-state-report +chmod 0755 /etc/periodic/daily/mulga-eks-etcd-snapshot + +# K3s server config — empty skeleton; cloud-init / first-boot fills in the +# per-cluster fields (cluster-cidr, service-cidr, token-file, etc). +# Webhook-token-auth is NOT wired in the skeleton: enabling it before the +# eks-token-webhook binary is the real implementation (currently a 503 stub, +# replaced by cs-eks-6c) would block /healthz from anonymous callers. The +# Sprint 6c orchestrator drops a webhook-config.yaml + restarts k3s once +# the binary is the real one. +mkdir -p /etc/rancher/k3s +cat > /etc/rancher/k3s/config.yaml.skel <<'EOF' +# Populated at first boot by cloud-init user-data via k3s-first-boot.sh. +# Fields documented at https://docs.k3s.io/installation/configuration. +cluster-init: true +disable: + - traefik + - servicelb + - local-storage +EOF + +# Sentinel file marker — k3s-first-boot self-disables after first success by +# checking for this path. Initial state is "not yet run". +touch /var/lib/spinifex-eks/first-boot.pending 2>/dev/null || { + mkdir -p /var/lib/spinifex-eks + touch /var/lib/spinifex-eks/first-boot.pending +} + +echo "[eks-server-setup] done"